WEB Debug + Trace

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-25 12:59:41 +03:00
parent 0779cd0901
commit e774bc8c9a
48 changed files with 3744 additions and 435 deletions
+15 -2
View File
@@ -30,6 +30,7 @@ use crate::startup::StartupTracker;
use crate::stats::Stats; use crate::stats::Stats;
use crate::transport::UpstreamManager; use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool; use crate::transport::middle_proxy::MePool;
use crate::web::trace::WebTraceStore;
mod config_edit; mod config_edit;
pub(crate) mod config_store; pub(crate) mod config_store;
@@ -47,6 +48,7 @@ mod runtime_stats;
mod runtime_watch; mod runtime_watch;
mod runtime_zero; mod runtime_zero;
mod users; mod users;
mod web_status;
use config_store::{ use config_store::{
current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk, current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk,
@@ -122,6 +124,7 @@ pub(super) struct ApiShared {
pub(super) proxy_shared: Arc<ProxySharedState>, pub(super) proxy_shared: Arc<ProxySharedState>,
pub(super) reload_control: ReloadControl, pub(super) reload_control: ReloadControl,
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>, pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
pub(super) web_trace: Arc<WebTraceStore>,
} }
impl ApiShared { impl ApiShared {
@@ -155,6 +158,7 @@ impl ApiShared {
proxy_shared: runtime.proxy_shared.clone(), proxy_shared: runtime.proxy_shared.clone(),
reload_control: self.reload_control.clone(), reload_control: self.reload_control.clone(),
active_runtime: self.active_runtime.clone(), active_runtime: self.active_runtime.clone(),
web_trace: self.web_trace.clone(),
} }
} }
} }
@@ -243,7 +247,8 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
| "/v1/runtime/tls-fingerprints" | "/v1/runtime/tls-fingerprints"
| "/v1/stats/users/active-ips" | "/v1/stats/users/active-ips"
| "/v1/stats/users/quota" | "/v1/stats/users/quota"
| "/v1/stats/users" => Some(ALLOW_GET), | "/v1/stats/users"
| "/web-status" => Some(ALLOW_GET),
"/v1/system/reload" => Some(ALLOW_POST), "/v1/system/reload" => Some(ALLOW_POST),
"/v1/users" => Some(ALLOW_GET_POST), "/v1/users" => Some(ALLOW_GET_POST),
"/v1/config" => Some(ALLOW_GET_PATCH), "/v1/config" => Some(ALLOW_GET_PATCH),
@@ -279,6 +284,7 @@ pub async fn serve(
reload_control: ReloadControl, reload_control: ReloadControl,
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>, mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>, mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
web_trace: Arc<WebTraceStore>,
) { ) {
let active_runtime = loop { let active_runtime = loop {
if let Some(active_runtime) = active_runtime_rx.borrow().clone() { if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
@@ -312,7 +318,7 @@ pub async fn serve(
} }
}; };
info!("API endpoint: http://{}/v1/*", listen); info!("API endpoint: http://{}/v1/* and /web-status", listen);
let runtime_state = Arc::new(ApiRuntimeState { let runtime_state = Arc::new(ApiRuntimeState {
process_started_at_epoch_secs, process_started_at_epoch_secs,
@@ -344,6 +350,7 @@ pub async fn serve(
proxy_shared, proxy_shared,
reload_control, reload_control,
active_runtime, active_runtime,
web_trace,
}); });
spawn_runtime_watchers( spawn_runtime_watchers(
@@ -492,6 +499,12 @@ async fn handle(
let result: Result<Response<Full<Bytes>>, ApiFailure> = async { let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
match (method.as_str(), normalized_path) { match (method.as_str(), normalized_path) {
("GET", "/web-status") => Ok(web_status::render(
query.as_deref(),
&shared.web_trace,
&cfg.web.debug,
)
.await),
("GET", "/v1/health") => { ("GET", "/v1/health") => {
let revision = current_revision(&shared.config_path).await?; let revision = current_revision(&shared.config_path).await?;
let data = HealthData { let data = HealthData {
+1
View File
@@ -112,6 +112,7 @@ fn reload_routes_expose_only_documented_methods_and_ids() {
Some(ALLOW_GET) Some(ALLOW_GET)
); );
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42)); assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
assert_eq!(allowed_methods_for_path("/web-status"), Some(ALLOW_GET));
assert_eq!( assert_eq!(
reload_status_route_id("/v1/system/reload/not-a-number"), reload_status_route_id("/v1/system/reload/not-a-number"),
None None
+540
View File
@@ -0,0 +1,540 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use base64::Engine as _;
use http_body_util::Full;
use hyper::body::Bytes;
use hyper::header::{self, HeaderValue};
use hyper::{Response, StatusCode};
use tokio::sync::OwnedSemaphorePermit;
use crate::config::WebDebugConfig;
use crate::web::trace::{
StoredTraceRecord, TraceRecord, TraceRecordKind, WebTraceStore,
};
const MAX_PAGE_BYTES: usize = 8 * 1024 * 1024;
const MAX_GROUPS: usize = 1024;
// Query parsing and matching remain independent from bounded HTML rendering.
mod query;
use query::{GroupBy, StatusQuery, client_ip, parse_query, record_matches};
struct GroupSummary {
count: usize,
latest_seq: u64,
}
struct RenderedPage {
html: String,
_permit: OwnedSemaphorePermit,
}
impl AsRef<[u8]> for RenderedPage {
fn as_ref(&self) -> &[u8] {
self.html.as_bytes()
}
}
/// Renders the authenticated server-side WEB debugging table.
pub(super) async fn render(
raw_query: Option<&str>,
store: &Arc<WebTraceStore>,
policy: &WebDebugConfig,
) -> Response<Full<Bytes>> {
store.apply_policy(policy);
let query = match parse_query(raw_query, policy) {
Ok(query) => query,
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
};
let Some(render_permit) = store.try_render_permit() else {
return html_error(
StatusCode::SERVICE_UNAVAILABLE,
"Renderer busy",
"Two WEB status pages are already rendering",
);
};
let now_millis = crate::web::trace::store_epoch_millis();
let since_millis = query
.record
.is_none()
.then(|| now_millis.saturating_sub(query.window_secs.saturating_mul(1000)))
.unwrap_or(0);
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
let status = store.status();
let mut html = String::with_capacity(MAX_PAGE_BYTES);
push_page_start(&mut html);
html.push_str("<h1>WEB status</h1>");
push_filter_form(&mut html, &query);
html.push_str("<section><h2>Store</h2><table><tbody>");
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
summary_row(&mut html, "body capture", body_mode(&status.policy));
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
summary_row(
&mut html,
"records",
&format!("{} / {}", status.records, status.records_capacity),
);
summary_row(
&mut html,
"bytes",
&format!("{} / {}", status.used_bytes, status.bytes_capacity),
);
summary_row(&mut html, "matched", &records.len().to_string());
summary_row(
&mut html,
"contention drops",
&status.contention_drops.to_string(),
);
summary_row(&mut html, "evictions", &status.evictions.to_string());
summary_row(
&mut html,
"byte truncations",
&status.byte_truncations.to_string(),
);
summary_row(
&mut html,
"sequence range",
&format!(
"{} .. {}",
option_u64(status.earliest_seq),
option_u64(status.latest_seq)
),
);
html.push_str("</tbody></table></section>");
if !query.group_by.is_empty() {
push_groups(&mut html, &records, &query.group_by);
}
push_records(&mut html, &records, &query);
html.push_str("</main></body></html>");
truncate_page(&mut html);
retained_html_response(StatusCode::OK, html, render_permit)
}
fn push_page_start(html: &mut String) {
html.push_str("<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><title>WEB status</title><style>body{font:14px system-ui,sans-serif;margin:0;background:#f5f7fa;color:#17202a}main{max-width:1600px;margin:auto;padding:20px}h1,h2{margin:.4em 0}section{background:#fff;border:1px solid #d9e0e7;border-radius:8px;padding:12px;margin:12px 0;overflow:auto}form{display:flex;flex-wrap:wrap;gap:8px;align-items:end}label{display:grid;gap:3px}input,select,button{font:inherit;padding:5px}table{border-collapse:collapse;width:100%}th,td{border:1px solid #d9e0e7;padding:5px;text-align:left;vertical-align:top}th{background:#edf2f7;position:sticky;top:0}code,pre{font:12px ui-monospace,monospace;white-space:pre-wrap;overflow-wrap:anywhere}details{max-width:900px}.muted{color:#657786}.bad{color:#a00}</style></head><body><main>");
}
fn push_filter_form(html: &mut String, query: &StatusQuery) {
html.push_str("<section><h2>Filters</h2><form method=\"get\" action=\"/web-status\">");
input(html, "window_secs", &query.window_secs.to_string());
input(html, "ip", &query.ip.map(|value| value.to_string()).unwrap_or_default());
input(
html,
"session",
&query.session.map(|value| value.to_string()).unwrap_or_default(),
);
input(
html,
"user_agent",
query.user_agent.as_deref().unwrap_or_default(),
);
input(html, "key", query.key.as_deref().unwrap_or_default());
input(html, "limit", &query.limit.to_string());
html.push_str("<label>group_by<select name=\"group_by\" multiple size=\"4\">");
for group in [GroupBy::Ip, GroupBy::Session, GroupBy::UserAgent, GroupBy::Key] {
html.push_str("<option value=\"");
html.push_str(group.as_str());
if query.group_by.contains(&group) {
html.push_str("\" selected>");
} else {
html.push_str("\">");
}
html.push_str(group.as_str());
html.push_str("</option>");
}
html.push_str("</select></label><button type=\"submit\">Observe</button></form></section>");
}
fn input(html: &mut String, name: &str, value: &str) {
html.push_str("<label>");
escape(html, name);
html.push_str("<input name=\"");
escape(html, name);
html.push_str("\" value=\"");
escape(html, value);
html.push_str("\"></label>");
}
fn summary_row(html: &mut String, name: &str, value: &str) {
html.push_str("<tr><th>");
escape(html, name);
html.push_str("</th><td>");
escape(html, value);
html.push_str("</td></tr>");
}
fn push_groups(
html: &mut String,
records: &[Arc<StoredTraceRecord>],
groups: &[GroupBy],
) {
let mut summaries = BTreeMap::<Vec<String>, GroupSummary>::new();
let mut overflow = 0usize;
for stored in records {
let values = groups
.iter()
.map(|group| group_value(&stored.record, *group))
.collect::<Vec<_>>();
if let Some(summary) = summaries.get_mut(&values) {
summary.count += 1;
summary.latest_seq = summary.latest_seq.max(stored.record.seq);
} else if summaries.len() < MAX_GROUPS {
summaries.insert(
values,
GroupSummary {
count: 1,
latest_seq: stored.record.seq,
},
);
} else {
overflow += 1;
}
}
let mut summaries = summaries.into_iter().collect::<Vec<_>>();
summaries.sort_by(|(left_values, left), (right_values, right)| {
right
.count
.cmp(&left.count)
.then_with(|| left_values.cmp(right_values))
});
html.push_str("<section><h2>Groups</h2><table><thead><tr>");
for group in groups {
html.push_str("<th>");
html.push_str(group.as_str());
html.push_str("</th>");
}
html.push_str("<th>records</th><th>latest seq</th></tr></thead><tbody>");
for (values, summary) in summaries {
html.push_str("<tr>");
for value in values {
html.push_str("<td>");
escape(html, &value);
html.push_str("</td>");
}
html.push_str("<td>");
html.push_str(&summary.count.to_string());
html.push_str("</td><td>");
html.push_str(&summary.latest_seq.to_string());
html.push_str("</td></tr>");
if html.len() >= MAX_PAGE_BYTES / 2 {
break;
}
}
if overflow != 0 {
html.push_str("<tr><td colspan=\"6\" class=\"muted\">Additional groups omitted: ");
html.push_str(&overflow.to_string());
html.push_str("</td></tr>");
}
html.push_str("</tbody></table></section>");
}
fn group_value(record: &TraceRecord, group: GroupBy) -> String {
match group {
GroupBy::Ip => client_ip(record).map(|value| value.to_string()),
GroupBy::Session => record.identity.session_id.map(|value| value.to_string()),
GroupBy::UserAgent => record.user_agent.clone(),
GroupBy::Key => record.identity.key_fingerprint.clone(),
}
.unwrap_or_else(|| "-".to_string())
}
fn push_records(html: &mut String, records: &[Arc<StoredTraceRecord>], query: &StatusQuery) {
html.push_str("<section><h2>Records</h2><table><thead><tr><th>seq</th><th>time</th><th>kind</th><th>route/event</th><th>method</th><th>status</th><th>IP</th><th>session</th><th>user / key</th><th>User-Agent</th><th>details</th></tr></thead><tbody>");
let mut shown = 0usize;
for stored in records.iter().take(query.limit) {
if html.len() >= MAX_PAGE_BYTES.saturating_sub(64 * 1024) {
break;
}
push_record(html, &stored.record);
shown += 1;
}
if shown == 0 {
html.push_str("<tr><td colspan=\"11\" class=\"muted\">No matching records</td></tr>");
}
html.push_str("</tbody></table>");
if records.len() > shown && shown != 0 {
let before = records[shown - 1].record.seq;
html.push_str("<p><a href=\"");
escape(html, &pagination_url(query, before));
html.push_str("\">Next page</a></p>");
}
html.push_str("</section>");
}
fn push_record(html: &mut String, record: &TraceRecord) {
html.push_str("<tr><td><a href=\"/web-status?record=");
html.push_str(&record.seq.to_string());
html.push_str("\">");
html.push_str(&record.seq.to_string());
html.push_str("</a></td><td>");
escape(html, &format_time(record.epoch_millis));
let (kind, route, method, status) = match &record.kind {
TraceRecordKind::Http(http) => (
"http",
http.route.as_str(),
http.method.as_str(),
http.status.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string()),
),
TraceRecordKind::Lifecycle(event) => (
"lifecycle",
event.event.as_str(),
"-",
event.reason.unwrap_or("-").to_string(),
),
};
for value in [kind, route, method, status.as_str()] {
html.push_str("</td><td>");
escape(html, value);
}
html.push_str("</td><td>");
escape(
html,
&client_ip(record)
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string()),
);
html.push_str("</td><td>");
escape(html, &option_u64(record.identity.session_id));
html.push_str("</td><td>");
escape(html, record.identity.user.as_deref().unwrap_or("-"));
html.push_str(" / ");
escape(
html,
record.identity.key_fingerprint.as_deref().unwrap_or("-"),
);
html.push_str("</td><td>");
escape(html, record.user_agent.as_deref().unwrap_or("-"));
html.push_str("</td><td><details><summary>request → response</summary>");
match &record.kind {
TraceRecordKind::Http(http) => {
html.push_str("<p><code>");
escape(html, &http.method);
html.push(' ');
escape(html, &http.path);
html.push_str("</code></p>");
push_headers(html, "request headers", &http.request_headers);
push_body(html, "request body", http.request_body.as_ref());
push_headers(html, "response headers", &http.response_headers);
push_body(html, "response body", http.response_body.as_ref());
if let Some(timings) = &http.timings {
html.push_str("<h3>timings</h3><pre>service/head accepted: 0 us\nrequest body: ");
html.push_str(&option_u64(timings.request_body_us));
html.push_str(" us\nresponse ready: ");
html.push_str(&option_u64(timings.response_ready_us));
html.push_str(" us\nresponse body consumed/polled: ");
html.push_str(&option_u64(timings.response_body_us));
html.push_str(" us\n(kernel flush and TCP ACK are not observed)</pre>");
}
if !http.frames.is_empty() {
html.push_str("<h3>frames</h3><table><tr><th>dir</th><th>type</th><th>stream/lane</th><th>payload</th><th>WINDOW</th><th>error</th></tr>");
for frame in &http.frames {
html.push_str("<tr>");
for value in [
frame.direction.as_str().to_string(),
frame.frame_type.unwrap_or("-").to_string(),
frame.stream_id.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
frame.payload_len.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
frame.window_delta.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
frame.parse_error.unwrap_or("-").to_string(),
] {
html.push_str("<td>");
escape(html, &value);
html.push_str("</td>");
}
html.push_str("</tr>");
}
html.push_str("</table>");
}
}
TraceRecordKind::Lifecycle(event) => {
html.push_str("<pre>event: ");
html.push_str(event.event.as_str());
html.push_str("\nstream: ");
html.push_str(&event.stream_id.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()));
html.push_str("\nreason: ");
html.push_str(event.reason.unwrap_or("-"));
html.push_str("</pre>");
}
}
html.push_str("</details></td></tr>");
}
fn push_headers(html: &mut String, title: &str, headers: &[crate::web::trace::TraceHeader]) {
html.push_str("<h3>");
escape(html, title);
html.push_str("</h3><pre>");
for header in headers {
escape(html, &header.name);
html.push_str(": ");
escape(html, header.value.as_deref().unwrap_or("[value omitted]"));
html.push('\n');
}
html.push_str("</pre>");
}
fn push_body(
html: &mut String,
title: &str,
body: Option<&crate::web::trace::TraceBodySnapshot>,
) {
html.push_str("<h3>");
escape(html, title);
html.push_str("</h3>");
let Some(body) = body else {
html.push_str("<p class=\"muted\">capture off</p>");
return;
};
html.push_str("<p>observed=");
html.push_str(&body.observed_bytes.to_string());
html.push_str(" captured=");
html.push_str(&body.captured.len().to_string());
html.push_str(" state=");
html.push_str(body.state.as_str());
html.push_str(" truncated=");
html.push_str(yes_no(body.truncated));
html.push_str("</p><pre>");
let available = MAX_PAGE_BYTES.saturating_sub(html.len()).saturating_sub(4096);
let raw_limit = available.saturating_mul(3) / 4;
let shown = body.captured.len().min(raw_limit);
base64::engine::general_purpose::STANDARD
.encode_string(&body.captured[..shown], html);
if shown < body.captured.len() {
html.push_str("\n[page output truncated]");
}
html.push_str("</pre>");
}
fn pagination_url(query: &StatusQuery, before_seq: u64) -> String {
let mut serializer = url::form_urlencoded::Serializer::new(String::from("/web-status?"));
serializer.append_pair("window_secs", &query.window_secs.to_string());
if let Some(ip) = query.ip {
serializer.append_pair("ip", &ip.to_string());
}
if let Some(session) = query.session {
serializer.append_pair("session", &session.to_string());
}
if let Some(user_agent) = &query.user_agent {
serializer.append_pair("user_agent", user_agent);
}
if let Some(key) = &query.key {
serializer.append_pair("key", key);
}
for group in &query.group_by {
serializer.append_pair("group_by", group.as_str());
}
serializer.append_pair("limit", &query.limit.to_string());
serializer.append_pair("before_seq", &before_seq.to_string());
serializer.finish()
}
fn format_time(epoch_millis: u64) -> String {
chrono::DateTime::from_timestamp_millis(epoch_millis as i64)
.map(|value| value.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
.unwrap_or_else(|| epoch_millis.to_string())
}
fn option_u64(value: Option<u64>) -> String {
value.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string())
}
fn body_mode(policy: &WebDebugConfig) -> &'static str {
match policy.body_capture {
crate::config::WebDebugBodyCapture::Off => "off",
crate::config::WebDebugBodyCapture::Metadata => "metadata",
crate::config::WebDebugBodyCapture::Prefix => "prefix",
crate::config::WebDebugBodyCapture::Full => "full",
}
}
fn yes_no(value: bool) -> &'static str {
if value { "yes" } else { "no" }
}
fn escape(output: &mut String, value: &str) {
for character in value.chars() {
match character {
'&' => output.push_str("&amp;"),
'<' => output.push_str("&lt;"),
'>' => output.push_str("&gt;"),
'"' => output.push_str("&quot;"),
'\'' => output.push_str("&#39;"),
_ => output.push(character),
}
}
}
fn truncate_page(html: &mut String) {
const SUFFIX: &str = "[page output truncated]";
if html.len() <= MAX_PAGE_BYTES {
return;
}
let mut end = MAX_PAGE_BYTES.saturating_sub(SUFFIX.len());
while !html.is_char_boundary(end) {
end -= 1;
}
html.truncate(end);
html.push_str(SUFFIX);
}
fn html_error(status: StatusCode, title: &str, message: &str) -> Response<Full<Bytes>> {
let mut html = String::new();
push_page_start(&mut html);
html.push_str("<h1 class=\"bad\">");
escape(&mut html, title);
html.push_str("</h1><p>");
escape(&mut html, message);
html.push_str("</p></main></body></html>");
html_response(status, html)
}
fn html_response(status: StatusCode, html: String) -> Response<Full<Bytes>> {
html_bytes_response(status, Bytes::from(html))
}
fn retained_html_response(
status: StatusCode,
html: String,
permit: OwnedSemaphorePermit,
) -> Response<Full<Bytes>> {
html_bytes_response(
status,
Bytes::from_owner(RenderedPage {
html,
_permit: permit,
}),
)
}
fn html_bytes_response(status: StatusCode, html: Bytes) -> Response<Full<Bytes>> {
let mut response = Response::new(Full::new(html));
*response.status_mut() = status;
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("text/html; charset=utf-8"),
);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
response.headers_mut().insert(
header::CONTENT_SECURITY_POLICY,
HeaderValue::from_static("default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"),
);
response.headers_mut().insert(
header::REFERRER_POLICY,
HeaderValue::from_static("no-referrer"),
);
response.headers_mut().insert(
header::X_CONTENT_TYPE_OPTIONS,
HeaderValue::from_static("nosniff"),
);
response
.headers_mut()
.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY"));
response
}
#[cfg(test)]
#[path = "web_status/tests.rs"]
mod tests;
+179
View File
@@ -0,0 +1,179 @@
use std::collections::BTreeSet;
use std::net::IpAddr;
use crate::config::WebDebugConfig;
use crate::web::trace::TraceRecord;
const DEFAULT_LIMIT: usize = 200;
const MAX_LIMIT: usize = 1000;
/// Supported status-page grouping dimensions.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum GroupBy {
Ip,
Session,
UserAgent,
Key,
}
impl GroupBy {
fn parse(value: &str) -> Option<Self> {
match value {
"ip" => Some(Self::Ip),
"session" => Some(Self::Session),
"user_agent" => Some(Self::UserAgent),
"key" => Some(Self::Key),
_ => None,
}
}
/// Returns the canonical query and table label.
pub(super) const fn as_str(self) -> &'static str {
match self {
Self::Ip => "ip",
Self::Session => "session",
Self::UserAgent => "user_agent",
Self::Key => "key",
}
}
}
/// Validated bounded status-page filter and pagination state.
pub(super) struct StatusQuery {
pub(super) window_secs: u64,
pub(super) ip: Option<IpAddr>,
pub(super) session: Option<u64>,
pub(super) user_agent: Option<String>,
pub(super) key: Option<String>,
pub(super) group_by: Vec<GroupBy>,
pub(super) limit: usize,
pub(super) before_seq: Option<u64>,
pub(super) record: Option<u64>,
}
/// Parses a strict query without accepting unknown or ambiguous fields.
pub(super) fn parse_query(
raw: Option<&str>,
policy: &WebDebugConfig,
) -> Result<StatusQuery, String> {
let mut query = StatusQuery {
window_secs: policy.default_window_secs,
ip: None,
session: None,
user_agent: None,
key: None,
group_by: Vec::new(),
limit: DEFAULT_LIMIT,
before_seq: None,
record: None,
};
let mut seen = BTreeSet::new();
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
let name = name.as_ref();
let value = value.as_ref();
if name != "group_by" && !seen.insert(name.to_string()) {
return Err(format!("{name} must not repeat"));
}
match name {
"window_secs" => {
query.window_secs = parse_positive_u64(value, "window_secs")?;
}
"ip" => {
let parsed = value
.parse::<IpAddr>()
.map_err(|_| "ip must be a canonical IP address".to_string())?;
if parsed.to_string() != value {
return Err("ip must use canonical formatting".to_string());
}
query.ip = Some(parsed);
}
"session" => query.session = Some(parse_positive_u64(value, "session")?),
"user_agent" => {
if value.is_empty() || value.len() > 512 {
return Err("user_agent must contain 1..512 bytes".to_string());
}
query.user_agent = Some(value.to_string());
}
"key" => {
if value.is_empty() || value.len() > 64 {
return Err("key must contain 1..64 bytes".to_string());
}
query.key = Some(value.to_string());
}
"group_by" => {
let group = GroupBy::parse(value)
.ok_or_else(|| "group_by must be ip, session, user_agent, or key".to_string())?;
if query.group_by.contains(&group) {
return Err("group_by values must not repeat".to_string());
}
query.group_by.push(group);
}
"limit" => {
query.limit = value
.parse::<usize>()
.ok()
.filter(|value| (1..=MAX_LIMIT).contains(value))
.ok_or_else(|| "limit must be within 1..1000".to_string())?;
}
"before_seq" => {
query.before_seq = Some(parse_positive_u64(value, "before_seq")?);
}
"record" => query.record = Some(parse_positive_u64(value, "record")?),
_ => return Err(format!("unknown query field `{name}`")),
}
}
if query.window_secs > policy.max_window_secs {
return Err(format!(
"window_secs must not exceed {}",
policy.max_window_secs
));
}
Ok(query)
}
fn parse_positive_u64(value: &str, field: &str) -> Result<u64, String> {
value
.parse::<u64>()
.ok()
.filter(|value| *value != 0)
.ok_or_else(|| format!("{field} must be a positive integer"))
}
/// Applies the complete filter predicate to one immutable record.
pub(super) fn record_matches(
record: &TraceRecord,
query: &StatusQuery,
since_millis: u64,
) -> bool {
!(record.epoch_millis < since_millis
|| query.before_seq.is_some_and(|before| record.seq >= before)
|| query.record.is_some_and(|seq| record.seq != seq)
|| query.ip.is_some_and(|ip| client_ip(record) != Some(ip))
|| query
.session
.is_some_and(|session| record.identity.session_id != Some(session))
|| query.user_agent.as_ref().is_some_and(|needle| {
record
.user_agent
.as_deref()
.is_none_or(|value| !contains_ascii_case_insensitive(value, needle))
})
|| query.key.as_ref().is_some_and(|key| {
record.identity.user.as_deref() != Some(key)
&& record.identity.key_fingerprint.as_deref() != Some(key)
}))
}
fn contains_ascii_case_insensitive(value: &str, needle: &str) -> bool {
let needle = needle.as_bytes();
needle.is_empty()
|| value
.as_bytes()
.windows(needle.len())
.any(|window| window.eq_ignore_ascii_case(needle))
}
/// Returns the trusted effective address or direct peer fallback.
pub(super) fn client_ip(record: &TraceRecord) -> Option<IpAddr> {
record.effective_ip.or(record.peer_ip)
}
+82
View File
@@ -0,0 +1,82 @@
use http_body_util::BodyExt as _;
use super::*;
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
#[test]
fn query_rejects_noncanonical_ip_and_excessive_window() {
let policy = WebDebugConfig::default();
assert!(parse_query(Some("ip=2001%3A0db8%3A%3A1"), &policy).is_err());
assert!(parse_query(Some("window_secs=3601"), &policy).is_err());
assert!(parse_query(Some("session=1&session=2"), &policy).is_err());
}
#[test]
fn html_escaping_covers_active_markup_characters() {
let mut output = String::new();
escape(&mut output, "<script a='\"'>&");
assert_eq!(output, "&lt;script a=&#39;&quot;&#39;&gt;&amp;");
}
#[tokio::test]
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
let mut policy = WebDebugConfig::default();
policy.enabled = true;
let mut limits = crate::config::WebLimitsConfig::default();
limits.debug_records_capacity = 8;
limits.debug_bytes_global = 16 * 1024;
let store = WebTraceStore::new(policy.clone(), &limits);
store.record_lifecycle(
None,
Some("192.0.2.40".parse().unwrap()),
TraceIdentity {
session_id: Some(42),
user: Some("alice".to_string()),
key_fingerprint: Some("0123456789abcdef".to_string()),
},
TraceLifecycleEvent::SessionCreated,
None,
None,
);
let response = render(
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
&store,
&policy,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
assert!(response.headers().contains_key(header::CONTENT_SECURITY_POLICY));
let body = response.into_body().collect().await.unwrap().to_bytes();
let body = std::str::from_utf8(&body).unwrap();
assert!(body.contains("session_created"));
assert!(body.contains("0123456789abcdef"));
assert!(body.contains("192.0.2.40"));
}
#[tokio::test]
async fn render_permits_remain_owned_by_inflight_response_bodies() {
let policy = WebDebugConfig::default();
let limits = crate::config::WebLimitsConfig::default();
let store = WebTraceStore::new(policy.clone(), &limits);
let first = render(None, &store, &policy).await;
let second = render(None, &store, &policy).await;
let busy = render(None, &store, &policy).await;
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
drop(first);
let admitted = render(None, &store, &policy).await;
assert_eq!(admitted.status(), StatusCode::OK);
drop(second);
drop(admitted);
}
#[test]
fn page_truncation_preserves_utf8_boundary_and_cap() {
let mut html = "я".repeat(MAX_PAGE_BYTES);
truncate_page(&mut html);
assert!(html.len() <= MAX_PAGE_BYTES);
assert!(html.ends_with("[page output truncated]"));
}
+1 -1
View File
@@ -36,7 +36,7 @@ use super::load::{LoadedConfig, ProxyConfig};
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::config::{ use crate::config::{
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel, CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
MeWriterPickMode, MeWriterPickMode, WebDebugConfig, web_debug_fits_limits,
}; };
#[cfg(test)] #[cfg(test)]
use crate::config::{ListenerConfig, SynLimitMode}; use crate::config::{ListenerConfig, SynLimitMode};
+5
View File
@@ -89,6 +89,7 @@ pub struct HotFields {
pub user_max_unique_ips_global_each: usize, pub user_max_unique_ips_global_each: usize,
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode, pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
pub user_max_unique_ips_window_secs: u64, pub user_max_unique_ips_window_secs: u64,
pub web_debug: WebDebugConfig,
} }
impl HotFields { impl HotFields {
@@ -218,6 +219,7 @@ impl HotFields {
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each, user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode, user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs, user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
web_debug: cfg.web.debug.clone(),
} }
} }
} }
@@ -340,6 +342,9 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
let process_limits = cfg.web.limits.clone(); let process_limits = cfg.web.limits.clone();
cfg.web = new.web.clone(); cfg.web = new.web.clone();
cfg.web.limits = process_limits; cfg.web.limits = process_limits;
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
cfg.web.debug = old.web.debug.clone();
}
if cfg.rebuild_runtime_user_auth().is_err() { if cfg.rebuild_runtime_user_auth().is_err() {
cfg.runtime_user_auth = None; cfg.runtime_user_auth = None;
} }
+9
View File
@@ -496,4 +496,13 @@ pub(super) fn log_changes(
new_hot.user_max_unique_ips_window_secs new_hot.user_max_unique_ips_window_secs
); );
} }
if old_hot.web_debug != new_hot.web_debug {
info!(
"config reload: web.debug updated: enabled={} body_capture={:?} window={}..={}s",
new_hot.web_debug.enabled,
new_hot.web_debug.body_capture,
new_hot.web_debug.default_window_secs,
new_hot.web_debug.max_window_secs,
);
}
} }
+36
View File
@@ -85,6 +85,42 @@ fn bind_stale_mode_is_hot() {
); );
} }
#[test]
fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
let old = sample_config();
let mut new = old.clone();
new.web.debug.enabled = true;
new.web.debug.default_window_secs = 60;
new.web.limits.debug_records_capacity += 1;
let applied = overlay_hot_fields(&old, &new);
assert!(applied.web.debug.enabled);
assert_eq!(applied.web.debug.default_window_secs, 60);
assert_eq!(
applied.web.limits.debug_records_capacity,
old.web.limits.debug_records_capacity
);
assert_ne!(
HotFields::from_config(&old),
HotFields::from_config(&applied)
);
}
#[test]
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
let old = sample_config();
let mut new = old.clone();
new.web.limits.max_body_bytes = 4 * 1024 * 1024;
new.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
let applied = overlay_hot_fields(&old, &new);
assert_eq!(applied.web.limits.max_body_bytes, old.web.limits.max_body_bytes);
assert_eq!(
applied.web.debug.body_prefix_bytes,
old.web.debug.body_prefix_bytes
);
}
#[test] #[test]
fn keepalive_is_not_hot() { fn keepalive_is_not_hot() {
let old = sample_config(); let old = sample_config();
+11
View File
@@ -14,6 +14,7 @@ use sha2::{Digest, Sha256};
use super::*; use super::*;
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n"; const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
const WEB_DEBUG_FINGERPRINT_CONTEXT: &[u8] = b"telemt-web-debug-key-fingerprint-v1\0";
const MAX_WEB_STATIC_DEPTH: usize = 64; const MAX_WEB_STATIC_DEPTH: usize = 64;
/// Builds the immutable WEB routing and decoy snapshot for one generation. /// Builds the immutable WEB routing and decoy snapshot for one generation.
@@ -49,6 +50,8 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
client_secret(auth_entry.secret, profile.secret_mode); client_secret(auth_entry.secret, profile.secret_mode);
let capability = let capability =
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?; derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
let key_fingerprint =
debug_key_fingerprint(&client_secret[..client_secret_len]);
if !capabilities.insert(capability) { if !capabilities.insert(capability) {
return Err(ProxyError::Config(format!( return Err(ProxyError::Config(format!(
"WEB vhost `{}` contains profiles with the same client capability", "WEB vhost `{}` contains profiles with the same client capability",
@@ -62,6 +65,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
secret_mode: profile.secret_mode, secret_mode: profile.secret_mode,
carrier: config.web.carrier, carrier: config.web.carrier,
capability, capability,
key_fingerprint,
max_sessions: profile max_sessions: profile
.max_sessions .max_sessions
.unwrap_or(config.web.limits.max_sessions_global), .unwrap_or(config.web.limits.max_sessions_global),
@@ -93,6 +97,13 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
Ok(()) Ok(())
} }
fn debug_key_fingerprint(secret: &[u8]) -> String {
let mut digest = Sha256::new();
digest.update(WEB_DEBUG_FINGERPRINT_CONTEXT);
digest.update(secret);
hex::encode(&digest.finalize()[..8])
}
/// Derives the Telegram Desktop WEB capability for one exact secret and host. /// Derives the Telegram Desktop WEB capability for one exact secret and host.
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> { pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
let mut mac = Hmac::<Sha256>::new_from_slice(secret) let mut mac = Hmac::<Sha256>::new_from_slice(secret)
+16 -1
View File
@@ -259,7 +259,7 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
"web_trusted_proxy_cidrs", "web_trusted_proxy_cidrs",
]; ];
const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "limits", "timeouts", "vhosts"]; const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "debug", "limits", "timeouts", "vhosts"];
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
"max_header_bytes", "max_header_bytes",
@@ -290,6 +290,8 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
"max_static_files", "max_static_files",
"max_static_file_bytes", "max_static_file_bytes",
"max_static_bytes", "max_static_bytes",
"debug_records_capacity",
"debug_bytes_global",
"memory_envelope_bytes", "memory_envelope_bytes",
"new_bootstraps_per_minute", "new_bootstraps_per_minute",
"new_bootstraps_burst", "new_bootstraps_burst",
@@ -299,6 +301,19 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
"new_streams_burst", "new_streams_burst",
]; ];
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
"enabled",
"capture_lifecycle",
"capture_headers",
"capture_timings",
"capture_frames",
"body_capture",
"body_prefix_bytes",
"decoy_body_prefix_bytes",
"default_window_secs",
"max_window_secs",
];
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"header_secs", "header_secs",
"body_secs", "body_secs",
+8
View File
@@ -37,6 +37,7 @@ fn known_config_keys_for_suggestion() -> Vec<&'static str> {
LISTENER_CONFIG_KEYS, LISTENER_CONFIG_KEYS,
WEB_CONFIG_KEYS, WEB_CONFIG_KEYS,
WEB_LIMITS_CONFIG_KEYS, WEB_LIMITS_CONFIG_KEYS,
WEB_DEBUG_CONFIG_KEYS,
WEB_TIMEOUTS_CONFIG_KEYS, WEB_TIMEOUTS_CONFIG_KEYS,
WEB_VHOST_CONFIG_KEYS, WEB_VHOST_CONFIG_KEYS,
WEB_DECOY_CONFIG_KEYS, WEB_DECOY_CONFIG_KEYS,
@@ -241,6 +242,13 @@ pub(super) fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<Unkn
&["web", "limits"], &["web", "limits"],
WEB_LIMITS_CONFIG_KEYS, WEB_LIMITS_CONFIG_KEYS,
); );
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["web", "debug"],
WEB_DEBUG_CONFIG_KEYS,
);
check_known_table( check_known_table(
parsed_toml, parsed_toml,
&mut unknown, &mut unknown,
+9 -32
View File
@@ -2,6 +2,11 @@ use std::collections::HashSet;
use super::*; use super::*;
// Debug capture validation is independent from restart-only storage limits.
mod debug;
// Memory-envelope arithmetic remains isolated from protocol validation.
mod memory;
const WEB_FRAME_HEADER_BYTES: usize = 8; const WEB_FRAME_HEADER_BYTES: usize = 8;
const WEB_QUEUE_ITEM_COST: usize = 256; const WEB_QUEUE_ITEM_COST: usize = 256;
const WEB_CONTROL_EXTRA_ITEMS: usize = 16; const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
@@ -59,6 +64,7 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
} }
validate_limits(&config.web.limits)?; validate_limits(&config.web.limits)?;
debug::validate(&config.web.debug, &config.web.limits)?;
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 { if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2"); return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
} }
@@ -175,6 +181,8 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
("max_static_files", limits.max_static_files), ("max_static_files", limits.max_static_files),
("max_static_file_bytes", limits.max_static_file_bytes), ("max_static_file_bytes", limits.max_static_file_bytes),
("max_static_bytes", limits.max_static_bytes), ("max_static_bytes", limits.max_static_bytes),
("debug_records_capacity", limits.debug_records_capacity),
("debug_bytes_global", limits.debug_bytes_global),
("memory_envelope_bytes", limits.memory_envelope_bytes), ("memory_envelope_bytes", limits.memory_envelope_bytes),
]; ];
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) { if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
@@ -309,38 +317,7 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
"web.limits pending ceilings must preserve one uplink batch and downlink progress", "web.limits pending ceilings must preserve one uplink batch and downlink progress",
); );
} }
let body_reservation = limits memory::validate(limits)?;
.max_body_readers
.checked_mul(limits.max_body_bytes)
.ok_or_else(|| {
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
})?;
if body_reservation > limits.max_body_bytes_global
|| limits.max_body_bytes_global > u32::MAX as usize
{
return config_error(
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
);
}
let http_header_reservation = limits
.max_http_connections
.checked_mul(limits.max_header_bytes)
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
})?;
let reserved = limits
.pending_bytes_global
.checked_add(limits.max_body_bytes_global)
.and_then(|value| value.checked_add(limits.max_static_bytes))
.and_then(|value| value.checked_add(http_header_reservation))
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
if reserved > limits.memory_envelope_bytes
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
{
return config_error(
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
);
}
Ok(()) Ok(())
} }
+31
View File
@@ -0,0 +1,31 @@
use super::*;
const MAX_WEB_TRACE_WINDOW_SECS: u64 = 86_400;
const MIN_WEB_DEBUG_BYTES_GLOBAL: usize = 4096;
/// Validates hot debug policy independently from process-owned storage limits.
pub(super) fn validate(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> Result<()> {
if limits.debug_bytes_global < MIN_WEB_DEBUG_BYTES_GLOBAL {
return config_error("web.limits.debug_bytes_global must be at least 4096 bytes");
}
if policy.default_window_secs == 0
|| policy.max_window_secs == 0
|| policy.default_window_secs > policy.max_window_secs
|| policy.max_window_secs > MAX_WEB_TRACE_WINDOW_SECS
{
return config_error(
"web.debug windows must be non-zero, ordered, and no greater than 86400 seconds",
);
}
if policy.body_prefix_bytes > limits.max_body_bytes {
return config_error("web.debug.body_prefix_bytes must not exceed web.limits.max_body_bytes");
}
if policy.body_prefix_bytes > limits.debug_bytes_global
|| policy.decoy_body_prefix_bytes > limits.debug_bytes_global
{
return config_error(
"web.debug body prefixes must not exceed web.limits.debug_bytes_global",
);
}
Ok(())
}
+69
View File
@@ -0,0 +1,69 @@
use super::*;
const WEB_DEBUG_RENDERERS: usize = 2;
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
/// Validates process-wide body, header, queue, static, and debug reservations.
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
let body_reservation = limits
.max_body_readers
.checked_mul(limits.max_body_bytes)
.ok_or_else(|| {
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
})?;
if body_reservation > limits.max_body_bytes_global
|| limits.max_body_bytes_global > u32::MAX as usize
{
return config_error(
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
);
}
let http_header_reservation = limits
.max_http_connections
.checked_mul(limits.max_header_bytes)
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
})?;
let debug_ring_index = limits
.debug_records_capacity
.checked_mul(std::mem::size_of::<usize>())
.ok_or_else(|| ProxyError::Config("web.limits debug index overflowed usize".to_string()))?;
let status_pages = WEB_DEBUG_RENDERERS
.checked_mul(WEB_DEBUG_STATUS_PAGE_BYTES)
.ok_or_else(|| ProxyError::Config("web.debug status pages overflowed usize".to_string()))?;
let debug_reservation = limits
.debug_bytes_global
.checked_add(
debug_ring_index
.checked_mul(WEB_DEBUG_RENDERERS)
.ok_or_else(|| {
ProxyError::Config("web.debug snapshot indexes overflowed usize".to_string())
})?,
)
.and_then(|value| {
WEB_DEBUG_RENDERERS
.checked_mul(WEB_DEBUG_GROUP_SCRATCH_BYTES)
.and_then(|scratch| value.checked_add(scratch))
})
.ok_or_else(|| {
ProxyError::Config("web.debug reservations overflowed usize".to_string())
})?;
let reserved = limits
.pending_bytes_global
.checked_add(limits.max_body_bytes_global)
.and_then(|value| value.checked_add(limits.max_static_bytes))
.and_then(|value| value.checked_add(debug_ring_index))
.and_then(|value| value.checked_add(status_pages))
.and_then(|value| value.checked_add(debug_reservation))
.and_then(|value| value.checked_add(http_header_reservation))
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
if reserved > limits.memory_envelope_bytes
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
{
return config_error(
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
);
}
Ok(())
}
@@ -47,6 +47,53 @@ fn web_config_builds_canonical_runtime_snapshot() {
assert_eq!(vhost.profiles[0].max_sessions, 4); assert_eq!(vhost.profiles[0].max_sessions, 4);
assert_eq!(vhost.profiles[0].max_streams, 64); assert_eq!(vhost.profiles[0].max_streams, 64);
assert_eq!(vhost.profiles[0].max_streams_per_session, 16); assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
assert_eq!(vhost.profiles[0].key_fingerprint.len(), 16);
assert_ne!(vhost.profiles[0].key_fingerprint, "0001020304050607");
}
#[test]
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
let configured = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
);
let config = load_config_from_temp_toml(&configured);
assert!(config.web.debug.enabled);
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
assert_eq!(config.web.debug.default_window_secs, 180);
assert_eq!(config.web.debug.max_window_secs, 900);
let old_name = format!("[general]\nconfig_strict = true\n{}", WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.trace]\nenabled = true\n\n[[web.vhosts]]",
));
let error = load_config_error_from_temp_toml(&old_name);
assert!(error.contains("web.trace"));
}
#[test]
fn web_debug_prefix_and_window_validation_fail_closed() {
let oversized_prefix = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.debug]\nenabled = true\nbody_prefix_bytes = 2097153\n\n[[web.vhosts]]",
);
let error = load_config_error_from_temp_toml(&oversized_prefix);
assert!(error.contains("web.debug.body_prefix_bytes"));
let reversed_window = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.debug]\nenabled = true\ndefault_window_secs = 181\nmax_window_secs = 180\n\n[[web.vhosts]]",
);
let error = load_config_error_from_temp_toml(&reversed_window);
assert!(error.contains("web.debug windows"));
let undersized_store = WEB_CONFIG.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\n\n[web.limits]\ndebug_bytes_global = 4095",
);
let error = load_config_error_from_temp_toml(&undersized_store);
assert!(error.contains("debug_bytes_global must be at least 4096"));
} }
#[test] #[test]
+4
View File
@@ -24,6 +24,8 @@ mod network;
mod policies; mod policies;
mod server; mod server;
mod web; mod web;
// WEB debug capture policy is reusable by config reload and process storage.
mod web_debug;
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps}; pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
#[allow(unused_imports)] #[allow(unused_imports)]
@@ -52,6 +54,8 @@ pub use web::{
WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode, WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode,
WebTimeoutsConfig, WebVhostConfig, WebTimeoutsConfig, WebVhostConfig,
}; };
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
pub(crate) use web_debug::web_debug_fits_limits;
pub(crate) use web::{ pub(crate) use web::{
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset, WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
WebStaticSite, WebStaticSite,
+17
View File
@@ -6,6 +6,8 @@ use std::sync::Arc;
use bytes::Bytes; use bytes::Bytes;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use super::web_debug::WebDebugConfig;
/// Client-facing secret representation used to derive a WEB capability. /// Client-facing secret representation used to derive a WEB capability.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
@@ -175,6 +177,12 @@ pub struct WebLimitsConfig {
/// Maximum static snapshot bytes across all virtual hosts. /// Maximum static snapshot bytes across all virtual hosts.
#[serde(default = "default_web_max_static_bytes")] #[serde(default = "default_web_max_static_bytes")]
pub max_static_bytes: usize, pub max_static_bytes: usize,
/// Maximum retained WEB debug record count.
#[serde(default = "default_web_debug_records_capacity")]
pub debug_records_capacity: usize,
/// Process-wide retained and in-flight WEB debug byte ceiling.
#[serde(default = "default_web_debug_bytes_global")]
pub debug_bytes_global: usize,
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots. /// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
#[serde(default = "default_web_memory_envelope_bytes")] #[serde(default = "default_web_memory_envelope_bytes")]
pub memory_envelope_bytes: usize, pub memory_envelope_bytes: usize,
@@ -229,6 +237,8 @@ impl Default for WebLimitsConfig {
max_static_files: default_web_max_static_files(), max_static_files: default_web_max_static_files(),
max_static_file_bytes: default_web_max_static_file_bytes(), max_static_file_bytes: default_web_max_static_file_bytes(),
max_static_bytes: default_web_max_static_bytes(), max_static_bytes: default_web_max_static_bytes(),
debug_records_capacity: default_web_debug_records_capacity(),
debug_bytes_global: default_web_debug_bytes_global(),
memory_envelope_bytes: default_web_memory_envelope_bytes(), memory_envelope_bytes: default_web_memory_envelope_bytes(),
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(), new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
new_bootstraps_burst: default_web_new_bootstraps_burst(), new_bootstraps_burst: default_web_new_bootstraps_burst(),
@@ -300,6 +310,9 @@ pub struct WebConfig {
/// Hard process and protocol limits. /// Hard process and protocol limits.
#[serde(default)] #[serde(default)]
pub limits: WebLimitsConfig, pub limits: WebLimitsConfig,
/// Hot-reloadable bounded server-side debug policy.
#[serde(default)]
pub debug: WebDebugConfig,
/// WEB lifecycle deadlines. /// WEB lifecycle deadlines.
#[serde(default)] #[serde(default)]
pub timeouts: WebTimeoutsConfig, pub timeouts: WebTimeoutsConfig,
@@ -348,6 +361,8 @@ pub(crate) struct WebRuntimeProfile {
pub(crate) carrier: WebCarrier, pub(crate) carrier: WebCarrier,
/// HMAC-derived bridge capability. /// HMAC-derived bridge capability.
pub(crate) capability: [u8; 32], pub(crate) capability: [u8; 32],
/// Non-secret domain-separated client-secret fingerprint for debugging.
pub(crate) key_fingerprint: String,
/// Per-profile live session ceiling. /// Per-profile live session ceiling.
pub(crate) max_sessions: usize, pub(crate) max_sessions: usize,
/// Per-profile live logical-stream ceiling. /// Per-profile live logical-stream ceiling.
@@ -439,6 +454,8 @@ usize_default!(default_web_max_profiles, 32);
usize_default!(default_web_max_static_files, 4096); usize_default!(default_web_max_static_files, 4096);
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024); usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024); usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
usize_default!(default_web_debug_records_capacity, 65_536);
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024); usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
u32_default!(default_web_new_bootstraps_per_minute, 1200); u32_default!(default_web_new_bootstraps_per_minute, 1200);
u32_default!(default_web_new_bootstraps_burst, 256); u32_default!(default_web_new_bootstraps_burst, 256);
+100
View File
@@ -0,0 +1,100 @@
use serde::{Deserialize, Serialize};
use super::web::WebLimitsConfig;
/// Request and response body retention policy for WEB debugging.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WebDebugBodyCapture {
/// Omits body snapshots entirely.
Off,
/// Retains body lengths and completion state without payload bytes.
#[default]
Metadata,
/// Retains a bounded prefix of each body.
Prefix,
/// Retains complete bounded carrier bodies and bounded decoy prefixes.
Full,
}
/// Hot-reloadable WEB server-side debugging policy.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WebDebugConfig {
/// Enables process-owned WEB debug collection.
#[serde(default)]
pub enabled: bool,
/// Records typed bridge, session, stream, handshake, and relay events.
#[serde(default = "default_true")]
pub capture_lifecycle: bool,
/// Retains allowlisted header values and names of all other headers.
#[serde(default = "default_true")]
pub capture_headers: bool,
/// Retains request service and body-consumption timing points.
#[serde(default = "default_true")]
pub capture_timings: bool,
/// Parses carrier bodies into bounded frame metadata.
#[serde(default = "default_true")]
pub capture_frames: bool,
/// Controls request and response body byte retention.
#[serde(default)]
pub body_capture: WebDebugBodyCapture,
/// Maximum retained body prefix for recognized WEB requests.
#[serde(default = "default_body_prefix_bytes")]
pub body_prefix_bytes: usize,
/// Maximum retained body prefix for ordinary decoy traffic.
#[serde(default = "default_decoy_body_prefix_bytes")]
pub decoy_body_prefix_bytes: usize,
/// Default observation window presented by the status page.
#[serde(default = "default_window_secs")]
pub default_window_secs: u64,
/// Largest observation window accepted by the status page.
#[serde(default = "default_max_window_secs")]
pub max_window_secs: u64,
}
impl Default for WebDebugConfig {
fn default() -> Self {
Self {
enabled: false,
capture_lifecycle: true,
capture_headers: true,
capture_timings: true,
capture_frames: true,
body_capture: WebDebugBodyCapture::Metadata,
body_prefix_bytes: default_body_prefix_bytes(),
decoy_body_prefix_bytes: default_decoy_body_prefix_bytes(),
default_window_secs: default_window_secs(),
max_window_secs: default_max_window_secs(),
}
}
}
fn default_true() -> bool {
true
}
fn default_body_prefix_bytes() -> usize {
4096
}
fn default_decoy_body_prefix_bytes() -> usize {
4096
}
fn default_window_secs() -> u64 {
180
}
fn default_max_window_secs() -> u64 {
3600
}
/// Checks whether a hot debug policy fits restart-frozen process capacities.
pub(crate) fn web_debug_fits_limits(
policy: &WebDebugConfig,
limits: &WebLimitsConfig,
) -> bool {
policy.body_prefix_bytes <= limits.max_body_bytes
&& policy.body_prefix_bytes <= limits.debug_bytes_global
&& policy.decoy_body_prefix_bytes <= limits.debug_bytes_global
}
+14 -3
View File
@@ -14,6 +14,7 @@ use super::plan::{ListenerBindSpec, listener_bind_plan};
#[cfg(unix)] #[cfg(unix)]
use super::unix::UnixAcceptHandle; use super::unix::UnixAcceptHandle;
use crate::web::manager::WebProcessRuntime; use crate::web::manager::WebProcessRuntime;
use crate::web::trace::WebTraceStore;
/// Process-owned listener inventory and accept-task lifecycle controller. /// Process-owned listener inventory and accept-task lifecycle controller.
pub(crate) struct ListenerManager { pub(crate) struct ListenerManager {
@@ -43,12 +44,14 @@ impl ListenerManager {
pub(crate) fn start( pub(crate) fn start(
bound: BoundListeners, bound: BoundListeners,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>, active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
trace: Arc<WebTraceStore>,
) -> Self { ) -> Self {
let has_web = bound let has_web = bound
.listeners .listeners
.iter() .iter()
.any(|listener| listener.spec.transport == ListenerTransport::Web); .any(|listener| listener.spec.transport == ListenerTransport::Web);
let web_runtime = has_web.then(|| WebProcessRuntime::start(active_runtime.clone())); let web_runtime =
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
let mut slots = BTreeMap::new(); let mut slots = BTreeMap::new();
for listener in bound.listeners { for listener in bound.listeners {
let addr = listener.spec.addr; let addr = listener.spec.addr;
@@ -307,7 +310,11 @@ mod tests {
#[cfg(unix)] #[cfg(unix)]
unix_listener: None, unix_listener: None,
}; };
let mut manager = ListenerManager::start(bound, active_runtime); let trace = WebTraceStore::new(
runtime.config().web.debug.clone(),
&runtime.config().web.limits,
);
let mut manager = ListenerManager::start(bound, active_runtime, trace);
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap(); let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
let blocked_addr = blocker.local_addr().unwrap(); let blocked_addr = blocker.local_addr().unwrap();
let mut desired = ProxyConfig::default(); let mut desired = ProxyConfig::default();
@@ -330,7 +337,11 @@ mod tests {
#[cfg(unix)] #[cfg(unix)]
unix_listener: None, unix_listener: None,
}; };
let mut manager = ListenerManager::start(bound, active_runtime); let trace = WebTraceStore::new(
runtime.config().web.debug.clone(),
&runtime.config().web.limits,
);
let mut manager = ListenerManager::start(bound, active_runtime, trace);
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap(); let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
let new_addr = reservation.local_addr().unwrap(); let new_addr = reservation.local_addr().unwrap();
drop(reservation); drop(reservation);
+10 -1
View File
@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
use crate::synlimit_control; use crate::synlimit_control;
use crate::transport::UpstreamManager; use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool; use crate::transport::middle_proxy::MePool;
use crate::web::trace::WebTraceStore;
use super::{ use super::{
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks, bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
@@ -104,6 +105,7 @@ pub(super) async fn run_telemt_core(
config.access.user_rate_limits.clone(), config.access.user_rate_limits.clone(),
config.access.cidr_rate_limits.clone(), config.access.cidr_rate_limits.clone(),
); );
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>)); let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback; let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
@@ -154,6 +156,7 @@ pub(super) async fn run_telemt_core(
let reload_control_api = reload_control.clone(); let reload_control_api = reload_control.clone();
let active_runtime_rx_api = active_runtime_rx.clone(); let active_runtime_rx_api = active_runtime_rx.clone();
let runtime_watch_rx_api = runtime_watch_rx.clone(); let runtime_watch_rx_api = runtime_watch_rx.clone();
let web_trace_api = web_trace.clone();
tokio::spawn(async move { tokio::spawn(async move {
api::serve( api::serve(
listen, listen,
@@ -171,6 +174,7 @@ pub(super) async fn run_telemt_core(
reload_control_api, reload_control_api,
active_runtime_rx_api, active_runtime_rx_api,
runtime_watch_rx_api, runtime_watch_rx_api,
web_trace_api,
) )
.await; .await;
}); });
@@ -314,7 +318,11 @@ pub(super) async fn run_telemt_core(
active_runtime_tx.send_replace(Some(active_runtime.clone())); active_runtime_tx.send_replace(Some(active_runtime.clone()));
runtime_tasks::mark_runtime_ready(&startup_tracker).await; runtime_tasks::mark_runtime_ready(&startup_tracker).await;
let listener_manager = listeners::ListenerManager::start(bound, active_runtime.clone()); let listener_manager = listeners::ListenerManager::start(
bound,
active_runtime.clone(),
web_trace.clone(),
);
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn( let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
active_runtime.clone(), active_runtime.clone(),
reload_control, reload_control,
@@ -325,6 +333,7 @@ pub(super) async fn run_telemt_core(
runtime_log_filter, runtime_log_filter,
runtime_watch_tx, runtime_watch_tx,
listener_manager, listener_manager,
web_trace,
); );
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at); shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
+5
View File
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, warn}; use tracing::{info, warn};
use crate::stats::QuotaStore; use crate::stats::QuotaStore;
use crate::web::trace::WebTraceStore;
use super::generation::{RuntimeGeneration, RuntimeWatchState}; use super::generation::{RuntimeGeneration, RuntimeWatchState};
use super::listeners::{ListenerManager, PreparedListenerTransition}; use super::listeners::{ListenerManager, PreparedListenerTransition};
@@ -29,6 +30,7 @@ pub(crate) struct ReloadSupervisor {
runtime_log_filter: RuntimeLogFilter, runtime_log_filter: RuntimeLogFilter,
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>, runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
listener_manager: Arc<Mutex<ListenerManager>>, listener_manager: Arc<Mutex<ListenerManager>>,
web_trace: Arc<WebTraceStore>,
} }
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime. /// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
@@ -105,6 +107,7 @@ impl ReloadSupervisor {
runtime_log_filter: RuntimeLogFilter, runtime_log_filter: RuntimeLogFilter,
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>, runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
listener_manager: ListenerManager, listener_manager: ListenerManager,
web_trace: Arc<WebTraceStore>,
) -> ReloadSupervisorHandle { ) -> ReloadSupervisorHandle {
let listener_manager = Arc::new(Mutex::new(listener_manager)); let listener_manager = Arc::new(Mutex::new(listener_manager));
let supervisor = Self { let supervisor = Self {
@@ -117,6 +120,7 @@ impl ReloadSupervisor {
runtime_log_filter, runtime_log_filter,
runtime_watch_tx, runtime_watch_tx,
listener_manager: listener_manager.clone(), listener_manager: listener_manager.clone(),
web_trace,
}; };
let control = supervisor.control.clone(); let control = supervisor.control.clone();
let shutdown = CancellationToken::new(); let shutdown = CancellationToken::new();
@@ -309,6 +313,7 @@ impl ReloadSupervisor {
}; };
old_runtime.stop_accepting_sessions(); old_runtime.stop_accepting_sessions();
let replaced = self.active_runtime.swap(new_runtime.clone()); let replaced = self.active_runtime.swap(new_runtime.clone());
self.web_trace.apply_policy(&new_runtime.config().web.debug);
if let Some(pending) = pending_listener_transition { if let Some(pending) = pending_listener_transition {
self.listener_manager self.listener_manager
.lock() .lock()
+9
View File
@@ -34,6 +34,10 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None)); let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state())); let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone()))); let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
let web_trace = crate::web::trace::WebTraceStore::new(
old_runtime.config().web.debug.clone(),
&old_runtime.config().web.limits,
);
let supervisor = Arc::new(ReloadSupervisor { let supervisor = Arc::new(ReloadSupervisor {
active_runtime, active_runtime,
control: control.clone(), control: control.clone(),
@@ -44,6 +48,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
runtime_log_filter: runtime_log_filter(), runtime_log_filter: runtime_log_filter(),
runtime_watch_tx, runtime_watch_tx,
listener_manager, listener_manager,
web_trace,
}); });
let command = ReloadCommand { let command = ReloadCommand {
reload_id: accepted.reload_id, reload_id: accepted.reload_id,
@@ -306,6 +311,10 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
runtime_log_filter(), runtime_log_filter(),
runtime_watch_tx, runtime_watch_tx,
listener_manager, listener_manager,
crate::web::trace::WebTraceStore::new(
runtime.config().web.debug.clone(),
&runtime.config().web.limits,
),
); );
tokio::time::timeout(Duration::from_secs(1), handle.quiesce()) tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
+5 -1
View File
@@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::{RwLock, Semaphore, watch}; use tokio::sync::{RwLock, Semaphore, watch};
use crate::config::{ProxyConfig, ServerConfig}; use crate::config::{ProxyConfig, ServerConfig, web_debug_fits_limits};
use crate::crypto::SecureRandom; use crate::crypto::SecureRandom;
use crate::ip_tracker::UserIpTracker; use crate::ip_tracker::UserIpTracker;
use crate::network::probe::{decide_network_capabilities, run_probe}; use crate::network::probe::{decide_network_capabilities, run_probe};
@@ -415,6 +415,10 @@ pub(crate) fn resolve_reload_config(
effective.web = old.web.clone(); effective.web = old.web.clone();
} }
} }
if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
fields.push("web.debug".to_string());
effective.web.debug = old.web.debug.clone();
}
let runtime_changed = !configs_equal(old, &effective); let runtime_changed = !configs_equal(old, &effective);
ResolvedReloadConfig { ResolvedReloadConfig {
effective, effective,
+21
View File
@@ -170,6 +170,27 @@ fn web_allocation_limits_are_deferred_until_restart() {
assert!(!resolved.runtime_changed); assert!(!resolved.runtime_changed);
} }
#[test]
fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
let mut old = ProxyConfig::default();
old.rebuild_runtime_user_auth().unwrap();
old.rebuild_runtime_web().unwrap();
let mut desired = old.clone();
desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
let resolved = resolve_reload_config(&old, &desired);
assert_eq!(
resolved.deferred_process_fields,
vec!["web.limits".to_string(), "web.debug".to_string()]
);
assert_eq!(
resolved.effective.web.debug.body_prefix_bytes,
old.web.debug.body_prefix_bytes
);
}
#[test] #[test]
fn strict_middle_proxy_requires_a_prepared_pool() { fn strict_middle_proxy_requires_a_prepared_pool() {
assert!(strict_middle_proxy_unavailable(true, false, false)); assert!(strict_middle_proxy_unavailable(true, false, false));
+147 -163
View File
@@ -6,8 +6,7 @@ use std::time::{Duration, Instant};
use bytes::Bytes; use bytes::Bytes;
use http_body_util::combinators::UnsyncBoxBody; use http_body_util::combinators::UnsyncBoxBody;
use http_body_util::{BodyExt, Full}; use http_body_util::BodyExt;
use hyper::body::Incoming;
use hyper::header::{self, HeaderName, HeaderValue}; use hyper::header::{self, HeaderName, HeaderValue};
use hyper::server::conn::http1; use hyper::server::conn::http1;
use hyper::service::service_fn; use hyper::service::service_fn;
@@ -18,7 +17,7 @@ use parking_lot::Mutex;
use tokio::net::TcpStream; use tokio::net::TcpStream;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use crate::config::{WebCarrier, WebClientIpSource, WebRuntimeVhost}; use crate::config::{WebClientIpSource, WebRuntimeVhost};
use crate::web::bridge; use crate::web::bridge;
use crate::web::frame::{self, FrameType}; use crate::web::frame::{self, FrameType};
use crate::web::manager::{ManagerError, WebProcessRuntime}; use crate::web::manager::{ManagerError, WebProcessRuntime};
@@ -29,18 +28,34 @@ mod activity;
mod body; mod body;
// Decoy routing and upstream proxying are isolated from carrier authentication. // Decoy routing and upstream proxying are isolated from carrier authentication.
mod decoy; mod decoy;
// Downlink long-poll handling remains isolated from request routing.
mod down;
// Canonical request parsing rejects ambiguous credentials before routing. // Canonical request parsing rejects ambiguous credentials before routing.
mod request; mod request;
// Carrier response construction and lane-header helpers are shared by handlers.
mod response;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
// Enabled-debug integration coverage remains separate from carrier behavior tests.
#[cfg(test)]
#[path = "http/trace_tests.rs"]
mod trace_tests;
use activity::{ActivityBody, RequestActivity}; use activity::{ActivityBody, RequestActivity};
use body::{CollectBodyError, CollectedBody, collect_body}; use body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
use decoy::serve_decoy; use decoy::serve_decoy;
use down::handle_down;
use request::{ use request::{
bearer_token_hash, binary_content_type, bridge_candidate, canonical_request_host, bearer_token_hash, binary_content_type, bridge_candidate, canonical_request_host,
canonical_u64_header, client_ip, compatible_cookie_header, match_profile, canonical_u64_header, client_ip, compatible_cookie_header, match_profile,
}; };
use response::{
bad_gateway, carrier_empty, carrier_headers, carrier_lane, full_response, generic_not_found,
insert_header, service_unavailable,
};
use crate::web::trace::{
HttpTraceExchange, TraceDirection, TraceLifecycleEvent, TraceRoute,
};
type BoxError = Box<dyn Error + Send + Sync>; type BoxError = Box<dyn Error + Send + Sync>;
type HttpBody = UnsyncBoxBody<Bytes, BoxError>; type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
@@ -65,13 +80,18 @@ pub(crate) async fn serve_connection(
let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs); let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs);
let last_activity = Arc::new(Mutex::new(Instant::now())); let last_activity = Arc::new(Mutex::new(Instant::now()));
let service_last_activity = Arc::clone(&last_activity); let service_last_activity = Arc::clone(&last_activity);
let service = service_fn(move |request| { let service = service_fn(move |mut request| {
let runtime = Arc::clone(&runtime); let runtime = Arc::clone(&runtime);
let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs); let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs);
let last_activity = Arc::clone(&service_last_activity); let last_activity = Arc::clone(&service_last_activity);
let client_ip_source = client_ip_source; let client_ip_source = client_ip_source;
async move { async move {
let activity = RequestActivity::begin(last_activity); let activity = RequestActivity::begin(last_activity);
let trace = runtime.trace().begin_http(&request, peer.ip());
if let Some(trace) = &trace {
request.extensions_mut().insert(Arc::clone(trace));
}
let request = request.map(|body| RequestBody::new(body, trace.clone()));
let response = if let Some(_handler_permit) = runtime.try_http_handler() { let response = if let Some(_handler_permit) = runtime.try_http_handler() {
handle_request( handle_request(
request, request,
@@ -84,7 +104,11 @@ pub(crate) async fn serve_connection(
} else { } else {
service_unavailable() service_unavailable()
}; };
let response = response.map(|body| ActivityBody::new(body, activity).boxed_unsync()); if let Some(trace) = &trace {
trace.response_ready(&response);
}
let response = response
.map(|body| ActivityBody::new(body, activity, trace).boxed_unsync());
Ok::<_, Infallible>(response) Ok::<_, Infallible>(response)
} }
}); });
@@ -115,12 +139,18 @@ pub(crate) async fn serve_connection(
} }
async fn handle_request( async fn handle_request(
request: Request<Incoming>, request: Request<RequestBody>,
peer: SocketAddr, peer: SocketAddr,
client_ip_source: WebClientIpSource, client_ip_source: WebClientIpSource,
trusted_proxy_cidrs: &[IpNetwork], trusted_proxy_cidrs: &[IpNetwork],
runtime: Arc<WebProcessRuntime>, runtime: Arc<WebProcessRuntime>,
) -> HttpResponse { ) -> HttpResponse {
set_trace_route(&request, TraceRoute::Decoy);
if let Some(trace) = request_trace(&request)
&& let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs)
{
trace.set_effective_ip(client_ip);
}
let generation = runtime.active_generation(); let generation = runtime.active_generation();
let config = generation.config(); let config = generation.config();
let Some(web_runtime) = config.web.runtime.as_ref() else { let Some(web_runtime) = config.web.runtime.as_ref() else {
@@ -159,7 +189,7 @@ async fn handle_request(
} }
async fn handle_root( async fn handle_root(
mut request: Request<Incoming>, mut request: Request<RequestBody>,
peer: SocketAddr, peer: SocketAddr,
client_ip_source: WebClientIpSource, client_ip_source: WebClientIpSource,
trusted_proxy_cidrs: &[IpNetwork], trusted_proxy_cidrs: &[IpNetwork],
@@ -175,15 +205,34 @@ async fn handle_root(
strip_query(&mut request); strip_query(&mut request);
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
}; };
if let Some(trace) = request_trace(&request) {
trace.set_route(TraceRoute::Bridge);
trace.set_effective_ip(client_ip);
}
let carrier = profile.carrier; let carrier = profile.carrier;
let Ok(bootstrap) = runtime.issue_bootstrap(profile, client_ip) else { let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
strip_query(&mut request); Ok(bootstrap) => bootstrap,
return serve_decoy(request, vhost, true, &runtime).await; Err(error) => {
runtime.trace().record_profile_lifecycle(
client_ip,
None,
&profile,
TraceLifecycleEvent::BootstrapRejected,
None,
Some(manager_error_reason(error)),
);
strip_query(&mut request);
return serve_decoy(request, vhost, true, &runtime).await;
}
}; };
if let Some(trace) = request_trace(&request) {
trace.bind_profile(&profile, bootstrap.trace_session_id);
trace.register_redaction(bootstrap.token.as_bytes());
}
let generation = runtime.active_generation(); let generation = runtime.active_generation();
let page = bridge::render( let page = bridge::render(
&vhost.host, &vhost.host,
&bootstrap, &bootstrap.token,
generation.config().web.limits.carrier_batch_bytes, generation.config().web.limits.carrier_batch_bytes,
generation.config().web.limits.pending_bytes_per_session, generation.config().web.limits.pending_bytes_per_session,
generation.config().web.limits.pending_items_per_session, generation.config().web.limits.pending_items_per_session,
@@ -224,7 +273,7 @@ async fn handle_root(
} }
async fn handle_api( async fn handle_api(
request: Request<Incoming>, request: Request<RequestBody>,
peer: SocketAddr, peer: SocketAddr,
client_ip_source: WebClientIpSource, client_ip_source: WebClientIpSource,
trusted_proxy_cidrs: &[IpNetwork], trusted_proxy_cidrs: &[IpNetwork],
@@ -237,6 +286,9 @@ async fn handle_api(
let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs) else { let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs) else {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
}; };
if let Some(trace) = request_trace(&request) {
trace.set_effective_ip(client_ip);
}
let Some(token_hash) = bearer_token_hash(&request) else { let Some(token_hash) = bearer_token_hash(&request) else {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
}; };
@@ -249,7 +301,7 @@ async fn handle_api(
} }
async fn handle_session( async fn handle_session(
request: Request<Incoming>, request: Request<RequestBody>,
runtime: Arc<WebProcessRuntime>, runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>, vhost: Arc<WebRuntimeVhost>,
token_hash: crate::web::manager::TokenHash, token_hash: crate::web::manager::TokenHash,
@@ -262,6 +314,12 @@ async fn handle_session(
if request.headers().contains_key(header::CONTENT_TYPE) { if request.headers().contains_key(header::CONTENT_TYPE) {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
} }
if let Some(trace) = request_trace(&request)
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
{
trace.set_route(TraceRoute::Session);
trace.bind_identity(session.trace_identity());
}
let CollectedBody { let CollectedBody {
request, request,
body, body,
@@ -281,8 +339,14 @@ async fn handle_session(
if request.method() != Method::POST || !binary_content_type(&request) { if request.method() != Method::POST || !binary_content_type(&request) {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
} }
if !runtime.has_bootstrap(token_hash, &vhost.host) { let Some((trace_session_id, profile)) =
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
else {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
};
if let Some(trace) = request_trace(&request) {
trace.set_route(TraceRoute::Session);
trace.bind_profile(&profile, trace_session_id);
} }
let CollectedBody { let CollectedBody {
request, request,
@@ -295,9 +359,24 @@ async fn handle_session(
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
} }
}; };
if let Some(trace) = request_trace(&request) {
trace.record_frames(
TraceDirection::Request,
&body,
&runtime.active_generation().config().web.limits,
);
}
match runtime.create_session(token_hash, &vhost.host, client_ip, &body) { match runtime.create_session(token_hash, &vhost.host, client_ip, &body) {
Ok(result) => { Ok(result) => {
let welcome = frame::encode(FrameType::Welcome, 0, &[]); let welcome = frame::encode(FrameType::Welcome, 0, &[]);
if let Some(trace) = request_trace(&request) {
trace.register_redaction(result.token.as_bytes());
trace.record_frames(
TraceDirection::Response,
&welcome,
&runtime.active_generation().config().web.limits,
);
}
let mut response = full_response(StatusCode::OK, welcome); let mut response = full_response(StatusCode::OK, welcome);
carrier_headers(&mut response); carrier_headers(&mut response);
insert_header( insert_header(
@@ -315,15 +394,33 @@ async fn handle_session(
); );
response response
} }
Err(ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent) => { Err(error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent)) => {
runtime.trace().record_profile_lifecycle(
client_ip,
Some(trace_session_id),
&profile,
TraceLifecycleEvent::SessionRejected,
None,
Some(manager_error_reason(error)),
);
service_unavailable() service_unavailable()
} }
Err(_) => serve_decoy(request, vhost, true, &runtime).await, Err(error) => {
runtime.trace().record_profile_lifecycle(
client_ip,
Some(trace_session_id),
&profile,
TraceLifecycleEvent::SessionRejected,
None,
Some(manager_error_reason(error)),
);
serve_decoy(request, vhost, true, &runtime).await
}
} }
} }
async fn handle_up( async fn handle_up(
request: Request<Incoming>, request: Request<RequestBody>,
runtime: Arc<WebProcessRuntime>, runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>, vhost: Arc<WebRuntimeVhost>,
token_hash: crate::web::manager::TokenHash, token_hash: crate::web::manager::TokenHash,
@@ -338,6 +435,10 @@ async fn handle_up(
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else { let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
}; };
if let Some(trace) = request_trace(&request) {
trace.set_route(TraceRoute::Uplink);
trace.bind_identity(session.trace_identity());
}
let Some(lane_id) = carrier_lane(&request, session.carrier()) else { let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
}; };
@@ -358,6 +459,13 @@ async fn handle_up(
return serve_decoy(request, vhost, true, &runtime).await; return serve_decoy(request, vhost, true, &runtime).await;
} }
}; };
if let Some(trace) = request_trace(&request) {
trace.record_frames(
TraceDirection::Request,
&body,
&runtime.active_generation().config().web.limits,
);
}
let result = match lane_id { let result = match lane_id {
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body), Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
None => session.process_up(sequence, &body), None => session.process_up(sequence, &body),
@@ -379,151 +487,6 @@ async fn handle_up(
} }
} }
async fn handle_down(
request: Request<Incoming>,
runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>,
token_hash: crate::web::manager::TokenHash,
) -> HttpResponse {
if request.method() != Method::POST || request.headers().contains_key(header::CONTENT_TYPE) {
return serve_decoy(request, vhost, true, &runtime).await;
}
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let CollectedBody {
request,
body,
_body_budget,
} = match collect_body(request, &runtime, 1, true).await {
Ok(result) => result,
Err(CollectBodyError::Limit) => return service_unavailable(),
Err(CollectBodyError::Invalid(request)) => {
return serve_decoy(request, vhost, true, &runtime).await;
}
};
if !body.is_empty() {
return serve_decoy(request, vhost, true, &runtime).await;
}
let _lane_poll = if lane_id.is_some() {
let Some(permit) = runtime.try_lane_poll() else {
return service_unavailable();
};
Some(permit)
} else {
None
};
let result = match lane_id {
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
None => session.poll_down(cursor).await,
};
match result {
Ok(result) if result.body.is_empty() => {
let mut response = carrier_empty(StatusCode::NO_CONTENT);
insert_header(
&mut response,
HeaderName::from_static("x-down-cursor"),
&result.next_cursor.to_string(),
);
if result.lane_closed {
response.headers_mut().insert(
HeaderName::from_static("x-lane-closed"),
HeaderValue::from_static("1"),
);
}
response
}
Ok(result) => {
let mut response = full_response(StatusCode::OK, result.body);
carrier_headers(&mut response);
insert_header(
&mut response,
HeaderName::from_static("x-down-cursor"),
&result.next_cursor.to_string(),
);
response
}
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
service_unavailable()
}
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
}
}
fn carrier_lane<B>(request: &Request<B>, carrier: WebCarrier) -> Option<Option<u32>> {
match carrier {
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
.and_then(|value| u32::try_from(value).ok())
.filter(|value| *value <= frame::MAX_STREAM_ID)
.map(Some),
}
}
fn carrier_headers(response: &mut HttpResponse) {
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/octet-stream"),
);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
}
fn carrier_empty(status: StatusCode) -> HttpResponse {
let mut response = empty_response(status);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
response
}
fn service_unavailable() -> HttpResponse {
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
response
.headers_mut()
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
response
}
fn bad_gateway() -> HttpResponse {
full_response(
StatusCode::BAD_GATEWAY,
Bytes::from_static(b"site unavailable\n"),
)
}
fn generic_not_found() -> HttpResponse {
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
}
fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
let length = body.len();
let body = Full::new(body)
.map_err(|never| -> BoxError { match never {} })
.boxed_unsync();
let mut response = Response::new(body);
*response.status_mut() = status;
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
response
}
fn empty_response(status: StatusCode) -> HttpResponse {
full_response(status, Bytes::new())
}
fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
if let Ok(value) = HeaderValue::from_str(value) {
response.headers_mut().insert(name, value);
}
}
fn strip_query<B>(request: &mut Request<B>) { fn strip_query<B>(request: &mut Request<B>) {
if request.uri().query().is_some() if request.uri().query().is_some()
&& let Ok(uri) = request.uri().path().parse() && let Ok(uri) = request.uri().path().parse()
@@ -531,3 +494,24 @@ fn strip_query<B>(request: &mut Request<B>) {
*request.uri_mut() = uri; *request.uri_mut() = uri;
} }
} }
fn request_trace<B>(request: &Request<B>) -> Option<&Arc<HttpTraceExchange>> {
request.extensions().get::<Arc<HttpTraceExchange>>()
}
fn set_trace_route<B>(request: &Request<B>, route: TraceRoute) {
if let Some(trace) = request_trace(request) {
trace.set_route(route);
}
}
fn manager_error_reason(error: ManagerError) -> &'static str {
match error {
ManagerError::Authentication => "authentication",
ManagerError::Backpressure => "backpressure",
ManagerError::Limit => "limit",
ManagerError::Protocol => "protocol",
ManagerError::Concurrent => "concurrent",
ManagerError::Closed => "closed",
}
}
+45 -2
View File
@@ -8,6 +8,7 @@ use hyper::body::{Body, Frame, SizeHint};
use parking_lot::Mutex; use parking_lot::Mutex;
use super::{BoxError, HttpBody}; use super::{BoxError, HttpBody};
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
/// Request lifecycle guard that refreshes HTTP connection activity on completion. /// Request lifecycle guard that refreshes HTTP connection activity on completion.
pub(super) struct RequestActivity { pub(super) struct RequestActivity {
@@ -32,12 +33,33 @@ impl Drop for RequestActivity {
pub(super) struct ActivityBody { pub(super) struct ActivityBody {
inner: HttpBody, inner: HttpBody,
activity: RequestActivity, activity: RequestActivity,
trace: Option<Arc<HttpTraceExchange>>,
terminal: bool,
} }
impl ActivityBody { impl ActivityBody {
/// Binds one response body to its request activity guard. /// Binds one response body to its request activity guard.
pub(super) fn new(inner: HttpBody, activity: RequestActivity) -> Self { pub(super) fn new(
Self { inner, activity } inner: HttpBody,
activity: RequestActivity,
trace: Option<Arc<HttpTraceExchange>>,
) -> Self {
Self {
inner,
activity,
trace,
terminal: false,
}
}
fn finish(&mut self, state: TraceBodyState) {
if self.terminal {
return;
}
self.terminal = true;
if let Some(trace) = &self.trace {
trace.body_finished(TraceDirection::Response, state);
}
} }
} }
@@ -50,6 +72,21 @@ impl Body for ActivityBody {
context: &mut Context<'_>, context: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> { ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let result = Pin::new(&mut self.inner).poll_frame(context); let result = Pin::new(&mut self.inner).poll_frame(context);
match &result {
Poll::Ready(Some(Ok(frame))) => {
if let Some(data) = frame.data_ref()
&& let Some(trace) = &self.trace
{
trace.body_data(TraceDirection::Response, data);
}
if self.inner.is_end_stream() {
self.finish(TraceBodyState::Complete);
}
}
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
Poll::Pending => {}
}
if result.is_ready() { if result.is_ready() {
*self.activity.last_activity.lock() = Instant::now(); *self.activity.last_activity.lock() = Instant::now();
} }
@@ -64,3 +101,9 @@ impl Body for ActivityBody {
self.inner.size_hint() self.inner.size_hint()
} }
} }
impl Drop for ActivityBody {
fn drop(&mut self) {
self.finish(TraceBodyState::Aborted);
}
}
+78 -2
View File
@@ -1,11 +1,87 @@
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration; use std::time::Duration;
use bytes::Bytes; use bytes::Bytes;
use http_body_util::{BodyExt, Empty, Limited}; use http_body_util::{BodyExt, Empty, Limited};
use hyper::Request; use hyper::Request;
use hyper::body::{Body as _, Incoming}; use hyper::body::{Body, Frame, Incoming, SizeHint};
use crate::web::manager::WebProcessRuntime; use crate::web::manager::WebProcessRuntime;
use crate::web::trace::{
HttpTraceExchange, TraceBodyState, TraceDirection,
};
/// Incoming request body wrapper that observes frames without changing streaming semantics.
pub(super) struct RequestBody {
inner: Incoming,
trace: Option<Arc<HttpTraceExchange>>,
terminal: bool,
}
impl RequestBody {
/// Wraps one Hyper request body with optional enabled-only capture state.
pub(super) fn new(inner: Incoming, trace: Option<Arc<HttpTraceExchange>>) -> Self {
Self {
inner,
trace,
terminal: false,
}
}
fn finish(&mut self, state: TraceBodyState) {
if self.terminal {
return;
}
self.terminal = true;
if let Some(trace) = &self.trace {
trace.body_finished(TraceDirection::Request, state);
}
}
}
impl Body for RequestBody {
type Data = Bytes;
type Error = hyper::Error;
fn poll_frame(
mut self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let result = Pin::new(&mut self.inner).poll_frame(context);
match &result {
Poll::Ready(Some(Ok(frame))) => {
if let Some(data) = frame.data_ref()
&& let Some(trace) = &self.trace
{
trace.body_data(TraceDirection::Request, data);
}
if self.inner.is_end_stream() {
self.finish(TraceBodyState::Complete);
}
}
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
Poll::Pending => {}
}
result
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> SizeHint {
self.inner.size_hint()
}
}
impl Drop for RequestBody {
fn drop(&mut self) {
self.finish(TraceBodyState::Aborted);
}
}
/// Collected carrier request retaining its process-wide body reservation. /// Collected carrier request retaining its process-wide body reservation.
pub(super) struct CollectedBody { pub(super) struct CollectedBody {
@@ -29,7 +105,7 @@ pub(super) enum CollectBodyError {
/// Collects one bounded carrier body under reader, byte, and deadline ownership. /// Collects one bounded carrier body under reader, byte, and deadline ownership.
pub(super) async fn collect_body( pub(super) async fn collect_body(
request: Request<Incoming>, request: Request<RequestBody>,
runtime: &WebProcessRuntime, runtime: &WebProcessRuntime,
limit: usize, limit: usize,
allow_empty: bool, allow_empty: bool,
+1
View File
@@ -28,6 +28,7 @@ where
B: hyper::body::Body<Data = Bytes> + Send + 'static, B: hyper::body::Body<Data = Bytes> + Send + 'static,
B::Error: Error + Send + Sync + 'static, B::Error: Error + Send + Sync + 'static,
{ {
super::set_trace_route(&request, crate::web::trace::TraceRoute::Decoy);
if sanitize_transport { if sanitize_transport {
sanitize_transport_request(&mut request); sanitize_transport_request(&mut request);
} }
+106
View File
@@ -0,0 +1,106 @@
use std::sync::Arc;
use hyper::header::{self, HeaderName, HeaderValue};
use hyper::{Request, StatusCode};
use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
use super::decoy::serve_decoy;
use super::request::canonical_u64_header;
use super::{
HttpResponse, carrier_empty, carrier_headers, carrier_lane, full_response, insert_header,
request_trace, service_unavailable,
};
use crate::config::WebRuntimeVhost;
use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
use crate::web::trace::{TraceDirection, TraceRoute};
/// Handles one authenticated long-poll downlink exchange.
pub(super) async fn handle_down(
request: Request<RequestBody>,
runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>,
token_hash: TokenHash,
) -> HttpResponse {
if request.method() != hyper::Method::POST
|| request.headers().contains_key(header::CONTENT_TYPE)
{
return serve_decoy(request, vhost, true, &runtime).await;
}
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
if let Some(trace) = request_trace(&request) {
trace.set_route(TraceRoute::Downlink);
trace.bind_identity(session.trace_identity());
}
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let CollectedBody {
request,
body,
_body_budget,
} = match collect_body(request, &runtime, 1, true).await {
Ok(result) => result,
Err(CollectBodyError::Limit) => return service_unavailable(),
Err(CollectBodyError::Invalid(request)) => {
return serve_decoy(request, vhost, true, &runtime).await;
}
};
if !body.is_empty() {
return serve_decoy(request, vhost, true, &runtime).await;
}
let _lane_poll = if lane_id.is_some() {
let Some(permit) = runtime.try_lane_poll() else {
return service_unavailable();
};
Some(permit)
} else {
None
};
let result = match lane_id {
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
None => session.poll_down(cursor).await,
};
match result {
Ok(result) if result.body.is_empty() => {
let mut response = carrier_empty(StatusCode::NO_CONTENT);
insert_header(
&mut response,
HeaderName::from_static("x-down-cursor"),
&result.next_cursor.to_string(),
);
if result.lane_closed {
response.headers_mut().insert(
HeaderName::from_static("x-lane-closed"),
HeaderValue::from_static("1"),
);
}
response
}
Ok(result) => {
if let Some(trace) = request_trace(&request) {
trace.record_frames(
TraceDirection::Response,
&result.body,
&runtime.active_generation().config().web.limits,
);
}
let mut response = full_response(StatusCode::OK, result.body);
carrier_headers(&mut response);
insert_header(
&mut response,
HeaderName::from_static("x-down-cursor"),
&result.next_cursor.to_string(),
);
response
}
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
service_unavailable()
}
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
}
}
+88
View File
@@ -0,0 +1,88 @@
use bytes::Bytes;
use http_body_util::{BodyExt, Full};
use hyper::header::{self, HeaderName, HeaderValue};
use hyper::{Request, Response, StatusCode};
use super::request::canonical_u64_header;
use super::{BoxError, HttpResponse};
use crate::config::WebCarrier;
use crate::web::frame;
/// Validates and resolves the optional carrier lane header.
pub(super) fn carrier_lane<B>(
request: &Request<B>,
carrier: WebCarrier,
) -> Option<Option<u32>> {
match carrier {
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
.and_then(|value| u32::try_from(value).ok())
.filter(|value| *value <= frame::MAX_STREAM_ID)
.map(Some),
}
}
/// Applies common binary carrier response headers.
pub(super) fn carrier_headers(response: &mut HttpResponse) {
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/octet-stream"),
);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
}
/// Builds an empty no-store carrier response.
pub(super) fn carrier_empty(status: StatusCode) -> HttpResponse {
let mut response = empty_response(status);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
response
}
/// Builds the bounded retryable carrier saturation response.
pub(super) fn service_unavailable() -> HttpResponse {
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
response
.headers_mut()
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
response
}
/// Builds the ordinary decoy upstream failure response.
pub(super) fn bad_gateway() -> HttpResponse {
full_response(
StatusCode::BAD_GATEWAY,
Bytes::from_static(b"site unavailable\n"),
)
}
/// Builds the ordinary unmatched-host response.
pub(super) fn generic_not_found() -> HttpResponse {
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
}
/// Builds one in-memory response with an exact content length.
pub(super) fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
let length = body.len();
let body = Full::new(body)
.map_err(|never| -> BoxError { match never {} })
.boxed_unsync();
let mut response = Response::new(body);
*response.status_mut() = status;
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
response
}
fn empty_response(status: StatusCode) -> HttpResponse {
full_response(status, Bytes::new())
}
/// Inserts one validated dynamic response header value.
pub(super) fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
if let Ok(value) = HeaderValue::from_str(value) {
response.headers_mut().insert(name, value);
}
}
+4 -3
View File
@@ -17,7 +17,7 @@ use crate::maestro::generation::test_runtime_generation;
use crate::web::frame::{self, FrameType}; use crate::web::frame::{self, FrameType};
use crate::web::manager::WebProcessRuntime; use crate::web::manager::WebProcessRuntime;
fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig { pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
let profile = Arc::new(WebRuntimeProfile { let profile = Arc::new(WebRuntimeProfile {
host: "proxy.example.com".to_string(), host: "proxy.example.com".to_string(),
public_addr: "203.0.113.10:443".parse().unwrap(), public_addr: "203.0.113.10:443".parse().unwrap(),
@@ -25,6 +25,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
secret_mode: WebSecretMode::Plain, secret_mode: WebSecretMode::Plain,
carrier, carrier,
capability, capability,
key_fingerprint: "0000000000000000".to_string(),
max_sessions: 4, max_sessions: 4,
max_streams: 16, max_streams: 16,
max_streams_per_session: 4, max_streams_per_session: 4,
@@ -71,7 +72,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
config config
} }
async fn request( pub(super) async fn request(
listener: &TcpListener, listener: &TcpListener,
runtime: &Arc<WebProcessRuntime>, runtime: &Arc<WebProcessRuntime>,
request: Vec<u8>, request: Vec<u8>,
@@ -97,7 +98,7 @@ async fn request(
response response
} }
fn split_response(response: &[u8]) -> (&[u8], &[u8]) { pub(super) fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
let separator = response let separator = response
.windows(4) .windows(4)
.position(|window| window == b"\r\n\r\n") .position(|window| window == b"\r\n\r\n")
+75
View File
@@ -0,0 +1,75 @@
use std::sync::Arc;
use arc_swap::ArcSwap;
use base64::Engine as _;
use tokio::net::TcpListener;
use super::tests::{request, runtime_config, split_response};
use crate::config::{WebCarrier, WebDebugBodyCapture};
use crate::maestro::generation::test_runtime_generation;
use crate::web::manager::WebProcessRuntime;
use crate::web::trace::{TraceBodyState, TraceRecordKind, TraceRoute};
#[tokio::test]
async fn enabled_debug_records_bridge_request_response_without_credentials() {
let capability = [18u8; 32];
let mut config = runtime_config(capability, WebCarrier::Https);
config.web.debug.enabled = true;
config.web.debug.body_capture = WebDebugBodyCapture::Prefix;
config.web.debug.body_prefix_bytes = 4096;
let generation = test_runtime_generation(1, config);
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
let runtime = WebProcessRuntime::start(active_runtime);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
let root = format!(
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.50\r\nUser-Agent: debug-client/1\r\nConnection: close\r\n\r\n"
)
.into_bytes();
let response = request(&listener, &runtime, root).await;
let (_, body) = split_response(&response);
let body = std::str::from_utf8(body).unwrap();
let bootstrap = body
.split_once("bootstrap=\"")
.and_then(|(_, suffix)| suffix.split_once('"'))
.map(|(token, _)| token)
.unwrap();
let records = runtime
.trace()
.snapshot_matching(|record| matches!(&record.kind, TraceRecordKind::Http(_)));
assert_eq!(records.len(), 1);
let record = &records[0].record;
assert_eq!(record.effective_ip, Some("192.0.2.50".parse().unwrap()));
assert_eq!(record.user_agent.as_deref(), Some("debug-client/1"));
assert!(record.identity.session_id.is_some());
assert_eq!(record.identity.user.as_deref(), Some("alice"));
let TraceRecordKind::Http(http) = &record.kind else {
panic!("expected HTTP debug record");
};
assert_eq!(http.method, "GET");
assert_eq!(http.path, "/");
assert_eq!(http.route, TraceRoute::Bridge);
assert_eq!(http.status, Some(200));
assert_eq!(
http.response_body.as_ref().unwrap().state,
TraceBodyState::Complete
);
let captured_response = &http.response_body.as_ref().unwrap().captured;
assert!(
!captured_response
.windows(bootstrap.len())
.any(|value| value == bootstrap.as_bytes())
);
assert!(
captured_response
.windows(bootstrap.len())
.any(|value| value.iter().all(|byte| *byte == b'*'))
);
assert!(http.timings.as_ref().unwrap().response_ready_us.is_some());
assert!(http.timings.as_ref().unwrap().response_body_us.is_some());
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
+34 -223
View File
@@ -1,34 +1,27 @@
use std::future::Future; use std::future::Future;
use std::net::IpAddr;
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant}; use std::time::Duration;
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use parking_lot::Mutex; use parking_lot::Mutex;
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker; use tokio_util::task::TaskTracker;
use zeroize::Zeroizing;
use crate::config::{WebCarrier, WebLimitsConfig, WebRuntimeProfile}; use crate::config::{WebCarrier, WebLimitsConfig};
use crate::maestro::generation::RuntimeGeneration; use crate::maestro::generation::RuntimeGeneration;
use crate::web::frame; use crate::web::trace::WebTraceStore;
use crate::web::session::WebSession;
// Credential maps, quotas, and token-bucket helpers remain private to the manager. // Credential maps, quotas, and token-bucket helpers remain private to the manager.
mod state; mod state;
// Bootstrap credentials and idempotent session creation are isolated from queue accounting.
mod credentials;
// Stream admission and synthetic tuple ownership are process-scoped. // Stream admission and synthetic tuple ownership are process-scoped.
mod admission; mod admission;
// Shutdown and expiry work remain outside request-path coordination. // Shutdown and expiry work remain outside request-path coordination.
mod lifecycle; mod lifecycle;
use state::{ use state::{ManagerState, control_item_reserve};
Bootstrap, ManagerState, allow_rate, control_item_reserve, decrement_map,
evict_oldest_unused_bootstrap, matching_profile, new_unique_token, profile_key,
remove_expired_locked,
};
const TOKEN_BYTES: usize = 32; const TOKEN_BYTES: usize = 32;
const CLEANUP_INTERVAL: Duration = Duration::from_secs(1); const CLEANUP_INTERVAL: Duration = Duration::from_secs(1);
@@ -63,9 +56,18 @@ pub(crate) struct CreateResult {
pub(crate) carrier: WebCarrier, pub(crate) carrier: WebCarrier,
} }
/// Successful bridge bootstrap issuance result.
pub(crate) struct BootstrapResult {
/// Opaque one-use bootstrap credential.
pub(crate) token: String,
/// Process-unique non-secret trace identifier.
pub(crate) trace_session_id: u64,
}
/// Process-owned bounded WEB credential, session, and memory coordinator. /// Process-owned bounded WEB credential, session, and memory coordinator.
pub(crate) struct WebProcessRuntime { pub(crate) struct WebProcessRuntime {
active_runtime: Arc<ArcSwap<RuntimeGeneration>>, active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
trace: Arc<WebTraceStore>,
limits: WebLimitsConfig, limits: WebLimitsConfig,
state: Mutex<ManagerState>, state: Mutex<ManagerState>,
http_connections: Arc<Semaphore>, http_connections: Arc<Semaphore>,
@@ -89,10 +91,22 @@ pub(crate) struct WebProcessRuntime {
impl WebProcessRuntime { impl WebProcessRuntime {
/// Starts one process-scoped manager using immutable allocation ceilings. /// Starts one process-scoped manager using immutable allocation ceilings.
#[cfg(test)]
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> { pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
let config = active_runtime.load().config();
let trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
Self::start_with_trace(active_runtime, trace)
}
/// Starts one process-scoped manager with a shared API-visible trace store.
pub(crate) fn start_with_trace(
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
trace: Arc<WebTraceStore>,
) -> Arc<Self> {
let limits = active_runtime.load().config().web.limits.clone(); let limits = active_runtime.load().config().web.limits.clone();
let runtime = Arc::new(Self { let runtime = Arc::new(Self {
active_runtime, active_runtime,
trace,
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)), http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)), http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))), lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))),
@@ -125,6 +139,8 @@ impl WebProcessRuntime {
let Some(runtime) = weak.upgrade() else { let Some(runtime) = weak.upgrade() else {
break; break;
}; };
let policy = runtime.active_generation().config().web.debug.clone();
runtime.trace.apply_policy(&policy);
runtime.cleanup(); runtime.cleanup();
} }
} }
@@ -138,6 +154,11 @@ impl WebProcessRuntime {
self.active_runtime.load_full() self.active_runtime.load_full()
} }
/// Returns the process-owned WEB debug trace store.
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
&self.trace
}
/// Reserves one accepted HTTP connection. /// Reserves one accepted HTTP connection.
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> { pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok(); let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
@@ -211,216 +232,6 @@ impl WebProcessRuntime {
Some((reader, body)) Some((reader, body))
} }
/// Issues a one-use bootstrap credential for an active compatible profile.
pub(crate) fn issue_bootstrap(
&self,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
) -> std::result::Result<String, ManagerError> {
let generation = self.active_generation();
let config = generation.config();
let profile = config
.web
.runtime
.as_ref()
.and_then(|runtime| matching_profile(runtime, &profile))
.ok_or(ManagerError::Authentication)?;
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
return Err(ManagerError::Closed);
}
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
if state.closed
|| state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
|| !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
if state.bootstraps.len() >= self.limits.max_bootstraps_global
&& !evict_oldest_unused_bootstrap(&mut state)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
};
state.bootstraps.insert(
hash,
Bootstrap {
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
issued_at: now,
issuance_ip: client_ip,
profile,
body_digest: [0; TOKEN_BYTES],
session_token: Zeroizing::new(String::new()),
session: None,
used: false,
},
);
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
Ok(token)
}
/// Checks whether a bootstrap token is live before reading a request body.
pub(crate) fn has_bootstrap(&self, hash: TokenHash, host: &str) -> bool {
let now = Instant::now();
let state = self.state.lock();
state
.bootstraps
.get(&hash)
.is_some_and(|entry| entry.profile.host == host && now <= entry.expires_at)
}
/// Creates a session exactly once or replays the original successful result.
pub(crate) fn create_session(
self: &Arc<Self>,
bootstrap_hash: TokenHash,
host: &str,
client_ip: IpAddr,
body: &[u8],
) -> std::result::Result<CreateResult, ManagerError> {
if !frame::validate_hello(body, &self.limits) {
return Err(ManagerError::Protocol);
}
let body_digest: TokenHash = Sha256::digest(body).into();
let generation = self.active_generation();
let config = generation.config();
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
return Err(ManagerError::Authentication);
};
if entry.profile.host != host || now > entry.expires_at {
return Err(ManagerError::Authentication);
}
if entry.used {
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
if !digest_matches {
return Err(ManagerError::Authentication);
}
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
return Ok(CreateResult {
token: entry.session_token.as_str().to_owned(),
carrier: session.carrier(),
});
}
if state.closed || !config.web.enabled {
return Err(ManagerError::Closed);
}
let profile = config
.web
.runtime
.as_ref()
.and_then(|runtime| matching_profile(runtime, &entry.profile))
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
.ok_or(ManagerError::Authentication)?;
let profile_key = profile_key(&profile);
if state.sessions.len() >= self.limits.max_sessions_global
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
>= self.limits.max_sessions_per_ip
|| state
.sessions_per_profile
.get(&profile_key)
.copied()
.unwrap_or(0)
>= profile.max_sessions
|| !allow_rate(
&mut state.session_rate,
now,
self.limits.new_sessions_per_minute,
self.limits.new_sessions_burst,
)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
};
let session = WebSession::new(
Arc::downgrade(self),
session_hash,
client_ip,
profile,
profile_key,
self.limits.clone(),
config.web.timeouts.clone(),
);
state.sessions.insert(session_hash, Arc::clone(&session));
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
let entry = state
.bootstraps
.get_mut(&bootstrap_hash)
.ok_or(ManagerError::Authentication)?;
entry.used = true;
entry.body_digest = body_digest;
entry.session_token = Zeroizing::new(session_token.clone());
entry.session = Some(Arc::clone(&session));
let issuance_ip = entry.issuance_ip;
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
Ok(CreateResult {
token: session_token,
carrier: session.carrier(),
})
}
/// Resolves an authenticated session token.
pub(crate) fn get_session(
&self,
hash: TokenHash,
host: &str,
) -> std::result::Result<Arc<WebSession>, ManagerError> {
self.state
.lock()
.sessions
.get(&hash)
.cloned()
.filter(|session| session.matches_host(host))
.ok_or(ManagerError::Authentication)
}
/// Closes a live token and accepts bounded tombstone retries.
pub(crate) fn close_token(
&self,
hash: TokenHash,
host: &str,
) -> std::result::Result<(), ManagerError> {
let state = self.state.lock();
let session = state
.sessions
.get(&hash)
.filter(|session| session.matches_host(host))
.cloned();
let closed = state
.closed_tokens
.get(&hash)
.is_some_and(|closed| closed.host == host);
drop(state);
if let Some(session) = session {
session.close();
return Ok(());
}
closed.then_some(()).ok_or(ManagerError::Authentication)
}
/// Reserves bounded process-wide queue capacity for data or control traffic. /// Reserves bounded process-wide queue capacity for data or control traffic.
pub(crate) fn try_reserve_pending( pub(crate) fn try_reserve_pending(
&self, &self,
+280
View File
@@ -0,0 +1,280 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use zeroize::Zeroizing;
use super::state::{
Bootstrap, allow_rate, decrement_map, evict_oldest_unused_bootstrap, matching_profile,
new_unique_token, profile_key, remove_expired_locked,
};
use super::{
BootstrapResult, CreateResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime,
};
use crate::config::WebRuntimeProfile;
use crate::web::frame;
use crate::web::session::WebSession;
impl WebProcessRuntime {
/// Issues a one-use bootstrap credential for an active compatible profile.
pub(crate) fn issue_bootstrap(
&self,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
) -> std::result::Result<BootstrapResult, ManagerError> {
let generation = self.active_generation();
let config = generation.config();
let profile = config
.web
.runtime
.as_ref()
.and_then(|runtime| matching_profile(runtime, &profile))
.ok_or(ManagerError::Authentication)?;
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
return Err(ManagerError::Closed);
}
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
if state.closed
|| state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
|| !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
if state.bootstraps.len() >= self.limits.max_bootstraps_global
&& !evict_oldest_unused_bootstrap(&mut state)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
};
let trace_session_id = self.trace.next_session_id();
state.bootstraps.insert(
hash,
Bootstrap {
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
issued_at: now,
issuance_ip: client_ip,
profile,
trace_session_id,
body_digest: [0; TOKEN_BYTES],
session_token: Zeroizing::new(String::new()),
session: None,
used: false,
},
);
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
let profile = state
.bootstraps
.get(&hash)
.map(|entry| Arc::clone(&entry.profile))
.ok_or(ManagerError::Closed)?;
drop(state);
self.trace.record_profile_lifecycle(
client_ip,
Some(trace_session_id),
&profile,
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
None,
None,
);
Ok(BootstrapResult {
token,
trace_session_id,
})
}
/// Resolves non-secret bootstrap trace identity without exposing its credential.
pub(crate) fn bootstrap_trace_identity(
&self,
hash: TokenHash,
host: &str,
) -> Option<(u64, Arc<WebRuntimeProfile>)> {
let now = Instant::now();
self.state
.lock()
.bootstraps
.get(&hash)
.filter(|entry| entry.profile.host == host && now <= entry.expires_at)
.map(|entry| (entry.trace_session_id, Arc::clone(&entry.profile)))
}
/// Creates a session exactly once or replays the original successful result.
pub(crate) fn create_session(
self: &Arc<Self>,
bootstrap_hash: TokenHash,
host: &str,
client_ip: IpAddr,
body: &[u8],
) -> std::result::Result<CreateResult, ManagerError> {
if !frame::validate_hello(body, &self.limits) {
return Err(ManagerError::Protocol);
}
let body_digest: TokenHash = Sha256::digest(body).into();
let generation = self.active_generation();
let config = generation.config();
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
return Err(ManagerError::Authentication);
};
if entry.profile.host != host || now > entry.expires_at {
return Err(ManagerError::Authentication);
}
if entry.used {
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
if !digest_matches {
return Err(ManagerError::Authentication);
}
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
let result = CreateResult {
token: entry.session_token.as_str().to_owned(),
carrier: session.carrier(),
};
let identity = session.trace_identity();
drop(state);
self.trace.record_lifecycle(
None,
Some(client_ip),
identity,
crate::web::trace::TraceLifecycleEvent::SessionReplayed,
None,
None,
);
return Ok(result);
}
let trace_session_id = entry.trace_session_id;
let issued_profile = Arc::clone(&entry.profile);
if state.closed || !config.web.enabled {
return Err(ManagerError::Closed);
}
let profile = config
.web
.runtime
.as_ref()
.and_then(|runtime| matching_profile(runtime, &issued_profile))
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
.ok_or(ManagerError::Authentication)?;
let profile_key = profile_key(&profile);
if state.sessions.len() >= self.limits.max_sessions_global
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
>= self.limits.max_sessions_per_ip
|| state
.sessions_per_profile
.get(&profile_key)
.copied()
.unwrap_or(0)
>= profile.max_sessions
|| !allow_rate(
&mut state.session_rate,
now,
self.limits.new_sessions_per_minute,
self.limits.new_sessions_burst,
)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
};
let session = WebSession::new(
Arc::downgrade(self),
session_hash,
client_ip,
trace_session_id,
profile,
profile_key,
self.limits.clone(),
config.web.timeouts.clone(),
);
state.sessions.insert(session_hash, Arc::clone(&session));
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
let entry = state
.bootstraps
.get_mut(&bootstrap_hash)
.ok_or(ManagerError::Authentication)?;
entry.used = true;
entry.body_digest = body_digest;
entry.session_token = Zeroizing::new(session_token.clone());
entry.session = Some(Arc::clone(&session));
let issuance_ip = entry.issuance_ip;
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
let identity = session.trace_identity();
let result = CreateResult {
token: session_token,
carrier: session.carrier(),
};
drop(state);
self.trace.record_lifecycle(
None,
Some(client_ip),
identity,
crate::web::trace::TraceLifecycleEvent::SessionCreated,
None,
None,
);
Ok(result)
}
/// Resolves an authenticated session token.
pub(crate) fn get_session(
&self,
hash: TokenHash,
host: &str,
) -> std::result::Result<Arc<WebSession>, ManagerError> {
self.state
.lock()
.sessions
.get(&hash)
.cloned()
.filter(|session| session.matches_host(host))
.ok_or(ManagerError::Authentication)
}
/// Closes a live token and accepts bounded tombstone retries.
pub(crate) fn close_token(
&self,
hash: TokenHash,
host: &str,
) -> std::result::Result<(), ManagerError> {
let state = self.state.lock();
let session = state
.sessions
.get(&hash)
.filter(|session| session.matches_host(host))
.cloned();
let closed = state
.closed_tokens
.get(&hash)
.is_some_and(|closed| closed.host == host);
drop(state);
if let Some(session) = session {
session.close();
return Ok(());
}
closed.then_some(()).ok_or(ManagerError::Authentication)
}
}
+3
View File
@@ -22,6 +22,8 @@ pub(super) struct Bootstrap {
pub(super) issuance_ip: IpAddr, pub(super) issuance_ip: IpAddr,
/// Immutable profile selected during capability validation. /// Immutable profile selected during capability validation.
pub(super) profile: Arc<WebRuntimeProfile>, pub(super) profile: Arc<WebRuntimeProfile>,
/// Process-unique non-secret identifier shared by bootstrap and session traces.
pub(super) trace_session_id: u64,
/// Digest of the accepted HELLO body for idempotent retry matching. /// Digest of the accepted HELLO body for idempotent retry matching.
pub(super) body_digest: TokenHash, pub(super) body_digest: TokenHash,
/// Zeroizing copy returned only for an exact session-creation retry. /// Zeroizing copy returned only for an exact session-creation retry.
@@ -130,6 +132,7 @@ pub(super) fn matching_profile(
&& profile.secret_mode == expected.secret_mode && profile.secret_mode == expected.secret_mode
&& profile.carrier == expected.carrier && profile.carrier == expected.carrier
&& profile.capability == expected.capability && profile.capability == expected.capability
&& profile.key_fingerprint == expected.key_fingerprint
}) })
.cloned() .cloned()
} }
+2
View File
@@ -12,3 +12,5 @@ pub(crate) mod manager;
pub(crate) mod session; pub(crate) mod session;
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream. /// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
pub(crate) mod stream; pub(crate) mod stream;
/// Process-owned bounded WEB debugging records and capture lifecycle.
pub(crate) mod trace;
+32
View File
@@ -120,6 +120,7 @@ pub(crate) struct WebSession {
manager: std::sync::Weak<WebProcessRuntime>, manager: std::sync::Weak<WebProcessRuntime>,
token_hash: TokenHash, token_hash: TokenHash,
client_ip: IpAddr, client_ip: IpAddr,
trace_session_id: u64,
profile: Arc<WebRuntimeProfile>, profile: Arc<WebRuntimeProfile>,
profile_key: ProfileKey, profile_key: ProfileKey,
limits: WebLimitsConfig, limits: WebLimitsConfig,
@@ -150,6 +151,7 @@ impl WebSession {
manager: std::sync::Weak<WebProcessRuntime>, manager: std::sync::Weak<WebProcessRuntime>,
token_hash: TokenHash, token_hash: TokenHash,
client_ip: IpAddr, client_ip: IpAddr,
trace_session_id: u64,
profile: Arc<WebRuntimeProfile>, profile: Arc<WebRuntimeProfile>,
profile_key: ProfileKey, profile_key: ProfileKey,
limits: WebLimitsConfig, limits: WebLimitsConfig,
@@ -163,6 +165,7 @@ impl WebSession {
manager, manager,
token_hash, token_hash,
client_ip, client_ip,
trace_session_id,
profile, profile,
profile_key, profile_key,
limits, limits,
@@ -211,6 +214,30 @@ impl WebSession {
self.profile.carrier self.profile.carrier
} }
/// Returns a cloned non-secret identity only for enabled debug capture.
pub(crate) fn trace_identity(&self) -> crate::web::trace::TraceIdentity {
crate::web::trace::TraceIdentity::from_profile(self.trace_session_id, &self.profile)
}
/// Records one typed lifecycle event without exposing session credentials.
pub(super) fn trace_lifecycle(
&self,
event: crate::web::trace::TraceLifecycleEvent,
stream_id: Option<u32>,
reason: Option<&'static str>,
) {
if let Some(manager) = self.manager.upgrade() {
manager.trace().record_profile_lifecycle(
self.client_ip,
Some(self.trace_session_id),
&self.profile,
event,
stream_id,
reason,
);
}
}
/// Closes carrier state while relay tasks retain their admission until exit. /// Closes carrier state while relay tasks retain their admission until exit.
pub(crate) fn close(&self) { pub(crate) fn close(&self) {
let (data_bytes, data_items, control_bytes, control_items) = { let (data_bytes, data_items, control_bytes, control_items) = {
@@ -253,6 +280,11 @@ impl WebSession {
manager.release_pending(data_bytes, data_items, false); manager.release_pending(data_bytes, data_items, false);
manager.release_pending(control_bytes, control_items, true); manager.release_pending(control_bytes, control_items, true);
if !self.finished.swap(true, Ordering::AcqRel) { if !self.finished.swap(true, Ordering::AcqRel) {
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::SessionClosed,
None,
Some("closed"),
);
manager.session_finished( manager.session_finished(
self.token_hash, self.token_hash,
self.client_ip, self.client_ip,
+73 -1
View File
@@ -22,11 +22,21 @@ impl WebSession {
}; };
let generation = manager.active_generation(); let generation = manager.active_generation();
if !*generation.admission_rx.borrow() { if !*generation.admission_rx.borrow() {
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_id),
Some("admission_closed"),
);
self.stream_finished(stream_id, peer_port); self.stream_finished(stream_id, peer_port);
return; return;
} }
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else { let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
manager.record_stream_rejected(); manager.record_stream_rejected();
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_id),
Some("connection_limit"),
);
self.stream_finished(stream_id, peer_port); self.stream_finished(stream_id, peer_port);
return; return;
}; };
@@ -42,11 +52,17 @@ impl WebSession {
stream_id, stream_id,
peer_port, peer_port,
}; };
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamAdmitted,
Some(stream_id),
None,
);
let stream = WebLogicalStream::new(Arc::clone(&session), stream_id); let stream = WebLogicalStream::new(Arc::clone(&session), stream_id);
tokio::select! { tokio::select! {
_ = cancel.cancelled() => {} _ = cancel.cancelled() => {}
_ = run_stream( _ = run_stream(
Arc::clone(&session), Arc::clone(&session),
stream_id,
stream, stream,
deps, deps,
replay_checker, replay_checker,
@@ -55,6 +71,11 @@ impl WebSession {
} }
}); });
if !spawned { if !spawned {
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_id),
Some("generation_closed"),
);
self.tasks_live.fetch_sub(1, Ordering::AcqRel); self.tasks_live.fetch_sub(1, Ordering::AcqRel);
self.stream_finished(stream_id, peer_port); self.stream_finished(stream_id, peer_port);
self.tasks_done.notify_waiters(); self.tasks_done.notify_waiters();
@@ -100,6 +121,11 @@ struct StreamCompletion {
impl Drop for StreamCompletion { impl Drop for StreamCompletion {
fn drop(&mut self) { fn drop(&mut self) {
self.session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamClosed,
Some(self.stream_id),
None,
);
self.session.stream_finished(self.stream_id, self.peer_port); self.session.stream_finished(self.stream_id, self.peer_port);
if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 { if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 {
self.session.tasks_done.notify_waiters(); self.session.tasks_done.notify_waiters();
@@ -109,6 +135,7 @@ impl Drop for StreamCompletion {
async fn run_stream( async fn run_stream(
session: Arc<WebSession>, session: Arc<WebSession>,
stream_id: u32,
stream: WebLogicalStream, stream: WebLogicalStream,
deps: crate::proxy::authenticated::ClientRuntimeDeps, deps: crate::proxy::authenticated::ClientRuntimeDeps,
replay_checker: Arc<crate::stats::ReplayChecker>, replay_checker: Arc<crate::stats::ReplayChecker>,
@@ -129,14 +156,29 @@ async fn run_stream(
// first byte. Session and stream quotas bound this idle phase without // first byte. Session and stream quotas bound this idle phase without
// consuming the process-wide active-handshake budget. // consuming the process-wide active-handshake budget.
if reader.read_exact(&mut handshake[..1]).await.is_err() { if reader.read_exact(&mut handshake[..1]).await.is_err() {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
Some(stream_id),
Some("first_byte_io"),
);
deps.stats deps.stats
.increment_connects_bad_with_class("web_mtproto_handshake_io"); .increment_connects_bad_with_class("web_mtproto_handshake_io");
return; return;
} }
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamFirstByte,
Some(stream_id),
None,
);
let Some(manager) = session.manager.upgrade() else { let Some(manager) = session.manager.upgrade() else {
return; return;
}; };
let Some(handshake_permit) = manager.try_stream_handshake() else { let Some(handshake_permit) = manager.try_stream_handshake() else {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_id),
Some("handshake_limit"),
);
return; return;
}; };
let handshake_result = tokio::time::timeout( let handshake_result = tokio::time::timeout(
@@ -163,6 +205,11 @@ async fn run_stream(
drop(handshake_permit); drop(handshake_permit);
let (reader, writer, success) = match handshake_result { let (reader, writer, success) = match handshake_result {
Err(_) => { Err(_) => {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::HandshakeTimeout,
Some(stream_id),
Some("timeout"),
);
deps.stats deps.stats
.increment_connects_bad_with_class("web_mtproto_handshake_timeout"); .increment_connects_bad_with_class("web_mtproto_handshake_timeout");
deps.stats.increment_handshake_timeouts(); deps.stats.increment_handshake_timeouts();
@@ -170,20 +217,40 @@ async fn run_stream(
return; return;
} }
Ok(Err(_)) => { Ok(Err(_)) => {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
Some(stream_id),
Some("io"),
);
deps.stats deps.stats
.increment_connects_bad_with_class("web_mtproto_handshake_io"); .increment_connects_bad_with_class("web_mtproto_handshake_io");
return; return;
} }
Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => { Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::HandshakeSucceeded,
Some(stream_id),
None,
);
(reader, writer, success) (reader, writer, success)
} }
Ok(Ok(_)) => { Ok(Ok(_)) => {
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::HandshakeRejected,
Some(stream_id),
Some("bad_client"),
);
deps.stats deps.stats
.increment_connects_bad_with_class("web_mtproto_bad_client"); .increment_connects_bad_with_class("web_mtproto_bad_client");
return; return;
} }
}; };
let _ = run_authenticated( session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::RelayStarted,
Some(stream_id),
None,
);
let relay_result = run_authenticated(
reader, reader,
writer, writer,
success, success,
@@ -193,4 +260,9 @@ async fn run_stream(
ConntrackClosePolicy::Suppress, ConntrackClosePolicy::Suppress,
) )
.await; .await;
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::RelayEnded,
Some(stream_id),
Some(if relay_result.is_ok() { "completed" } else { "error" }),
);
} }
+2
View File
@@ -72,6 +72,7 @@ fn test_runtime_with_dc(
secret_mode: WebSecretMode::Plain, secret_mode: WebSecretMode::Plain,
carrier, carrier,
capability: [7; 32], capability: [7; 32],
key_fingerprint: "0000000000000000".to_string(),
max_sessions: 4, max_sessions: 4,
max_streams: 16, max_streams: 16,
max_streams_per_session: 4, max_streams_per_session: 4,
@@ -117,6 +118,7 @@ fn test_runtime_with_dc(
Arc::downgrade(&manager), Arc::downgrade(&manager),
[8; 32], [8; 32],
"192.0.2.10".parse().unwrap(), "192.0.2.10".parse().unwrap(),
1,
profile, profile,
[7; 32], [7; 32],
limits, limits,
+2
View File
@@ -420,6 +420,7 @@ mod tests {
secret_mode: WebSecretMode::Plain, secret_mode: WebSecretMode::Plain,
carrier: WebCarrier::Https, carrier: WebCarrier::Https,
capability: [0; 32], capability: [0; 32],
key_fingerprint: "0000000000000000".to_string(),
max_sessions: 1, max_sessions: 1,
max_streams: 1, max_streams: 1,
max_streams_per_session: 1, max_streams_per_session: 1,
@@ -428,6 +429,7 @@ mod tests {
std::sync::Weak::<WebProcessRuntime>::new(), std::sync::Weak::<WebProcessRuntime>::new(),
[1; 32], [1; 32],
"192.0.2.10".parse().unwrap(), "192.0.2.10".parse().unwrap(),
1,
profile, profile,
[2; 32], [2; 32],
WebLimitsConfig::default(), WebLimitsConfig::default(),
+2
View File
@@ -15,6 +15,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
secret_mode: WebSecretMode::Plain, secret_mode: WebSecretMode::Plain,
carrier: WebCarrier::HttpsLanes, carrier: WebCarrier::HttpsLanes,
capability: [0; 32], capability: [0; 32],
key_fingerprint: "0000000000000000".to_string(),
max_sessions: 1, max_sessions: 1,
max_streams: 2, max_streams: 2,
max_streams_per_session: 2, max_streams_per_session: 2,
@@ -23,6 +24,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
std::sync::Weak::<WebProcessRuntime>::new(), std::sync::Weak::<WebProcessRuntime>::new(),
[1; 32], [1; 32],
"192.0.2.10".parse().unwrap(), "192.0.2.10".parse().unwrap(),
1,
profile, profile,
[2; 32], [2; 32],
limits, limits,
+2
View File
@@ -342,6 +342,7 @@ mod tests {
secret_mode: WebSecretMode::Plain, secret_mode: WebSecretMode::Plain,
carrier: WebCarrier::Https, carrier: WebCarrier::Https,
capability: [0; 32], capability: [0; 32],
key_fingerprint: "0000000000000000".to_string(),
max_sessions: 1, max_sessions: 1,
max_streams: 1, max_streams: 1,
max_streams_per_session: 1, max_streams_per_session: 1,
@@ -350,6 +351,7 @@ mod tests {
std::sync::Weak::<WebProcessRuntime>::new(), std::sync::Weak::<WebProcessRuntime>::new(),
[1; 32], [1; 32],
"192.0.2.10".parse().unwrap(), "192.0.2.10".parse().unwrap(),
1,
profile, profile,
[2; 32], [2; 32],
WebLimitsConfig::default(), WebLimitsConfig::default(),
+487
View File
@@ -0,0 +1,487 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Instant;
use parking_lot::Mutex;
use zeroize::Zeroizing;
use super::sanitize::{
bounded_text, capture_limit, frame_error_name, frame_type_name, request_dynamic_bytes,
response_dynamic_bytes, sanitized_headers, scrub_body, sensitive_values,
};
use super::store::{WebTraceStore, epoch_millis};
use super::types::{
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceHttpRecord,
TraceIdentity, TraceRecord, TraceRecordKind, TraceRoute, TraceTimings,
};
use crate::config::{WebDebugBodyCapture, WebDebugConfig, WebLimitsConfig, WebRuntimeProfile};
use crate::web::frame::{self, FrameType};
const USER_AGENT_MAX_BYTES: usize = 512;
#[derive(Default)]
struct BodyCapture {
observed_bytes: u64,
captured: Vec<u8>,
truncated: bool,
state: Option<TraceBodyState>,
}
struct ExchangeState {
method: String,
path: String,
route: TraceRoute,
peer_ip: IpAddr,
effective_ip: Option<IpAddr>,
user_agent: Option<String>,
identity: TraceIdentity,
request_headers: Vec<TraceHeader>,
response_headers: Vec<TraceHeader>,
request_body: BodyCapture,
response_body: BodyCapture,
status: Option<u16>,
frames: Vec<TraceFrame>,
timings: TraceTimings,
redactions: Vec<Zeroizing<Vec<u8>>>,
body_capture_blocked: bool,
}
/// One in-flight request-to-response capture with process-wide byte leases.
pub(crate) struct HttpTraceExchange {
store: Arc<WebTraceStore>,
epoch: u64,
policy: Arc<WebDebugConfig>,
started: Instant,
started_epoch_millis: u64,
state: Mutex<ExchangeState>,
reserved: AtomicUsize,
committed: AtomicBool,
}
impl HttpTraceExchange {
/// Creates one enabled exchange after the store reserved its base record lease.
pub(super) fn new<B>(
store: Arc<WebTraceStore>,
epoch: u64,
policy: Arc<WebDebugConfig>,
request: &hyper::Request<B>,
peer_ip: IpAddr,
base_reservation: usize,
) -> Arc<Self> {
let started = Instant::now();
let started_epoch_millis = epoch_millis();
let dynamic = request_dynamic_bytes(request, &policy);
let dynamic_reserved = store.try_reserve(dynamic);
let (method, path, user_agent, request_headers, redactions) = if dynamic_reserved {
(
request.method().as_str().to_string(),
request.uri().path().to_string(),
request
.headers()
.get(hyper::header::USER_AGENT)
.map(|value| bounded_text(value.as_bytes(), USER_AGENT_MAX_BYTES)),
policy
.capture_headers
.then(|| sanitized_headers(request.headers()))
.unwrap_or_default(),
sensitive_values(request.headers(), request.uri().query()),
)
} else {
(
bounded_text(request.method().as_str().as_bytes(), 32),
bounded_text(request.uri().path().as_bytes(), 512),
None,
Vec::new(),
Vec::new(),
)
};
Arc::new(Self {
store,
epoch,
policy,
started,
started_epoch_millis,
state: Mutex::new(ExchangeState {
method,
path,
route: TraceRoute::Unknown,
peer_ip,
effective_ip: None,
user_agent,
identity: TraceIdentity::default(),
request_headers,
response_headers: Vec::new(),
request_body: BodyCapture::default(),
response_body: BodyCapture::default(),
status: None,
frames: Vec::new(),
timings: TraceTimings::default(),
redactions,
body_capture_blocked: !dynamic_reserved,
}),
reserved: AtomicUsize::new(
base_reservation + if dynamic_reserved { dynamic } else { 0 },
),
committed: AtomicBool::new(false),
})
}
/// Sets the final request route before body polling or decoy forwarding.
pub(crate) fn set_route(&self, route: TraceRoute) {
self.state.lock().route = route;
}
/// Sets the trusted effective client address after proxy-header validation.
pub(crate) fn set_effective_ip(&self, client_ip: IpAddr) {
self.state.lock().effective_ip = Some(client_ip);
}
/// Binds non-secret profile and process session identity.
pub(crate) fn bind_profile(&self, profile: &WebRuntimeProfile, session_id: u64) {
let dynamic = profile.user.len().saturating_add(profile.key_fingerprint.len());
let mut state = self.state.lock();
state.identity.session_id = Some(session_id);
if self.reserve(dynamic) {
state.identity.user = Some(profile.user.clone());
state.identity.key_fingerprint = Some(profile.key_fingerprint.clone());
}
}
/// Binds an already resolved non-secret session identity.
pub(crate) fn bind_identity(&self, identity: TraceIdentity) {
let dynamic = identity.user.as_ref().map_or(0, String::len).saturating_add(
identity
.key_fingerprint
.as_ref()
.map_or(0, String::len),
);
let mut state = self.state.lock();
state.identity.session_id = identity.session_id;
if self.reserve(dynamic) {
state.identity.user = identity.user;
state.identity.key_fingerprint = identity.key_fingerprint;
}
}
/// Registers an ephemeral credential for body scrubbing before commit.
pub(crate) fn register_redaction(&self, value: &[u8]) {
if value.is_empty() {
return;
}
if !self.reserve(value.len()) {
self.block_body_capture();
return;
}
self.state
.lock()
.redactions
.push(Zeroizing::new(value.to_vec()));
}
/// Captures response status and sanitized headers at handler completion.
pub(crate) fn response_ready<B>(&self, response: &hyper::Response<B>) {
let dynamic = response_dynamic_bytes(response, &self.policy);
let reserved = self.reserve(dynamic);
let mut state = self.state.lock();
state.status = Some(response.status().as_u16());
if self.policy.capture_headers && reserved {
state.response_headers = sanitized_headers(response.headers());
}
if reserved {
state
.redactions
.extend(sensitive_values(response.headers(), None));
} else if dynamic != 0 {
state.body_capture_blocked = true;
state.request_body.captured.clear();
state.response_body.captured.clear();
state.request_body.truncated = true;
state.response_body.truncated = true;
}
if self.policy.capture_timings {
state.timings.response_ready_us = Some(self.elapsed_us());
}
}
/// Appends one body data frame without changing the proxied bytes.
pub(crate) fn body_data(&self, direction: TraceDirection, data: &[u8]) {
let mut state = self.state.lock();
let route = state.route;
let body_capture_blocked = state.body_capture_blocked;
let body = match direction {
TraceDirection::Request => &mut state.request_body,
TraceDirection::Response => &mut state.response_body,
};
body.observed_bytes = body
.observed_bytes
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
if data.is_empty() {
return;
}
if body_capture_blocked {
body.truncated |= !data.is_empty();
return;
}
let Some(limit) = capture_limit(
&self.policy,
route,
self.store.max_carrier_body_bytes(),
) else {
return;
};
if body.captured.len() >= limit {
if !data.is_empty() && !body.truncated {
self.store.record_truncation();
}
body.truncated |= !data.is_empty();
return;
}
if body.captured.capacity() == 0 {
if !self.reserve(limit) {
body.truncated = true;
return;
}
body.captured = Vec::with_capacity(limit);
}
let take = data.len().min(limit - body.captured.len());
body.captured.extend_from_slice(&data[..take]);
if take < data.len() {
if !body.truncated {
self.store.record_truncation();
}
body.truncated = true;
}
}
/// Marks one request or response body terminal state.
pub(crate) fn body_finished(&self, direction: TraceDirection, terminal: TraceBodyState) {
let mut state = self.state.lock();
let body = match direction {
TraceDirection::Request => &mut state.request_body,
TraceDirection::Response => &mut state.response_body,
};
if body.state.is_none() {
body.state = Some(terminal);
}
if self.policy.capture_timings {
match direction {
TraceDirection::Request => state.timings.request_body_us = Some(self.elapsed_us()),
TraceDirection::Response => state.timings.response_body_us = Some(self.elapsed_us()),
}
}
drop(state);
if direction == TraceDirection::Response {
self.commit();
}
}
/// Attaches parsed carrier frame metadata from one complete bounded body.
pub(crate) fn record_frames(
&self,
direction: TraceDirection,
body: &[u8],
limits: &WebLimitsConfig,
) {
if !self.policy.capture_frames {
return;
}
let estimated_frames = body
.len()
.div_ceil(frame::HEADER_BYTES)
.clamp(1, limits.max_frames_per_body);
let reservation = estimated_frames.saturating_mul(std::mem::size_of::<TraceFrame>());
if !self.reserve(reservation) {
return;
}
let frames = match frame::parse_all(body, limits) {
Ok(frames) => frames
.into_iter()
.map(|frame| TraceFrame {
direction,
frame_type: Some(frame_type_name(frame.frame_type)),
stream_id: Some(frame.stream_id),
payload_len: Some(frame.payload.len()),
window_delta: (frame.frame_type == FrameType::Window)
.then(|| frame::window_amount(frame.payload).ok())
.flatten(),
parse_error: None,
})
.collect::<Vec<_>>(),
Err(error) => vec![TraceFrame {
direction,
frame_type: None,
stream_id: None,
payload_len: None,
window_delta: None,
parse_error: Some(frame_error_name(error)),
}],
};
self.state.lock().frames.extend(frames);
}
/// Commits once after response body consumption or drop.
pub(crate) fn commit(&self) {
if self.committed.swap(true, Ordering::AcqRel) {
return;
}
let reserved = self.reserved.load(Ordering::Acquire);
let record = self.build_record();
if !self.store.try_commit(record, reserved, self.epoch) {
self.store.release(reserved);
}
}
fn reserve(&self, bytes: usize) -> bool {
if bytes == 0 {
return true;
}
if self.store.try_reserve(bytes) {
self.reserved.fetch_add(bytes, Ordering::AcqRel);
true
} else {
false
}
}
fn block_body_capture(&self) {
let mut state = self.state.lock();
state.body_capture_blocked = true;
state.request_body.captured.clear();
state.response_body.captured.clear();
state.request_body.truncated = true;
state.response_body.truncated = true;
}
fn build_record(&self) -> TraceRecord {
let mut state = self.state.lock();
let redactions = std::mem::take(&mut state.redactions);
scrub_body(&mut state.request_body.captured, &redactions);
scrub_body(&mut state.response_body.captured, &redactions);
let request_body = body_snapshot(&self.policy, &mut state.request_body);
let response_body = body_snapshot(&self.policy, &mut state.response_body);
TraceRecord {
seq: self.store.next_record_seq(),
epoch_millis: self.started_epoch_millis,
peer_ip: Some(state.peer_ip),
effective_ip: state.effective_ip,
user_agent: state.user_agent.take(),
identity: std::mem::take(&mut state.identity),
kind: TraceRecordKind::Http(TraceHttpRecord {
method: std::mem::take(&mut state.method),
path: std::mem::take(&mut state.path),
route: state.route,
request_headers: std::mem::take(&mut state.request_headers),
response_headers: std::mem::take(&mut state.response_headers),
request_body,
status: state.status,
response_body,
frames: std::mem::take(&mut state.frames),
timings: self
.policy
.capture_timings
.then(|| std::mem::take(&mut state.timings)),
}),
}
}
fn elapsed_us(&self) -> u64 {
self.started
.elapsed()
.as_micros()
.min(u128::from(u64::MAX)) as u64
}
}
impl Drop for HttpTraceExchange {
fn drop(&mut self) {
if !self.committed.load(Ordering::Acquire) {
self.body_finished(TraceDirection::Request, TraceBodyState::Aborted);
self.body_finished(TraceDirection::Response, TraceBodyState::Aborted);
}
}
}
fn body_snapshot(
policy: &WebDebugConfig,
body: &mut BodyCapture,
) -> Option<TraceBodySnapshot> {
(policy.body_capture != WebDebugBodyCapture::Off).then(|| TraceBodySnapshot {
observed_bytes: body.observed_bytes,
captured: std::mem::take(&mut body.captured),
truncated: body.truncated,
state: body.state.unwrap_or(TraceBodyState::Aborted),
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn request_response_capture_redacts_credentials_and_omits_query() {
let request_token = "request-token-0123456789";
let capability = "capability-0123456789";
let response_token = "response-token-0123456789";
let request = hyper::Request::builder()
.uri(format!("/?bridge={capability}"))
.header("authorization", format!("Bearer {request_token}"))
.body(())
.unwrap();
let mut policy = WebDebugConfig::default();
policy.enabled = true;
policy.body_capture = WebDebugBodyCapture::Prefix;
policy.body_prefix_bytes = 256;
let mut limits = WebLimitsConfig::default();
limits.debug_records_capacity = 4;
limits.debug_bytes_global = 16 * 1024;
let store = WebTraceStore::new(policy, &limits);
let exchange = store
.begin_http(&request, "192.0.2.30".parse().unwrap())
.unwrap();
exchange.set_route(TraceRoute::Bridge);
exchange.body_data(
TraceDirection::Request,
format!("{request_token}:{capability}").as_bytes(),
);
exchange.body_finished(TraceDirection::Request, TraceBodyState::Complete);
let response = hyper::Response::builder()
.status(hyper::StatusCode::OK)
.header("x-session-token", response_token)
.body(())
.unwrap();
exchange.response_ready(&response);
exchange.body_data(TraceDirection::Response, response_token.as_bytes());
exchange.body_finished(TraceDirection::Response, TraceBodyState::Complete);
let records = store.snapshot_matching(|_| true);
assert_eq!(records.len(), 1);
let TraceRecordKind::Http(http) = &records[0].record.kind else {
panic!("expected HTTP debug record");
};
assert_eq!(http.path, "/");
assert_eq!(http.route, TraceRoute::Bridge);
assert!(
http.request_headers
.iter()
.any(|header| header.name == "authorization" && header.value.is_none())
);
assert!(
http.response_headers
.iter()
.any(|header| header.name == "x-session-token" && header.value.is_none())
);
let request_body = http.request_body.as_ref().unwrap();
let response_body = http.response_body.as_ref().unwrap();
for secret in [request_token.as_bytes(), capability.as_bytes()] {
assert!(!request_body.captured.windows(secret.len()).any(|value| value == secret));
}
assert!(
!response_body
.captured
.windows(response_token.len())
.any(|value| value == response_token.as_bytes())
);
}
}
+17
View File
@@ -0,0 +1,17 @@
//! Process-owned bounded WEB request, frame, and lifecycle debug recording.
// HTTP exchange ownership binds byte leases to one request and response.
mod exchange;
// Header, credential, body, and frame sanitization stays independent from storage.
mod sanitize;
// The process-wide store owns policy epochs, ring eviction, and render admission.
mod store;
// Closed record types define the status-page data contract.
mod types;
pub(crate) use exchange::HttpTraceExchange;
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
pub(crate) use types::{
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceHeader, TraceIdentity,
TraceLifecycleEvent, TraceRecord, TraceRecordKind, TraceRoute,
};
+297
View File
@@ -0,0 +1,297 @@
use hyper::header;
use zeroize::Zeroizing;
use super::types::{TraceHeader, TraceRoute};
use crate::config::{WebDebugBodyCapture, WebDebugConfig};
use crate::web::frame::{FrameError, FrameType};
const USER_AGENT_MAX_BYTES: usize = 512;
const MIN_REDACTION_BYTES: usize = 8;
const MAX_REDACTION_BYTES: usize = 512;
const MAX_HEADER_REDACTIONS: usize = 16;
/// Calculates a conservative request metadata and credential lease.
pub(super) fn request_dynamic_bytes<B>(
request: &hyper::Request<B>,
policy: &WebDebugConfig,
) -> usize {
request
.method()
.as_str()
.len()
.saturating_add(request.uri().path().len())
.saturating_add(
request
.headers()
.get(header::USER_AGENT)
.map_or(0, |value| lossy_text_reservation(value.as_bytes(), USER_AGENT_MAX_BYTES)),
)
.saturating_add(
policy
.capture_headers
.then(|| sanitized_header_bytes(request.headers()))
.unwrap_or(0),
)
.saturating_add(sensitive_value_bytes(
request.headers(),
request.uri().query(),
))
}
/// Calculates a conservative response metadata and credential lease.
pub(super) fn response_dynamic_bytes<B>(
response: &hyper::Response<B>,
policy: &WebDebugConfig,
) -> usize {
policy
.capture_headers
.then(|| sanitized_header_bytes(response.headers()))
.unwrap_or(0)
.saturating_add(sensitive_value_bytes(response.headers(), None))
}
/// Copies header names and only allowlisted bounded values.
pub(super) fn sanitized_headers(headers: &hyper::HeaderMap) -> Vec<TraceHeader> {
headers
.iter()
.map(|(name, value)| TraceHeader {
name: name.as_str().to_string(),
value: header_value_allowed(name)
.then(|| bounded_text(value.as_bytes(), 4096)),
})
.collect()
}
/// Copies a bounded set of ephemeral credentials into zeroizing scrub patterns.
pub(super) fn sensitive_values(
headers: &hyper::HeaderMap,
query: Option<&str>,
) -> Vec<Zeroizing<Vec<u8>>> {
let mut values = Vec::new();
for name in [
"authorization",
"proxy-authorization",
"x-session-token",
"sec-websocket-protocol",
] {
for value in headers.get_all(name) {
push_redaction(&mut values, value.as_bytes());
if let Some(token) = value.as_bytes().strip_prefix(b"Bearer ") {
push_redaction(&mut values, token);
}
if values.len() >= MAX_HEADER_REDACTIONS {
break;
}
}
if values.len() >= MAX_HEADER_REDACTIONS {
break;
}
}
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge=")) {
push_redaction(&mut values, capability.as_bytes());
}
values
}
/// Converts at most `limit` raw bytes into loss-tolerant display text.
pub(super) fn bounded_text(value: &[u8], limit: usize) -> String {
String::from_utf8_lossy(&value[..value.len().min(limit)]).into_owned()
}
/// Overwrites complete credentials and a prefix cut by body truncation.
pub(super) fn scrub_body(body: &mut [u8], redactions: &[Zeroizing<Vec<u8>>]) {
for secret in redactions.iter().filter(|secret| !secret.is_empty()) {
if secret.len() <= body.len() {
for offset in 0..=body.len() - secret.len() {
if body[offset..].starts_with(secret) {
body[offset..offset + secret.len()].fill(b'*');
}
}
}
let overlap = secret.len().min(body.len());
for length in (1..=overlap).rev() {
if body.ends_with(&secret[..length]) {
let start = body.len() - length;
body[start..].fill(b'*');
break;
}
}
}
}
/// Resolves the route-sensitive retained body allocation ceiling.
pub(super) fn capture_limit(
policy: &WebDebugConfig,
route: TraceRoute,
max_carrier_body_bytes: usize,
) -> Option<usize> {
match policy.body_capture {
WebDebugBodyCapture::Off | WebDebugBodyCapture::Metadata => None,
WebDebugBodyCapture::Prefix => Some(if decoy_route(route) {
policy.decoy_body_prefix_bytes
} else {
policy.body_prefix_bytes
}),
WebDebugBodyCapture::Full => Some(if decoy_route(route) {
policy.decoy_body_prefix_bytes
} else {
max_carrier_body_bytes
}),
}
}
/// Returns a closed display label for one parsed frame type.
pub(super) fn frame_type_name(frame_type: FrameType) -> &'static str {
match frame_type {
FrameType::Open => "OPEN",
FrameType::Data => "DATA",
FrameType::Close => "CLOSE",
FrameType::Window => "WINDOW",
FrameType::Ping => "PING",
FrameType::Pong => "PONG",
FrameType::Hello => "HELLO",
FrameType::Welcome => "WELCOME",
FrameType::Bye => "BYE",
}
}
/// Returns a closed display label for one frame parse failure.
pub(super) fn frame_error_name(error: FrameError) -> &'static str {
match error {
FrameError::EmptyBatch => "empty_batch",
FrameError::TooManyFrames => "too_many_frames",
FrameError::Incomplete => "incomplete",
FrameError::PayloadLimit => "payload_limit",
FrameError::UnknownType => "unknown_type",
FrameError::InvalidShape => "invalid_shape",
}
}
fn sanitized_header_bytes(headers: &hyper::HeaderMap) -> usize {
headers.iter().fold(0usize, |total, (name, value)| {
total
.saturating_add(std::mem::size_of::<TraceHeader>())
.saturating_add(name.as_str().len())
.saturating_add(
header_value_allowed(name)
.then(|| lossy_text_reservation(value.as_bytes(), 4096))
.unwrap_or(0),
)
})
}
fn header_value_allowed(name: &hyper::header::HeaderName) -> bool {
matches!(
name.as_str(),
"host"
| "user-agent"
| "content-type"
| "content-length"
| "origin"
| "x-forwarded-for"
| "x-up-seq"
| "x-up-ack"
| "x-down-cursor"
| "x-lane-id"
| "x-lane-closed"
| "x-carrier-mode"
| "retry-after"
| "cache-control"
| "etag"
| "if-none-match"
| "accept"
| "accept-encoding"
)
}
fn sensitive_value_bytes(headers: &hyper::HeaderMap, query: Option<&str>) -> usize {
let mut total = 0usize;
let mut retained = 0usize;
for name in [
"authorization",
"proxy-authorization",
"x-session-token",
"sec-websocket-protocol",
] {
for value in headers.get_all(name) {
for candidate in [
Some(value.as_bytes()),
value.as_bytes().strip_prefix(b"Bearer "),
]
.into_iter()
.flatten()
{
if (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&candidate.len()) {
total = total
.saturating_add(std::mem::size_of::<Vec<u8>>())
.saturating_add(candidate.len());
retained += 1;
if retained >= MAX_HEADER_REDACTIONS {
break;
}
}
}
if retained >= MAX_HEADER_REDACTIONS {
break;
}
}
if retained >= MAX_HEADER_REDACTIONS {
break;
}
}
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge="))
&& (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&capability.len())
{
total = total
.saturating_add(std::mem::size_of::<Vec<u8>>())
.saturating_add(capability.len());
}
total
}
fn lossy_text_reservation(value: &[u8], limit: usize) -> usize {
value.len().min(limit).saturating_mul(3)
}
fn decoy_route(route: TraceRoute) -> bool {
matches!(route, TraceRoute::Unknown | TraceRoute::Decoy)
}
fn push_redaction(values: &mut Vec<Zeroizing<Vec<u8>>>, value: &[u8]) {
if value.len() < MIN_REDACTION_BYTES || value.len() > MAX_REDACTION_BYTES {
return;
}
if !values.iter().any(|existing| existing.as_slice() == value) {
values.push(Zeroizing::new(value.to_vec()));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn full_capture_keeps_decoy_bodies_prefix_bounded() {
let mut policy = WebDebugConfig::default();
policy.body_capture = WebDebugBodyCapture::Full;
policy.decoy_body_prefix_bytes = 123;
assert_eq!(capture_limit(&policy, TraceRoute::Decoy, 4096), Some(123));
assert_eq!(capture_limit(&policy, TraceRoute::Uplink, 4096), Some(4096));
}
#[test]
fn scrub_removes_complete_and_prefix_truncated_credentials() {
let secret = Zeroizing::new(b"credential-value".to_vec());
let mut complete = b"before credential-value after".to_vec();
scrub_body(&mut complete, std::slice::from_ref(&secret));
assert!(
!complete
.windows(secret.len())
.any(|value| value == secret.as_slice())
);
let mut truncated = b"before credent".to_vec();
scrub_body(&mut truncated, &[secret]);
assert!(truncated.ends_with(b"*******"));
}
}
+442
View File
@@ -0,0 +1,442 @@
use std::collections::VecDeque;
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use arc_swap::ArcSwap;
use parking_lot::Mutex;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use super::exchange::HttpTraceExchange;
use super::types::{
TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind,
};
use crate::config::{WebDebugConfig, WebLimitsConfig};
const BASE_RECORD_RESERVATION: usize = 1024;
struct RingState {
records: VecDeque<Arc<StoredTraceRecord>>,
}
/// One retained record whose lease survives status-page snapshots.
pub(crate) struct StoredTraceRecord {
/// Immutable trace record.
pub(crate) record: TraceRecord,
bytes: usize,
used_bytes: Arc<AtomicUsize>,
}
impl Drop for StoredTraceRecord {
fn drop(&mut self) {
self.used_bytes.fetch_sub(self.bytes, Ordering::AcqRel);
}
}
/// Point-in-time process-owned trace counters and ring bounds.
pub(crate) struct TraceStoreStatus {
/// Current debug policy.
pub(crate) policy: Arc<WebDebugConfig>,
/// Retained record count.
pub(crate) records: usize,
/// Configured record capacity.
pub(crate) records_capacity: usize,
/// Retained plus in-flight byte leases.
pub(crate) used_bytes: usize,
/// Configured byte capacity.
pub(crate) bytes_capacity: usize,
/// Records dropped on commit-lock contention.
pub(crate) contention_drops: u64,
/// Oldest records evicted by ring capacity.
pub(crate) evictions: u64,
/// Body or metadata captures truncated by policy or byte capacity.
pub(crate) byte_truncations: u64,
/// Earliest retained record sequence.
pub(crate) earliest_seq: Option<u64>,
/// Latest retained record sequence.
pub(crate) latest_seq: Option<u64>,
}
/// Process-owned bounded WEB debug trace store.
pub(crate) struct WebTraceStore {
policy: ArcSwap<WebDebugConfig>,
policy_update: Mutex<()>,
enabled: AtomicBool,
epoch: AtomicU64,
records_capacity: usize,
bytes_capacity: usize,
max_carrier_body_bytes: usize,
used_bytes: Arc<AtomicUsize>,
ring: Mutex<RingState>,
next_record_seq: AtomicU64,
next_session_id: AtomicU64,
contention_drops: AtomicU64,
evictions: AtomicU64,
byte_truncations: AtomicU64,
renders: Arc<Semaphore>,
}
impl WebTraceStore {
/// Creates a process-owned store from restart-only capacity and initial policy.
pub(crate) fn new(policy: WebDebugConfig, limits: &WebLimitsConfig) -> Arc<Self> {
Arc::new(Self {
enabled: AtomicBool::new(policy.enabled),
policy: ArcSwap::from_pointee(policy),
policy_update: Mutex::new(()),
epoch: AtomicU64::new(1),
records_capacity: limits.debug_records_capacity,
bytes_capacity: limits.debug_bytes_global,
max_carrier_body_bytes: limits.max_body_bytes,
used_bytes: Arc::new(AtomicUsize::new(0)),
ring: Mutex::new(RingState {
records: VecDeque::with_capacity(limits.debug_records_capacity),
}),
next_record_seq: AtomicU64::new(1),
next_session_id: AtomicU64::new(1),
contention_drops: AtomicU64::new(0),
evictions: AtomicU64::new(0),
byte_truncations: AtomicU64::new(0),
renders: Arc::new(Semaphore::new(2)),
})
}
/// Applies one hot policy and clears incompatible retained records.
pub(crate) fn apply_policy(&self, policy: &WebDebugConfig) {
let _policy_update = self.policy_update.lock();
let current = self.policy.load_full();
if current.as_ref() == policy {
return;
}
let capture_changed = current.enabled != policy.enabled
|| current.capture_lifecycle != policy.capture_lifecycle
|| current.capture_headers != policy.capture_headers
|| current.capture_timings != policy.capture_timings
|| current.capture_frames != policy.capture_frames
|| current.body_capture != policy.body_capture
|| current.body_prefix_bytes != policy.body_prefix_bytes
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
self.policy.store(Arc::new(policy.clone()));
self.enabled.store(policy.enabled, Ordering::Release);
if capture_changed {
self.epoch.fetch_add(1, Ordering::AcqRel);
self.ring.lock().records.clear();
}
}
/// Allocates a process-unique monotonic WEB session trace identifier.
pub(crate) fn next_session_id(&self) -> u64 {
self.next_session_id.fetch_add(1, Ordering::Relaxed)
}
/// Starts one HTTP exchange only when debugging is enabled and budgeted.
pub(crate) fn begin_http<B>(
self: &Arc<Self>,
request: &hyper::Request<B>,
peer_ip: IpAddr,
) -> Option<Arc<HttpTraceExchange>> {
if !self.enabled.load(Ordering::Acquire) {
return None;
}
let epoch = self.epoch.load(Ordering::Acquire);
let policy = self.policy.load_full();
if !policy.enabled || !self.try_reserve_record(BASE_RECORD_RESERVATION) {
return None;
}
Some(HttpTraceExchange::new(
Arc::clone(self),
epoch,
policy,
request,
peer_ip,
BASE_RECORD_RESERVATION,
))
}
/// Records one typed lifecycle event without retaining dynamic error strings.
pub(crate) fn record_lifecycle(
&self,
peer_ip: Option<IpAddr>,
effective_ip: Option<IpAddr>,
identity: TraceIdentity,
event: TraceLifecycleEvent,
stream_id: Option<u32>,
reason: Option<&'static str>,
) {
if !self.enabled.load(Ordering::Acquire) {
return;
}
let epoch = self.epoch.load(Ordering::Acquire);
let policy = self.policy.load_full();
let identity_bytes = identity
.user
.as_ref()
.map_or(0, String::len)
.checked_add(
identity
.key_fingerprint
.as_ref()
.map_or(0, String::len),
);
let Some(reservation) = identity_bytes
.and_then(|bytes| BASE_RECORD_RESERVATION.checked_add(bytes))
else {
self.record_truncation();
return;
};
if !policy.enabled
|| !policy.capture_lifecycle
|| !self.try_reserve_record(reservation)
{
return;
}
let record = TraceRecord {
seq: self.next_record_seq(),
epoch_millis: epoch_millis(),
peer_ip,
effective_ip,
user_agent: None,
identity,
kind: TraceRecordKind::Lifecycle(TraceLifecycleRecord {
event,
stream_id,
reason,
}),
};
if !self.try_commit(record, reservation, epoch) {
self.release(reservation);
}
}
/// Records lifecycle identity from a profile only after enabled-policy checks.
pub(crate) fn record_profile_lifecycle(
&self,
effective_ip: IpAddr,
session_id: Option<u64>,
profile: &crate::config::WebRuntimeProfile,
event: TraceLifecycleEvent,
stream_id: Option<u32>,
reason: Option<&'static str>,
) {
if !self.enabled.load(Ordering::Acquire) || !self.policy.load().capture_lifecycle {
return;
}
self.record_lifecycle(
None,
Some(effective_ip),
TraceIdentity::from_optional_profile(session_id, profile),
event,
stream_id,
reason,
);
}
/// Returns records matching the supplied predicate in newest-first order.
pub(crate) fn snapshot_matching<F>(&self, mut matches: F) -> Vec<Arc<StoredTraceRecord>>
where
F: FnMut(&TraceRecord) -> bool,
{
self.ring
.lock()
.records
.iter()
.rev()
.filter(|record| matches(&record.record))
.cloned()
.collect()
}
/// Returns current bounds, counters, and retained sequence range.
pub(crate) fn status(&self) -> TraceStoreStatus {
let ring = self.ring.lock();
TraceStoreStatus {
policy: self.policy.load_full(),
records: ring.records.len(),
records_capacity: self.records_capacity,
used_bytes: self.used_bytes.load(Ordering::Acquire),
bytes_capacity: self.bytes_capacity,
contention_drops: self.contention_drops.load(Ordering::Relaxed),
evictions: self.evictions.load(Ordering::Relaxed),
byte_truncations: self.byte_truncations.load(Ordering::Relaxed),
earliest_seq: ring.records.front().map(|record| record.record.seq),
latest_seq: ring.records.back().map(|record| record.record.seq),
}
}
/// Reserves one of two bounded concurrent status-page render slots.
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.renders).try_acquire_owned().ok()
}
/// Returns the current capture-policy epoch.
pub(super) fn policy_epoch(&self) -> u64 {
self.epoch.load(Ordering::Acquire)
}
/// Returns the restart-frozen recognized carrier body ceiling.
pub(super) fn max_carrier_body_bytes(&self) -> usize {
self.max_carrier_body_bytes
}
/// Atomically reserves debug bytes or records one truncation.
pub(super) fn try_reserve(&self, bytes: usize) -> bool {
if self.try_reserve_inner(bytes) {
true
} else {
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
false
}
}
fn try_reserve_inner(&self, bytes: usize) -> bool {
let mut current = self.used_bytes.load(Ordering::Acquire);
loop {
let Some(next) = current.checked_add(bytes) else {
return false;
};
if next > self.bytes_capacity {
return false;
}
match self.used_bytes.compare_exchange_weak(
current,
next,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return true,
Err(actual) => current = actual,
}
}
}
fn try_reserve_record(&self, bytes: usize) -> bool {
if self.try_reserve_inner(bytes) {
return true;
}
let Some(mut ring) = self.ring.try_lock() else {
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
return false;
};
while let Some(record) = ring.records.pop_front() {
self.evictions.fetch_add(1, Ordering::Relaxed);
drop(record);
if self.try_reserve_inner(bytes) {
return true;
}
}
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
false
}
/// Releases an in-flight lease that was not transferred into a record.
pub(super) fn release(&self, bytes: usize) {
self.used_bytes.fetch_sub(bytes, Ordering::AcqRel);
}
/// Increments the closed truncation counter.
pub(super) fn record_truncation(&self) {
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
}
/// Allocates a process-monotonic record sequence number.
pub(super) fn next_record_seq(&self) -> u64 {
self.next_record_seq.fetch_add(1, Ordering::Relaxed)
}
/// Attempts one non-blocking ring commit under the originating policy epoch.
pub(super) fn try_commit(&self, record: TraceRecord, bytes: usize, epoch: u64) -> bool {
if epoch != self.policy_epoch() {
return false;
}
let Some(mut ring) = self.ring.try_lock() else {
self.contention_drops.fetch_add(1, Ordering::Relaxed);
return false;
};
if epoch != self.policy_epoch() {
return false;
}
while ring.records.len() >= self.records_capacity {
ring.records.pop_front();
self.evictions.fetch_add(1, Ordering::Relaxed);
}
ring.records.push_back(Arc::new(StoredTraceRecord {
record,
bytes,
used_bytes: Arc::clone(&self.used_bytes),
}));
true
}
}
/// Returns Unix epoch milliseconds with saturation for stored display timestamps.
pub(crate) fn epoch_millis() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.min(u128::from(u64::MAX)) as u64
}
#[cfg(test)]
mod tests {
use super::*;
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
let mut policy = WebDebugConfig::default();
policy.enabled = true;
let mut limits = WebLimitsConfig::default();
limits.debug_records_capacity = records_capacity;
limits.debug_bytes_global = bytes_capacity;
WebTraceStore::new(policy, &limits)
}
#[test]
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
let store = store(2, 4 * BASE_RECORD_RESERVATION);
for _ in 0..3 {
store.record_lifecycle(
None,
Some("192.0.2.10".parse().unwrap()),
TraceIdentity::default(),
TraceLifecycleEvent::BridgeIssued,
None,
None,
);
}
let snapshot = store.snapshot_matching(|_| true);
assert_eq!(
snapshot
.iter()
.map(|record| record.record.seq)
.collect::<Vec<_>>(),
vec![3, 2]
);
assert_eq!(store.status().evictions, 1);
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
let policy = WebDebugConfig::default();
store.apply_policy(&policy);
assert_eq!(store.status().records, 0);
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
drop(snapshot);
assert_eq!(store.status().used_bytes, 0);
}
#[test]
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
let store = store(4, 8 * BASE_RECORD_RESERVATION);
let request = hyper::Request::builder().uri("/").body(()).unwrap();
let exchange = store
.begin_http(&request, "192.0.2.20".parse().unwrap())
.unwrap();
let mut changed = WebDebugConfig::default();
changed.enabled = true;
changed.capture_headers = false;
store.apply_policy(&changed);
exchange.commit();
assert_eq!(store.status().records, 0);
assert_eq!(store.status().used_bytes, 0);
}
}
+281
View File
@@ -0,0 +1,281 @@
use std::net::IpAddr;
/// WEB HTTP route classification retained by the debug trace.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum TraceRoute {
/// Routing has not completed yet.
Unknown,
/// Ordinary static or upstream fallback traffic.
Decoy,
/// Authenticated bridge page issuance.
Bridge,
/// Bootstrap-to-session exchange.
Session,
/// Carrier uplink exchange.
Uplink,
/// Carrier downlink exchange.
Downlink,
}
impl TraceRoute {
/// Returns the stable status-page label.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::Decoy => "decoy",
Self::Bridge => "bridge",
Self::Session => "session",
Self::Uplink => "uplink",
Self::Downlink => "downlink",
}
}
}
/// Direction of captured HTTP body or carrier frame data.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum TraceDirection {
/// Client-to-server data.
Request,
/// Server-to-client data.
Response,
}
impl TraceDirection {
/// Returns the stable status-page label.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Request => "request",
Self::Response => "response",
}
}
}
/// Terminal observation state of one HTTP body.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum TraceBodyState {
/// The body completed normally.
Complete,
/// Body polling returned an error.
Error,
/// The body was dropped before a terminal poll.
Aborted,
}
impl TraceBodyState {
/// Returns the stable status-page label.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Complete => "complete",
Self::Error => "error",
Self::Aborted => "aborted",
}
}
}
/// One sanitized HTTP header.
#[derive(Debug)]
pub(crate) struct TraceHeader {
/// Lowercase header name.
pub(crate) name: String,
/// Allowlisted value, or `None` when only the name may be retained.
pub(crate) value: Option<String>,
}
/// One bounded HTTP body observation.
#[derive(Debug)]
pub(crate) struct TraceBodySnapshot {
/// Total bytes observed on the wire-facing Hyper body.
pub(crate) observed_bytes: u64,
/// Retained prefix or full bounded body bytes.
pub(crate) captured: Vec<u8>,
/// Indicates that bytes were omitted by policy or capacity.
pub(crate) truncated: bool,
/// Terminal body state.
pub(crate) state: TraceBodyState,
}
/// One parsed carrier frame without payload retention.
#[derive(Debug)]
pub(crate) struct TraceFrame {
/// HTTP body direction containing the frame.
pub(crate) direction: TraceDirection,
/// Stable protocol frame type, when the header parsed.
pub(crate) frame_type: Option<&'static str>,
/// Logical stream or lane identifier.
pub(crate) stream_id: Option<u32>,
/// Frame payload length.
pub(crate) payload_len: Option<usize>,
/// Decoded non-zero WINDOW delta.
pub(crate) window_delta: Option<u32>,
/// Closed parse or shape error category.
pub(crate) parse_error: Option<&'static str>,
}
/// Request lifecycle timing points measured from service entry.
#[derive(Debug, Default)]
pub(crate) struct TraceTimings {
/// Request body terminal poll time.
pub(crate) request_body_us: Option<u64>,
/// Handler response-ready time.
pub(crate) response_ready_us: Option<u64>,
/// Response body terminal poll or drop time.
pub(crate) response_body_us: Option<u64>,
}
/// Stable non-secret WEB trace identity.
#[derive(Clone, Debug, Default)]
pub(crate) struct TraceIdentity {
/// Process-unique monotonic WEB session identifier.
pub(crate) session_id: Option<u64>,
/// Exact access user name.
pub(crate) user: Option<String>,
/// Domain-separated client-secret fingerprint.
pub(crate) key_fingerprint: Option<String>,
}
impl TraceIdentity {
/// Builds a non-secret identity from one validated runtime profile.
pub(crate) fn from_profile(
session_id: u64,
profile: &crate::config::WebRuntimeProfile,
) -> Self {
Self::from_optional_profile(Some(session_id), profile)
}
/// Builds a profile identity when admission failed before session allocation.
pub(crate) fn from_optional_profile(
session_id: Option<u64>,
profile: &crate::config::WebRuntimeProfile,
) -> Self {
Self {
session_id,
user: Some(profile.user.clone()),
key_fingerprint: Some(profile.key_fingerprint.clone()),
}
}
}
/// Complete request-to-response WEB HTTP trace.
#[derive(Debug)]
pub(crate) struct TraceHttpRecord {
/// HTTP method.
pub(crate) method: String,
/// URI path without query material.
pub(crate) path: String,
/// Final route classification.
pub(crate) route: TraceRoute,
/// Sanitized request headers.
pub(crate) request_headers: Vec<TraceHeader>,
/// Sanitized response headers.
pub(crate) response_headers: Vec<TraceHeader>,
/// Request body observation when enabled by policy.
pub(crate) request_body: Option<TraceBodySnapshot>,
/// Response status when a response head was produced.
pub(crate) status: Option<u16>,
/// Response body observation when enabled by policy.
pub(crate) response_body: Option<TraceBodySnapshot>,
/// Parsed carrier frame metadata.
pub(crate) frames: Vec<TraceFrame>,
/// Monotonic request timing points.
pub(crate) timings: Option<TraceTimings>,
}
/// Closed WEB lifecycle event category.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum TraceLifecycleEvent {
/// A bridge bootstrap was issued.
BridgeIssued,
/// Bootstrap or bridge admission was rejected.
BootstrapRejected,
/// A new session was created.
SessionCreated,
/// An idempotent session creation was replayed.
SessionReplayed,
/// Session creation was rejected.
SessionRejected,
/// A session closed.
SessionClosed,
/// A logical stream was admitted.
StreamAdmitted,
/// A logical stream was rejected.
StreamRejected,
/// A logical stream delivered its first inner byte.
StreamFirstByte,
/// An admitted logical stream released its relay and tuple ownership.
StreamClosed,
/// The inner MTProxy handshake succeeded.
HandshakeSucceeded,
/// The inner MTProxy handshake timed out.
HandshakeTimeout,
/// The inner MTProxy handshake failed on I/O.
HandshakeIo,
/// The inner MTProxy handshake was rejected.
HandshakeRejected,
/// Authenticated relay started.
RelayStarted,
/// Authenticated relay ended.
RelayEnded,
}
impl TraceLifecycleEvent {
/// Returns the stable status-page label.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::BridgeIssued => "bridge_issued",
Self::BootstrapRejected => "bootstrap_rejected",
Self::SessionCreated => "session_created",
Self::SessionReplayed => "session_replayed",
Self::SessionRejected => "session_rejected",
Self::SessionClosed => "session_closed",
Self::StreamAdmitted => "stream_admitted",
Self::StreamRejected => "stream_rejected",
Self::StreamFirstByte => "stream_first_byte",
Self::StreamClosed => "stream_closed",
Self::HandshakeSucceeded => "handshake_succeeded",
Self::HandshakeTimeout => "handshake_timeout",
Self::HandshakeIo => "handshake_io",
Self::HandshakeRejected => "handshake_rejected",
Self::RelayStarted => "relay_started",
Self::RelayEnded => "relay_ended",
}
}
}
/// One typed WEB lifecycle observation.
#[derive(Debug)]
pub(crate) struct TraceLifecycleRecord {
/// Closed event category.
pub(crate) event: TraceLifecycleEvent,
/// Logical stream identifier when applicable.
pub(crate) stream_id: Option<u32>,
/// Closed outcome or rejection reason.
pub(crate) reason: Option<&'static str>,
}
/// Trace record payload variant.
#[derive(Debug)]
pub(crate) enum TraceRecordKind {
/// HTTP request-to-response exchange.
Http(TraceHttpRecord),
/// Session or stream lifecycle event.
Lifecycle(TraceLifecycleRecord),
}
/// Common stored WEB trace record.
#[derive(Debug)]
pub(crate) struct TraceRecord {
/// Process-unique monotonic record sequence.
pub(crate) seq: u64,
/// Event completion time in Unix epoch milliseconds.
pub(crate) epoch_millis: u64,
/// Direct TCP peer accepted by the WEB listener.
pub(crate) peer_ip: Option<IpAddr>,
/// Trusted effective client identity.
pub(crate) effective_ip: Option<IpAddr>,
/// Bounded user-agent value retained for filtering.
pub(crate) user_agent: Option<String>,
/// Non-secret WEB identity.
pub(crate) identity: TraceIdentity,
/// Typed record payload.
pub(crate) kind: TraceRecordKind,
}