mirror of
https://github.com/telemt/telemt.git
synced 2026-09-05 18:16:06 +03:00
WEB Debug + Trace
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
+15
-2
@@ -30,6 +30,7 @@ use crate::startup::StartupTracker;
|
|||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
mod config_edit;
|
mod config_edit;
|
||||||
pub(crate) mod config_store;
|
pub(crate) mod config_store;
|
||||||
@@ -47,6 +48,7 @@ mod runtime_stats;
|
|||||||
mod runtime_watch;
|
mod runtime_watch;
|
||||||
mod runtime_zero;
|
mod runtime_zero;
|
||||||
mod users;
|
mod users;
|
||||||
|
mod web_status;
|
||||||
|
|
||||||
use config_store::{
|
use config_store::{
|
||||||
current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk,
|
current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk,
|
||||||
@@ -122,6 +124,7 @@ pub(super) struct ApiShared {
|
|||||||
pub(super) proxy_shared: Arc<ProxySharedState>,
|
pub(super) proxy_shared: Arc<ProxySharedState>,
|
||||||
pub(super) reload_control: ReloadControl,
|
pub(super) reload_control: ReloadControl,
|
||||||
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
|
pub(super) web_trace: Arc<WebTraceStore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiShared {
|
impl ApiShared {
|
||||||
@@ -155,6 +158,7 @@ impl ApiShared {
|
|||||||
proxy_shared: runtime.proxy_shared.clone(),
|
proxy_shared: runtime.proxy_shared.clone(),
|
||||||
reload_control: self.reload_control.clone(),
|
reload_control: self.reload_control.clone(),
|
||||||
active_runtime: self.active_runtime.clone(),
|
active_runtime: self.active_runtime.clone(),
|
||||||
|
web_trace: self.web_trace.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -243,7 +247,8 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
|||||||
| "/v1/runtime/tls-fingerprints"
|
| "/v1/runtime/tls-fingerprints"
|
||||||
| "/v1/stats/users/active-ips"
|
| "/v1/stats/users/active-ips"
|
||||||
| "/v1/stats/users/quota"
|
| "/v1/stats/users/quota"
|
||||||
| "/v1/stats/users" => Some(ALLOW_GET),
|
| "/v1/stats/users"
|
||||||
|
| "/web-status" => Some(ALLOW_GET),
|
||||||
"/v1/system/reload" => Some(ALLOW_POST),
|
"/v1/system/reload" => Some(ALLOW_POST),
|
||||||
"/v1/users" => Some(ALLOW_GET_POST),
|
"/v1/users" => Some(ALLOW_GET_POST),
|
||||||
"/v1/config" => Some(ALLOW_GET_PATCH),
|
"/v1/config" => Some(ALLOW_GET_PATCH),
|
||||||
@@ -279,6 +284,7 @@ pub async fn serve(
|
|||||||
reload_control: ReloadControl,
|
reload_control: ReloadControl,
|
||||||
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
||||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||||
|
web_trace: Arc<WebTraceStore>,
|
||||||
) {
|
) {
|
||||||
let active_runtime = loop {
|
let active_runtime = loop {
|
||||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||||
@@ -312,7 +318,7 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
info!("API endpoint: http://{}/v1/*", listen);
|
info!("API endpoint: http://{}/v1/* and /web-status", listen);
|
||||||
|
|
||||||
let runtime_state = Arc::new(ApiRuntimeState {
|
let runtime_state = Arc::new(ApiRuntimeState {
|
||||||
process_started_at_epoch_secs,
|
process_started_at_epoch_secs,
|
||||||
@@ -344,6 +350,7 @@ pub async fn serve(
|
|||||||
proxy_shared,
|
proxy_shared,
|
||||||
reload_control,
|
reload_control,
|
||||||
active_runtime,
|
active_runtime,
|
||||||
|
web_trace,
|
||||||
});
|
});
|
||||||
|
|
||||||
spawn_runtime_watchers(
|
spawn_runtime_watchers(
|
||||||
@@ -492,6 +499,12 @@ async fn handle(
|
|||||||
|
|
||||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||||
match (method.as_str(), normalized_path) {
|
match (method.as_str(), normalized_path) {
|
||||||
|
("GET", "/web-status") => Ok(web_status::render(
|
||||||
|
query.as_deref(),
|
||||||
|
&shared.web_trace,
|
||||||
|
&cfg.web.debug,
|
||||||
|
)
|
||||||
|
.await),
|
||||||
("GET", "/v1/health") => {
|
("GET", "/v1/health") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let data = HealthData {
|
let data = HealthData {
|
||||||
|
|||||||
@@ -112,6 +112,7 @@ fn reload_routes_expose_only_documented_methods_and_ids() {
|
|||||||
Some(ALLOW_GET)
|
Some(ALLOW_GET)
|
||||||
);
|
);
|
||||||
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
||||||
|
assert_eq!(allowed_methods_for_path("/web-status"), Some(ALLOW_GET));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
reload_status_route_id("/v1/system/reload/not-a-number"),
|
reload_status_route_id("/v1/system/reload/not-a-number"),
|
||||||
None
|
None
|
||||||
|
|||||||
@@ -0,0 +1,540 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use base64::Engine as _;
|
||||||
|
use http_body_util::Full;
|
||||||
|
use hyper::body::Bytes;
|
||||||
|
use hyper::header::{self, HeaderValue};
|
||||||
|
use hyper::{Response, StatusCode};
|
||||||
|
use tokio::sync::OwnedSemaphorePermit;
|
||||||
|
|
||||||
|
use crate::config::WebDebugConfig;
|
||||||
|
use crate::web::trace::{
|
||||||
|
StoredTraceRecord, TraceRecord, TraceRecordKind, WebTraceStore,
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAX_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
const MAX_GROUPS: usize = 1024;
|
||||||
|
|
||||||
|
// Query parsing and matching remain independent from bounded HTML rendering.
|
||||||
|
mod query;
|
||||||
|
|
||||||
|
use query::{GroupBy, StatusQuery, client_ip, parse_query, record_matches};
|
||||||
|
|
||||||
|
struct GroupSummary {
|
||||||
|
count: usize,
|
||||||
|
latest_seq: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RenderedPage {
|
||||||
|
html: String,
|
||||||
|
_permit: OwnedSemaphorePermit,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsRef<[u8]> for RenderedPage {
|
||||||
|
fn as_ref(&self) -> &[u8] {
|
||||||
|
self.html.as_bytes()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Renders the authenticated server-side WEB debugging table.
|
||||||
|
pub(super) async fn render(
|
||||||
|
raw_query: Option<&str>,
|
||||||
|
store: &Arc<WebTraceStore>,
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
) -> Response<Full<Bytes>> {
|
||||||
|
store.apply_policy(policy);
|
||||||
|
let query = match parse_query(raw_query, policy) {
|
||||||
|
Ok(query) => query,
|
||||||
|
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
||||||
|
};
|
||||||
|
let Some(render_permit) = store.try_render_permit() else {
|
||||||
|
return html_error(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"Renderer busy",
|
||||||
|
"Two WEB status pages are already rendering",
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let now_millis = crate::web::trace::store_epoch_millis();
|
||||||
|
let since_millis = query
|
||||||
|
.record
|
||||||
|
.is_none()
|
||||||
|
.then(|| now_millis.saturating_sub(query.window_secs.saturating_mul(1000)))
|
||||||
|
.unwrap_or(0);
|
||||||
|
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||||
|
let status = store.status();
|
||||||
|
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||||
|
push_page_start(&mut html);
|
||||||
|
html.push_str("<h1>WEB status</h1>");
|
||||||
|
push_filter_form(&mut html, &query);
|
||||||
|
html.push_str("<section><h2>Store</h2><table><tbody>");
|
||||||
|
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
|
||||||
|
summary_row(&mut html, "body capture", body_mode(&status.policy));
|
||||||
|
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"records",
|
||||||
|
&format!("{} / {}", status.records, status.records_capacity),
|
||||||
|
);
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"bytes",
|
||||||
|
&format!("{} / {}", status.used_bytes, status.bytes_capacity),
|
||||||
|
);
|
||||||
|
summary_row(&mut html, "matched", &records.len().to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"contention drops",
|
||||||
|
&status.contention_drops.to_string(),
|
||||||
|
);
|
||||||
|
summary_row(&mut html, "evictions", &status.evictions.to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"byte truncations",
|
||||||
|
&status.byte_truncations.to_string(),
|
||||||
|
);
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"sequence range",
|
||||||
|
&format!(
|
||||||
|
"{} .. {}",
|
||||||
|
option_u64(status.earliest_seq),
|
||||||
|
option_u64(status.latest_seq)
|
||||||
|
),
|
||||||
|
);
|
||||||
|
html.push_str("</tbody></table></section>");
|
||||||
|
if !query.group_by.is_empty() {
|
||||||
|
push_groups(&mut html, &records, &query.group_by);
|
||||||
|
}
|
||||||
|
push_records(&mut html, &records, &query);
|
||||||
|
html.push_str("</main></body></html>");
|
||||||
|
truncate_page(&mut html);
|
||||||
|
retained_html_response(StatusCode::OK, html, render_permit)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_page_start(html: &mut String) {
|
||||||
|
html.push_str("<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><title>WEB status</title><style>body{font:14px system-ui,sans-serif;margin:0;background:#f5f7fa;color:#17202a}main{max-width:1600px;margin:auto;padding:20px}h1,h2{margin:.4em 0}section{background:#fff;border:1px solid #d9e0e7;border-radius:8px;padding:12px;margin:12px 0;overflow:auto}form{display:flex;flex-wrap:wrap;gap:8px;align-items:end}label{display:grid;gap:3px}input,select,button{font:inherit;padding:5px}table{border-collapse:collapse;width:100%}th,td{border:1px solid #d9e0e7;padding:5px;text-align:left;vertical-align:top}th{background:#edf2f7;position:sticky;top:0}code,pre{font:12px ui-monospace,monospace;white-space:pre-wrap;overflow-wrap:anywhere}details{max-width:900px}.muted{color:#657786}.bad{color:#a00}</style></head><body><main>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_filter_form(html: &mut String, query: &StatusQuery) {
|
||||||
|
html.push_str("<section><h2>Filters</h2><form method=\"get\" action=\"/web-status\">");
|
||||||
|
input(html, "window_secs", &query.window_secs.to_string());
|
||||||
|
input(html, "ip", &query.ip.map(|value| value.to_string()).unwrap_or_default());
|
||||||
|
input(
|
||||||
|
html,
|
||||||
|
"session",
|
||||||
|
&query.session.map(|value| value.to_string()).unwrap_or_default(),
|
||||||
|
);
|
||||||
|
input(
|
||||||
|
html,
|
||||||
|
"user_agent",
|
||||||
|
query.user_agent.as_deref().unwrap_or_default(),
|
||||||
|
);
|
||||||
|
input(html, "key", query.key.as_deref().unwrap_or_default());
|
||||||
|
input(html, "limit", &query.limit.to_string());
|
||||||
|
html.push_str("<label>group_by<select name=\"group_by\" multiple size=\"4\">");
|
||||||
|
for group in [GroupBy::Ip, GroupBy::Session, GroupBy::UserAgent, GroupBy::Key] {
|
||||||
|
html.push_str("<option value=\"");
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
if query.group_by.contains(&group) {
|
||||||
|
html.push_str("\" selected>");
|
||||||
|
} else {
|
||||||
|
html.push_str("\">");
|
||||||
|
}
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
html.push_str("</option>");
|
||||||
|
}
|
||||||
|
html.push_str("</select></label><button type=\"submit\">Observe</button></form></section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn input(html: &mut String, name: &str, value: &str) {
|
||||||
|
html.push_str("<label>");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("<input name=\"");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("\" value=\"");
|
||||||
|
escape(html, value);
|
||||||
|
html.push_str("\"></label>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summary_row(html: &mut String, name: &str, value: &str) {
|
||||||
|
html.push_str("<tr><th>");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("</th><td>");
|
||||||
|
escape(html, value);
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_groups(
|
||||||
|
html: &mut String,
|
||||||
|
records: &[Arc<StoredTraceRecord>],
|
||||||
|
groups: &[GroupBy],
|
||||||
|
) {
|
||||||
|
let mut summaries = BTreeMap::<Vec<String>, GroupSummary>::new();
|
||||||
|
let mut overflow = 0usize;
|
||||||
|
for stored in records {
|
||||||
|
let values = groups
|
||||||
|
.iter()
|
||||||
|
.map(|group| group_value(&stored.record, *group))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if let Some(summary) = summaries.get_mut(&values) {
|
||||||
|
summary.count += 1;
|
||||||
|
summary.latest_seq = summary.latest_seq.max(stored.record.seq);
|
||||||
|
} else if summaries.len() < MAX_GROUPS {
|
||||||
|
summaries.insert(
|
||||||
|
values,
|
||||||
|
GroupSummary {
|
||||||
|
count: 1,
|
||||||
|
latest_seq: stored.record.seq,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
overflow += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut summaries = summaries.into_iter().collect::<Vec<_>>();
|
||||||
|
summaries.sort_by(|(left_values, left), (right_values, right)| {
|
||||||
|
right
|
||||||
|
.count
|
||||||
|
.cmp(&left.count)
|
||||||
|
.then_with(|| left_values.cmp(right_values))
|
||||||
|
});
|
||||||
|
html.push_str("<section><h2>Groups</h2><table><thead><tr>");
|
||||||
|
for group in groups {
|
||||||
|
html.push_str("<th>");
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
html.push_str("</th>");
|
||||||
|
}
|
||||||
|
html.push_str("<th>records</th><th>latest seq</th></tr></thead><tbody>");
|
||||||
|
for (values, summary) in summaries {
|
||||||
|
html.push_str("<tr>");
|
||||||
|
for value in values {
|
||||||
|
html.push_str("<td>");
|
||||||
|
escape(html, &value);
|
||||||
|
html.push_str("</td>");
|
||||||
|
}
|
||||||
|
html.push_str("<td>");
|
||||||
|
html.push_str(&summary.count.to_string());
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
html.push_str(&summary.latest_seq.to_string());
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
if html.len() >= MAX_PAGE_BYTES / 2 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if overflow != 0 {
|
||||||
|
html.push_str("<tr><td colspan=\"6\" class=\"muted\">Additional groups omitted: ");
|
||||||
|
html.push_str(&overflow.to_string());
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</tbody></table></section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn group_value(record: &TraceRecord, group: GroupBy) -> String {
|
||||||
|
match group {
|
||||||
|
GroupBy::Ip => client_ip(record).map(|value| value.to_string()),
|
||||||
|
GroupBy::Session => record.identity.session_id.map(|value| value.to_string()),
|
||||||
|
GroupBy::UserAgent => record.user_agent.clone(),
|
||||||
|
GroupBy::Key => record.identity.key_fingerprint.clone(),
|
||||||
|
}
|
||||||
|
.unwrap_or_else(|| "-".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_records(html: &mut String, records: &[Arc<StoredTraceRecord>], query: &StatusQuery) {
|
||||||
|
html.push_str("<section><h2>Records</h2><table><thead><tr><th>seq</th><th>time</th><th>kind</th><th>route/event</th><th>method</th><th>status</th><th>IP</th><th>session</th><th>user / key</th><th>User-Agent</th><th>details</th></tr></thead><tbody>");
|
||||||
|
let mut shown = 0usize;
|
||||||
|
for stored in records.iter().take(query.limit) {
|
||||||
|
if html.len() >= MAX_PAGE_BYTES.saturating_sub(64 * 1024) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
push_record(html, &stored.record);
|
||||||
|
shown += 1;
|
||||||
|
}
|
||||||
|
if shown == 0 {
|
||||||
|
html.push_str("<tr><td colspan=\"11\" class=\"muted\">No matching records</td></tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</tbody></table>");
|
||||||
|
if records.len() > shown && shown != 0 {
|
||||||
|
let before = records[shown - 1].record.seq;
|
||||||
|
html.push_str("<p><a href=\"");
|
||||||
|
escape(html, &pagination_url(query, before));
|
||||||
|
html.push_str("\">Next page</a></p>");
|
||||||
|
}
|
||||||
|
html.push_str("</section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_record(html: &mut String, record: &TraceRecord) {
|
||||||
|
html.push_str("<tr><td><a href=\"/web-status?record=");
|
||||||
|
html.push_str(&record.seq.to_string());
|
||||||
|
html.push_str("\">");
|
||||||
|
html.push_str(&record.seq.to_string());
|
||||||
|
html.push_str("</a></td><td>");
|
||||||
|
escape(html, &format_time(record.epoch_millis));
|
||||||
|
let (kind, route, method, status) = match &record.kind {
|
||||||
|
TraceRecordKind::Http(http) => (
|
||||||
|
"http",
|
||||||
|
http.route.as_str(),
|
||||||
|
http.method.as_str(),
|
||||||
|
http.status.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string()),
|
||||||
|
),
|
||||||
|
TraceRecordKind::Lifecycle(event) => (
|
||||||
|
"lifecycle",
|
||||||
|
event.event.as_str(),
|
||||||
|
"-",
|
||||||
|
event.reason.unwrap_or("-").to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
for value in [kind, route, method, status.as_str()] {
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, value);
|
||||||
|
}
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(
|
||||||
|
html,
|
||||||
|
&client_ip(record)
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, &option_u64(record.identity.session_id));
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, record.identity.user.as_deref().unwrap_or("-"));
|
||||||
|
html.push_str(" / ");
|
||||||
|
escape(
|
||||||
|
html,
|
||||||
|
record.identity.key_fingerprint.as_deref().unwrap_or("-"),
|
||||||
|
);
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, record.user_agent.as_deref().unwrap_or("-"));
|
||||||
|
html.push_str("</td><td><details><summary>request → response</summary>");
|
||||||
|
match &record.kind {
|
||||||
|
TraceRecordKind::Http(http) => {
|
||||||
|
html.push_str("<p><code>");
|
||||||
|
escape(html, &http.method);
|
||||||
|
html.push(' ');
|
||||||
|
escape(html, &http.path);
|
||||||
|
html.push_str("</code></p>");
|
||||||
|
push_headers(html, "request headers", &http.request_headers);
|
||||||
|
push_body(html, "request body", http.request_body.as_ref());
|
||||||
|
push_headers(html, "response headers", &http.response_headers);
|
||||||
|
push_body(html, "response body", http.response_body.as_ref());
|
||||||
|
if let Some(timings) = &http.timings {
|
||||||
|
html.push_str("<h3>timings</h3><pre>service/head accepted: 0 us\nrequest body: ");
|
||||||
|
html.push_str(&option_u64(timings.request_body_us));
|
||||||
|
html.push_str(" us\nresponse ready: ");
|
||||||
|
html.push_str(&option_u64(timings.response_ready_us));
|
||||||
|
html.push_str(" us\nresponse body consumed/polled: ");
|
||||||
|
html.push_str(&option_u64(timings.response_body_us));
|
||||||
|
html.push_str(" us\n(kernel flush and TCP ACK are not observed)</pre>");
|
||||||
|
}
|
||||||
|
if !http.frames.is_empty() {
|
||||||
|
html.push_str("<h3>frames</h3><table><tr><th>dir</th><th>type</th><th>stream/lane</th><th>payload</th><th>WINDOW</th><th>error</th></tr>");
|
||||||
|
for frame in &http.frames {
|
||||||
|
html.push_str("<tr>");
|
||||||
|
for value in [
|
||||||
|
frame.direction.as_str().to_string(),
|
||||||
|
frame.frame_type.unwrap_or("-").to_string(),
|
||||||
|
frame.stream_id.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame.payload_len.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame.window_delta.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame.parse_error.unwrap_or("-").to_string(),
|
||||||
|
] {
|
||||||
|
html.push_str("<td>");
|
||||||
|
escape(html, &value);
|
||||||
|
html.push_str("</td>");
|
||||||
|
}
|
||||||
|
html.push_str("</tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</table>");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TraceRecordKind::Lifecycle(event) => {
|
||||||
|
html.push_str("<pre>event: ");
|
||||||
|
html.push_str(event.event.as_str());
|
||||||
|
html.push_str("\nstream: ");
|
||||||
|
html.push_str(&event.stream_id.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()));
|
||||||
|
html.push_str("\nreason: ");
|
||||||
|
html.push_str(event.reason.unwrap_or("-"));
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
html.push_str("</details></td></tr>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_headers(html: &mut String, title: &str, headers: &[crate::web::trace::TraceHeader]) {
|
||||||
|
html.push_str("<h3>");
|
||||||
|
escape(html, title);
|
||||||
|
html.push_str("</h3><pre>");
|
||||||
|
for header in headers {
|
||||||
|
escape(html, &header.name);
|
||||||
|
html.push_str(": ");
|
||||||
|
escape(html, header.value.as_deref().unwrap_or("[value omitted]"));
|
||||||
|
html.push('\n');
|
||||||
|
}
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_body(
|
||||||
|
html: &mut String,
|
||||||
|
title: &str,
|
||||||
|
body: Option<&crate::web::trace::TraceBodySnapshot>,
|
||||||
|
) {
|
||||||
|
html.push_str("<h3>");
|
||||||
|
escape(html, title);
|
||||||
|
html.push_str("</h3>");
|
||||||
|
let Some(body) = body else {
|
||||||
|
html.push_str("<p class=\"muted\">capture off</p>");
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
html.push_str("<p>observed=");
|
||||||
|
html.push_str(&body.observed_bytes.to_string());
|
||||||
|
html.push_str(" captured=");
|
||||||
|
html.push_str(&body.captured.len().to_string());
|
||||||
|
html.push_str(" state=");
|
||||||
|
html.push_str(body.state.as_str());
|
||||||
|
html.push_str(" truncated=");
|
||||||
|
html.push_str(yes_no(body.truncated));
|
||||||
|
html.push_str("</p><pre>");
|
||||||
|
let available = MAX_PAGE_BYTES.saturating_sub(html.len()).saturating_sub(4096);
|
||||||
|
let raw_limit = available.saturating_mul(3) / 4;
|
||||||
|
let shown = body.captured.len().min(raw_limit);
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.encode_string(&body.captured[..shown], html);
|
||||||
|
if shown < body.captured.len() {
|
||||||
|
html.push_str("\n[page output truncated]");
|
||||||
|
}
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pagination_url(query: &StatusQuery, before_seq: u64) -> String {
|
||||||
|
let mut serializer = url::form_urlencoded::Serializer::new(String::from("/web-status?"));
|
||||||
|
serializer.append_pair("window_secs", &query.window_secs.to_string());
|
||||||
|
if let Some(ip) = query.ip {
|
||||||
|
serializer.append_pair("ip", &ip.to_string());
|
||||||
|
}
|
||||||
|
if let Some(session) = query.session {
|
||||||
|
serializer.append_pair("session", &session.to_string());
|
||||||
|
}
|
||||||
|
if let Some(user_agent) = &query.user_agent {
|
||||||
|
serializer.append_pair("user_agent", user_agent);
|
||||||
|
}
|
||||||
|
if let Some(key) = &query.key {
|
||||||
|
serializer.append_pair("key", key);
|
||||||
|
}
|
||||||
|
for group in &query.group_by {
|
||||||
|
serializer.append_pair("group_by", group.as_str());
|
||||||
|
}
|
||||||
|
serializer.append_pair("limit", &query.limit.to_string());
|
||||||
|
serializer.append_pair("before_seq", &before_seq.to_string());
|
||||||
|
serializer.finish()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_time(epoch_millis: u64) -> String {
|
||||||
|
chrono::DateTime::from_timestamp_millis(epoch_millis as i64)
|
||||||
|
.map(|value| value.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
|
||||||
|
.unwrap_or_else(|| epoch_millis.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn option_u64(value: Option<u64>) -> String {
|
||||||
|
value.map(|value| value.to_string()).unwrap_or_else(|| "-".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn body_mode(policy: &WebDebugConfig) -> &'static str {
|
||||||
|
match policy.body_capture {
|
||||||
|
crate::config::WebDebugBodyCapture::Off => "off",
|
||||||
|
crate::config::WebDebugBodyCapture::Metadata => "metadata",
|
||||||
|
crate::config::WebDebugBodyCapture::Prefix => "prefix",
|
||||||
|
crate::config::WebDebugBodyCapture::Full => "full",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn yes_no(value: bool) -> &'static str {
|
||||||
|
if value { "yes" } else { "no" }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn escape(output: &mut String, value: &str) {
|
||||||
|
for character in value.chars() {
|
||||||
|
match character {
|
||||||
|
'&' => output.push_str("&"),
|
||||||
|
'<' => output.push_str("<"),
|
||||||
|
'>' => output.push_str(">"),
|
||||||
|
'"' => output.push_str("""),
|
||||||
|
'\'' => output.push_str("'"),
|
||||||
|
_ => output.push(character),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_page(html: &mut String) {
|
||||||
|
const SUFFIX: &str = "[page output truncated]";
|
||||||
|
if html.len() <= MAX_PAGE_BYTES {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut end = MAX_PAGE_BYTES.saturating_sub(SUFFIX.len());
|
||||||
|
while !html.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
html.truncate(end);
|
||||||
|
html.push_str(SUFFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_error(status: StatusCode, title: &str, message: &str) -> Response<Full<Bytes>> {
|
||||||
|
let mut html = String::new();
|
||||||
|
push_page_start(&mut html);
|
||||||
|
html.push_str("<h1 class=\"bad\">");
|
||||||
|
escape(&mut html, title);
|
||||||
|
html.push_str("</h1><p>");
|
||||||
|
escape(&mut html, message);
|
||||||
|
html.push_str("</p></main></body></html>");
|
||||||
|
html_response(status, html)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_response(status: StatusCode, html: String) -> Response<Full<Bytes>> {
|
||||||
|
html_bytes_response(status, Bytes::from(html))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retained_html_response(
|
||||||
|
status: StatusCode,
|
||||||
|
html: String,
|
||||||
|
permit: OwnedSemaphorePermit,
|
||||||
|
) -> Response<Full<Bytes>> {
|
||||||
|
html_bytes_response(
|
||||||
|
status,
|
||||||
|
Bytes::from_owner(RenderedPage {
|
||||||
|
html,
|
||||||
|
_permit: permit,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_bytes_response(status: StatusCode, html: Bytes) -> Response<Full<Bytes>> {
|
||||||
|
let mut response = Response::new(Full::new(html));
|
||||||
|
*response.status_mut() = status;
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_TYPE,
|
||||||
|
HeaderValue::from_static("text/html; charset=utf-8"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_SECURITY_POLICY,
|
||||||
|
HeaderValue::from_static("default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"),
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::REFERRER_POLICY,
|
||||||
|
HeaderValue::from_static("no-referrer"),
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY"));
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "web_status/tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
|
||||||
|
use crate::config::WebDebugConfig;
|
||||||
|
use crate::web::trace::TraceRecord;
|
||||||
|
|
||||||
|
const DEFAULT_LIMIT: usize = 200;
|
||||||
|
const MAX_LIMIT: usize = 1000;
|
||||||
|
|
||||||
|
/// Supported status-page grouping dimensions.
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(super) enum GroupBy {
|
||||||
|
Ip,
|
||||||
|
Session,
|
||||||
|
UserAgent,
|
||||||
|
Key,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GroupBy {
|
||||||
|
fn parse(value: &str) -> Option<Self> {
|
||||||
|
match value {
|
||||||
|
"ip" => Some(Self::Ip),
|
||||||
|
"session" => Some(Self::Session),
|
||||||
|
"user_agent" => Some(Self::UserAgent),
|
||||||
|
"key" => Some(Self::Key),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the canonical query and table label.
|
||||||
|
pub(super) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Ip => "ip",
|
||||||
|
Self::Session => "session",
|
||||||
|
Self::UserAgent => "user_agent",
|
||||||
|
Self::Key => "key",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validated bounded status-page filter and pagination state.
|
||||||
|
pub(super) struct StatusQuery {
|
||||||
|
pub(super) window_secs: u64,
|
||||||
|
pub(super) ip: Option<IpAddr>,
|
||||||
|
pub(super) session: Option<u64>,
|
||||||
|
pub(super) user_agent: Option<String>,
|
||||||
|
pub(super) key: Option<String>,
|
||||||
|
pub(super) group_by: Vec<GroupBy>,
|
||||||
|
pub(super) limit: usize,
|
||||||
|
pub(super) before_seq: Option<u64>,
|
||||||
|
pub(super) record: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses a strict query without accepting unknown or ambiguous fields.
|
||||||
|
pub(super) fn parse_query(
|
||||||
|
raw: Option<&str>,
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
) -> Result<StatusQuery, String> {
|
||||||
|
let mut query = StatusQuery {
|
||||||
|
window_secs: policy.default_window_secs,
|
||||||
|
ip: None,
|
||||||
|
session: None,
|
||||||
|
user_agent: None,
|
||||||
|
key: None,
|
||||||
|
group_by: Vec::new(),
|
||||||
|
limit: DEFAULT_LIMIT,
|
||||||
|
before_seq: None,
|
||||||
|
record: None,
|
||||||
|
};
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||||
|
let name = name.as_ref();
|
||||||
|
let value = value.as_ref();
|
||||||
|
if name != "group_by" && !seen.insert(name.to_string()) {
|
||||||
|
return Err(format!("{name} must not repeat"));
|
||||||
|
}
|
||||||
|
match name {
|
||||||
|
"window_secs" => {
|
||||||
|
query.window_secs = parse_positive_u64(value, "window_secs")?;
|
||||||
|
}
|
||||||
|
"ip" => {
|
||||||
|
let parsed = value
|
||||||
|
.parse::<IpAddr>()
|
||||||
|
.map_err(|_| "ip must be a canonical IP address".to_string())?;
|
||||||
|
if parsed.to_string() != value {
|
||||||
|
return Err("ip must use canonical formatting".to_string());
|
||||||
|
}
|
||||||
|
query.ip = Some(parsed);
|
||||||
|
}
|
||||||
|
"session" => query.session = Some(parse_positive_u64(value, "session")?),
|
||||||
|
"user_agent" => {
|
||||||
|
if value.is_empty() || value.len() > 512 {
|
||||||
|
return Err("user_agent must contain 1..512 bytes".to_string());
|
||||||
|
}
|
||||||
|
query.user_agent = Some(value.to_string());
|
||||||
|
}
|
||||||
|
"key" => {
|
||||||
|
if value.is_empty() || value.len() > 64 {
|
||||||
|
return Err("key must contain 1..64 bytes".to_string());
|
||||||
|
}
|
||||||
|
query.key = Some(value.to_string());
|
||||||
|
}
|
||||||
|
"group_by" => {
|
||||||
|
let group = GroupBy::parse(value)
|
||||||
|
.ok_or_else(|| "group_by must be ip, session, user_agent, or key".to_string())?;
|
||||||
|
if query.group_by.contains(&group) {
|
||||||
|
return Err("group_by values must not repeat".to_string());
|
||||||
|
}
|
||||||
|
query.group_by.push(group);
|
||||||
|
}
|
||||||
|
"limit" => {
|
||||||
|
query.limit = value
|
||||||
|
.parse::<usize>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| (1..=MAX_LIMIT).contains(value))
|
||||||
|
.ok_or_else(|| "limit must be within 1..1000".to_string())?;
|
||||||
|
}
|
||||||
|
"before_seq" => {
|
||||||
|
query.before_seq = Some(parse_positive_u64(value, "before_seq")?);
|
||||||
|
}
|
||||||
|
"record" => query.record = Some(parse_positive_u64(value, "record")?),
|
||||||
|
_ => return Err(format!("unknown query field `{name}`")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if query.window_secs > policy.max_window_secs {
|
||||||
|
return Err(format!(
|
||||||
|
"window_secs must not exceed {}",
|
||||||
|
policy.max_window_secs
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_positive_u64(value: &str, field: &str) -> Result<u64, String> {
|
||||||
|
value
|
||||||
|
.parse::<u64>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| *value != 0)
|
||||||
|
.ok_or_else(|| format!("{field} must be a positive integer"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies the complete filter predicate to one immutable record.
|
||||||
|
pub(super) fn record_matches(
|
||||||
|
record: &TraceRecord,
|
||||||
|
query: &StatusQuery,
|
||||||
|
since_millis: u64,
|
||||||
|
) -> bool {
|
||||||
|
!(record.epoch_millis < since_millis
|
||||||
|
|| query.before_seq.is_some_and(|before| record.seq >= before)
|
||||||
|
|| query.record.is_some_and(|seq| record.seq != seq)
|
||||||
|
|| query.ip.is_some_and(|ip| client_ip(record) != Some(ip))
|
||||||
|
|| query
|
||||||
|
.session
|
||||||
|
.is_some_and(|session| record.identity.session_id != Some(session))
|
||||||
|
|| query.user_agent.as_ref().is_some_and(|needle| {
|
||||||
|
record
|
||||||
|
.user_agent
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| !contains_ascii_case_insensitive(value, needle))
|
||||||
|
})
|
||||||
|
|| query.key.as_ref().is_some_and(|key| {
|
||||||
|
record.identity.user.as_deref() != Some(key)
|
||||||
|
&& record.identity.key_fingerprint.as_deref() != Some(key)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contains_ascii_case_insensitive(value: &str, needle: &str) -> bool {
|
||||||
|
let needle = needle.as_bytes();
|
||||||
|
needle.is_empty()
|
||||||
|
|| value
|
||||||
|
.as_bytes()
|
||||||
|
.windows(needle.len())
|
||||||
|
.any(|window| window.eq_ignore_ascii_case(needle))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the trusted effective address or direct peer fallback.
|
||||||
|
pub(super) fn client_ip(record: &TraceRecord) -> Option<IpAddr> {
|
||||||
|
record.effective_ip.or(record.peer_ip)
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
use http_body_util::BodyExt as _;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn query_rejects_noncanonical_ip_and_excessive_window() {
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
assert!(parse_query(Some("ip=2001%3A0db8%3A%3A1"), &policy).is_err());
|
||||||
|
assert!(parse_query(Some("window_secs=3601"), &policy).is_err());
|
||||||
|
assert!(parse_query(Some("session=1&session=2"), &policy).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn html_escaping_covers_active_markup_characters() {
|
||||||
|
let mut output = String::new();
|
||||||
|
escape(&mut output, "<script a='\"'>&");
|
||||||
|
assert_eq!(output, "<script a='"'>&");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||||
|
let mut policy = WebDebugConfig::default();
|
||||||
|
policy.enabled = true;
|
||||||
|
let mut limits = crate::config::WebLimitsConfig::default();
|
||||||
|
limits.debug_records_capacity = 8;
|
||||||
|
limits.debug_bytes_global = 16 * 1024;
|
||||||
|
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||||
|
store.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.40".parse().unwrap()),
|
||||||
|
TraceIdentity {
|
||||||
|
session_id: Some(42),
|
||||||
|
user: Some("alice".to_string()),
|
||||||
|
key_fingerprint: Some("0123456789abcdef".to_string()),
|
||||||
|
},
|
||||||
|
TraceLifecycleEvent::SessionCreated,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = render(
|
||||||
|
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
||||||
|
&store,
|
||||||
|
&policy,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||||
|
assert!(response.headers().contains_key(header::CONTENT_SECURITY_POLICY));
|
||||||
|
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||||
|
let body = std::str::from_utf8(&body).unwrap();
|
||||||
|
assert!(body.contains("session_created"));
|
||||||
|
assert!(body.contains("0123456789abcdef"));
|
||||||
|
assert!(body.contains("192.0.2.40"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
let limits = crate::config::WebLimitsConfig::default();
|
||||||
|
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||||
|
|
||||||
|
let first = render(None, &store, &policy).await;
|
||||||
|
let second = render(None, &store, &policy).await;
|
||||||
|
let busy = render(None, &store, &policy).await;
|
||||||
|
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
|
||||||
|
drop(first);
|
||||||
|
let admitted = render(None, &store, &policy).await;
|
||||||
|
assert_eq!(admitted.status(), StatusCode::OK);
|
||||||
|
drop(second);
|
||||||
|
drop(admitted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||||
|
let mut html = "я".repeat(MAX_PAGE_BYTES);
|
||||||
|
truncate_page(&mut html);
|
||||||
|
assert!(html.len() <= MAX_PAGE_BYTES);
|
||||||
|
assert!(html.ends_with("[page output truncated]"));
|
||||||
|
}
|
||||||
@@ -36,7 +36,7 @@ use super::load::{LoadedConfig, ProxyConfig};
|
|||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
use crate::config::{
|
use crate::config::{
|
||||||
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
||||||
MeWriterPickMode,
|
MeWriterPickMode, WebDebugConfig, web_debug_fits_limits,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::config::{ListenerConfig, SynLimitMode};
|
use crate::config::{ListenerConfig, SynLimitMode};
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ pub struct HotFields {
|
|||||||
pub user_max_unique_ips_global_each: usize,
|
pub user_max_unique_ips_global_each: usize,
|
||||||
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
||||||
pub user_max_unique_ips_window_secs: u64,
|
pub user_max_unique_ips_window_secs: u64,
|
||||||
|
pub web_debug: WebDebugConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HotFields {
|
impl HotFields {
|
||||||
@@ -218,6 +219,7 @@ impl HotFields {
|
|||||||
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
||||||
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
||||||
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||||
|
web_debug: cfg.web.debug.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -340,6 +342,9 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
|||||||
let process_limits = cfg.web.limits.clone();
|
let process_limits = cfg.web.limits.clone();
|
||||||
cfg.web = new.web.clone();
|
cfg.web = new.web.clone();
|
||||||
cfg.web.limits = process_limits;
|
cfg.web.limits = process_limits;
|
||||||
|
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
|
||||||
|
cfg.web.debug = old.web.debug.clone();
|
||||||
|
}
|
||||||
if cfg.rebuild_runtime_user_auth().is_err() {
|
if cfg.rebuild_runtime_user_auth().is_err() {
|
||||||
cfg.runtime_user_auth = None;
|
cfg.runtime_user_auth = None;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -496,4 +496,13 @@ pub(super) fn log_changes(
|
|||||||
new_hot.user_max_unique_ips_window_secs
|
new_hot.user_max_unique_ips_window_secs
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if old_hot.web_debug != new_hot.web_debug {
|
||||||
|
info!(
|
||||||
|
"config reload: web.debug updated: enabled={} body_capture={:?} window={}..={}s",
|
||||||
|
new_hot.web_debug.enabled,
|
||||||
|
new_hot.web_debug.body_capture,
|
||||||
|
new_hot.web_debug.default_window_secs,
|
||||||
|
new_hot.web_debug.max_window_secs,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,6 +85,42 @@ fn bind_stale_mode_is_hot() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
|
||||||
|
let old = sample_config();
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.debug.enabled = true;
|
||||||
|
new.web.debug.default_window_secs = 60;
|
||||||
|
new.web.limits.debug_records_capacity += 1;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
assert!(applied.web.debug.enabled);
|
||||||
|
assert_eq!(applied.web.debug.default_window_secs, 60);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.debug_records_capacity,
|
||||||
|
old.web.limits.debug_records_capacity
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
HotFields::from_config(&old),
|
||||||
|
HotFields::from_config(&applied)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
|
||||||
|
let old = sample_config();
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||||
|
new.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
assert_eq!(applied.web.limits.max_body_bytes, old.web.limits.max_body_bytes);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.debug.body_prefix_bytes,
|
||||||
|
old.web.debug.body_prefix_bytes
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keepalive_is_not_hot() {
|
fn keepalive_is_not_hot() {
|
||||||
let old = sample_config();
|
let old = sample_config();
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use sha2::{Digest, Sha256};
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
|
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
|
||||||
|
const WEB_DEBUG_FINGERPRINT_CONTEXT: &[u8] = b"telemt-web-debug-key-fingerprint-v1\0";
|
||||||
const MAX_WEB_STATIC_DEPTH: usize = 64;
|
const MAX_WEB_STATIC_DEPTH: usize = 64;
|
||||||
|
|
||||||
/// Builds the immutable WEB routing and decoy snapshot for one generation.
|
/// Builds the immutable WEB routing and decoy snapshot for one generation.
|
||||||
@@ -49,6 +50,8 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
client_secret(auth_entry.secret, profile.secret_mode);
|
client_secret(auth_entry.secret, profile.secret_mode);
|
||||||
let capability =
|
let capability =
|
||||||
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
|
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
|
||||||
|
let key_fingerprint =
|
||||||
|
debug_key_fingerprint(&client_secret[..client_secret_len]);
|
||||||
if !capabilities.insert(capability) {
|
if !capabilities.insert(capability) {
|
||||||
return Err(ProxyError::Config(format!(
|
return Err(ProxyError::Config(format!(
|
||||||
"WEB vhost `{}` contains profiles with the same client capability",
|
"WEB vhost `{}` contains profiles with the same client capability",
|
||||||
@@ -62,6 +65,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
secret_mode: profile.secret_mode,
|
secret_mode: profile.secret_mode,
|
||||||
carrier: config.web.carrier,
|
carrier: config.web.carrier,
|
||||||
capability,
|
capability,
|
||||||
|
key_fingerprint,
|
||||||
max_sessions: profile
|
max_sessions: profile
|
||||||
.max_sessions
|
.max_sessions
|
||||||
.unwrap_or(config.web.limits.max_sessions_global),
|
.unwrap_or(config.web.limits.max_sessions_global),
|
||||||
@@ -93,6 +97,13 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn debug_key_fingerprint(secret: &[u8]) -> String {
|
||||||
|
let mut digest = Sha256::new();
|
||||||
|
digest.update(WEB_DEBUG_FINGERPRINT_CONTEXT);
|
||||||
|
digest.update(secret);
|
||||||
|
hex::encode(&digest.finalize()[..8])
|
||||||
|
}
|
||||||
|
|
||||||
/// Derives the Telegram Desktop WEB capability for one exact secret and host.
|
/// Derives the Telegram Desktop WEB capability for one exact secret and host.
|
||||||
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
|
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
|
||||||
let mut mac = Hmac::<Sha256>::new_from_slice(secret)
|
let mut mac = Hmac::<Sha256>::new_from_slice(secret)
|
||||||
|
|||||||
@@ -259,7 +259,7 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
|||||||
"web_trusted_proxy_cidrs",
|
"web_trusted_proxy_cidrs",
|
||||||
];
|
];
|
||||||
|
|
||||||
const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "limits", "timeouts", "vhosts"];
|
const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "debug", "limits", "timeouts", "vhosts"];
|
||||||
|
|
||||||
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||||
"max_header_bytes",
|
"max_header_bytes",
|
||||||
@@ -290,6 +290,8 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"max_static_files",
|
"max_static_files",
|
||||||
"max_static_file_bytes",
|
"max_static_file_bytes",
|
||||||
"max_static_bytes",
|
"max_static_bytes",
|
||||||
|
"debug_records_capacity",
|
||||||
|
"debug_bytes_global",
|
||||||
"memory_envelope_bytes",
|
"memory_envelope_bytes",
|
||||||
"new_bootstraps_per_minute",
|
"new_bootstraps_per_minute",
|
||||||
"new_bootstraps_burst",
|
"new_bootstraps_burst",
|
||||||
@@ -299,6 +301,19 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"new_streams_burst",
|
"new_streams_burst",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"enabled",
|
||||||
|
"capture_lifecycle",
|
||||||
|
"capture_headers",
|
||||||
|
"capture_timings",
|
||||||
|
"capture_frames",
|
||||||
|
"body_capture",
|
||||||
|
"body_prefix_bytes",
|
||||||
|
"decoy_body_prefix_bytes",
|
||||||
|
"default_window_secs",
|
||||||
|
"max_window_secs",
|
||||||
|
];
|
||||||
|
|
||||||
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||||
"header_secs",
|
"header_secs",
|
||||||
"body_secs",
|
"body_secs",
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ fn known_config_keys_for_suggestion() -> Vec<&'static str> {
|
|||||||
LISTENER_CONFIG_KEYS,
|
LISTENER_CONFIG_KEYS,
|
||||||
WEB_CONFIG_KEYS,
|
WEB_CONFIG_KEYS,
|
||||||
WEB_LIMITS_CONFIG_KEYS,
|
WEB_LIMITS_CONFIG_KEYS,
|
||||||
|
WEB_DEBUG_CONFIG_KEYS,
|
||||||
WEB_TIMEOUTS_CONFIG_KEYS,
|
WEB_TIMEOUTS_CONFIG_KEYS,
|
||||||
WEB_VHOST_CONFIG_KEYS,
|
WEB_VHOST_CONFIG_KEYS,
|
||||||
WEB_DECOY_CONFIG_KEYS,
|
WEB_DECOY_CONFIG_KEYS,
|
||||||
@@ -241,6 +242,13 @@ pub(super) fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<Unkn
|
|||||||
&["web", "limits"],
|
&["web", "limits"],
|
||||||
WEB_LIMITS_CONFIG_KEYS,
|
WEB_LIMITS_CONFIG_KEYS,
|
||||||
);
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["web", "debug"],
|
||||||
|
WEB_DEBUG_CONFIG_KEYS,
|
||||||
|
);
|
||||||
check_known_table(
|
check_known_table(
|
||||||
parsed_toml,
|
parsed_toml,
|
||||||
&mut unknown,
|
&mut unknown,
|
||||||
|
|||||||
@@ -2,6 +2,11 @@ use std::collections::HashSet;
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// Debug capture validation is independent from restart-only storage limits.
|
||||||
|
mod debug;
|
||||||
|
// Memory-envelope arithmetic remains isolated from protocol validation.
|
||||||
|
mod memory;
|
||||||
|
|
||||||
const WEB_FRAME_HEADER_BYTES: usize = 8;
|
const WEB_FRAME_HEADER_BYTES: usize = 8;
|
||||||
const WEB_QUEUE_ITEM_COST: usize = 256;
|
const WEB_QUEUE_ITEM_COST: usize = 256;
|
||||||
const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
|
const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
|
||||||
@@ -59,6 +64,7 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
validate_limits(&config.web.limits)?;
|
validate_limits(&config.web.limits)?;
|
||||||
|
debug::validate(&config.web.debug, &config.web.limits)?;
|
||||||
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
|
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
|
||||||
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
|
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
|
||||||
}
|
}
|
||||||
@@ -175,6 +181,8 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
("max_static_files", limits.max_static_files),
|
("max_static_files", limits.max_static_files),
|
||||||
("max_static_file_bytes", limits.max_static_file_bytes),
|
("max_static_file_bytes", limits.max_static_file_bytes),
|
||||||
("max_static_bytes", limits.max_static_bytes),
|
("max_static_bytes", limits.max_static_bytes),
|
||||||
|
("debug_records_capacity", limits.debug_records_capacity),
|
||||||
|
("debug_bytes_global", limits.debug_bytes_global),
|
||||||
("memory_envelope_bytes", limits.memory_envelope_bytes),
|
("memory_envelope_bytes", limits.memory_envelope_bytes),
|
||||||
];
|
];
|
||||||
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
|
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
|
||||||
@@ -309,38 +317,7 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
"web.limits pending ceilings must preserve one uplink batch and downlink progress",
|
"web.limits pending ceilings must preserve one uplink batch and downlink progress",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let body_reservation = limits
|
memory::validate(limits)?;
|
||||||
.max_body_readers
|
|
||||||
.checked_mul(limits.max_body_bytes)
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
|
||||||
})?;
|
|
||||||
if body_reservation > limits.max_body_bytes_global
|
|
||||||
|| limits.max_body_bytes_global > u32::MAX as usize
|
|
||||||
{
|
|
||||||
return config_error(
|
|
||||||
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let http_header_reservation = limits
|
|
||||||
.max_http_connections
|
|
||||||
.checked_mul(limits.max_header_bytes)
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
|
||||||
})?;
|
|
||||||
let reserved = limits
|
|
||||||
.pending_bytes_global
|
|
||||||
.checked_add(limits.max_body_bytes_global)
|
|
||||||
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
|
||||||
.and_then(|value| value.checked_add(http_header_reservation))
|
|
||||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
|
||||||
if reserved > limits.memory_envelope_bytes
|
|
||||||
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
|
||||||
{
|
|
||||||
return config_error(
|
|
||||||
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const MAX_WEB_TRACE_WINDOW_SECS: u64 = 86_400;
|
||||||
|
const MIN_WEB_DEBUG_BYTES_GLOBAL: usize = 4096;
|
||||||
|
|
||||||
|
/// Validates hot debug policy independently from process-owned storage limits.
|
||||||
|
pub(super) fn validate(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> Result<()> {
|
||||||
|
if limits.debug_bytes_global < MIN_WEB_DEBUG_BYTES_GLOBAL {
|
||||||
|
return config_error("web.limits.debug_bytes_global must be at least 4096 bytes");
|
||||||
|
}
|
||||||
|
if policy.default_window_secs == 0
|
||||||
|
|| policy.max_window_secs == 0
|
||||||
|
|| policy.default_window_secs > policy.max_window_secs
|
||||||
|
|| policy.max_window_secs > MAX_WEB_TRACE_WINDOW_SECS
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.debug windows must be non-zero, ordered, and no greater than 86400 seconds",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if policy.body_prefix_bytes > limits.max_body_bytes {
|
||||||
|
return config_error("web.debug.body_prefix_bytes must not exceed web.limits.max_body_bytes");
|
||||||
|
}
|
||||||
|
if policy.body_prefix_bytes > limits.debug_bytes_global
|
||||||
|
|| policy.decoy_body_prefix_bytes > limits.debug_bytes_global
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.debug body prefixes must not exceed web.limits.debug_bytes_global",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const WEB_DEBUG_RENDERERS: usize = 2;
|
||||||
|
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Validates process-wide body, header, queue, static, and debug reservations.
|
||||||
|
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||||
|
let body_reservation = limits
|
||||||
|
.max_body_readers
|
||||||
|
.checked_mul(limits.max_body_bytes)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
if body_reservation > limits.max_body_bytes_global
|
||||||
|
|| limits.max_body_bytes_global > u32::MAX as usize
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let http_header_reservation = limits
|
||||||
|
.max_http_connections
|
||||||
|
.checked_mul(limits.max_header_bytes)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
||||||
|
})?;
|
||||||
|
let debug_ring_index = limits
|
||||||
|
.debug_records_capacity
|
||||||
|
.checked_mul(std::mem::size_of::<usize>())
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.limits debug index overflowed usize".to_string()))?;
|
||||||
|
let status_pages = WEB_DEBUG_RENDERERS
|
||||||
|
.checked_mul(WEB_DEBUG_STATUS_PAGE_BYTES)
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.debug status pages overflowed usize".to_string()))?;
|
||||||
|
let debug_reservation = limits
|
||||||
|
.debug_bytes_global
|
||||||
|
.checked_add(
|
||||||
|
debug_ring_index
|
||||||
|
.checked_mul(WEB_DEBUG_RENDERERS)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.debug snapshot indexes overflowed usize".to_string())
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
.and_then(|value| {
|
||||||
|
WEB_DEBUG_RENDERERS
|
||||||
|
.checked_mul(WEB_DEBUG_GROUP_SCRATCH_BYTES)
|
||||||
|
.and_then(|scratch| value.checked_add(scratch))
|
||||||
|
})
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.debug reservations overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
let reserved = limits
|
||||||
|
.pending_bytes_global
|
||||||
|
.checked_add(limits.max_body_bytes_global)
|
||||||
|
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
||||||
|
.and_then(|value| value.checked_add(debug_ring_index))
|
||||||
|
.and_then(|value| value.checked_add(status_pages))
|
||||||
|
.and_then(|value| value.checked_add(debug_reservation))
|
||||||
|
.and_then(|value| value.checked_add(http_header_reservation))
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||||
|
if reserved > limits.memory_envelope_bytes
|
||||||
|
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -47,6 +47,53 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
|||||||
assert_eq!(vhost.profiles[0].max_sessions, 4);
|
assert_eq!(vhost.profiles[0].max_sessions, 4);
|
||||||
assert_eq!(vhost.profiles[0].max_streams, 64);
|
assert_eq!(vhost.profiles[0].max_streams, 64);
|
||||||
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
||||||
|
assert_eq!(vhost.profiles[0].key_fingerprint.len(), 16);
|
||||||
|
assert_ne!(vhost.profiles[0].key_fingerprint, "0001020304050607");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
assert!(config.web.debug.enabled);
|
||||||
|
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
|
||||||
|
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
|
||||||
|
assert_eq!(config.web.debug.default_window_secs, 180);
|
||||||
|
assert_eq!(config.web.debug.max_window_secs, 900);
|
||||||
|
|
||||||
|
let old_name = format!("[general]\nconfig_strict = true\n{}", WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.trace]\nenabled = true\n\n[[web.vhosts]]",
|
||||||
|
));
|
||||||
|
let error = load_config_error_from_temp_toml(&old_name);
|
||||||
|
assert!(error.contains("web.trace"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_and_window_validation_fail_closed() {
|
||||||
|
let oversized_prefix = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\nbody_prefix_bytes = 2097153\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&oversized_prefix);
|
||||||
|
assert!(error.contains("web.debug.body_prefix_bytes"));
|
||||||
|
|
||||||
|
let reversed_window = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\ndefault_window_secs = 181\nmax_window_secs = 180\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&reversed_window);
|
||||||
|
assert!(error.contains("web.debug windows"));
|
||||||
|
|
||||||
|
let undersized_store = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\n\n[web.limits]\ndebug_bytes_global = 4095",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&undersized_store);
|
||||||
|
assert!(error.contains("debug_bytes_global must be at least 4096"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ mod network;
|
|||||||
mod policies;
|
mod policies;
|
||||||
mod server;
|
mod server;
|
||||||
mod web;
|
mod web;
|
||||||
|
// WEB debug capture policy is reusable by config reload and process storage.
|
||||||
|
mod web_debug;
|
||||||
|
|
||||||
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
|
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
@@ -52,6 +54,8 @@ pub use web::{
|
|||||||
WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
||||||
WebTimeoutsConfig, WebVhostConfig,
|
WebTimeoutsConfig, WebVhostConfig,
|
||||||
};
|
};
|
||||||
|
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||||
|
pub(crate) use web_debug::web_debug_fits_limits;
|
||||||
pub(crate) use web::{
|
pub(crate) use web::{
|
||||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||||
WebStaticSite,
|
WebStaticSite,
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ use std::sync::Arc;
|
|||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::web_debug::WebDebugConfig;
|
||||||
|
|
||||||
/// Client-facing secret representation used to derive a WEB capability.
|
/// Client-facing secret representation used to derive a WEB capability.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
@@ -175,6 +177,12 @@ pub struct WebLimitsConfig {
|
|||||||
/// Maximum static snapshot bytes across all virtual hosts.
|
/// Maximum static snapshot bytes across all virtual hosts.
|
||||||
#[serde(default = "default_web_max_static_bytes")]
|
#[serde(default = "default_web_max_static_bytes")]
|
||||||
pub max_static_bytes: usize,
|
pub max_static_bytes: usize,
|
||||||
|
/// Maximum retained WEB debug record count.
|
||||||
|
#[serde(default = "default_web_debug_records_capacity")]
|
||||||
|
pub debug_records_capacity: usize,
|
||||||
|
/// Process-wide retained and in-flight WEB debug byte ceiling.
|
||||||
|
#[serde(default = "default_web_debug_bytes_global")]
|
||||||
|
pub debug_bytes_global: usize,
|
||||||
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
|
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
|
||||||
#[serde(default = "default_web_memory_envelope_bytes")]
|
#[serde(default = "default_web_memory_envelope_bytes")]
|
||||||
pub memory_envelope_bytes: usize,
|
pub memory_envelope_bytes: usize,
|
||||||
@@ -229,6 +237,8 @@ impl Default for WebLimitsConfig {
|
|||||||
max_static_files: default_web_max_static_files(),
|
max_static_files: default_web_max_static_files(),
|
||||||
max_static_file_bytes: default_web_max_static_file_bytes(),
|
max_static_file_bytes: default_web_max_static_file_bytes(),
|
||||||
max_static_bytes: default_web_max_static_bytes(),
|
max_static_bytes: default_web_max_static_bytes(),
|
||||||
|
debug_records_capacity: default_web_debug_records_capacity(),
|
||||||
|
debug_bytes_global: default_web_debug_bytes_global(),
|
||||||
memory_envelope_bytes: default_web_memory_envelope_bytes(),
|
memory_envelope_bytes: default_web_memory_envelope_bytes(),
|
||||||
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
|
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
|
||||||
new_bootstraps_burst: default_web_new_bootstraps_burst(),
|
new_bootstraps_burst: default_web_new_bootstraps_burst(),
|
||||||
@@ -300,6 +310,9 @@ pub struct WebConfig {
|
|||||||
/// Hard process and protocol limits.
|
/// Hard process and protocol limits.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub limits: WebLimitsConfig,
|
pub limits: WebLimitsConfig,
|
||||||
|
/// Hot-reloadable bounded server-side debug policy.
|
||||||
|
#[serde(default)]
|
||||||
|
pub debug: WebDebugConfig,
|
||||||
/// WEB lifecycle deadlines.
|
/// WEB lifecycle deadlines.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timeouts: WebTimeoutsConfig,
|
pub timeouts: WebTimeoutsConfig,
|
||||||
@@ -348,6 +361,8 @@ pub(crate) struct WebRuntimeProfile {
|
|||||||
pub(crate) carrier: WebCarrier,
|
pub(crate) carrier: WebCarrier,
|
||||||
/// HMAC-derived bridge capability.
|
/// HMAC-derived bridge capability.
|
||||||
pub(crate) capability: [u8; 32],
|
pub(crate) capability: [u8; 32],
|
||||||
|
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||||
|
pub(crate) key_fingerprint: String,
|
||||||
/// Per-profile live session ceiling.
|
/// Per-profile live session ceiling.
|
||||||
pub(crate) max_sessions: usize,
|
pub(crate) max_sessions: usize,
|
||||||
/// Per-profile live logical-stream ceiling.
|
/// Per-profile live logical-stream ceiling.
|
||||||
@@ -439,6 +454,8 @@ usize_default!(default_web_max_profiles, 32);
|
|||||||
usize_default!(default_web_max_static_files, 4096);
|
usize_default!(default_web_max_static_files, 4096);
|
||||||
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
||||||
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_debug_records_capacity, 65_536);
|
||||||
|
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
|
||||||
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
|
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
|
||||||
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
||||||
u32_default!(default_web_new_bootstraps_burst, 256);
|
u32_default!(default_web_new_bootstraps_burst, 256);
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::web::WebLimitsConfig;
|
||||||
|
|
||||||
|
/// Request and response body retention policy for WEB debugging.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebDebugBodyCapture {
|
||||||
|
/// Omits body snapshots entirely.
|
||||||
|
Off,
|
||||||
|
/// Retains body lengths and completion state without payload bytes.
|
||||||
|
#[default]
|
||||||
|
Metadata,
|
||||||
|
/// Retains a bounded prefix of each body.
|
||||||
|
Prefix,
|
||||||
|
/// Retains complete bounded carrier bodies and bounded decoy prefixes.
|
||||||
|
Full,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hot-reloadable WEB server-side debugging policy.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct WebDebugConfig {
|
||||||
|
/// Enables process-owned WEB debug collection.
|
||||||
|
#[serde(default)]
|
||||||
|
pub enabled: bool,
|
||||||
|
/// Records typed bridge, session, stream, handshake, and relay events.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_lifecycle: bool,
|
||||||
|
/// Retains allowlisted header values and names of all other headers.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_headers: bool,
|
||||||
|
/// Retains request service and body-consumption timing points.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_timings: bool,
|
||||||
|
/// Parses carrier bodies into bounded frame metadata.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_frames: bool,
|
||||||
|
/// Controls request and response body byte retention.
|
||||||
|
#[serde(default)]
|
||||||
|
pub body_capture: WebDebugBodyCapture,
|
||||||
|
/// Maximum retained body prefix for recognized WEB requests.
|
||||||
|
#[serde(default = "default_body_prefix_bytes")]
|
||||||
|
pub body_prefix_bytes: usize,
|
||||||
|
/// Maximum retained body prefix for ordinary decoy traffic.
|
||||||
|
#[serde(default = "default_decoy_body_prefix_bytes")]
|
||||||
|
pub decoy_body_prefix_bytes: usize,
|
||||||
|
/// Default observation window presented by the status page.
|
||||||
|
#[serde(default = "default_window_secs")]
|
||||||
|
pub default_window_secs: u64,
|
||||||
|
/// Largest observation window accepted by the status page.
|
||||||
|
#[serde(default = "default_max_window_secs")]
|
||||||
|
pub max_window_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WebDebugConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: false,
|
||||||
|
capture_lifecycle: true,
|
||||||
|
capture_headers: true,
|
||||||
|
capture_timings: true,
|
||||||
|
capture_frames: true,
|
||||||
|
body_capture: WebDebugBodyCapture::Metadata,
|
||||||
|
body_prefix_bytes: default_body_prefix_bytes(),
|
||||||
|
decoy_body_prefix_bytes: default_decoy_body_prefix_bytes(),
|
||||||
|
default_window_secs: default_window_secs(),
|
||||||
|
max_window_secs: default_max_window_secs(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_true() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_body_prefix_bytes() -> usize {
|
||||||
|
4096
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_decoy_body_prefix_bytes() -> usize {
|
||||||
|
4096
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_window_secs() -> u64 {
|
||||||
|
180
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_max_window_secs() -> u64 {
|
||||||
|
3600
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Checks whether a hot debug policy fits restart-frozen process capacities.
|
||||||
|
pub(crate) fn web_debug_fits_limits(
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
limits: &WebLimitsConfig,
|
||||||
|
) -> bool {
|
||||||
|
policy.body_prefix_bytes <= limits.max_body_bytes
|
||||||
|
&& policy.body_prefix_bytes <= limits.debug_bytes_global
|
||||||
|
&& policy.decoy_body_prefix_bytes <= limits.debug_bytes_global
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ use super::plan::{ListenerBindSpec, listener_bind_plan};
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use super::unix::UnixAcceptHandle;
|
use super::unix::UnixAcceptHandle;
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::manager::WebProcessRuntime;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
/// Process-owned listener inventory and accept-task lifecycle controller.
|
/// Process-owned listener inventory and accept-task lifecycle controller.
|
||||||
pub(crate) struct ListenerManager {
|
pub(crate) struct ListenerManager {
|
||||||
@@ -43,12 +44,14 @@ impl ListenerManager {
|
|||||||
pub(crate) fn start(
|
pub(crate) fn start(
|
||||||
bound: BoundListeners,
|
bound: BoundListeners,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
|
trace: Arc<WebTraceStore>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let has_web = bound
|
let has_web = bound
|
||||||
.listeners
|
.listeners
|
||||||
.iter()
|
.iter()
|
||||||
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
||||||
let web_runtime = has_web.then(|| WebProcessRuntime::start(active_runtime.clone()));
|
let web_runtime =
|
||||||
|
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
||||||
let mut slots = BTreeMap::new();
|
let mut slots = BTreeMap::new();
|
||||||
for listener in bound.listeners {
|
for listener in bound.listeners {
|
||||||
let addr = listener.spec.addr;
|
let addr = listener.spec.addr;
|
||||||
@@ -307,7 +310,11 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix_listener: None,
|
unix_listener: None,
|
||||||
};
|
};
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
let trace = WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
);
|
||||||
|
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||||
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let blocked_addr = blocker.local_addr().unwrap();
|
let blocked_addr = blocker.local_addr().unwrap();
|
||||||
let mut desired = ProxyConfig::default();
|
let mut desired = ProxyConfig::default();
|
||||||
@@ -330,7 +337,11 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix_listener: None,
|
unix_listener: None,
|
||||||
};
|
};
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
let trace = WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
);
|
||||||
|
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||||
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let new_addr = reservation.local_addr().unwrap();
|
let new_addr = reservation.local_addr().unwrap();
|
||||||
drop(reservation);
|
drop(reservation);
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
|
|||||||
use crate::synlimit_control;
|
use crate::synlimit_control;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
||||||
@@ -104,6 +105,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
config.access.user_rate_limits.clone(),
|
config.access.user_rate_limits.clone(),
|
||||||
config.access.cidr_rate_limits.clone(),
|
config.access.cidr_rate_limits.clone(),
|
||||||
);
|
);
|
||||||
|
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||||
|
|
||||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||||
@@ -154,6 +156,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
let reload_control_api = reload_control.clone();
|
let reload_control_api = reload_control.clone();
|
||||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||||
|
let web_trace_api = web_trace.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
api::serve(
|
api::serve(
|
||||||
listen,
|
listen,
|
||||||
@@ -171,6 +174,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
reload_control_api,
|
reload_control_api,
|
||||||
active_runtime_rx_api,
|
active_runtime_rx_api,
|
||||||
runtime_watch_rx_api,
|
runtime_watch_rx_api,
|
||||||
|
web_trace_api,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -314,7 +318,11 @@ pub(super) async fn run_telemt_core(
|
|||||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||||
|
|
||||||
let listener_manager = listeners::ListenerManager::start(bound, active_runtime.clone());
|
let listener_manager = listeners::ListenerManager::start(
|
||||||
|
bound,
|
||||||
|
active_runtime.clone(),
|
||||||
|
web_trace.clone(),
|
||||||
|
);
|
||||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||||
active_runtime.clone(),
|
active_runtime.clone(),
|
||||||
reload_control,
|
reload_control,
|
||||||
@@ -325,6 +333,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
runtime_log_filter,
|
runtime_log_filter,
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
web_trace,
|
||||||
);
|
);
|
||||||
|
|
||||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::stats::QuotaStore;
|
use crate::stats::QuotaStore;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||||
use super::listeners::{ListenerManager, PreparedListenerTransition};
|
use super::listeners::{ListenerManager, PreparedListenerTransition};
|
||||||
@@ -29,6 +30,7 @@ pub(crate) struct ReloadSupervisor {
|
|||||||
runtime_log_filter: RuntimeLogFilter,
|
runtime_log_filter: RuntimeLogFilter,
|
||||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||||
listener_manager: Arc<Mutex<ListenerManager>>,
|
listener_manager: Arc<Mutex<ListenerManager>>,
|
||||||
|
web_trace: Arc<WebTraceStore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
||||||
@@ -105,6 +107,7 @@ impl ReloadSupervisor {
|
|||||||
runtime_log_filter: RuntimeLogFilter,
|
runtime_log_filter: RuntimeLogFilter,
|
||||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||||
listener_manager: ListenerManager,
|
listener_manager: ListenerManager,
|
||||||
|
web_trace: Arc<WebTraceStore>,
|
||||||
) -> ReloadSupervisorHandle {
|
) -> ReloadSupervisorHandle {
|
||||||
let listener_manager = Arc::new(Mutex::new(listener_manager));
|
let listener_manager = Arc::new(Mutex::new(listener_manager));
|
||||||
let supervisor = Self {
|
let supervisor = Self {
|
||||||
@@ -117,6 +120,7 @@ impl ReloadSupervisor {
|
|||||||
runtime_log_filter,
|
runtime_log_filter,
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager: listener_manager.clone(),
|
listener_manager: listener_manager.clone(),
|
||||||
|
web_trace,
|
||||||
};
|
};
|
||||||
let control = supervisor.control.clone();
|
let control = supervisor.control.clone();
|
||||||
let shutdown = CancellationToken::new();
|
let shutdown = CancellationToken::new();
|
||||||
@@ -309,6 +313,7 @@ impl ReloadSupervisor {
|
|||||||
};
|
};
|
||||||
old_runtime.stop_accepting_sessions();
|
old_runtime.stop_accepting_sessions();
|
||||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||||
|
self.web_trace.apply_policy(&new_runtime.config().web.debug);
|
||||||
if let Some(pending) = pending_listener_transition {
|
if let Some(pending) = pending_listener_transition {
|
||||||
self.listener_manager
|
self.listener_manager
|
||||||
.lock()
|
.lock()
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
|||||||
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
||||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
||||||
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
|
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
|
||||||
|
let web_trace = crate::web::trace::WebTraceStore::new(
|
||||||
|
old_runtime.config().web.debug.clone(),
|
||||||
|
&old_runtime.config().web.limits,
|
||||||
|
);
|
||||||
let supervisor = Arc::new(ReloadSupervisor {
|
let supervisor = Arc::new(ReloadSupervisor {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
control: control.clone(),
|
control: control.clone(),
|
||||||
@@ -44,6 +48,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
|||||||
runtime_log_filter: runtime_log_filter(),
|
runtime_log_filter: runtime_log_filter(),
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
web_trace,
|
||||||
});
|
});
|
||||||
let command = ReloadCommand {
|
let command = ReloadCommand {
|
||||||
reload_id: accepted.reload_id,
|
reload_id: accepted.reload_id,
|
||||||
@@ -306,6 +311,10 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
|||||||
runtime_log_filter(),
|
runtime_log_filter(),
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
crate::web::trace::WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|||||||
|
|
||||||
use tokio::sync::{RwLock, Semaphore, watch};
|
use tokio::sync::{RwLock, Semaphore, watch};
|
||||||
|
|
||||||
use crate::config::{ProxyConfig, ServerConfig};
|
use crate::config::{ProxyConfig, ServerConfig, web_debug_fits_limits};
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::network::probe::{decide_network_capabilities, run_probe};
|
use crate::network::probe::{decide_network_capabilities, run_probe};
|
||||||
@@ -415,6 +415,10 @@ pub(crate) fn resolve_reload_config(
|
|||||||
effective.web = old.web.clone();
|
effective.web = old.web.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
|
||||||
|
fields.push("web.debug".to_string());
|
||||||
|
effective.web.debug = old.web.debug.clone();
|
||||||
|
}
|
||||||
let runtime_changed = !configs_equal(old, &effective);
|
let runtime_changed = !configs_equal(old, &effective);
|
||||||
ResolvedReloadConfig {
|
ResolvedReloadConfig {
|
||||||
effective,
|
effective,
|
||||||
|
|||||||
@@ -170,6 +170,27 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
|||||||
assert!(!resolved.runtime_changed);
|
assert!(!resolved.runtime_changed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
|
||||||
|
let mut old = ProxyConfig::default();
|
||||||
|
old.rebuild_runtime_user_auth().unwrap();
|
||||||
|
old.rebuild_runtime_web().unwrap();
|
||||||
|
let mut desired = old.clone();
|
||||||
|
desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||||
|
desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||||
|
|
||||||
|
let resolved = resolve_reload_config(&old, &desired);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolved.deferred_process_fields,
|
||||||
|
vec!["web.limits".to_string(), "web.debug".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolved.effective.web.debug.body_prefix_bytes,
|
||||||
|
old.web.debug.body_prefix_bytes
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn strict_middle_proxy_requires_a_prepared_pool() {
|
fn strict_middle_proxy_requires_a_prepared_pool() {
|
||||||
assert!(strict_middle_proxy_unavailable(true, false, false));
|
assert!(strict_middle_proxy_unavailable(true, false, false));
|
||||||
|
|||||||
+147
-163
@@ -6,8 +6,7 @@ use std::time::{Duration, Instant};
|
|||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http_body_util::combinators::UnsyncBoxBody;
|
use http_body_util::combinators::UnsyncBoxBody;
|
||||||
use http_body_util::{BodyExt, Full};
|
use http_body_util::BodyExt;
|
||||||
use hyper::body::Incoming;
|
|
||||||
use hyper::header::{self, HeaderName, HeaderValue};
|
use hyper::header::{self, HeaderName, HeaderValue};
|
||||||
use hyper::server::conn::http1;
|
use hyper::server::conn::http1;
|
||||||
use hyper::service::service_fn;
|
use hyper::service::service_fn;
|
||||||
@@ -18,7 +17,7 @@ use parking_lot::Mutex;
|
|||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use crate::config::{WebCarrier, WebClientIpSource, WebRuntimeVhost};
|
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
||||||
use crate::web::bridge;
|
use crate::web::bridge;
|
||||||
use crate::web::frame::{self, FrameType};
|
use crate::web::frame::{self, FrameType};
|
||||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||||
@@ -29,18 +28,34 @@ mod activity;
|
|||||||
mod body;
|
mod body;
|
||||||
// Decoy routing and upstream proxying are isolated from carrier authentication.
|
// Decoy routing and upstream proxying are isolated from carrier authentication.
|
||||||
mod decoy;
|
mod decoy;
|
||||||
|
// Downlink long-poll handling remains isolated from request routing.
|
||||||
|
mod down;
|
||||||
// Canonical request parsing rejects ambiguous credentials before routing.
|
// Canonical request parsing rejects ambiguous credentials before routing.
|
||||||
mod request;
|
mod request;
|
||||||
|
// Carrier response construction and lane-header helpers are shared by handlers.
|
||||||
|
mod response;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
// Enabled-debug integration coverage remains separate from carrier behavior tests.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "http/trace_tests.rs"]
|
||||||
|
mod trace_tests;
|
||||||
|
|
||||||
use activity::{ActivityBody, RequestActivity};
|
use activity::{ActivityBody, RequestActivity};
|
||||||
use body::{CollectBodyError, CollectedBody, collect_body};
|
use body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||||
use decoy::serve_decoy;
|
use decoy::serve_decoy;
|
||||||
|
use down::handle_down;
|
||||||
use request::{
|
use request::{
|
||||||
bearer_token_hash, binary_content_type, bridge_candidate, canonical_request_host,
|
bearer_token_hash, binary_content_type, bridge_candidate, canonical_request_host,
|
||||||
canonical_u64_header, client_ip, compatible_cookie_header, match_profile,
|
canonical_u64_header, client_ip, compatible_cookie_header, match_profile,
|
||||||
};
|
};
|
||||||
|
use response::{
|
||||||
|
bad_gateway, carrier_empty, carrier_headers, carrier_lane, full_response, generic_not_found,
|
||||||
|
insert_header, service_unavailable,
|
||||||
|
};
|
||||||
|
use crate::web::trace::{
|
||||||
|
HttpTraceExchange, TraceDirection, TraceLifecycleEvent, TraceRoute,
|
||||||
|
};
|
||||||
|
|
||||||
type BoxError = Box<dyn Error + Send + Sync>;
|
type BoxError = Box<dyn Error + Send + Sync>;
|
||||||
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
||||||
@@ -65,13 +80,18 @@ pub(crate) async fn serve_connection(
|
|||||||
let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs);
|
let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs);
|
||||||
let last_activity = Arc::new(Mutex::new(Instant::now()));
|
let last_activity = Arc::new(Mutex::new(Instant::now()));
|
||||||
let service_last_activity = Arc::clone(&last_activity);
|
let service_last_activity = Arc::clone(&last_activity);
|
||||||
let service = service_fn(move |request| {
|
let service = service_fn(move |mut request| {
|
||||||
let runtime = Arc::clone(&runtime);
|
let runtime = Arc::clone(&runtime);
|
||||||
let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs);
|
let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs);
|
||||||
let last_activity = Arc::clone(&service_last_activity);
|
let last_activity = Arc::clone(&service_last_activity);
|
||||||
let client_ip_source = client_ip_source;
|
let client_ip_source = client_ip_source;
|
||||||
async move {
|
async move {
|
||||||
let activity = RequestActivity::begin(last_activity);
|
let activity = RequestActivity::begin(last_activity);
|
||||||
|
let trace = runtime.trace().begin_http(&request, peer.ip());
|
||||||
|
if let Some(trace) = &trace {
|
||||||
|
request.extensions_mut().insert(Arc::clone(trace));
|
||||||
|
}
|
||||||
|
let request = request.map(|body| RequestBody::new(body, trace.clone()));
|
||||||
let response = if let Some(_handler_permit) = runtime.try_http_handler() {
|
let response = if let Some(_handler_permit) = runtime.try_http_handler() {
|
||||||
handle_request(
|
handle_request(
|
||||||
request,
|
request,
|
||||||
@@ -84,7 +104,11 @@ pub(crate) async fn serve_connection(
|
|||||||
} else {
|
} else {
|
||||||
service_unavailable()
|
service_unavailable()
|
||||||
};
|
};
|
||||||
let response = response.map(|body| ActivityBody::new(body, activity).boxed_unsync());
|
if let Some(trace) = &trace {
|
||||||
|
trace.response_ready(&response);
|
||||||
|
}
|
||||||
|
let response = response
|
||||||
|
.map(|body| ActivityBody::new(body, activity, trace).boxed_unsync());
|
||||||
Ok::<_, Infallible>(response)
|
Ok::<_, Infallible>(response)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -115,12 +139,18 @@ pub(crate) async fn serve_connection(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_request(
|
async fn handle_request(
|
||||||
request: Request<Incoming>,
|
request: Request<RequestBody>,
|
||||||
peer: SocketAddr,
|
peer: SocketAddr,
|
||||||
client_ip_source: WebClientIpSource,
|
client_ip_source: WebClientIpSource,
|
||||||
trusted_proxy_cidrs: &[IpNetwork],
|
trusted_proxy_cidrs: &[IpNetwork],
|
||||||
runtime: Arc<WebProcessRuntime>,
|
runtime: Arc<WebProcessRuntime>,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
|
set_trace_route(&request, TraceRoute::Decoy);
|
||||||
|
if let Some(trace) = request_trace(&request)
|
||||||
|
&& let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs)
|
||||||
|
{
|
||||||
|
trace.set_effective_ip(client_ip);
|
||||||
|
}
|
||||||
let generation = runtime.active_generation();
|
let generation = runtime.active_generation();
|
||||||
let config = generation.config();
|
let config = generation.config();
|
||||||
let Some(web_runtime) = config.web.runtime.as_ref() else {
|
let Some(web_runtime) = config.web.runtime.as_ref() else {
|
||||||
@@ -159,7 +189,7 @@ async fn handle_request(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_root(
|
async fn handle_root(
|
||||||
mut request: Request<Incoming>,
|
mut request: Request<RequestBody>,
|
||||||
peer: SocketAddr,
|
peer: SocketAddr,
|
||||||
client_ip_source: WebClientIpSource,
|
client_ip_source: WebClientIpSource,
|
||||||
trusted_proxy_cidrs: &[IpNetwork],
|
trusted_proxy_cidrs: &[IpNetwork],
|
||||||
@@ -175,15 +205,34 @@ async fn handle_root(
|
|||||||
strip_query(&mut request);
|
strip_query(&mut request);
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Bridge);
|
||||||
|
trace.set_effective_ip(client_ip);
|
||||||
|
}
|
||||||
let carrier = profile.carrier;
|
let carrier = profile.carrier;
|
||||||
let Ok(bootstrap) = runtime.issue_bootstrap(profile, client_ip) else {
|
let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
|
||||||
strip_query(&mut request);
|
Ok(bootstrap) => bootstrap,
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
Err(error) => {
|
||||||
|
runtime.trace().record_profile_lifecycle(
|
||||||
|
client_ip,
|
||||||
|
None,
|
||||||
|
&profile,
|
||||||
|
TraceLifecycleEvent::BootstrapRejected,
|
||||||
|
None,
|
||||||
|
Some(manager_error_reason(error)),
|
||||||
|
);
|
||||||
|
strip_query(&mut request);
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.bind_profile(&profile, bootstrap.trace_session_id);
|
||||||
|
trace.register_redaction(bootstrap.token.as_bytes());
|
||||||
|
}
|
||||||
let generation = runtime.active_generation();
|
let generation = runtime.active_generation();
|
||||||
let page = bridge::render(
|
let page = bridge::render(
|
||||||
&vhost.host,
|
&vhost.host,
|
||||||
&bootstrap,
|
&bootstrap.token,
|
||||||
generation.config().web.limits.carrier_batch_bytes,
|
generation.config().web.limits.carrier_batch_bytes,
|
||||||
generation.config().web.limits.pending_bytes_per_session,
|
generation.config().web.limits.pending_bytes_per_session,
|
||||||
generation.config().web.limits.pending_items_per_session,
|
generation.config().web.limits.pending_items_per_session,
|
||||||
@@ -224,7 +273,7 @@ async fn handle_root(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_api(
|
async fn handle_api(
|
||||||
request: Request<Incoming>,
|
request: Request<RequestBody>,
|
||||||
peer: SocketAddr,
|
peer: SocketAddr,
|
||||||
client_ip_source: WebClientIpSource,
|
client_ip_source: WebClientIpSource,
|
||||||
trusted_proxy_cidrs: &[IpNetwork],
|
trusted_proxy_cidrs: &[IpNetwork],
|
||||||
@@ -237,6 +286,9 @@ async fn handle_api(
|
|||||||
let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs) else {
|
let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_effective_ip(client_ip);
|
||||||
|
}
|
||||||
let Some(token_hash) = bearer_token_hash(&request) else {
|
let Some(token_hash) = bearer_token_hash(&request) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
@@ -249,7 +301,7 @@ async fn handle_api(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_session(
|
async fn handle_session(
|
||||||
request: Request<Incoming>,
|
request: Request<RequestBody>,
|
||||||
runtime: Arc<WebProcessRuntime>,
|
runtime: Arc<WebProcessRuntime>,
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
token_hash: crate::web::manager::TokenHash,
|
token_hash: crate::web::manager::TokenHash,
|
||||||
@@ -262,6 +314,12 @@ async fn handle_session(
|
|||||||
if request.headers().contains_key(header::CONTENT_TYPE) {
|
if request.headers().contains_key(header::CONTENT_TYPE) {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
|
if let Some(trace) = request_trace(&request)
|
||||||
|
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
|
||||||
|
{
|
||||||
|
trace.set_route(TraceRoute::Session);
|
||||||
|
trace.bind_identity(session.trace_identity());
|
||||||
|
}
|
||||||
let CollectedBody {
|
let CollectedBody {
|
||||||
request,
|
request,
|
||||||
body,
|
body,
|
||||||
@@ -281,8 +339,14 @@ async fn handle_session(
|
|||||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
if request.method() != Method::POST || !binary_content_type(&request) {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
if !runtime.has_bootstrap(token_hash, &vhost.host) {
|
let Some((trace_session_id, profile)) =
|
||||||
|
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||||
|
else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Session);
|
||||||
|
trace.bind_profile(&profile, trace_session_id);
|
||||||
}
|
}
|
||||||
let CollectedBody {
|
let CollectedBody {
|
||||||
request,
|
request,
|
||||||
@@ -295,9 +359,24 @@ async fn handle_session(
|
|||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.record_frames(
|
||||||
|
TraceDirection::Request,
|
||||||
|
&body,
|
||||||
|
&runtime.active_generation().config().web.limits,
|
||||||
|
);
|
||||||
|
}
|
||||||
match runtime.create_session(token_hash, &vhost.host, client_ip, &body) {
|
match runtime.create_session(token_hash, &vhost.host, client_ip, &body) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
|
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.register_redaction(result.token.as_bytes());
|
||||||
|
trace.record_frames(
|
||||||
|
TraceDirection::Response,
|
||||||
|
&welcome,
|
||||||
|
&runtime.active_generation().config().web.limits,
|
||||||
|
);
|
||||||
|
}
|
||||||
let mut response = full_response(StatusCode::OK, welcome);
|
let mut response = full_response(StatusCode::OK, welcome);
|
||||||
carrier_headers(&mut response);
|
carrier_headers(&mut response);
|
||||||
insert_header(
|
insert_header(
|
||||||
@@ -315,15 +394,33 @@ async fn handle_session(
|
|||||||
);
|
);
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
Err(ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent) => {
|
Err(error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent)) => {
|
||||||
|
runtime.trace().record_profile_lifecycle(
|
||||||
|
client_ip,
|
||||||
|
Some(trace_session_id),
|
||||||
|
&profile,
|
||||||
|
TraceLifecycleEvent::SessionRejected,
|
||||||
|
None,
|
||||||
|
Some(manager_error_reason(error)),
|
||||||
|
);
|
||||||
service_unavailable()
|
service_unavailable()
|
||||||
}
|
}
|
||||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
Err(error) => {
|
||||||
|
runtime.trace().record_profile_lifecycle(
|
||||||
|
client_ip,
|
||||||
|
Some(trace_session_id),
|
||||||
|
&profile,
|
||||||
|
TraceLifecycleEvent::SessionRejected,
|
||||||
|
None,
|
||||||
|
Some(manager_error_reason(error)),
|
||||||
|
);
|
||||||
|
serve_decoy(request, vhost, true, &runtime).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_up(
|
async fn handle_up(
|
||||||
request: Request<Incoming>,
|
request: Request<RequestBody>,
|
||||||
runtime: Arc<WebProcessRuntime>,
|
runtime: Arc<WebProcessRuntime>,
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
token_hash: crate::web::manager::TokenHash,
|
token_hash: crate::web::manager::TokenHash,
|
||||||
@@ -338,6 +435,10 @@ async fn handle_up(
|
|||||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Uplink);
|
||||||
|
trace.bind_identity(session.trace_identity());
|
||||||
|
}
|
||||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
@@ -358,6 +459,13 @@ async fn handle_up(
|
|||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.record_frames(
|
||||||
|
TraceDirection::Request,
|
||||||
|
&body,
|
||||||
|
&runtime.active_generation().config().web.limits,
|
||||||
|
);
|
||||||
|
}
|
||||||
let result = match lane_id {
|
let result = match lane_id {
|
||||||
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
||||||
None => session.process_up(sequence, &body),
|
None => session.process_up(sequence, &body),
|
||||||
@@ -379,151 +487,6 @@ async fn handle_up(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_down(
|
|
||||||
request: Request<Incoming>,
|
|
||||||
runtime: Arc<WebProcessRuntime>,
|
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
|
||||||
token_hash: crate::web::manager::TokenHash,
|
|
||||||
) -> HttpResponse {
|
|
||||||
if request.method() != Method::POST || request.headers().contains_key(header::CONTENT_TYPE) {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let CollectedBody {
|
|
||||||
request,
|
|
||||||
body,
|
|
||||||
_body_budget,
|
|
||||||
} = match collect_body(request, &runtime, 1, true).await {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
|
||||||
Err(CollectBodyError::Invalid(request)) => {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if !body.is_empty() {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
let _lane_poll = if lane_id.is_some() {
|
|
||||||
let Some(permit) = runtime.try_lane_poll() else {
|
|
||||||
return service_unavailable();
|
|
||||||
};
|
|
||||||
Some(permit)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let result = match lane_id {
|
|
||||||
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
|
||||||
None => session.poll_down(cursor).await,
|
|
||||||
};
|
|
||||||
match result {
|
|
||||||
Ok(result) if result.body.is_empty() => {
|
|
||||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
|
||||||
insert_header(
|
|
||||||
&mut response,
|
|
||||||
HeaderName::from_static("x-down-cursor"),
|
|
||||||
&result.next_cursor.to_string(),
|
|
||||||
);
|
|
||||||
if result.lane_closed {
|
|
||||||
response.headers_mut().insert(
|
|
||||||
HeaderName::from_static("x-lane-closed"),
|
|
||||||
HeaderValue::from_static("1"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
response
|
|
||||||
}
|
|
||||||
Ok(result) => {
|
|
||||||
let mut response = full_response(StatusCode::OK, result.body);
|
|
||||||
carrier_headers(&mut response);
|
|
||||||
insert_header(
|
|
||||||
&mut response,
|
|
||||||
HeaderName::from_static("x-down-cursor"),
|
|
||||||
&result.next_cursor.to_string(),
|
|
||||||
);
|
|
||||||
response
|
|
||||||
}
|
|
||||||
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
|
|
||||||
service_unavailable()
|
|
||||||
}
|
|
||||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn carrier_lane<B>(request: &Request<B>, carrier: WebCarrier) -> Option<Option<u32>> {
|
|
||||||
match carrier {
|
|
||||||
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
|
|
||||||
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
|
|
||||||
.and_then(|value| u32::try_from(value).ok())
|
|
||||||
.filter(|value| *value <= frame::MAX_STREAM_ID)
|
|
||||||
.map(Some),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn carrier_headers(response: &mut HttpResponse) {
|
|
||||||
response.headers_mut().insert(
|
|
||||||
header::CONTENT_TYPE,
|
|
||||||
HeaderValue::from_static("application/octet-stream"),
|
|
||||||
);
|
|
||||||
response
|
|
||||||
.headers_mut()
|
|
||||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn carrier_empty(status: StatusCode) -> HttpResponse {
|
|
||||||
let mut response = empty_response(status);
|
|
||||||
response
|
|
||||||
.headers_mut()
|
|
||||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
|
||||||
response
|
|
||||||
}
|
|
||||||
|
|
||||||
fn service_unavailable() -> HttpResponse {
|
|
||||||
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
|
|
||||||
response
|
|
||||||
.headers_mut()
|
|
||||||
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
|
|
||||||
response
|
|
||||||
}
|
|
||||||
|
|
||||||
fn bad_gateway() -> HttpResponse {
|
|
||||||
full_response(
|
|
||||||
StatusCode::BAD_GATEWAY,
|
|
||||||
Bytes::from_static(b"site unavailable\n"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn generic_not_found() -> HttpResponse {
|
|
||||||
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
|
|
||||||
let length = body.len();
|
|
||||||
let body = Full::new(body)
|
|
||||||
.map_err(|never| -> BoxError { match never {} })
|
|
||||||
.boxed_unsync();
|
|
||||||
let mut response = Response::new(body);
|
|
||||||
*response.status_mut() = status;
|
|
||||||
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
|
|
||||||
response
|
|
||||||
}
|
|
||||||
|
|
||||||
fn empty_response(status: StatusCode) -> HttpResponse {
|
|
||||||
full_response(status, Bytes::new())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
|
|
||||||
if let Ok(value) = HeaderValue::from_str(value) {
|
|
||||||
response.headers_mut().insert(name, value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn strip_query<B>(request: &mut Request<B>) {
|
fn strip_query<B>(request: &mut Request<B>) {
|
||||||
if request.uri().query().is_some()
|
if request.uri().query().is_some()
|
||||||
&& let Ok(uri) = request.uri().path().parse()
|
&& let Ok(uri) = request.uri().path().parse()
|
||||||
@@ -531,3 +494,24 @@ fn strip_query<B>(request: &mut Request<B>) {
|
|||||||
*request.uri_mut() = uri;
|
*request.uri_mut() = uri;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn request_trace<B>(request: &Request<B>) -> Option<&Arc<HttpTraceExchange>> {
|
||||||
|
request.extensions().get::<Arc<HttpTraceExchange>>()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_trace_route<B>(request: &Request<B>, route: TraceRoute) {
|
||||||
|
if let Some(trace) = request_trace(request) {
|
||||||
|
trace.set_route(route);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn manager_error_reason(error: ManagerError) -> &'static str {
|
||||||
|
match error {
|
||||||
|
ManagerError::Authentication => "authentication",
|
||||||
|
ManagerError::Backpressure => "backpressure",
|
||||||
|
ManagerError::Limit => "limit",
|
||||||
|
ManagerError::Protocol => "protocol",
|
||||||
|
ManagerError::Concurrent => "concurrent",
|
||||||
|
ManagerError::Closed => "closed",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use hyper::body::{Body, Frame, SizeHint};
|
|||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
use super::{BoxError, HttpBody};
|
use super::{BoxError, HttpBody};
|
||||||
|
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
|
||||||
|
|
||||||
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
|
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
|
||||||
pub(super) struct RequestActivity {
|
pub(super) struct RequestActivity {
|
||||||
@@ -32,12 +33,33 @@ impl Drop for RequestActivity {
|
|||||||
pub(super) struct ActivityBody {
|
pub(super) struct ActivityBody {
|
||||||
inner: HttpBody,
|
inner: HttpBody,
|
||||||
activity: RequestActivity,
|
activity: RequestActivity,
|
||||||
|
trace: Option<Arc<HttpTraceExchange>>,
|
||||||
|
terminal: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ActivityBody {
|
impl ActivityBody {
|
||||||
/// Binds one response body to its request activity guard.
|
/// Binds one response body to its request activity guard.
|
||||||
pub(super) fn new(inner: HttpBody, activity: RequestActivity) -> Self {
|
pub(super) fn new(
|
||||||
Self { inner, activity }
|
inner: HttpBody,
|
||||||
|
activity: RequestActivity,
|
||||||
|
trace: Option<Arc<HttpTraceExchange>>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
activity,
|
||||||
|
trace,
|
||||||
|
terminal: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self, state: TraceBodyState) {
|
||||||
|
if self.terminal {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.terminal = true;
|
||||||
|
if let Some(trace) = &self.trace {
|
||||||
|
trace.body_finished(TraceDirection::Response, state);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,6 +72,21 @@ impl Body for ActivityBody {
|
|||||||
context: &mut Context<'_>,
|
context: &mut Context<'_>,
|
||||||
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||||
let result = Pin::new(&mut self.inner).poll_frame(context);
|
let result = Pin::new(&mut self.inner).poll_frame(context);
|
||||||
|
match &result {
|
||||||
|
Poll::Ready(Some(Ok(frame))) => {
|
||||||
|
if let Some(data) = frame.data_ref()
|
||||||
|
&& let Some(trace) = &self.trace
|
||||||
|
{
|
||||||
|
trace.body_data(TraceDirection::Response, data);
|
||||||
|
}
|
||||||
|
if self.inner.is_end_stream() {
|
||||||
|
self.finish(TraceBodyState::Complete);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
|
||||||
|
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
|
||||||
|
Poll::Pending => {}
|
||||||
|
}
|
||||||
if result.is_ready() {
|
if result.is_ready() {
|
||||||
*self.activity.last_activity.lock() = Instant::now();
|
*self.activity.last_activity.lock() = Instant::now();
|
||||||
}
|
}
|
||||||
@@ -64,3 +101,9 @@ impl Body for ActivityBody {
|
|||||||
self.inner.size_hint()
|
self.inner.size_hint()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Drop for ActivityBody {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.finish(TraceBodyState::Aborted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+78
-2
@@ -1,11 +1,87 @@
|
|||||||
|
use std::pin::Pin;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::task::{Context, Poll};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http_body_util::{BodyExt, Empty, Limited};
|
use http_body_util::{BodyExt, Empty, Limited};
|
||||||
use hyper::Request;
|
use hyper::Request;
|
||||||
use hyper::body::{Body as _, Incoming};
|
use hyper::body::{Body, Frame, Incoming, SizeHint};
|
||||||
|
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::manager::WebProcessRuntime;
|
||||||
|
use crate::web::trace::{
|
||||||
|
HttpTraceExchange, TraceBodyState, TraceDirection,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Incoming request body wrapper that observes frames without changing streaming semantics.
|
||||||
|
pub(super) struct RequestBody {
|
||||||
|
inner: Incoming,
|
||||||
|
trace: Option<Arc<HttpTraceExchange>>,
|
||||||
|
terminal: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RequestBody {
|
||||||
|
/// Wraps one Hyper request body with optional enabled-only capture state.
|
||||||
|
pub(super) fn new(inner: Incoming, trace: Option<Arc<HttpTraceExchange>>) -> Self {
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
trace,
|
||||||
|
terminal: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self, state: TraceBodyState) {
|
||||||
|
if self.terminal {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.terminal = true;
|
||||||
|
if let Some(trace) = &self.trace {
|
||||||
|
trace.body_finished(TraceDirection::Request, state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Body for RequestBody {
|
||||||
|
type Data = Bytes;
|
||||||
|
type Error = hyper::Error;
|
||||||
|
|
||||||
|
fn poll_frame(
|
||||||
|
mut self: Pin<&mut Self>,
|
||||||
|
context: &mut Context<'_>,
|
||||||
|
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||||
|
let result = Pin::new(&mut self.inner).poll_frame(context);
|
||||||
|
match &result {
|
||||||
|
Poll::Ready(Some(Ok(frame))) => {
|
||||||
|
if let Some(data) = frame.data_ref()
|
||||||
|
&& let Some(trace) = &self.trace
|
||||||
|
{
|
||||||
|
trace.body_data(TraceDirection::Request, data);
|
||||||
|
}
|
||||||
|
if self.inner.is_end_stream() {
|
||||||
|
self.finish(TraceBodyState::Complete);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
|
||||||
|
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
|
||||||
|
Poll::Pending => {}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_end_stream(&self) -> bool {
|
||||||
|
self.inner.is_end_stream()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn size_hint(&self) -> SizeHint {
|
||||||
|
self.inner.size_hint()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RequestBody {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.finish(TraceBodyState::Aborted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Collected carrier request retaining its process-wide body reservation.
|
/// Collected carrier request retaining its process-wide body reservation.
|
||||||
pub(super) struct CollectedBody {
|
pub(super) struct CollectedBody {
|
||||||
@@ -29,7 +105,7 @@ pub(super) enum CollectBodyError {
|
|||||||
|
|
||||||
/// Collects one bounded carrier body under reader, byte, and deadline ownership.
|
/// Collects one bounded carrier body under reader, byte, and deadline ownership.
|
||||||
pub(super) async fn collect_body(
|
pub(super) async fn collect_body(
|
||||||
request: Request<Incoming>,
|
request: Request<RequestBody>,
|
||||||
runtime: &WebProcessRuntime,
|
runtime: &WebProcessRuntime,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
allow_empty: bool,
|
allow_empty: bool,
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ where
|
|||||||
B: hyper::body::Body<Data = Bytes> + Send + 'static,
|
B: hyper::body::Body<Data = Bytes> + Send + 'static,
|
||||||
B::Error: Error + Send + Sync + 'static,
|
B::Error: Error + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
|
super::set_trace_route(&request, crate::web::trace::TraceRoute::Decoy);
|
||||||
if sanitize_transport {
|
if sanitize_transport {
|
||||||
sanitize_transport_request(&mut request);
|
sanitize_transport_request(&mut request);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use hyper::header::{self, HeaderName, HeaderValue};
|
||||||
|
use hyper::{Request, StatusCode};
|
||||||
|
|
||||||
|
use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||||
|
use super::decoy::serve_decoy;
|
||||||
|
use super::request::canonical_u64_header;
|
||||||
|
use super::{
|
||||||
|
HttpResponse, carrier_empty, carrier_headers, carrier_lane, full_response, insert_header,
|
||||||
|
request_trace, service_unavailable,
|
||||||
|
};
|
||||||
|
use crate::config::WebRuntimeVhost;
|
||||||
|
use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
|
||||||
|
use crate::web::trace::{TraceDirection, TraceRoute};
|
||||||
|
|
||||||
|
/// Handles one authenticated long-poll downlink exchange.
|
||||||
|
pub(super) async fn handle_down(
|
||||||
|
request: Request<RequestBody>,
|
||||||
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
|
token_hash: TokenHash,
|
||||||
|
) -> HttpResponse {
|
||||||
|
if request.method() != hyper::Method::POST
|
||||||
|
|| request.headers().contains_key(header::CONTENT_TYPE)
|
||||||
|
{
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Downlink);
|
||||||
|
trace.bind_identity(session.trace_identity());
|
||||||
|
}
|
||||||
|
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
let CollectedBody {
|
||||||
|
request,
|
||||||
|
body,
|
||||||
|
_body_budget,
|
||||||
|
} = match collect_body(request, &runtime, 1, true).await {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||||
|
Err(CollectBodyError::Invalid(request)) => {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !body.is_empty() {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
let _lane_poll = if lane_id.is_some() {
|
||||||
|
let Some(permit) = runtime.try_lane_poll() else {
|
||||||
|
return service_unavailable();
|
||||||
|
};
|
||||||
|
Some(permit)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let result = match lane_id {
|
||||||
|
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
||||||
|
None => session.poll_down(cursor).await,
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(result) if result.body.is_empty() => {
|
||||||
|
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-down-cursor"),
|
||||||
|
&result.next_cursor.to_string(),
|
||||||
|
);
|
||||||
|
if result.lane_closed {
|
||||||
|
response.headers_mut().insert(
|
||||||
|
HeaderName::from_static("x-lane-closed"),
|
||||||
|
HeaderValue::from_static("1"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
response
|
||||||
|
}
|
||||||
|
Ok(result) => {
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.record_frames(
|
||||||
|
TraceDirection::Response,
|
||||||
|
&result.body,
|
||||||
|
&runtime.active_generation().config().web.limits,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut response = full_response(StatusCode::OK, result.body);
|
||||||
|
carrier_headers(&mut response);
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-down-cursor"),
|
||||||
|
&result.next_cursor.to_string(),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
|
||||||
|
service_unavailable()
|
||||||
|
}
|
||||||
|
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
use bytes::Bytes;
|
||||||
|
use http_body_util::{BodyExt, Full};
|
||||||
|
use hyper::header::{self, HeaderName, HeaderValue};
|
||||||
|
use hyper::{Request, Response, StatusCode};
|
||||||
|
|
||||||
|
use super::request::canonical_u64_header;
|
||||||
|
use super::{BoxError, HttpResponse};
|
||||||
|
use crate::config::WebCarrier;
|
||||||
|
use crate::web::frame;
|
||||||
|
|
||||||
|
/// Validates and resolves the optional carrier lane header.
|
||||||
|
pub(super) fn carrier_lane<B>(
|
||||||
|
request: &Request<B>,
|
||||||
|
carrier: WebCarrier,
|
||||||
|
) -> Option<Option<u32>> {
|
||||||
|
match carrier {
|
||||||
|
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
|
||||||
|
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
|
||||||
|
.and_then(|value| u32::try_from(value).ok())
|
||||||
|
.filter(|value| *value <= frame::MAX_STREAM_ID)
|
||||||
|
.map(Some),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies common binary carrier response headers.
|
||||||
|
pub(super) fn carrier_headers(response: &mut HttpResponse) {
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_TYPE,
|
||||||
|
HeaderValue::from_static("application/octet-stream"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds an empty no-store carrier response.
|
||||||
|
pub(super) fn carrier_empty(status: StatusCode) -> HttpResponse {
|
||||||
|
let mut response = empty_response(status);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the bounded retryable carrier saturation response.
|
||||||
|
pub(super) fn service_unavailable() -> HttpResponse {
|
||||||
|
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the ordinary decoy upstream failure response.
|
||||||
|
pub(super) fn bad_gateway() -> HttpResponse {
|
||||||
|
full_response(
|
||||||
|
StatusCode::BAD_GATEWAY,
|
||||||
|
Bytes::from_static(b"site unavailable\n"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the ordinary unmatched-host response.
|
||||||
|
pub(super) fn generic_not_found() -> HttpResponse {
|
||||||
|
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds one in-memory response with an exact content length.
|
||||||
|
pub(super) fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
|
||||||
|
let length = body.len();
|
||||||
|
let body = Full::new(body)
|
||||||
|
.map_err(|never| -> BoxError { match never {} })
|
||||||
|
.boxed_unsync();
|
||||||
|
let mut response = Response::new(body);
|
||||||
|
*response.status_mut() = status;
|
||||||
|
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
fn empty_response(status: StatusCode) -> HttpResponse {
|
||||||
|
full_response(status, Bytes::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Inserts one validated dynamic response header value.
|
||||||
|
pub(super) fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
|
||||||
|
if let Ok(value) = HeaderValue::from_str(value) {
|
||||||
|
response.headers_mut().insert(name, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,7 +17,7 @@ use crate::maestro::generation::test_runtime_generation;
|
|||||||
use crate::web::frame::{self, FrameType};
|
use crate::web::frame::{self, FrameType};
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::manager::WebProcessRuntime;
|
||||||
|
|
||||||
fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||||
let profile = Arc::new(WebRuntimeProfile {
|
let profile = Arc::new(WebRuntimeProfile {
|
||||||
host: "proxy.example.com".to_string(),
|
host: "proxy.example.com".to_string(),
|
||||||
public_addr: "203.0.113.10:443".parse().unwrap(),
|
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||||
@@ -25,6 +25,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
|||||||
secret_mode: WebSecretMode::Plain,
|
secret_mode: WebSecretMode::Plain,
|
||||||
carrier,
|
carrier,
|
||||||
capability,
|
capability,
|
||||||
|
key_fingerprint: "0000000000000000".to_string(),
|
||||||
max_sessions: 4,
|
max_sessions: 4,
|
||||||
max_streams: 16,
|
max_streams: 16,
|
||||||
max_streams_per_session: 4,
|
max_streams_per_session: 4,
|
||||||
@@ -71,7 +72,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
|||||||
config
|
config
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn request(
|
pub(super) async fn request(
|
||||||
listener: &TcpListener,
|
listener: &TcpListener,
|
||||||
runtime: &Arc<WebProcessRuntime>,
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
request: Vec<u8>,
|
request: Vec<u8>,
|
||||||
@@ -97,7 +98,7 @@ async fn request(
|
|||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
|
pub(super) fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
|
||||||
let separator = response
|
let separator = response
|
||||||
.windows(4)
|
.windows(4)
|
||||||
.position(|window| window == b"\r\n\r\n")
|
.position(|window| window == b"\r\n\r\n")
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use base64::Engine as _;
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
use super::tests::{request, runtime_config, split_response};
|
||||||
|
use crate::config::{WebCarrier, WebDebugBodyCapture};
|
||||||
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::manager::WebProcessRuntime;
|
||||||
|
use crate::web::trace::{TraceBodyState, TraceRecordKind, TraceRoute};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn enabled_debug_records_bridge_request_response_without_credentials() {
|
||||||
|
let capability = [18u8; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.debug.enabled = true;
|
||||||
|
config.web.debug.body_capture = WebDebugBodyCapture::Prefix;
|
||||||
|
config.web.debug.body_prefix_bytes = 4096;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(active_runtime);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let root = format!(
|
||||||
|
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.50\r\nUser-Agent: debug-client/1\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
|
||||||
|
let response = request(&listener, &runtime, root).await;
|
||||||
|
let (_, body) = split_response(&response);
|
||||||
|
let body = std::str::from_utf8(body).unwrap();
|
||||||
|
let bootstrap = body
|
||||||
|
.split_once("bootstrap=\"")
|
||||||
|
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||||
|
.map(|(token, _)| token)
|
||||||
|
.unwrap();
|
||||||
|
let records = runtime
|
||||||
|
.trace()
|
||||||
|
.snapshot_matching(|record| matches!(&record.kind, TraceRecordKind::Http(_)));
|
||||||
|
assert_eq!(records.len(), 1);
|
||||||
|
let record = &records[0].record;
|
||||||
|
assert_eq!(record.effective_ip, Some("192.0.2.50".parse().unwrap()));
|
||||||
|
assert_eq!(record.user_agent.as_deref(), Some("debug-client/1"));
|
||||||
|
assert!(record.identity.session_id.is_some());
|
||||||
|
assert_eq!(record.identity.user.as_deref(), Some("alice"));
|
||||||
|
let TraceRecordKind::Http(http) = &record.kind else {
|
||||||
|
panic!("expected HTTP debug record");
|
||||||
|
};
|
||||||
|
assert_eq!(http.method, "GET");
|
||||||
|
assert_eq!(http.path, "/");
|
||||||
|
assert_eq!(http.route, TraceRoute::Bridge);
|
||||||
|
assert_eq!(http.status, Some(200));
|
||||||
|
assert_eq!(
|
||||||
|
http.response_body.as_ref().unwrap().state,
|
||||||
|
TraceBodyState::Complete
|
||||||
|
);
|
||||||
|
let captured_response = &http.response_body.as_ref().unwrap().captured;
|
||||||
|
assert!(
|
||||||
|
!captured_response
|
||||||
|
.windows(bootstrap.len())
|
||||||
|
.any(|value| value == bootstrap.as_bytes())
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
captured_response
|
||||||
|
.windows(bootstrap.len())
|
||||||
|
.any(|value| value.iter().all(|byte| *byte == b'*'))
|
||||||
|
);
|
||||||
|
assert!(http.timings.as_ref().unwrap().response_ready_us.is_some());
|
||||||
|
assert!(http.timings.as_ref().unwrap().response_body_us.is_some());
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
+34
-223
@@ -1,34 +1,27 @@
|
|||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::net::IpAddr;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::Duration;
|
||||||
|
|
||||||
use arc_swap::ArcSwap;
|
use arc_swap::ArcSwap;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use sha2::{Digest, Sha256};
|
|
||||||
use subtle::ConstantTimeEq;
|
|
||||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tokio_util::task::TaskTracker;
|
use tokio_util::task::TaskTracker;
|
||||||
use zeroize::Zeroizing;
|
|
||||||
|
|
||||||
use crate::config::{WebCarrier, WebLimitsConfig, WebRuntimeProfile};
|
use crate::config::{WebCarrier, WebLimitsConfig};
|
||||||
use crate::maestro::generation::RuntimeGeneration;
|
use crate::maestro::generation::RuntimeGeneration;
|
||||||
use crate::web::frame;
|
use crate::web::trace::WebTraceStore;
|
||||||
use crate::web::session::WebSession;
|
|
||||||
|
|
||||||
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
|
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
|
||||||
mod state;
|
mod state;
|
||||||
|
// Bootstrap credentials and idempotent session creation are isolated from queue accounting.
|
||||||
|
mod credentials;
|
||||||
// Stream admission and synthetic tuple ownership are process-scoped.
|
// Stream admission and synthetic tuple ownership are process-scoped.
|
||||||
mod admission;
|
mod admission;
|
||||||
// Shutdown and expiry work remain outside request-path coordination.
|
// Shutdown and expiry work remain outside request-path coordination.
|
||||||
mod lifecycle;
|
mod lifecycle;
|
||||||
use state::{
|
use state::{ManagerState, control_item_reserve};
|
||||||
Bootstrap, ManagerState, allow_rate, control_item_reserve, decrement_map,
|
|
||||||
evict_oldest_unused_bootstrap, matching_profile, new_unique_token, profile_key,
|
|
||||||
remove_expired_locked,
|
|
||||||
};
|
|
||||||
|
|
||||||
const TOKEN_BYTES: usize = 32;
|
const TOKEN_BYTES: usize = 32;
|
||||||
const CLEANUP_INTERVAL: Duration = Duration::from_secs(1);
|
const CLEANUP_INTERVAL: Duration = Duration::from_secs(1);
|
||||||
@@ -63,9 +56,18 @@ pub(crate) struct CreateResult {
|
|||||||
pub(crate) carrier: WebCarrier,
|
pub(crate) carrier: WebCarrier,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Successful bridge bootstrap issuance result.
|
||||||
|
pub(crate) struct BootstrapResult {
|
||||||
|
/// Opaque one-use bootstrap credential.
|
||||||
|
pub(crate) token: String,
|
||||||
|
/// Process-unique non-secret trace identifier.
|
||||||
|
pub(crate) trace_session_id: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
||||||
pub(crate) struct WebProcessRuntime {
|
pub(crate) struct WebProcessRuntime {
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
|
trace: Arc<WebTraceStore>,
|
||||||
limits: WebLimitsConfig,
|
limits: WebLimitsConfig,
|
||||||
state: Mutex<ManagerState>,
|
state: Mutex<ManagerState>,
|
||||||
http_connections: Arc<Semaphore>,
|
http_connections: Arc<Semaphore>,
|
||||||
@@ -89,10 +91,22 @@ pub(crate) struct WebProcessRuntime {
|
|||||||
|
|
||||||
impl WebProcessRuntime {
|
impl WebProcessRuntime {
|
||||||
/// Starts one process-scoped manager using immutable allocation ceilings.
|
/// Starts one process-scoped manager using immutable allocation ceilings.
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
|
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
|
||||||
|
let config = active_runtime.load().config();
|
||||||
|
let trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||||
|
Self::start_with_trace(active_runtime, trace)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts one process-scoped manager with a shared API-visible trace store.
|
||||||
|
pub(crate) fn start_with_trace(
|
||||||
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
|
trace: Arc<WebTraceStore>,
|
||||||
|
) -> Arc<Self> {
|
||||||
let limits = active_runtime.load().config().web.limits.clone();
|
let limits = active_runtime.load().config().web.limits.clone();
|
||||||
let runtime = Arc::new(Self {
|
let runtime = Arc::new(Self {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
|
trace,
|
||||||
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
||||||
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
|
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
|
||||||
lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))),
|
lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))),
|
||||||
@@ -125,6 +139,8 @@ impl WebProcessRuntime {
|
|||||||
let Some(runtime) = weak.upgrade() else {
|
let Some(runtime) = weak.upgrade() else {
|
||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
|
let policy = runtime.active_generation().config().web.debug.clone();
|
||||||
|
runtime.trace.apply_policy(&policy);
|
||||||
runtime.cleanup();
|
runtime.cleanup();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -138,6 +154,11 @@ impl WebProcessRuntime {
|
|||||||
self.active_runtime.load_full()
|
self.active_runtime.load_full()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the process-owned WEB debug trace store.
|
||||||
|
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
|
||||||
|
&self.trace
|
||||||
|
}
|
||||||
|
|
||||||
/// Reserves one accepted HTTP connection.
|
/// Reserves one accepted HTTP connection.
|
||||||
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
|
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
|
||||||
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
|
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
|
||||||
@@ -211,216 +232,6 @@ impl WebProcessRuntime {
|
|||||||
Some((reader, body))
|
Some((reader, body))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Issues a one-use bootstrap credential for an active compatible profile.
|
|
||||||
pub(crate) fn issue_bootstrap(
|
|
||||||
&self,
|
|
||||||
profile: Arc<WebRuntimeProfile>,
|
|
||||||
client_ip: IpAddr,
|
|
||||||
) -> std::result::Result<String, ManagerError> {
|
|
||||||
let generation = self.active_generation();
|
|
||||||
let config = generation.config();
|
|
||||||
let profile = config
|
|
||||||
.web
|
|
||||||
.runtime
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|runtime| matching_profile(runtime, &profile))
|
|
||||||
.ok_or(ManagerError::Authentication)?;
|
|
||||||
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
|
|
||||||
return Err(ManagerError::Closed);
|
|
||||||
}
|
|
||||||
let now = Instant::now();
|
|
||||||
let mut state = self.state.lock();
|
|
||||||
remove_expired_locked(&mut state, now);
|
|
||||||
if state.closed
|
|
||||||
|| state
|
|
||||||
.bootstraps_per_ip
|
|
||||||
.get(&client_ip)
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(0)
|
|
||||||
>= self.limits.max_bootstraps_per_ip
|
|
||||||
|| !allow_rate(
|
|
||||||
&mut state.bootstrap_rate,
|
|
||||||
now,
|
|
||||||
self.limits.new_bootstraps_per_minute,
|
|
||||||
self.limits.new_bootstraps_burst,
|
|
||||||
)
|
|
||||||
{
|
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
|
||||||
return Err(ManagerError::Limit);
|
|
||||||
}
|
|
||||||
if state.bootstraps.len() >= self.limits.max_bootstraps_global
|
|
||||||
&& !evict_oldest_unused_bootstrap(&mut state)
|
|
||||||
{
|
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
|
||||||
return Err(ManagerError::Limit);
|
|
||||||
}
|
|
||||||
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
|
||||||
return Err(ManagerError::Limit);
|
|
||||||
};
|
|
||||||
state.bootstraps.insert(
|
|
||||||
hash,
|
|
||||||
Bootstrap {
|
|
||||||
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
|
||||||
issued_at: now,
|
|
||||||
issuance_ip: client_ip,
|
|
||||||
profile,
|
|
||||||
body_digest: [0; TOKEN_BYTES],
|
|
||||||
session_token: Zeroizing::new(String::new()),
|
|
||||||
session: None,
|
|
||||||
used: false,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
|
||||||
Ok(token)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Checks whether a bootstrap token is live before reading a request body.
|
|
||||||
pub(crate) fn has_bootstrap(&self, hash: TokenHash, host: &str) -> bool {
|
|
||||||
let now = Instant::now();
|
|
||||||
let state = self.state.lock();
|
|
||||||
state
|
|
||||||
.bootstraps
|
|
||||||
.get(&hash)
|
|
||||||
.is_some_and(|entry| entry.profile.host == host && now <= entry.expires_at)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates a session exactly once or replays the original successful result.
|
|
||||||
pub(crate) fn create_session(
|
|
||||||
self: &Arc<Self>,
|
|
||||||
bootstrap_hash: TokenHash,
|
|
||||||
host: &str,
|
|
||||||
client_ip: IpAddr,
|
|
||||||
body: &[u8],
|
|
||||||
) -> std::result::Result<CreateResult, ManagerError> {
|
|
||||||
if !frame::validate_hello(body, &self.limits) {
|
|
||||||
return Err(ManagerError::Protocol);
|
|
||||||
}
|
|
||||||
let body_digest: TokenHash = Sha256::digest(body).into();
|
|
||||||
let generation = self.active_generation();
|
|
||||||
let config = generation.config();
|
|
||||||
let now = Instant::now();
|
|
||||||
let mut state = self.state.lock();
|
|
||||||
remove_expired_locked(&mut state, now);
|
|
||||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
|
||||||
return Err(ManagerError::Authentication);
|
|
||||||
};
|
|
||||||
if entry.profile.host != host || now > entry.expires_at {
|
|
||||||
return Err(ManagerError::Authentication);
|
|
||||||
}
|
|
||||||
if entry.used {
|
|
||||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
|
||||||
if !digest_matches {
|
|
||||||
return Err(ManagerError::Authentication);
|
|
||||||
}
|
|
||||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
|
||||||
return Ok(CreateResult {
|
|
||||||
token: entry.session_token.as_str().to_owned(),
|
|
||||||
carrier: session.carrier(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if state.closed || !config.web.enabled {
|
|
||||||
return Err(ManagerError::Closed);
|
|
||||||
}
|
|
||||||
let profile = config
|
|
||||||
.web
|
|
||||||
.runtime
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|runtime| matching_profile(runtime, &entry.profile))
|
|
||||||
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
|
||||||
.ok_or(ManagerError::Authentication)?;
|
|
||||||
let profile_key = profile_key(&profile);
|
|
||||||
if state.sessions.len() >= self.limits.max_sessions_global
|
|
||||||
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
|
||||||
>= self.limits.max_sessions_per_ip
|
|
||||||
|| state
|
|
||||||
.sessions_per_profile
|
|
||||||
.get(&profile_key)
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(0)
|
|
||||||
>= profile.max_sessions
|
|
||||||
|| !allow_rate(
|
|
||||||
&mut state.session_rate,
|
|
||||||
now,
|
|
||||||
self.limits.new_sessions_per_minute,
|
|
||||||
self.limits.new_sessions_burst,
|
|
||||||
)
|
|
||||||
{
|
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
|
||||||
return Err(ManagerError::Limit);
|
|
||||||
}
|
|
||||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
|
||||||
return Err(ManagerError::Limit);
|
|
||||||
};
|
|
||||||
let session = WebSession::new(
|
|
||||||
Arc::downgrade(self),
|
|
||||||
session_hash,
|
|
||||||
client_ip,
|
|
||||||
profile,
|
|
||||||
profile_key,
|
|
||||||
self.limits.clone(),
|
|
||||||
config.web.timeouts.clone(),
|
|
||||||
);
|
|
||||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
|
||||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
|
||||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
|
||||||
let entry = state
|
|
||||||
.bootstraps
|
|
||||||
.get_mut(&bootstrap_hash)
|
|
||||||
.ok_or(ManagerError::Authentication)?;
|
|
||||||
entry.used = true;
|
|
||||||
entry.body_digest = body_digest;
|
|
||||||
entry.session_token = Zeroizing::new(session_token.clone());
|
|
||||||
entry.session = Some(Arc::clone(&session));
|
|
||||||
let issuance_ip = entry.issuance_ip;
|
|
||||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
|
||||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
|
||||||
Ok(CreateResult {
|
|
||||||
token: session_token,
|
|
||||||
carrier: session.carrier(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Resolves an authenticated session token.
|
|
||||||
pub(crate) fn get_session(
|
|
||||||
&self,
|
|
||||||
hash: TokenHash,
|
|
||||||
host: &str,
|
|
||||||
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
|
||||||
self.state
|
|
||||||
.lock()
|
|
||||||
.sessions
|
|
||||||
.get(&hash)
|
|
||||||
.cloned()
|
|
||||||
.filter(|session| session.matches_host(host))
|
|
||||||
.ok_or(ManagerError::Authentication)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Closes a live token and accepts bounded tombstone retries.
|
|
||||||
pub(crate) fn close_token(
|
|
||||||
&self,
|
|
||||||
hash: TokenHash,
|
|
||||||
host: &str,
|
|
||||||
) -> std::result::Result<(), ManagerError> {
|
|
||||||
let state = self.state.lock();
|
|
||||||
let session = state
|
|
||||||
.sessions
|
|
||||||
.get(&hash)
|
|
||||||
.filter(|session| session.matches_host(host))
|
|
||||||
.cloned();
|
|
||||||
let closed = state
|
|
||||||
.closed_tokens
|
|
||||||
.get(&hash)
|
|
||||||
.is_some_and(|closed| closed.host == host);
|
|
||||||
drop(state);
|
|
||||||
if let Some(session) = session {
|
|
||||||
session.close();
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
closed.then_some(()).ok_or(ManagerError::Authentication)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reserves bounded process-wide queue capacity for data or control traffic.
|
/// Reserves bounded process-wide queue capacity for data or control traffic.
|
||||||
pub(crate) fn try_reserve_pending(
|
pub(crate) fn try_reserve_pending(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -0,0 +1,280 @@
|
|||||||
|
use std::net::IpAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use subtle::ConstantTimeEq;
|
||||||
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
|
use super::state::{
|
||||||
|
Bootstrap, allow_rate, decrement_map, evict_oldest_unused_bootstrap, matching_profile,
|
||||||
|
new_unique_token, profile_key, remove_expired_locked,
|
||||||
|
};
|
||||||
|
use super::{
|
||||||
|
BootstrapResult, CreateResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime,
|
||||||
|
};
|
||||||
|
use crate::config::WebRuntimeProfile;
|
||||||
|
use crate::web::frame;
|
||||||
|
use crate::web::session::WebSession;
|
||||||
|
|
||||||
|
impl WebProcessRuntime {
|
||||||
|
/// Issues a one-use bootstrap credential for an active compatible profile.
|
||||||
|
pub(crate) fn issue_bootstrap(
|
||||||
|
&self,
|
||||||
|
profile: Arc<WebRuntimeProfile>,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
|
let generation = self.active_generation();
|
||||||
|
let config = generation.config();
|
||||||
|
let profile = config
|
||||||
|
.web
|
||||||
|
.runtime
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|runtime| matching_profile(runtime, &profile))
|
||||||
|
.ok_or(ManagerError::Authentication)?;
|
||||||
|
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
|
||||||
|
return Err(ManagerError::Closed);
|
||||||
|
}
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
remove_expired_locked(&mut state, now);
|
||||||
|
if state.closed
|
||||||
|
|| state
|
||||||
|
.bootstraps_per_ip
|
||||||
|
.get(&client_ip)
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0)
|
||||||
|
>= self.limits.max_bootstraps_per_ip
|
||||||
|
|| !allow_rate(
|
||||||
|
&mut state.bootstrap_rate,
|
||||||
|
now,
|
||||||
|
self.limits.new_bootstraps_per_minute,
|
||||||
|
self.limits.new_bootstraps_burst,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return Err(ManagerError::Limit);
|
||||||
|
}
|
||||||
|
if state.bootstraps.len() >= self.limits.max_bootstraps_global
|
||||||
|
&& !evict_oldest_unused_bootstrap(&mut state)
|
||||||
|
{
|
||||||
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return Err(ManagerError::Limit);
|
||||||
|
}
|
||||||
|
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
||||||
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return Err(ManagerError::Limit);
|
||||||
|
};
|
||||||
|
let trace_session_id = self.trace.next_session_id();
|
||||||
|
state.bootstraps.insert(
|
||||||
|
hash,
|
||||||
|
Bootstrap {
|
||||||
|
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
||||||
|
issued_at: now,
|
||||||
|
issuance_ip: client_ip,
|
||||||
|
profile,
|
||||||
|
trace_session_id,
|
||||||
|
body_digest: [0; TOKEN_BYTES],
|
||||||
|
session_token: Zeroizing::new(String::new()),
|
||||||
|
session: None,
|
||||||
|
used: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
|
let profile = state
|
||||||
|
.bootstraps
|
||||||
|
.get(&hash)
|
||||||
|
.map(|entry| Arc::clone(&entry.profile))
|
||||||
|
.ok_or(ManagerError::Closed)?;
|
||||||
|
drop(state);
|
||||||
|
self.trace.record_profile_lifecycle(
|
||||||
|
client_ip,
|
||||||
|
Some(trace_session_id),
|
||||||
|
&profile,
|
||||||
|
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
Ok(BootstrapResult {
|
||||||
|
token,
|
||||||
|
trace_session_id,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves non-secret bootstrap trace identity without exposing its credential.
|
||||||
|
pub(crate) fn bootstrap_trace_identity(
|
||||||
|
&self,
|
||||||
|
hash: TokenHash,
|
||||||
|
host: &str,
|
||||||
|
) -> Option<(u64, Arc<WebRuntimeProfile>)> {
|
||||||
|
let now = Instant::now();
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.bootstraps
|
||||||
|
.get(&hash)
|
||||||
|
.filter(|entry| entry.profile.host == host && now <= entry.expires_at)
|
||||||
|
.map(|entry| (entry.trace_session_id, Arc::clone(&entry.profile)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a session exactly once or replays the original successful result.
|
||||||
|
pub(crate) fn create_session(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
bootstrap_hash: TokenHash,
|
||||||
|
host: &str,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
body: &[u8],
|
||||||
|
) -> std::result::Result<CreateResult, ManagerError> {
|
||||||
|
if !frame::validate_hello(body, &self.limits) {
|
||||||
|
return Err(ManagerError::Protocol);
|
||||||
|
}
|
||||||
|
let body_digest: TokenHash = Sha256::digest(body).into();
|
||||||
|
let generation = self.active_generation();
|
||||||
|
let config = generation.config();
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
remove_expired_locked(&mut state, now);
|
||||||
|
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||||
|
return Err(ManagerError::Authentication);
|
||||||
|
};
|
||||||
|
if entry.profile.host != host || now > entry.expires_at {
|
||||||
|
return Err(ManagerError::Authentication);
|
||||||
|
}
|
||||||
|
if entry.used {
|
||||||
|
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||||
|
if !digest_matches {
|
||||||
|
return Err(ManagerError::Authentication);
|
||||||
|
}
|
||||||
|
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||||
|
let result = CreateResult {
|
||||||
|
token: entry.session_token.as_str().to_owned(),
|
||||||
|
carrier: session.carrier(),
|
||||||
|
};
|
||||||
|
let identity = session.trace_identity();
|
||||||
|
drop(state);
|
||||||
|
self.trace.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some(client_ip),
|
||||||
|
identity,
|
||||||
|
crate::web::trace::TraceLifecycleEvent::SessionReplayed,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
let trace_session_id = entry.trace_session_id;
|
||||||
|
let issued_profile = Arc::clone(&entry.profile);
|
||||||
|
if state.closed || !config.web.enabled {
|
||||||
|
return Err(ManagerError::Closed);
|
||||||
|
}
|
||||||
|
let profile = config
|
||||||
|
.web
|
||||||
|
.runtime
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|runtime| matching_profile(runtime, &issued_profile))
|
||||||
|
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
||||||
|
.ok_or(ManagerError::Authentication)?;
|
||||||
|
let profile_key = profile_key(&profile);
|
||||||
|
if state.sessions.len() >= self.limits.max_sessions_global
|
||||||
|
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||||
|
>= self.limits.max_sessions_per_ip
|
||||||
|
|| state
|
||||||
|
.sessions_per_profile
|
||||||
|
.get(&profile_key)
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0)
|
||||||
|
>= profile.max_sessions
|
||||||
|
|| !allow_rate(
|
||||||
|
&mut state.session_rate,
|
||||||
|
now,
|
||||||
|
self.limits.new_sessions_per_minute,
|
||||||
|
self.limits.new_sessions_burst,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return Err(ManagerError::Limit);
|
||||||
|
}
|
||||||
|
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||||
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return Err(ManagerError::Limit);
|
||||||
|
};
|
||||||
|
let session = WebSession::new(
|
||||||
|
Arc::downgrade(self),
|
||||||
|
session_hash,
|
||||||
|
client_ip,
|
||||||
|
trace_session_id,
|
||||||
|
profile,
|
||||||
|
profile_key,
|
||||||
|
self.limits.clone(),
|
||||||
|
config.web.timeouts.clone(),
|
||||||
|
);
|
||||||
|
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||||
|
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
|
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||||
|
let entry = state
|
||||||
|
.bootstraps
|
||||||
|
.get_mut(&bootstrap_hash)
|
||||||
|
.ok_or(ManagerError::Authentication)?;
|
||||||
|
entry.used = true;
|
||||||
|
entry.body_digest = body_digest;
|
||||||
|
entry.session_token = Zeroizing::new(session_token.clone());
|
||||||
|
entry.session = Some(Arc::clone(&session));
|
||||||
|
let issuance_ip = entry.issuance_ip;
|
||||||
|
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||||
|
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let identity = session.trace_identity();
|
||||||
|
let result = CreateResult {
|
||||||
|
token: session_token,
|
||||||
|
carrier: session.carrier(),
|
||||||
|
};
|
||||||
|
drop(state);
|
||||||
|
self.trace.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some(client_ip),
|
||||||
|
identity,
|
||||||
|
crate::web::trace::TraceLifecycleEvent::SessionCreated,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves an authenticated session token.
|
||||||
|
pub(crate) fn get_session(
|
||||||
|
&self,
|
||||||
|
hash: TokenHash,
|
||||||
|
host: &str,
|
||||||
|
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.sessions
|
||||||
|
.get(&hash)
|
||||||
|
.cloned()
|
||||||
|
.filter(|session| session.matches_host(host))
|
||||||
|
.ok_or(ManagerError::Authentication)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Closes a live token and accepts bounded tombstone retries.
|
||||||
|
pub(crate) fn close_token(
|
||||||
|
&self,
|
||||||
|
hash: TokenHash,
|
||||||
|
host: &str,
|
||||||
|
) -> std::result::Result<(), ManagerError> {
|
||||||
|
let state = self.state.lock();
|
||||||
|
let session = state
|
||||||
|
.sessions
|
||||||
|
.get(&hash)
|
||||||
|
.filter(|session| session.matches_host(host))
|
||||||
|
.cloned();
|
||||||
|
let closed = state
|
||||||
|
.closed_tokens
|
||||||
|
.get(&hash)
|
||||||
|
.is_some_and(|closed| closed.host == host);
|
||||||
|
drop(state);
|
||||||
|
if let Some(session) = session {
|
||||||
|
session.close();
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
closed.then_some(()).ok_or(ManagerError::Authentication)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ pub(super) struct Bootstrap {
|
|||||||
pub(super) issuance_ip: IpAddr,
|
pub(super) issuance_ip: IpAddr,
|
||||||
/// Immutable profile selected during capability validation.
|
/// Immutable profile selected during capability validation.
|
||||||
pub(super) profile: Arc<WebRuntimeProfile>,
|
pub(super) profile: Arc<WebRuntimeProfile>,
|
||||||
|
/// Process-unique non-secret identifier shared by bootstrap and session traces.
|
||||||
|
pub(super) trace_session_id: u64,
|
||||||
/// Digest of the accepted HELLO body for idempotent retry matching.
|
/// Digest of the accepted HELLO body for idempotent retry matching.
|
||||||
pub(super) body_digest: TokenHash,
|
pub(super) body_digest: TokenHash,
|
||||||
/// Zeroizing copy returned only for an exact session-creation retry.
|
/// Zeroizing copy returned only for an exact session-creation retry.
|
||||||
@@ -130,6 +132,7 @@ pub(super) fn matching_profile(
|
|||||||
&& profile.secret_mode == expected.secret_mode
|
&& profile.secret_mode == expected.secret_mode
|
||||||
&& profile.carrier == expected.carrier
|
&& profile.carrier == expected.carrier
|
||||||
&& profile.capability == expected.capability
|
&& profile.capability == expected.capability
|
||||||
|
&& profile.key_fingerprint == expected.key_fingerprint
|
||||||
})
|
})
|
||||||
.cloned()
|
.cloned()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,3 +12,5 @@ pub(crate) mod manager;
|
|||||||
pub(crate) mod session;
|
pub(crate) mod session;
|
||||||
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
|
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
|
||||||
pub(crate) mod stream;
|
pub(crate) mod stream;
|
||||||
|
/// Process-owned bounded WEB debugging records and capture lifecycle.
|
||||||
|
pub(crate) mod trace;
|
||||||
|
|||||||
@@ -120,6 +120,7 @@ pub(crate) struct WebSession {
|
|||||||
manager: std::sync::Weak<WebProcessRuntime>,
|
manager: std::sync::Weak<WebProcessRuntime>,
|
||||||
token_hash: TokenHash,
|
token_hash: TokenHash,
|
||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
|
trace_session_id: u64,
|
||||||
profile: Arc<WebRuntimeProfile>,
|
profile: Arc<WebRuntimeProfile>,
|
||||||
profile_key: ProfileKey,
|
profile_key: ProfileKey,
|
||||||
limits: WebLimitsConfig,
|
limits: WebLimitsConfig,
|
||||||
@@ -150,6 +151,7 @@ impl WebSession {
|
|||||||
manager: std::sync::Weak<WebProcessRuntime>,
|
manager: std::sync::Weak<WebProcessRuntime>,
|
||||||
token_hash: TokenHash,
|
token_hash: TokenHash,
|
||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
|
trace_session_id: u64,
|
||||||
profile: Arc<WebRuntimeProfile>,
|
profile: Arc<WebRuntimeProfile>,
|
||||||
profile_key: ProfileKey,
|
profile_key: ProfileKey,
|
||||||
limits: WebLimitsConfig,
|
limits: WebLimitsConfig,
|
||||||
@@ -163,6 +165,7 @@ impl WebSession {
|
|||||||
manager,
|
manager,
|
||||||
token_hash,
|
token_hash,
|
||||||
client_ip,
|
client_ip,
|
||||||
|
trace_session_id,
|
||||||
profile,
|
profile,
|
||||||
profile_key,
|
profile_key,
|
||||||
limits,
|
limits,
|
||||||
@@ -211,6 +214,30 @@ impl WebSession {
|
|||||||
self.profile.carrier
|
self.profile.carrier
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns a cloned non-secret identity only for enabled debug capture.
|
||||||
|
pub(crate) fn trace_identity(&self) -> crate::web::trace::TraceIdentity {
|
||||||
|
crate::web::trace::TraceIdentity::from_profile(self.trace_session_id, &self.profile)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one typed lifecycle event without exposing session credentials.
|
||||||
|
pub(super) fn trace_lifecycle(
|
||||||
|
&self,
|
||||||
|
event: crate::web::trace::TraceLifecycleEvent,
|
||||||
|
stream_id: Option<u32>,
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
) {
|
||||||
|
if let Some(manager) = self.manager.upgrade() {
|
||||||
|
manager.trace().record_profile_lifecycle(
|
||||||
|
self.client_ip,
|
||||||
|
Some(self.trace_session_id),
|
||||||
|
&self.profile,
|
||||||
|
event,
|
||||||
|
stream_id,
|
||||||
|
reason,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||||
pub(crate) fn close(&self) {
|
pub(crate) fn close(&self) {
|
||||||
let (data_bytes, data_items, control_bytes, control_items) = {
|
let (data_bytes, data_items, control_bytes, control_items) = {
|
||||||
@@ -253,6 +280,11 @@ impl WebSession {
|
|||||||
manager.release_pending(data_bytes, data_items, false);
|
manager.release_pending(data_bytes, data_items, false);
|
||||||
manager.release_pending(control_bytes, control_items, true);
|
manager.release_pending(control_bytes, control_items, true);
|
||||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||||
|
self.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||||
|
None,
|
||||||
|
Some("closed"),
|
||||||
|
);
|
||||||
manager.session_finished(
|
manager.session_finished(
|
||||||
self.token_hash,
|
self.token_hash,
|
||||||
self.client_ip,
|
self.client_ip,
|
||||||
|
|||||||
@@ -22,11 +22,21 @@ impl WebSession {
|
|||||||
};
|
};
|
||||||
let generation = manager.active_generation();
|
let generation = manager.active_generation();
|
||||||
if !*generation.admission_rx.borrow() {
|
if !*generation.admission_rx.borrow() {
|
||||||
|
self.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("admission_closed"),
|
||||||
|
);
|
||||||
self.stream_finished(stream_id, peer_port);
|
self.stream_finished(stream_id, peer_port);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
|
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
|
||||||
manager.record_stream_rejected();
|
manager.record_stream_rejected();
|
||||||
|
self.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("connection_limit"),
|
||||||
|
);
|
||||||
self.stream_finished(stream_id, peer_port);
|
self.stream_finished(stream_id, peer_port);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -42,11 +52,17 @@ impl WebSession {
|
|||||||
stream_id,
|
stream_id,
|
||||||
peer_port,
|
peer_port,
|
||||||
};
|
};
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamAdmitted,
|
||||||
|
Some(stream_id),
|
||||||
|
None,
|
||||||
|
);
|
||||||
let stream = WebLogicalStream::new(Arc::clone(&session), stream_id);
|
let stream = WebLogicalStream::new(Arc::clone(&session), stream_id);
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = cancel.cancelled() => {}
|
_ = cancel.cancelled() => {}
|
||||||
_ = run_stream(
|
_ = run_stream(
|
||||||
Arc::clone(&session),
|
Arc::clone(&session),
|
||||||
|
stream_id,
|
||||||
stream,
|
stream,
|
||||||
deps,
|
deps,
|
||||||
replay_checker,
|
replay_checker,
|
||||||
@@ -55,6 +71,11 @@ impl WebSession {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
if !spawned {
|
if !spawned {
|
||||||
|
self.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("generation_closed"),
|
||||||
|
);
|
||||||
self.tasks_live.fetch_sub(1, Ordering::AcqRel);
|
self.tasks_live.fetch_sub(1, Ordering::AcqRel);
|
||||||
self.stream_finished(stream_id, peer_port);
|
self.stream_finished(stream_id, peer_port);
|
||||||
self.tasks_done.notify_waiters();
|
self.tasks_done.notify_waiters();
|
||||||
@@ -100,6 +121,11 @@ struct StreamCompletion {
|
|||||||
|
|
||||||
impl Drop for StreamCompletion {
|
impl Drop for StreamCompletion {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
|
self.session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamClosed,
|
||||||
|
Some(self.stream_id),
|
||||||
|
None,
|
||||||
|
);
|
||||||
self.session.stream_finished(self.stream_id, self.peer_port);
|
self.session.stream_finished(self.stream_id, self.peer_port);
|
||||||
if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 {
|
if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 {
|
||||||
self.session.tasks_done.notify_waiters();
|
self.session.tasks_done.notify_waiters();
|
||||||
@@ -109,6 +135,7 @@ impl Drop for StreamCompletion {
|
|||||||
|
|
||||||
async fn run_stream(
|
async fn run_stream(
|
||||||
session: Arc<WebSession>,
|
session: Arc<WebSession>,
|
||||||
|
stream_id: u32,
|
||||||
stream: WebLogicalStream,
|
stream: WebLogicalStream,
|
||||||
deps: crate::proxy::authenticated::ClientRuntimeDeps,
|
deps: crate::proxy::authenticated::ClientRuntimeDeps,
|
||||||
replay_checker: Arc<crate::stats::ReplayChecker>,
|
replay_checker: Arc<crate::stats::ReplayChecker>,
|
||||||
@@ -129,14 +156,29 @@ async fn run_stream(
|
|||||||
// first byte. Session and stream quotas bound this idle phase without
|
// first byte. Session and stream quotas bound this idle phase without
|
||||||
// consuming the process-wide active-handshake budget.
|
// consuming the process-wide active-handshake budget.
|
||||||
if reader.read_exact(&mut handshake[..1]).await.is_err() {
|
if reader.read_exact(&mut handshake[..1]).await.is_err() {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("first_byte_io"),
|
||||||
|
);
|
||||||
deps.stats
|
deps.stats
|
||||||
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamFirstByte,
|
||||||
|
Some(stream_id),
|
||||||
|
None,
|
||||||
|
);
|
||||||
let Some(manager) = session.manager.upgrade() else {
|
let Some(manager) = session.manager.upgrade() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("handshake_limit"),
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let handshake_result = tokio::time::timeout(
|
let handshake_result = tokio::time::timeout(
|
||||||
@@ -163,6 +205,11 @@ async fn run_stream(
|
|||||||
drop(handshake_permit);
|
drop(handshake_permit);
|
||||||
let (reader, writer, success) = match handshake_result {
|
let (reader, writer, success) = match handshake_result {
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::HandshakeTimeout,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("timeout"),
|
||||||
|
);
|
||||||
deps.stats
|
deps.stats
|
||||||
.increment_connects_bad_with_class("web_mtproto_handshake_timeout");
|
.increment_connects_bad_with_class("web_mtproto_handshake_timeout");
|
||||||
deps.stats.increment_handshake_timeouts();
|
deps.stats.increment_handshake_timeouts();
|
||||||
@@ -170,20 +217,40 @@ async fn run_stream(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Ok(Err(_)) => {
|
Ok(Err(_)) => {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("io"),
|
||||||
|
);
|
||||||
deps.stats
|
deps.stats
|
||||||
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => {
|
Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::HandshakeSucceeded,
|
||||||
|
Some(stream_id),
|
||||||
|
None,
|
||||||
|
);
|
||||||
(reader, writer, success)
|
(reader, writer, success)
|
||||||
}
|
}
|
||||||
Ok(Ok(_)) => {
|
Ok(Ok(_)) => {
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::HandshakeRejected,
|
||||||
|
Some(stream_id),
|
||||||
|
Some("bad_client"),
|
||||||
|
);
|
||||||
deps.stats
|
deps.stats
|
||||||
.increment_connects_bad_with_class("web_mtproto_bad_client");
|
.increment_connects_bad_with_class("web_mtproto_bad_client");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let _ = run_authenticated(
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::RelayStarted,
|
||||||
|
Some(stream_id),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let relay_result = run_authenticated(
|
||||||
reader,
|
reader,
|
||||||
writer,
|
writer,
|
||||||
success,
|
success,
|
||||||
@@ -193,4 +260,9 @@ async fn run_stream(
|
|||||||
ConntrackClosePolicy::Suppress,
|
ConntrackClosePolicy::Suppress,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
session.trace_lifecycle(
|
||||||
|
crate::web::trace::TraceLifecycleEvent::RelayEnded,
|
||||||
|
Some(stream_id),
|
||||||
|
Some(if relay_result.is_ok() { "completed" } else { "error" }),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ fn test_runtime_with_dc(
|
|||||||
secret_mode: WebSecretMode::Plain,
|
secret_mode: WebSecretMode::Plain,
|
||||||
carrier,
|
carrier,
|
||||||
capability: [7; 32],
|
capability: [7; 32],
|
||||||
|
key_fingerprint: "0000000000000000".to_string(),
|
||||||
max_sessions: 4,
|
max_sessions: 4,
|
||||||
max_streams: 16,
|
max_streams: 16,
|
||||||
max_streams_per_session: 4,
|
max_streams_per_session: 4,
|
||||||
@@ -117,6 +118,7 @@ fn test_runtime_with_dc(
|
|||||||
Arc::downgrade(&manager),
|
Arc::downgrade(&manager),
|
||||||
[8; 32],
|
[8; 32],
|
||||||
"192.0.2.10".parse().unwrap(),
|
"192.0.2.10".parse().unwrap(),
|
||||||
|
1,
|
||||||
profile,
|
profile,
|
||||||
[7; 32],
|
[7; 32],
|
||||||
limits,
|
limits,
|
||||||
|
|||||||
@@ -420,6 +420,7 @@ mod tests {
|
|||||||
secret_mode: WebSecretMode::Plain,
|
secret_mode: WebSecretMode::Plain,
|
||||||
carrier: WebCarrier::Https,
|
carrier: WebCarrier::Https,
|
||||||
capability: [0; 32],
|
capability: [0; 32],
|
||||||
|
key_fingerprint: "0000000000000000".to_string(),
|
||||||
max_sessions: 1,
|
max_sessions: 1,
|
||||||
max_streams: 1,
|
max_streams: 1,
|
||||||
max_streams_per_session: 1,
|
max_streams_per_session: 1,
|
||||||
@@ -428,6 +429,7 @@ mod tests {
|
|||||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||||
[1; 32],
|
[1; 32],
|
||||||
"192.0.2.10".parse().unwrap(),
|
"192.0.2.10".parse().unwrap(),
|
||||||
|
1,
|
||||||
profile,
|
profile,
|
||||||
[2; 32],
|
[2; 32],
|
||||||
WebLimitsConfig::default(),
|
WebLimitsConfig::default(),
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
|||||||
secret_mode: WebSecretMode::Plain,
|
secret_mode: WebSecretMode::Plain,
|
||||||
carrier: WebCarrier::HttpsLanes,
|
carrier: WebCarrier::HttpsLanes,
|
||||||
capability: [0; 32],
|
capability: [0; 32],
|
||||||
|
key_fingerprint: "0000000000000000".to_string(),
|
||||||
max_sessions: 1,
|
max_sessions: 1,
|
||||||
max_streams: 2,
|
max_streams: 2,
|
||||||
max_streams_per_session: 2,
|
max_streams_per_session: 2,
|
||||||
@@ -23,6 +24,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
|||||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||||
[1; 32],
|
[1; 32],
|
||||||
"192.0.2.10".parse().unwrap(),
|
"192.0.2.10".parse().unwrap(),
|
||||||
|
1,
|
||||||
profile,
|
profile,
|
||||||
[2; 32],
|
[2; 32],
|
||||||
limits,
|
limits,
|
||||||
|
|||||||
@@ -342,6 +342,7 @@ mod tests {
|
|||||||
secret_mode: WebSecretMode::Plain,
|
secret_mode: WebSecretMode::Plain,
|
||||||
carrier: WebCarrier::Https,
|
carrier: WebCarrier::Https,
|
||||||
capability: [0; 32],
|
capability: [0; 32],
|
||||||
|
key_fingerprint: "0000000000000000".to_string(),
|
||||||
max_sessions: 1,
|
max_sessions: 1,
|
||||||
max_streams: 1,
|
max_streams: 1,
|
||||||
max_streams_per_session: 1,
|
max_streams_per_session: 1,
|
||||||
@@ -350,6 +351,7 @@ mod tests {
|
|||||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||||
[1; 32],
|
[1; 32],
|
||||||
"192.0.2.10".parse().unwrap(),
|
"192.0.2.10".parse().unwrap(),
|
||||||
|
1,
|
||||||
profile,
|
profile,
|
||||||
[2; 32],
|
[2; 32],
|
||||||
WebLimitsConfig::default(),
|
WebLimitsConfig::default(),
|
||||||
|
|||||||
@@ -0,0 +1,487 @@
|
|||||||
|
use std::net::IpAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
|
use super::sanitize::{
|
||||||
|
bounded_text, capture_limit, frame_error_name, frame_type_name, request_dynamic_bytes,
|
||||||
|
response_dynamic_bytes, sanitized_headers, scrub_body, sensitive_values,
|
||||||
|
};
|
||||||
|
use super::store::{WebTraceStore, epoch_millis};
|
||||||
|
use super::types::{
|
||||||
|
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceHttpRecord,
|
||||||
|
TraceIdentity, TraceRecord, TraceRecordKind, TraceRoute, TraceTimings,
|
||||||
|
};
|
||||||
|
use crate::config::{WebDebugBodyCapture, WebDebugConfig, WebLimitsConfig, WebRuntimeProfile};
|
||||||
|
use crate::web::frame::{self, FrameType};
|
||||||
|
|
||||||
|
const USER_AGENT_MAX_BYTES: usize = 512;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct BodyCapture {
|
||||||
|
observed_bytes: u64,
|
||||||
|
captured: Vec<u8>,
|
||||||
|
truncated: bool,
|
||||||
|
state: Option<TraceBodyState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ExchangeState {
|
||||||
|
method: String,
|
||||||
|
path: String,
|
||||||
|
route: TraceRoute,
|
||||||
|
peer_ip: IpAddr,
|
||||||
|
effective_ip: Option<IpAddr>,
|
||||||
|
user_agent: Option<String>,
|
||||||
|
identity: TraceIdentity,
|
||||||
|
request_headers: Vec<TraceHeader>,
|
||||||
|
response_headers: Vec<TraceHeader>,
|
||||||
|
request_body: BodyCapture,
|
||||||
|
response_body: BodyCapture,
|
||||||
|
status: Option<u16>,
|
||||||
|
frames: Vec<TraceFrame>,
|
||||||
|
timings: TraceTimings,
|
||||||
|
redactions: Vec<Zeroizing<Vec<u8>>>,
|
||||||
|
body_capture_blocked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One in-flight request-to-response capture with process-wide byte leases.
|
||||||
|
pub(crate) struct HttpTraceExchange {
|
||||||
|
store: Arc<WebTraceStore>,
|
||||||
|
epoch: u64,
|
||||||
|
policy: Arc<WebDebugConfig>,
|
||||||
|
started: Instant,
|
||||||
|
started_epoch_millis: u64,
|
||||||
|
state: Mutex<ExchangeState>,
|
||||||
|
reserved: AtomicUsize,
|
||||||
|
committed: AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpTraceExchange {
|
||||||
|
/// Creates one enabled exchange after the store reserved its base record lease.
|
||||||
|
pub(super) fn new<B>(
|
||||||
|
store: Arc<WebTraceStore>,
|
||||||
|
epoch: u64,
|
||||||
|
policy: Arc<WebDebugConfig>,
|
||||||
|
request: &hyper::Request<B>,
|
||||||
|
peer_ip: IpAddr,
|
||||||
|
base_reservation: usize,
|
||||||
|
) -> Arc<Self> {
|
||||||
|
let started = Instant::now();
|
||||||
|
let started_epoch_millis = epoch_millis();
|
||||||
|
let dynamic = request_dynamic_bytes(request, &policy);
|
||||||
|
let dynamic_reserved = store.try_reserve(dynamic);
|
||||||
|
let (method, path, user_agent, request_headers, redactions) = if dynamic_reserved {
|
||||||
|
(
|
||||||
|
request.method().as_str().to_string(),
|
||||||
|
request.uri().path().to_string(),
|
||||||
|
request
|
||||||
|
.headers()
|
||||||
|
.get(hyper::header::USER_AGENT)
|
||||||
|
.map(|value| bounded_text(value.as_bytes(), USER_AGENT_MAX_BYTES)),
|
||||||
|
policy
|
||||||
|
.capture_headers
|
||||||
|
.then(|| sanitized_headers(request.headers()))
|
||||||
|
.unwrap_or_default(),
|
||||||
|
sensitive_values(request.headers(), request.uri().query()),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
bounded_text(request.method().as_str().as_bytes(), 32),
|
||||||
|
bounded_text(request.uri().path().as_bytes(), 512),
|
||||||
|
None,
|
||||||
|
Vec::new(),
|
||||||
|
Vec::new(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
Arc::new(Self {
|
||||||
|
store,
|
||||||
|
epoch,
|
||||||
|
policy,
|
||||||
|
started,
|
||||||
|
started_epoch_millis,
|
||||||
|
state: Mutex::new(ExchangeState {
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
route: TraceRoute::Unknown,
|
||||||
|
peer_ip,
|
||||||
|
effective_ip: None,
|
||||||
|
user_agent,
|
||||||
|
identity: TraceIdentity::default(),
|
||||||
|
request_headers,
|
||||||
|
response_headers: Vec::new(),
|
||||||
|
request_body: BodyCapture::default(),
|
||||||
|
response_body: BodyCapture::default(),
|
||||||
|
status: None,
|
||||||
|
frames: Vec::new(),
|
||||||
|
timings: TraceTimings::default(),
|
||||||
|
redactions,
|
||||||
|
body_capture_blocked: !dynamic_reserved,
|
||||||
|
}),
|
||||||
|
reserved: AtomicUsize::new(
|
||||||
|
base_reservation + if dynamic_reserved { dynamic } else { 0 },
|
||||||
|
),
|
||||||
|
committed: AtomicBool::new(false),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sets the final request route before body polling or decoy forwarding.
|
||||||
|
pub(crate) fn set_route(&self, route: TraceRoute) {
|
||||||
|
self.state.lock().route = route;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sets the trusted effective client address after proxy-header validation.
|
||||||
|
pub(crate) fn set_effective_ip(&self, client_ip: IpAddr) {
|
||||||
|
self.state.lock().effective_ip = Some(client_ip);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Binds non-secret profile and process session identity.
|
||||||
|
pub(crate) fn bind_profile(&self, profile: &WebRuntimeProfile, session_id: u64) {
|
||||||
|
let dynamic = profile.user.len().saturating_add(profile.key_fingerprint.len());
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
state.identity.session_id = Some(session_id);
|
||||||
|
if self.reserve(dynamic) {
|
||||||
|
state.identity.user = Some(profile.user.clone());
|
||||||
|
state.identity.key_fingerprint = Some(profile.key_fingerprint.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Binds an already resolved non-secret session identity.
|
||||||
|
pub(crate) fn bind_identity(&self, identity: TraceIdentity) {
|
||||||
|
let dynamic = identity.user.as_ref().map_or(0, String::len).saturating_add(
|
||||||
|
identity
|
||||||
|
.key_fingerprint
|
||||||
|
.as_ref()
|
||||||
|
.map_or(0, String::len),
|
||||||
|
);
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
state.identity.session_id = identity.session_id;
|
||||||
|
if self.reserve(dynamic) {
|
||||||
|
state.identity.user = identity.user;
|
||||||
|
state.identity.key_fingerprint = identity.key_fingerprint;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Registers an ephemeral credential for body scrubbing before commit.
|
||||||
|
pub(crate) fn register_redaction(&self, value: &[u8]) {
|
||||||
|
if value.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if !self.reserve(value.len()) {
|
||||||
|
self.block_body_capture();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.redactions
|
||||||
|
.push(Zeroizing::new(value.to_vec()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captures response status and sanitized headers at handler completion.
|
||||||
|
pub(crate) fn response_ready<B>(&self, response: &hyper::Response<B>) {
|
||||||
|
let dynamic = response_dynamic_bytes(response, &self.policy);
|
||||||
|
let reserved = self.reserve(dynamic);
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
state.status = Some(response.status().as_u16());
|
||||||
|
if self.policy.capture_headers && reserved {
|
||||||
|
state.response_headers = sanitized_headers(response.headers());
|
||||||
|
}
|
||||||
|
if reserved {
|
||||||
|
state
|
||||||
|
.redactions
|
||||||
|
.extend(sensitive_values(response.headers(), None));
|
||||||
|
} else if dynamic != 0 {
|
||||||
|
state.body_capture_blocked = true;
|
||||||
|
state.request_body.captured.clear();
|
||||||
|
state.response_body.captured.clear();
|
||||||
|
state.request_body.truncated = true;
|
||||||
|
state.response_body.truncated = true;
|
||||||
|
}
|
||||||
|
if self.policy.capture_timings {
|
||||||
|
state.timings.response_ready_us = Some(self.elapsed_us());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Appends one body data frame without changing the proxied bytes.
|
||||||
|
pub(crate) fn body_data(&self, direction: TraceDirection, data: &[u8]) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let route = state.route;
|
||||||
|
let body_capture_blocked = state.body_capture_blocked;
|
||||||
|
let body = match direction {
|
||||||
|
TraceDirection::Request => &mut state.request_body,
|
||||||
|
TraceDirection::Response => &mut state.response_body,
|
||||||
|
};
|
||||||
|
body.observed_bytes = body
|
||||||
|
.observed_bytes
|
||||||
|
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
|
||||||
|
if data.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if body_capture_blocked {
|
||||||
|
body.truncated |= !data.is_empty();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(limit) = capture_limit(
|
||||||
|
&self.policy,
|
||||||
|
route,
|
||||||
|
self.store.max_carrier_body_bytes(),
|
||||||
|
) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if body.captured.len() >= limit {
|
||||||
|
if !data.is_empty() && !body.truncated {
|
||||||
|
self.store.record_truncation();
|
||||||
|
}
|
||||||
|
body.truncated |= !data.is_empty();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if body.captured.capacity() == 0 {
|
||||||
|
if !self.reserve(limit) {
|
||||||
|
body.truncated = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
body.captured = Vec::with_capacity(limit);
|
||||||
|
}
|
||||||
|
let take = data.len().min(limit - body.captured.len());
|
||||||
|
body.captured.extend_from_slice(&data[..take]);
|
||||||
|
if take < data.len() {
|
||||||
|
if !body.truncated {
|
||||||
|
self.store.record_truncation();
|
||||||
|
}
|
||||||
|
body.truncated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Marks one request or response body terminal state.
|
||||||
|
pub(crate) fn body_finished(&self, direction: TraceDirection, terminal: TraceBodyState) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let body = match direction {
|
||||||
|
TraceDirection::Request => &mut state.request_body,
|
||||||
|
TraceDirection::Response => &mut state.response_body,
|
||||||
|
};
|
||||||
|
if body.state.is_none() {
|
||||||
|
body.state = Some(terminal);
|
||||||
|
}
|
||||||
|
if self.policy.capture_timings {
|
||||||
|
match direction {
|
||||||
|
TraceDirection::Request => state.timings.request_body_us = Some(self.elapsed_us()),
|
||||||
|
TraceDirection::Response => state.timings.response_body_us = Some(self.elapsed_us()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(state);
|
||||||
|
if direction == TraceDirection::Response {
|
||||||
|
self.commit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attaches parsed carrier frame metadata from one complete bounded body.
|
||||||
|
pub(crate) fn record_frames(
|
||||||
|
&self,
|
||||||
|
direction: TraceDirection,
|
||||||
|
body: &[u8],
|
||||||
|
limits: &WebLimitsConfig,
|
||||||
|
) {
|
||||||
|
if !self.policy.capture_frames {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let estimated_frames = body
|
||||||
|
.len()
|
||||||
|
.div_ceil(frame::HEADER_BYTES)
|
||||||
|
.clamp(1, limits.max_frames_per_body);
|
||||||
|
let reservation = estimated_frames.saturating_mul(std::mem::size_of::<TraceFrame>());
|
||||||
|
if !self.reserve(reservation) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let frames = match frame::parse_all(body, limits) {
|
||||||
|
Ok(frames) => frames
|
||||||
|
.into_iter()
|
||||||
|
.map(|frame| TraceFrame {
|
||||||
|
direction,
|
||||||
|
frame_type: Some(frame_type_name(frame.frame_type)),
|
||||||
|
stream_id: Some(frame.stream_id),
|
||||||
|
payload_len: Some(frame.payload.len()),
|
||||||
|
window_delta: (frame.frame_type == FrameType::Window)
|
||||||
|
.then(|| frame::window_amount(frame.payload).ok())
|
||||||
|
.flatten(),
|
||||||
|
parse_error: None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
Err(error) => vec![TraceFrame {
|
||||||
|
direction,
|
||||||
|
frame_type: None,
|
||||||
|
stream_id: None,
|
||||||
|
payload_len: None,
|
||||||
|
window_delta: None,
|
||||||
|
parse_error: Some(frame_error_name(error)),
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
self.state.lock().frames.extend(frames);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Commits once after response body consumption or drop.
|
||||||
|
pub(crate) fn commit(&self) {
|
||||||
|
if self.committed.swap(true, Ordering::AcqRel) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let reserved = self.reserved.load(Ordering::Acquire);
|
||||||
|
let record = self.build_record();
|
||||||
|
if !self.store.try_commit(record, reserved, self.epoch) {
|
||||||
|
self.store.release(reserved);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reserve(&self, bytes: usize) -> bool {
|
||||||
|
if bytes == 0 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if self.store.try_reserve(bytes) {
|
||||||
|
self.reserved.fetch_add(bytes, Ordering::AcqRel);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn block_body_capture(&self) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
state.body_capture_blocked = true;
|
||||||
|
state.request_body.captured.clear();
|
||||||
|
state.response_body.captured.clear();
|
||||||
|
state.request_body.truncated = true;
|
||||||
|
state.response_body.truncated = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_record(&self) -> TraceRecord {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let redactions = std::mem::take(&mut state.redactions);
|
||||||
|
scrub_body(&mut state.request_body.captured, &redactions);
|
||||||
|
scrub_body(&mut state.response_body.captured, &redactions);
|
||||||
|
let request_body = body_snapshot(&self.policy, &mut state.request_body);
|
||||||
|
let response_body = body_snapshot(&self.policy, &mut state.response_body);
|
||||||
|
TraceRecord {
|
||||||
|
seq: self.store.next_record_seq(),
|
||||||
|
epoch_millis: self.started_epoch_millis,
|
||||||
|
peer_ip: Some(state.peer_ip),
|
||||||
|
effective_ip: state.effective_ip,
|
||||||
|
user_agent: state.user_agent.take(),
|
||||||
|
identity: std::mem::take(&mut state.identity),
|
||||||
|
kind: TraceRecordKind::Http(TraceHttpRecord {
|
||||||
|
method: std::mem::take(&mut state.method),
|
||||||
|
path: std::mem::take(&mut state.path),
|
||||||
|
route: state.route,
|
||||||
|
request_headers: std::mem::take(&mut state.request_headers),
|
||||||
|
response_headers: std::mem::take(&mut state.response_headers),
|
||||||
|
request_body,
|
||||||
|
status: state.status,
|
||||||
|
response_body,
|
||||||
|
frames: std::mem::take(&mut state.frames),
|
||||||
|
timings: self
|
||||||
|
.policy
|
||||||
|
.capture_timings
|
||||||
|
.then(|| std::mem::take(&mut state.timings)),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_us(&self) -> u64 {
|
||||||
|
self.started
|
||||||
|
.elapsed()
|
||||||
|
.as_micros()
|
||||||
|
.min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for HttpTraceExchange {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.committed.load(Ordering::Acquire) {
|
||||||
|
self.body_finished(TraceDirection::Request, TraceBodyState::Aborted);
|
||||||
|
self.body_finished(TraceDirection::Response, TraceBodyState::Aborted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn body_snapshot(
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
body: &mut BodyCapture,
|
||||||
|
) -> Option<TraceBodySnapshot> {
|
||||||
|
(policy.body_capture != WebDebugBodyCapture::Off).then(|| TraceBodySnapshot {
|
||||||
|
observed_bytes: body.observed_bytes,
|
||||||
|
captured: std::mem::take(&mut body.captured),
|
||||||
|
truncated: body.truncated,
|
||||||
|
state: body.state.unwrap_or(TraceBodyState::Aborted),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn request_response_capture_redacts_credentials_and_omits_query() {
|
||||||
|
let request_token = "request-token-0123456789";
|
||||||
|
let capability = "capability-0123456789";
|
||||||
|
let response_token = "response-token-0123456789";
|
||||||
|
let request = hyper::Request::builder()
|
||||||
|
.uri(format!("/?bridge={capability}"))
|
||||||
|
.header("authorization", format!("Bearer {request_token}"))
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
let mut policy = WebDebugConfig::default();
|
||||||
|
policy.enabled = true;
|
||||||
|
policy.body_capture = WebDebugBodyCapture::Prefix;
|
||||||
|
policy.body_prefix_bytes = 256;
|
||||||
|
let mut limits = WebLimitsConfig::default();
|
||||||
|
limits.debug_records_capacity = 4;
|
||||||
|
limits.debug_bytes_global = 16 * 1024;
|
||||||
|
let store = WebTraceStore::new(policy, &limits);
|
||||||
|
let exchange = store
|
||||||
|
.begin_http(&request, "192.0.2.30".parse().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
exchange.set_route(TraceRoute::Bridge);
|
||||||
|
exchange.body_data(
|
||||||
|
TraceDirection::Request,
|
||||||
|
format!("{request_token}:{capability}").as_bytes(),
|
||||||
|
);
|
||||||
|
exchange.body_finished(TraceDirection::Request, TraceBodyState::Complete);
|
||||||
|
|
||||||
|
let response = hyper::Response::builder()
|
||||||
|
.status(hyper::StatusCode::OK)
|
||||||
|
.header("x-session-token", response_token)
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
exchange.response_ready(&response);
|
||||||
|
exchange.body_data(TraceDirection::Response, response_token.as_bytes());
|
||||||
|
exchange.body_finished(TraceDirection::Response, TraceBodyState::Complete);
|
||||||
|
|
||||||
|
let records = store.snapshot_matching(|_| true);
|
||||||
|
assert_eq!(records.len(), 1);
|
||||||
|
let TraceRecordKind::Http(http) = &records[0].record.kind else {
|
||||||
|
panic!("expected HTTP debug record");
|
||||||
|
};
|
||||||
|
assert_eq!(http.path, "/");
|
||||||
|
assert_eq!(http.route, TraceRoute::Bridge);
|
||||||
|
assert!(
|
||||||
|
http.request_headers
|
||||||
|
.iter()
|
||||||
|
.any(|header| header.name == "authorization" && header.value.is_none())
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
http.response_headers
|
||||||
|
.iter()
|
||||||
|
.any(|header| header.name == "x-session-token" && header.value.is_none())
|
||||||
|
);
|
||||||
|
let request_body = http.request_body.as_ref().unwrap();
|
||||||
|
let response_body = http.response_body.as_ref().unwrap();
|
||||||
|
for secret in [request_token.as_bytes(), capability.as_bytes()] {
|
||||||
|
assert!(!request_body.captured.windows(secret.len()).any(|value| value == secret));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!response_body
|
||||||
|
.captured
|
||||||
|
.windows(response_token.len())
|
||||||
|
.any(|value| value == response_token.as_bytes())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
//! Process-owned bounded WEB request, frame, and lifecycle debug recording.
|
||||||
|
|
||||||
|
// HTTP exchange ownership binds byte leases to one request and response.
|
||||||
|
mod exchange;
|
||||||
|
// Header, credential, body, and frame sanitization stays independent from storage.
|
||||||
|
mod sanitize;
|
||||||
|
// The process-wide store owns policy epochs, ring eviction, and render admission.
|
||||||
|
mod store;
|
||||||
|
// Closed record types define the status-page data contract.
|
||||||
|
mod types;
|
||||||
|
|
||||||
|
pub(crate) use exchange::HttpTraceExchange;
|
||||||
|
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
|
||||||
|
pub(crate) use types::{
|
||||||
|
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceHeader, TraceIdentity,
|
||||||
|
TraceLifecycleEvent, TraceRecord, TraceRecordKind, TraceRoute,
|
||||||
|
};
|
||||||
@@ -0,0 +1,297 @@
|
|||||||
|
use hyper::header;
|
||||||
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
|
use super::types::{TraceHeader, TraceRoute};
|
||||||
|
use crate::config::{WebDebugBodyCapture, WebDebugConfig};
|
||||||
|
use crate::web::frame::{FrameError, FrameType};
|
||||||
|
|
||||||
|
const USER_AGENT_MAX_BYTES: usize = 512;
|
||||||
|
const MIN_REDACTION_BYTES: usize = 8;
|
||||||
|
const MAX_REDACTION_BYTES: usize = 512;
|
||||||
|
const MAX_HEADER_REDACTIONS: usize = 16;
|
||||||
|
|
||||||
|
/// Calculates a conservative request metadata and credential lease.
|
||||||
|
pub(super) fn request_dynamic_bytes<B>(
|
||||||
|
request: &hyper::Request<B>,
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
) -> usize {
|
||||||
|
request
|
||||||
|
.method()
|
||||||
|
.as_str()
|
||||||
|
.len()
|
||||||
|
.saturating_add(request.uri().path().len())
|
||||||
|
.saturating_add(
|
||||||
|
request
|
||||||
|
.headers()
|
||||||
|
.get(header::USER_AGENT)
|
||||||
|
.map_or(0, |value| lossy_text_reservation(value.as_bytes(), USER_AGENT_MAX_BYTES)),
|
||||||
|
)
|
||||||
|
.saturating_add(
|
||||||
|
policy
|
||||||
|
.capture_headers
|
||||||
|
.then(|| sanitized_header_bytes(request.headers()))
|
||||||
|
.unwrap_or(0),
|
||||||
|
)
|
||||||
|
.saturating_add(sensitive_value_bytes(
|
||||||
|
request.headers(),
|
||||||
|
request.uri().query(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Calculates a conservative response metadata and credential lease.
|
||||||
|
pub(super) fn response_dynamic_bytes<B>(
|
||||||
|
response: &hyper::Response<B>,
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
) -> usize {
|
||||||
|
policy
|
||||||
|
.capture_headers
|
||||||
|
.then(|| sanitized_header_bytes(response.headers()))
|
||||||
|
.unwrap_or(0)
|
||||||
|
.saturating_add(sensitive_value_bytes(response.headers(), None))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copies header names and only allowlisted bounded values.
|
||||||
|
pub(super) fn sanitized_headers(headers: &hyper::HeaderMap) -> Vec<TraceHeader> {
|
||||||
|
headers
|
||||||
|
.iter()
|
||||||
|
.map(|(name, value)| TraceHeader {
|
||||||
|
name: name.as_str().to_string(),
|
||||||
|
value: header_value_allowed(name)
|
||||||
|
.then(|| bounded_text(value.as_bytes(), 4096)),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copies a bounded set of ephemeral credentials into zeroizing scrub patterns.
|
||||||
|
pub(super) fn sensitive_values(
|
||||||
|
headers: &hyper::HeaderMap,
|
||||||
|
query: Option<&str>,
|
||||||
|
) -> Vec<Zeroizing<Vec<u8>>> {
|
||||||
|
let mut values = Vec::new();
|
||||||
|
for name in [
|
||||||
|
"authorization",
|
||||||
|
"proxy-authorization",
|
||||||
|
"x-session-token",
|
||||||
|
"sec-websocket-protocol",
|
||||||
|
] {
|
||||||
|
for value in headers.get_all(name) {
|
||||||
|
push_redaction(&mut values, value.as_bytes());
|
||||||
|
if let Some(token) = value.as_bytes().strip_prefix(b"Bearer ") {
|
||||||
|
push_redaction(&mut values, token);
|
||||||
|
}
|
||||||
|
if values.len() >= MAX_HEADER_REDACTIONS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if values.len() >= MAX_HEADER_REDACTIONS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge=")) {
|
||||||
|
push_redaction(&mut values, capability.as_bytes());
|
||||||
|
}
|
||||||
|
values
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Converts at most `limit` raw bytes into loss-tolerant display text.
|
||||||
|
pub(super) fn bounded_text(value: &[u8], limit: usize) -> String {
|
||||||
|
String::from_utf8_lossy(&value[..value.len().min(limit)]).into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Overwrites complete credentials and a prefix cut by body truncation.
|
||||||
|
pub(super) fn scrub_body(body: &mut [u8], redactions: &[Zeroizing<Vec<u8>>]) {
|
||||||
|
for secret in redactions.iter().filter(|secret| !secret.is_empty()) {
|
||||||
|
if secret.len() <= body.len() {
|
||||||
|
for offset in 0..=body.len() - secret.len() {
|
||||||
|
if body[offset..].starts_with(secret) {
|
||||||
|
body[offset..offset + secret.len()].fill(b'*');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let overlap = secret.len().min(body.len());
|
||||||
|
for length in (1..=overlap).rev() {
|
||||||
|
if body.ends_with(&secret[..length]) {
|
||||||
|
let start = body.len() - length;
|
||||||
|
body[start..].fill(b'*');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves the route-sensitive retained body allocation ceiling.
|
||||||
|
pub(super) fn capture_limit(
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
route: TraceRoute,
|
||||||
|
max_carrier_body_bytes: usize,
|
||||||
|
) -> Option<usize> {
|
||||||
|
match policy.body_capture {
|
||||||
|
WebDebugBodyCapture::Off | WebDebugBodyCapture::Metadata => None,
|
||||||
|
WebDebugBodyCapture::Prefix => Some(if decoy_route(route) {
|
||||||
|
policy.decoy_body_prefix_bytes
|
||||||
|
} else {
|
||||||
|
policy.body_prefix_bytes
|
||||||
|
}),
|
||||||
|
WebDebugBodyCapture::Full => Some(if decoy_route(route) {
|
||||||
|
policy.decoy_body_prefix_bytes
|
||||||
|
} else {
|
||||||
|
max_carrier_body_bytes
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a closed display label for one parsed frame type.
|
||||||
|
pub(super) fn frame_type_name(frame_type: FrameType) -> &'static str {
|
||||||
|
match frame_type {
|
||||||
|
FrameType::Open => "OPEN",
|
||||||
|
FrameType::Data => "DATA",
|
||||||
|
FrameType::Close => "CLOSE",
|
||||||
|
FrameType::Window => "WINDOW",
|
||||||
|
FrameType::Ping => "PING",
|
||||||
|
FrameType::Pong => "PONG",
|
||||||
|
FrameType::Hello => "HELLO",
|
||||||
|
FrameType::Welcome => "WELCOME",
|
||||||
|
FrameType::Bye => "BYE",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a closed display label for one frame parse failure.
|
||||||
|
pub(super) fn frame_error_name(error: FrameError) -> &'static str {
|
||||||
|
match error {
|
||||||
|
FrameError::EmptyBatch => "empty_batch",
|
||||||
|
FrameError::TooManyFrames => "too_many_frames",
|
||||||
|
FrameError::Incomplete => "incomplete",
|
||||||
|
FrameError::PayloadLimit => "payload_limit",
|
||||||
|
FrameError::UnknownType => "unknown_type",
|
||||||
|
FrameError::InvalidShape => "invalid_shape",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitized_header_bytes(headers: &hyper::HeaderMap) -> usize {
|
||||||
|
headers.iter().fold(0usize, |total, (name, value)| {
|
||||||
|
total
|
||||||
|
.saturating_add(std::mem::size_of::<TraceHeader>())
|
||||||
|
.saturating_add(name.as_str().len())
|
||||||
|
.saturating_add(
|
||||||
|
header_value_allowed(name)
|
||||||
|
.then(|| lossy_text_reservation(value.as_bytes(), 4096))
|
||||||
|
.unwrap_or(0),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn header_value_allowed(name: &hyper::header::HeaderName) -> bool {
|
||||||
|
matches!(
|
||||||
|
name.as_str(),
|
||||||
|
"host"
|
||||||
|
| "user-agent"
|
||||||
|
| "content-type"
|
||||||
|
| "content-length"
|
||||||
|
| "origin"
|
||||||
|
| "x-forwarded-for"
|
||||||
|
| "x-up-seq"
|
||||||
|
| "x-up-ack"
|
||||||
|
| "x-down-cursor"
|
||||||
|
| "x-lane-id"
|
||||||
|
| "x-lane-closed"
|
||||||
|
| "x-carrier-mode"
|
||||||
|
| "retry-after"
|
||||||
|
| "cache-control"
|
||||||
|
| "etag"
|
||||||
|
| "if-none-match"
|
||||||
|
| "accept"
|
||||||
|
| "accept-encoding"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sensitive_value_bytes(headers: &hyper::HeaderMap, query: Option<&str>) -> usize {
|
||||||
|
let mut total = 0usize;
|
||||||
|
let mut retained = 0usize;
|
||||||
|
for name in [
|
||||||
|
"authorization",
|
||||||
|
"proxy-authorization",
|
||||||
|
"x-session-token",
|
||||||
|
"sec-websocket-protocol",
|
||||||
|
] {
|
||||||
|
for value in headers.get_all(name) {
|
||||||
|
for candidate in [
|
||||||
|
Some(value.as_bytes()),
|
||||||
|
value.as_bytes().strip_prefix(b"Bearer "),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
{
|
||||||
|
if (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&candidate.len()) {
|
||||||
|
total = total
|
||||||
|
.saturating_add(std::mem::size_of::<Vec<u8>>())
|
||||||
|
.saturating_add(candidate.len());
|
||||||
|
retained += 1;
|
||||||
|
if retained >= MAX_HEADER_REDACTIONS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if retained >= MAX_HEADER_REDACTIONS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if retained >= MAX_HEADER_REDACTIONS {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge="))
|
||||||
|
&& (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&capability.len())
|
||||||
|
{
|
||||||
|
total = total
|
||||||
|
.saturating_add(std::mem::size_of::<Vec<u8>>())
|
||||||
|
.saturating_add(capability.len());
|
||||||
|
}
|
||||||
|
total
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lossy_text_reservation(value: &[u8], limit: usize) -> usize {
|
||||||
|
value.len().min(limit).saturating_mul(3)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decoy_route(route: TraceRoute) -> bool {
|
||||||
|
matches!(route, TraceRoute::Unknown | TraceRoute::Decoy)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_redaction(values: &mut Vec<Zeroizing<Vec<u8>>>, value: &[u8]) {
|
||||||
|
if value.len() < MIN_REDACTION_BYTES || value.len() > MAX_REDACTION_BYTES {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if !values.iter().any(|existing| existing.as_slice() == value) {
|
||||||
|
values.push(Zeroizing::new(value.to_vec()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn full_capture_keeps_decoy_bodies_prefix_bounded() {
|
||||||
|
let mut policy = WebDebugConfig::default();
|
||||||
|
policy.body_capture = WebDebugBodyCapture::Full;
|
||||||
|
policy.decoy_body_prefix_bytes = 123;
|
||||||
|
assert_eq!(capture_limit(&policy, TraceRoute::Decoy, 4096), Some(123));
|
||||||
|
assert_eq!(capture_limit(&policy, TraceRoute::Uplink, 4096), Some(4096));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scrub_removes_complete_and_prefix_truncated_credentials() {
|
||||||
|
let secret = Zeroizing::new(b"credential-value".to_vec());
|
||||||
|
let mut complete = b"before credential-value after".to_vec();
|
||||||
|
scrub_body(&mut complete, std::slice::from_ref(&secret));
|
||||||
|
assert!(
|
||||||
|
!complete
|
||||||
|
.windows(secret.len())
|
||||||
|
.any(|value| value == secret.as_slice())
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut truncated = b"before credent".to_vec();
|
||||||
|
scrub_body(&mut truncated, &[secret]);
|
||||||
|
assert!(truncated.ends_with(b"*******"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,442 @@
|
|||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||||
|
|
||||||
|
use super::exchange::HttpTraceExchange;
|
||||||
|
use super::types::{
|
||||||
|
TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind,
|
||||||
|
};
|
||||||
|
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
||||||
|
|
||||||
|
const BASE_RECORD_RESERVATION: usize = 1024;
|
||||||
|
|
||||||
|
struct RingState {
|
||||||
|
records: VecDeque<Arc<StoredTraceRecord>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One retained record whose lease survives status-page snapshots.
|
||||||
|
pub(crate) struct StoredTraceRecord {
|
||||||
|
/// Immutable trace record.
|
||||||
|
pub(crate) record: TraceRecord,
|
||||||
|
bytes: usize,
|
||||||
|
used_bytes: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for StoredTraceRecord {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.used_bytes.fetch_sub(self.bytes, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Point-in-time process-owned trace counters and ring bounds.
|
||||||
|
pub(crate) struct TraceStoreStatus {
|
||||||
|
/// Current debug policy.
|
||||||
|
pub(crate) policy: Arc<WebDebugConfig>,
|
||||||
|
/// Retained record count.
|
||||||
|
pub(crate) records: usize,
|
||||||
|
/// Configured record capacity.
|
||||||
|
pub(crate) records_capacity: usize,
|
||||||
|
/// Retained plus in-flight byte leases.
|
||||||
|
pub(crate) used_bytes: usize,
|
||||||
|
/// Configured byte capacity.
|
||||||
|
pub(crate) bytes_capacity: usize,
|
||||||
|
/// Records dropped on commit-lock contention.
|
||||||
|
pub(crate) contention_drops: u64,
|
||||||
|
/// Oldest records evicted by ring capacity.
|
||||||
|
pub(crate) evictions: u64,
|
||||||
|
/// Body or metadata captures truncated by policy or byte capacity.
|
||||||
|
pub(crate) byte_truncations: u64,
|
||||||
|
/// Earliest retained record sequence.
|
||||||
|
pub(crate) earliest_seq: Option<u64>,
|
||||||
|
/// Latest retained record sequence.
|
||||||
|
pub(crate) latest_seq: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process-owned bounded WEB debug trace store.
|
||||||
|
pub(crate) struct WebTraceStore {
|
||||||
|
policy: ArcSwap<WebDebugConfig>,
|
||||||
|
policy_update: Mutex<()>,
|
||||||
|
enabled: AtomicBool,
|
||||||
|
epoch: AtomicU64,
|
||||||
|
records_capacity: usize,
|
||||||
|
bytes_capacity: usize,
|
||||||
|
max_carrier_body_bytes: usize,
|
||||||
|
used_bytes: Arc<AtomicUsize>,
|
||||||
|
ring: Mutex<RingState>,
|
||||||
|
next_record_seq: AtomicU64,
|
||||||
|
next_session_id: AtomicU64,
|
||||||
|
contention_drops: AtomicU64,
|
||||||
|
evictions: AtomicU64,
|
||||||
|
byte_truncations: AtomicU64,
|
||||||
|
renders: Arc<Semaphore>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebTraceStore {
|
||||||
|
/// Creates a process-owned store from restart-only capacity and initial policy.
|
||||||
|
pub(crate) fn new(policy: WebDebugConfig, limits: &WebLimitsConfig) -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
enabled: AtomicBool::new(policy.enabled),
|
||||||
|
policy: ArcSwap::from_pointee(policy),
|
||||||
|
policy_update: Mutex::new(()),
|
||||||
|
epoch: AtomicU64::new(1),
|
||||||
|
records_capacity: limits.debug_records_capacity,
|
||||||
|
bytes_capacity: limits.debug_bytes_global,
|
||||||
|
max_carrier_body_bytes: limits.max_body_bytes,
|
||||||
|
used_bytes: Arc::new(AtomicUsize::new(0)),
|
||||||
|
ring: Mutex::new(RingState {
|
||||||
|
records: VecDeque::with_capacity(limits.debug_records_capacity),
|
||||||
|
}),
|
||||||
|
next_record_seq: AtomicU64::new(1),
|
||||||
|
next_session_id: AtomicU64::new(1),
|
||||||
|
contention_drops: AtomicU64::new(0),
|
||||||
|
evictions: AtomicU64::new(0),
|
||||||
|
byte_truncations: AtomicU64::new(0),
|
||||||
|
renders: Arc::new(Semaphore::new(2)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies one hot policy and clears incompatible retained records.
|
||||||
|
pub(crate) fn apply_policy(&self, policy: &WebDebugConfig) {
|
||||||
|
let _policy_update = self.policy_update.lock();
|
||||||
|
let current = self.policy.load_full();
|
||||||
|
if current.as_ref() == policy {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let capture_changed = current.enabled != policy.enabled
|
||||||
|
|| current.capture_lifecycle != policy.capture_lifecycle
|
||||||
|
|| current.capture_headers != policy.capture_headers
|
||||||
|
|| current.capture_timings != policy.capture_timings
|
||||||
|
|| current.capture_frames != policy.capture_frames
|
||||||
|
|| current.body_capture != policy.body_capture
|
||||||
|
|| current.body_prefix_bytes != policy.body_prefix_bytes
|
||||||
|
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
|
||||||
|
self.policy.store(Arc::new(policy.clone()));
|
||||||
|
self.enabled.store(policy.enabled, Ordering::Release);
|
||||||
|
if capture_changed {
|
||||||
|
self.epoch.fetch_add(1, Ordering::AcqRel);
|
||||||
|
self.ring.lock().records.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allocates a process-unique monotonic WEB session trace identifier.
|
||||||
|
pub(crate) fn next_session_id(&self) -> u64 {
|
||||||
|
self.next_session_id.fetch_add(1, Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts one HTTP exchange only when debugging is enabled and budgeted.
|
||||||
|
pub(crate) fn begin_http<B>(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
request: &hyper::Request<B>,
|
||||||
|
peer_ip: IpAddr,
|
||||||
|
) -> Option<Arc<HttpTraceExchange>> {
|
||||||
|
if !self.enabled.load(Ordering::Acquire) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let epoch = self.epoch.load(Ordering::Acquire);
|
||||||
|
let policy = self.policy.load_full();
|
||||||
|
if !policy.enabled || !self.try_reserve_record(BASE_RECORD_RESERVATION) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(HttpTraceExchange::new(
|
||||||
|
Arc::clone(self),
|
||||||
|
epoch,
|
||||||
|
policy,
|
||||||
|
request,
|
||||||
|
peer_ip,
|
||||||
|
BASE_RECORD_RESERVATION,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one typed lifecycle event without retaining dynamic error strings.
|
||||||
|
pub(crate) fn record_lifecycle(
|
||||||
|
&self,
|
||||||
|
peer_ip: Option<IpAddr>,
|
||||||
|
effective_ip: Option<IpAddr>,
|
||||||
|
identity: TraceIdentity,
|
||||||
|
event: TraceLifecycleEvent,
|
||||||
|
stream_id: Option<u32>,
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
) {
|
||||||
|
if !self.enabled.load(Ordering::Acquire) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let epoch = self.epoch.load(Ordering::Acquire);
|
||||||
|
let policy = self.policy.load_full();
|
||||||
|
let identity_bytes = identity
|
||||||
|
.user
|
||||||
|
.as_ref()
|
||||||
|
.map_or(0, String::len)
|
||||||
|
.checked_add(
|
||||||
|
identity
|
||||||
|
.key_fingerprint
|
||||||
|
.as_ref()
|
||||||
|
.map_or(0, String::len),
|
||||||
|
);
|
||||||
|
let Some(reservation) = identity_bytes
|
||||||
|
.and_then(|bytes| BASE_RECORD_RESERVATION.checked_add(bytes))
|
||||||
|
else {
|
||||||
|
self.record_truncation();
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !policy.enabled
|
||||||
|
|| !policy.capture_lifecycle
|
||||||
|
|| !self.try_reserve_record(reservation)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let record = TraceRecord {
|
||||||
|
seq: self.next_record_seq(),
|
||||||
|
epoch_millis: epoch_millis(),
|
||||||
|
peer_ip,
|
||||||
|
effective_ip,
|
||||||
|
user_agent: None,
|
||||||
|
identity,
|
||||||
|
kind: TraceRecordKind::Lifecycle(TraceLifecycleRecord {
|
||||||
|
event,
|
||||||
|
stream_id,
|
||||||
|
reason,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
if !self.try_commit(record, reservation, epoch) {
|
||||||
|
self.release(reservation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records lifecycle identity from a profile only after enabled-policy checks.
|
||||||
|
pub(crate) fn record_profile_lifecycle(
|
||||||
|
&self,
|
||||||
|
effective_ip: IpAddr,
|
||||||
|
session_id: Option<u64>,
|
||||||
|
profile: &crate::config::WebRuntimeProfile,
|
||||||
|
event: TraceLifecycleEvent,
|
||||||
|
stream_id: Option<u32>,
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
) {
|
||||||
|
if !self.enabled.load(Ordering::Acquire) || !self.policy.load().capture_lifecycle {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some(effective_ip),
|
||||||
|
TraceIdentity::from_optional_profile(session_id, profile),
|
||||||
|
event,
|
||||||
|
stream_id,
|
||||||
|
reason,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns records matching the supplied predicate in newest-first order.
|
||||||
|
pub(crate) fn snapshot_matching<F>(&self, mut matches: F) -> Vec<Arc<StoredTraceRecord>>
|
||||||
|
where
|
||||||
|
F: FnMut(&TraceRecord) -> bool,
|
||||||
|
{
|
||||||
|
self.ring
|
||||||
|
.lock()
|
||||||
|
.records
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.filter(|record| matches(&record.record))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns current bounds, counters, and retained sequence range.
|
||||||
|
pub(crate) fn status(&self) -> TraceStoreStatus {
|
||||||
|
let ring = self.ring.lock();
|
||||||
|
TraceStoreStatus {
|
||||||
|
policy: self.policy.load_full(),
|
||||||
|
records: ring.records.len(),
|
||||||
|
records_capacity: self.records_capacity,
|
||||||
|
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||||
|
bytes_capacity: self.bytes_capacity,
|
||||||
|
contention_drops: self.contention_drops.load(Ordering::Relaxed),
|
||||||
|
evictions: self.evictions.load(Ordering::Relaxed),
|
||||||
|
byte_truncations: self.byte_truncations.load(Ordering::Relaxed),
|
||||||
|
earliest_seq: ring.records.front().map(|record| record.record.seq),
|
||||||
|
latest_seq: ring.records.back().map(|record| record.record.seq),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reserves one of two bounded concurrent status-page render slots.
|
||||||
|
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
|
||||||
|
Arc::clone(&self.renders).try_acquire_owned().ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the current capture-policy epoch.
|
||||||
|
pub(super) fn policy_epoch(&self) -> u64 {
|
||||||
|
self.epoch.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the restart-frozen recognized carrier body ceiling.
|
||||||
|
pub(super) fn max_carrier_body_bytes(&self) -> usize {
|
||||||
|
self.max_carrier_body_bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Atomically reserves debug bytes or records one truncation.
|
||||||
|
pub(super) fn try_reserve(&self, bytes: usize) -> bool {
|
||||||
|
if self.try_reserve_inner(bytes) {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_reserve_inner(&self, bytes: usize) -> bool {
|
||||||
|
let mut current = self.used_bytes.load(Ordering::Acquire);
|
||||||
|
loop {
|
||||||
|
let Some(next) = current.checked_add(bytes) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if next > self.bytes_capacity {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
match self.used_bytes.compare_exchange_weak(
|
||||||
|
current,
|
||||||
|
next,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Acquire,
|
||||||
|
) {
|
||||||
|
Ok(_) => return true,
|
||||||
|
Err(actual) => current = actual,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_reserve_record(&self, bytes: usize) -> bool {
|
||||||
|
if self.try_reserve_inner(bytes) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let Some(mut ring) = self.ring.try_lock() else {
|
||||||
|
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
while let Some(record) = ring.records.pop_front() {
|
||||||
|
self.evictions.fetch_add(1, Ordering::Relaxed);
|
||||||
|
drop(record);
|
||||||
|
if self.try_reserve_inner(bytes) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Releases an in-flight lease that was not transferred into a record.
|
||||||
|
pub(super) fn release(&self, bytes: usize) {
|
||||||
|
self.used_bytes.fetch_sub(bytes, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Increments the closed truncation counter.
|
||||||
|
pub(super) fn record_truncation(&self) {
|
||||||
|
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allocates a process-monotonic record sequence number.
|
||||||
|
pub(super) fn next_record_seq(&self) -> u64 {
|
||||||
|
self.next_record_seq.fetch_add(1, Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attempts one non-blocking ring commit under the originating policy epoch.
|
||||||
|
pub(super) fn try_commit(&self, record: TraceRecord, bytes: usize, epoch: u64) -> bool {
|
||||||
|
if epoch != self.policy_epoch() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(mut ring) = self.ring.try_lock() else {
|
||||||
|
self.contention_drops.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if epoch != self.policy_epoch() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
while ring.records.len() >= self.records_capacity {
|
||||||
|
ring.records.pop_front();
|
||||||
|
self.evictions.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
ring.records.push_back(Arc::new(StoredTraceRecord {
|
||||||
|
record,
|
||||||
|
bytes,
|
||||||
|
used_bytes: Arc::clone(&self.used_bytes),
|
||||||
|
}));
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns Unix epoch milliseconds with saturation for stored display timestamps.
|
||||||
|
pub(crate) fn epoch_millis() -> u64 {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis()
|
||||||
|
.min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||||
|
let mut policy = WebDebugConfig::default();
|
||||||
|
policy.enabled = true;
|
||||||
|
let mut limits = WebLimitsConfig::default();
|
||||||
|
limits.debug_records_capacity = records_capacity;
|
||||||
|
limits.debug_bytes_global = bytes_capacity;
|
||||||
|
WebTraceStore::new(policy, &limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
||||||
|
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||||
|
for _ in 0..3 {
|
||||||
|
store.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.10".parse().unwrap()),
|
||||||
|
TraceIdentity::default(),
|
||||||
|
TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = store.snapshot_matching(|_| true);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot
|
||||||
|
.iter()
|
||||||
|
.map(|record| record.record.seq)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec![3, 2]
|
||||||
|
);
|
||||||
|
assert_eq!(store.status().evictions, 1);
|
||||||
|
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||||
|
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
store.apply_policy(&policy);
|
||||||
|
assert_eq!(store.status().records, 0);
|
||||||
|
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||||
|
drop(snapshot);
|
||||||
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
||||||
|
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||||
|
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||||
|
let exchange = store
|
||||||
|
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut changed = WebDebugConfig::default();
|
||||||
|
changed.enabled = true;
|
||||||
|
changed.capture_headers = false;
|
||||||
|
store.apply_policy(&changed);
|
||||||
|
exchange.commit();
|
||||||
|
|
||||||
|
assert_eq!(store.status().records, 0);
|
||||||
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
use std::net::IpAddr;
|
||||||
|
|
||||||
|
/// WEB HTTP route classification retained by the debug trace.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum TraceRoute {
|
||||||
|
/// Routing has not completed yet.
|
||||||
|
Unknown,
|
||||||
|
/// Ordinary static or upstream fallback traffic.
|
||||||
|
Decoy,
|
||||||
|
/// Authenticated bridge page issuance.
|
||||||
|
Bridge,
|
||||||
|
/// Bootstrap-to-session exchange.
|
||||||
|
Session,
|
||||||
|
/// Carrier uplink exchange.
|
||||||
|
Uplink,
|
||||||
|
/// Carrier downlink exchange.
|
||||||
|
Downlink,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceRoute {
|
||||||
|
/// Returns the stable status-page label.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Unknown => "unknown",
|
||||||
|
Self::Decoy => "decoy",
|
||||||
|
Self::Bridge => "bridge",
|
||||||
|
Self::Session => "session",
|
||||||
|
Self::Uplink => "uplink",
|
||||||
|
Self::Downlink => "downlink",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Direction of captured HTTP body or carrier frame data.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum TraceDirection {
|
||||||
|
/// Client-to-server data.
|
||||||
|
Request,
|
||||||
|
/// Server-to-client data.
|
||||||
|
Response,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceDirection {
|
||||||
|
/// Returns the stable status-page label.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Request => "request",
|
||||||
|
Self::Response => "response",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Terminal observation state of one HTTP body.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum TraceBodyState {
|
||||||
|
/// The body completed normally.
|
||||||
|
Complete,
|
||||||
|
/// Body polling returned an error.
|
||||||
|
Error,
|
||||||
|
/// The body was dropped before a terminal poll.
|
||||||
|
Aborted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceBodyState {
|
||||||
|
/// Returns the stable status-page label.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Complete => "complete",
|
||||||
|
Self::Error => "error",
|
||||||
|
Self::Aborted => "aborted",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One sanitized HTTP header.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceHeader {
|
||||||
|
/// Lowercase header name.
|
||||||
|
pub(crate) name: String,
|
||||||
|
/// Allowlisted value, or `None` when only the name may be retained.
|
||||||
|
pub(crate) value: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One bounded HTTP body observation.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceBodySnapshot {
|
||||||
|
/// Total bytes observed on the wire-facing Hyper body.
|
||||||
|
pub(crate) observed_bytes: u64,
|
||||||
|
/// Retained prefix or full bounded body bytes.
|
||||||
|
pub(crate) captured: Vec<u8>,
|
||||||
|
/// Indicates that bytes were omitted by policy or capacity.
|
||||||
|
pub(crate) truncated: bool,
|
||||||
|
/// Terminal body state.
|
||||||
|
pub(crate) state: TraceBodyState,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One parsed carrier frame without payload retention.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceFrame {
|
||||||
|
/// HTTP body direction containing the frame.
|
||||||
|
pub(crate) direction: TraceDirection,
|
||||||
|
/// Stable protocol frame type, when the header parsed.
|
||||||
|
pub(crate) frame_type: Option<&'static str>,
|
||||||
|
/// Logical stream or lane identifier.
|
||||||
|
pub(crate) stream_id: Option<u32>,
|
||||||
|
/// Frame payload length.
|
||||||
|
pub(crate) payload_len: Option<usize>,
|
||||||
|
/// Decoded non-zero WINDOW delta.
|
||||||
|
pub(crate) window_delta: Option<u32>,
|
||||||
|
/// Closed parse or shape error category.
|
||||||
|
pub(crate) parse_error: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Request lifecycle timing points measured from service entry.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(crate) struct TraceTimings {
|
||||||
|
/// Request body terminal poll time.
|
||||||
|
pub(crate) request_body_us: Option<u64>,
|
||||||
|
/// Handler response-ready time.
|
||||||
|
pub(crate) response_ready_us: Option<u64>,
|
||||||
|
/// Response body terminal poll or drop time.
|
||||||
|
pub(crate) response_body_us: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stable non-secret WEB trace identity.
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub(crate) struct TraceIdentity {
|
||||||
|
/// Process-unique monotonic WEB session identifier.
|
||||||
|
pub(crate) session_id: Option<u64>,
|
||||||
|
/// Exact access user name.
|
||||||
|
pub(crate) user: Option<String>,
|
||||||
|
/// Domain-separated client-secret fingerprint.
|
||||||
|
pub(crate) key_fingerprint: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceIdentity {
|
||||||
|
/// Builds a non-secret identity from one validated runtime profile.
|
||||||
|
pub(crate) fn from_profile(
|
||||||
|
session_id: u64,
|
||||||
|
profile: &crate::config::WebRuntimeProfile,
|
||||||
|
) -> Self {
|
||||||
|
Self::from_optional_profile(Some(session_id), profile)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a profile identity when admission failed before session allocation.
|
||||||
|
pub(crate) fn from_optional_profile(
|
||||||
|
session_id: Option<u64>,
|
||||||
|
profile: &crate::config::WebRuntimeProfile,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
session_id,
|
||||||
|
user: Some(profile.user.clone()),
|
||||||
|
key_fingerprint: Some(profile.key_fingerprint.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Complete request-to-response WEB HTTP trace.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceHttpRecord {
|
||||||
|
/// HTTP method.
|
||||||
|
pub(crate) method: String,
|
||||||
|
/// URI path without query material.
|
||||||
|
pub(crate) path: String,
|
||||||
|
/// Final route classification.
|
||||||
|
pub(crate) route: TraceRoute,
|
||||||
|
/// Sanitized request headers.
|
||||||
|
pub(crate) request_headers: Vec<TraceHeader>,
|
||||||
|
/// Sanitized response headers.
|
||||||
|
pub(crate) response_headers: Vec<TraceHeader>,
|
||||||
|
/// Request body observation when enabled by policy.
|
||||||
|
pub(crate) request_body: Option<TraceBodySnapshot>,
|
||||||
|
/// Response status when a response head was produced.
|
||||||
|
pub(crate) status: Option<u16>,
|
||||||
|
/// Response body observation when enabled by policy.
|
||||||
|
pub(crate) response_body: Option<TraceBodySnapshot>,
|
||||||
|
/// Parsed carrier frame metadata.
|
||||||
|
pub(crate) frames: Vec<TraceFrame>,
|
||||||
|
/// Monotonic request timing points.
|
||||||
|
pub(crate) timings: Option<TraceTimings>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Closed WEB lifecycle event category.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum TraceLifecycleEvent {
|
||||||
|
/// A bridge bootstrap was issued.
|
||||||
|
BridgeIssued,
|
||||||
|
/// Bootstrap or bridge admission was rejected.
|
||||||
|
BootstrapRejected,
|
||||||
|
/// A new session was created.
|
||||||
|
SessionCreated,
|
||||||
|
/// An idempotent session creation was replayed.
|
||||||
|
SessionReplayed,
|
||||||
|
/// Session creation was rejected.
|
||||||
|
SessionRejected,
|
||||||
|
/// A session closed.
|
||||||
|
SessionClosed,
|
||||||
|
/// A logical stream was admitted.
|
||||||
|
StreamAdmitted,
|
||||||
|
/// A logical stream was rejected.
|
||||||
|
StreamRejected,
|
||||||
|
/// A logical stream delivered its first inner byte.
|
||||||
|
StreamFirstByte,
|
||||||
|
/// An admitted logical stream released its relay and tuple ownership.
|
||||||
|
StreamClosed,
|
||||||
|
/// The inner MTProxy handshake succeeded.
|
||||||
|
HandshakeSucceeded,
|
||||||
|
/// The inner MTProxy handshake timed out.
|
||||||
|
HandshakeTimeout,
|
||||||
|
/// The inner MTProxy handshake failed on I/O.
|
||||||
|
HandshakeIo,
|
||||||
|
/// The inner MTProxy handshake was rejected.
|
||||||
|
HandshakeRejected,
|
||||||
|
/// Authenticated relay started.
|
||||||
|
RelayStarted,
|
||||||
|
/// Authenticated relay ended.
|
||||||
|
RelayEnded,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TraceLifecycleEvent {
|
||||||
|
/// Returns the stable status-page label.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::BridgeIssued => "bridge_issued",
|
||||||
|
Self::BootstrapRejected => "bootstrap_rejected",
|
||||||
|
Self::SessionCreated => "session_created",
|
||||||
|
Self::SessionReplayed => "session_replayed",
|
||||||
|
Self::SessionRejected => "session_rejected",
|
||||||
|
Self::SessionClosed => "session_closed",
|
||||||
|
Self::StreamAdmitted => "stream_admitted",
|
||||||
|
Self::StreamRejected => "stream_rejected",
|
||||||
|
Self::StreamFirstByte => "stream_first_byte",
|
||||||
|
Self::StreamClosed => "stream_closed",
|
||||||
|
Self::HandshakeSucceeded => "handshake_succeeded",
|
||||||
|
Self::HandshakeTimeout => "handshake_timeout",
|
||||||
|
Self::HandshakeIo => "handshake_io",
|
||||||
|
Self::HandshakeRejected => "handshake_rejected",
|
||||||
|
Self::RelayStarted => "relay_started",
|
||||||
|
Self::RelayEnded => "relay_ended",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One typed WEB lifecycle observation.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceLifecycleRecord {
|
||||||
|
/// Closed event category.
|
||||||
|
pub(crate) event: TraceLifecycleEvent,
|
||||||
|
/// Logical stream identifier when applicable.
|
||||||
|
pub(crate) stream_id: Option<u32>,
|
||||||
|
/// Closed outcome or rejection reason.
|
||||||
|
pub(crate) reason: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trace record payload variant.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) enum TraceRecordKind {
|
||||||
|
/// HTTP request-to-response exchange.
|
||||||
|
Http(TraceHttpRecord),
|
||||||
|
/// Session or stream lifecycle event.
|
||||||
|
Lifecycle(TraceLifecycleRecord),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Common stored WEB trace record.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct TraceRecord {
|
||||||
|
/// Process-unique monotonic record sequence.
|
||||||
|
pub(crate) seq: u64,
|
||||||
|
/// Event completion time in Unix epoch milliseconds.
|
||||||
|
pub(crate) epoch_millis: u64,
|
||||||
|
/// Direct TCP peer accepted by the WEB listener.
|
||||||
|
pub(crate) peer_ip: Option<IpAddr>,
|
||||||
|
/// Trusted effective client identity.
|
||||||
|
pub(crate) effective_ip: Option<IpAddr>,
|
||||||
|
/// Bounded user-agent value retained for filtering.
|
||||||
|
pub(crate) user_agent: Option<String>,
|
||||||
|
/// Non-secret WEB identity.
|
||||||
|
pub(crate) identity: TraceIdentity,
|
||||||
|
/// Typed record payload.
|
||||||
|
pub(crate) kind: TraceRecordKind,
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user