mirror of
https://github.com/telemt/telemt.git
synced 2026-09-05 18:16:06 +03:00
Merge pull request #907 from telemt/flow-carriers
Bounded Debugging + Websocket Carriers + Carriers Negotiation
This commit is contained in:
Generated
+1
-1
@@ -2900,7 +2900,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "telemt"
|
||||
version = "3.5.3"
|
||||
version = "3.5.4"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"anyhow",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.5.3"
|
||||
version = "3.5.4"
|
||||
edition = "2024"
|
||||
|
||||
[features]
|
||||
|
||||
+7
-20
@@ -18,7 +18,7 @@ mod details;
|
||||
// Query parsing and matching remain independent from bounded HTML rendering.
|
||||
mod query;
|
||||
|
||||
use details::{push_body, push_frames, push_headers};
|
||||
use details::{push_body, push_frames, push_headers, push_lifecycle};
|
||||
use query::{GroupBy, StatusQuery, client_ip, parse_query, record_matches};
|
||||
|
||||
struct GroupSummary {
|
||||
@@ -56,11 +56,11 @@ pub(super) async fn render(
|
||||
);
|
||||
};
|
||||
let now_millis = crate::web::trace::store_epoch_millis();
|
||||
let since_millis = query
|
||||
.record
|
||||
.is_none()
|
||||
.then(|| now_millis.saturating_sub(query.window_secs.saturating_mul(1000)))
|
||||
.unwrap_or(0);
|
||||
let since_millis = if query.record.is_none() {
|
||||
now_millis.saturating_sub(query.window_secs.saturating_mul(1000))
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||
let status = store.status();
|
||||
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||
@@ -367,20 +367,7 @@ fn push_record(html: &mut String, record: &TraceRecord) {
|
||||
push_body(html, "message body", message.body.as_ref());
|
||||
push_frames(html, &message.frames);
|
||||
}
|
||||
TraceRecordKind::Lifecycle(event) => {
|
||||
html.push_str("<pre>event: ");
|
||||
html.push_str(event.event.as_str());
|
||||
html.push_str("\nstream: ");
|
||||
html.push_str(
|
||||
&event
|
||||
.stream_id
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("\nreason: ");
|
||||
html.push_str(event.reason.unwrap_or("-"));
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
TraceRecordKind::Lifecycle(event) => push_lifecycle(html, event),
|
||||
}
|
||||
html.push_str("</details></td></tr>");
|
||||
}
|
||||
|
||||
@@ -84,3 +84,34 @@ pub(super) fn push_body(
|
||||
}
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
|
||||
pub(super) fn push_lifecycle(html: &mut String, event: &crate::web::trace::TraceLifecycleRecord) {
|
||||
html.push_str("<pre>event: ");
|
||||
html.push_str(event.event.as_str());
|
||||
html.push_str("\nstream: ");
|
||||
html.push_str(
|
||||
&event
|
||||
.stream_id
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("\nreason: ");
|
||||
html.push_str(event.reason.unwrap_or("-"));
|
||||
if let Some(carrier) = &event.carrier {
|
||||
html.push_str("\nclient class: ");
|
||||
html.push_str(carrier.client_class);
|
||||
html.push_str("\ncarrier: ");
|
||||
html.push_str(carrier.carrier.as_str());
|
||||
html.push_str("\nattempt: ");
|
||||
html.push_str(&carrier.attempt.to_string());
|
||||
html.push_str("\nscores: https=");
|
||||
html.push_str(&carrier.scores[0].to_string());
|
||||
html.push_str(" https-lanes=");
|
||||
html.push_str(&carrier.scores[1].to_string());
|
||||
html.push_str(" websocket=");
|
||||
html.push_str(&carrier.scores[2].to_string());
|
||||
html.push_str(" websocket-lanes=");
|
||||
html.push_str(&carrier.scores[3].to_string());
|
||||
}
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
|
||||
@@ -20,11 +20,15 @@ fn html_escaping_covers_active_markup_characters() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
let mut limits = crate::config::WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 8;
|
||||
limits.debug_bytes_global = 16 * 1024;
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = crate::config::WebLimitsConfig {
|
||||
debug_records_capacity: 8,
|
||||
debug_bytes_global: 16 * 1024,
|
||||
..Default::default()
|
||||
};
|
||||
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
@@ -79,7 +83,7 @@ async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
||||
|
||||
#[test]
|
||||
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||
let mut html = "я".repeat(MAX_PAGE_BYTES);
|
||||
let mut html = "\u{044f}".repeat(MAX_PAGE_BYTES);
|
||||
truncate_page(&mut html);
|
||||
assert!(html.len() <= MAX_PAGE_BYTES);
|
||||
assert!(html.ends_with("[page output truncated]"));
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
||||
//! | `network` | `dns_overrides` | Applied immediately |
|
||||
//! | `access` | All user/quota fields | Effective immediately |
|
||||
//! | `web` | Carrier, timing, and debug policy | Applied to newly issued sessions |
|
||||
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
||||
|
||||
@@ -30,6 +30,23 @@ fn write_reload_config(path: &Path, ad_tag: Option<&str>, server_port: Option<u1
|
||||
std::fs::write(path, config).unwrap();
|
||||
}
|
||||
|
||||
fn write_web_reload_config(path: &Path, carriers: &str, carrier_learning: bool) {
|
||||
let config = format!(
|
||||
r#"
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
|
||||
[web]
|
||||
carriers = {carriers}
|
||||
carrier_learning = {carrier_learning}
|
||||
"#,
|
||||
);
|
||||
std::fs::write(path, config).unwrap();
|
||||
}
|
||||
|
||||
fn temp_config_path(prefix: &str) -> PathBuf {
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -264,6 +281,29 @@ fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_publishes_web_negotiation_policy_outside_hot_field_reporting() {
|
||||
let path = temp_config_path("telemt_web_negotiation_reload");
|
||||
|
||||
write_web_reload_config(&path, "false", true);
|
||||
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||
.unwrap()
|
||||
.rendered_hash;
|
||||
let (config_tx, _config_rx) = watch::channel(initial_cfg.clone());
|
||||
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
|
||||
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||
|
||||
write_web_reload_config(&path, "[\"websocket\", \"https\"]", false);
|
||||
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||
|
||||
let applied = config_tx.borrow().clone();
|
||||
assert!(applied.web.carrier_negotiation_enabled());
|
||||
assert!(!applied.web.carrier_learning);
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_sni_change_requires_restart() {
|
||||
// censorship.* is not in overlay_hot_fields -> restart.
|
||||
|
||||
@@ -164,7 +164,7 @@ pub(super) fn reload_config(
|
||||
let old_hot = HotFields::from_config(&old_cfg);
|
||||
let applied_hot = HotFields::from_config(&applied_cfg);
|
||||
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
||||
let hot_changed = old_hot != applied_hot;
|
||||
let hot_changed = !config_equal(&old_cfg, &applied_cfg);
|
||||
|
||||
if non_hot_changed {
|
||||
warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed);
|
||||
|
||||
@@ -27,6 +27,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
let mut static_files = 0usize;
|
||||
let mut static_bytes = 0usize;
|
||||
|
||||
let carrier_candidates: Arc<[WebCarrier]> = config.web.carrier_candidates().into();
|
||||
for vhost in &config.web.vhosts {
|
||||
let decoy = build_decoy(
|
||||
vhost,
|
||||
@@ -63,6 +64,14 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
user: profile.user.clone(),
|
||||
secret_mode: profile.secret_mode,
|
||||
carrier: config.web.carrier,
|
||||
carrier_negotiation_enabled: config.web.carrier_negotiation_enabled(),
|
||||
carrier_learning: config.web.carrier_negotiation_enabled()
|
||||
&& config.web.carrier_learning,
|
||||
carriers: Arc::clone(&carrier_candidates),
|
||||
carrier_negotiation_deadlines_secs: config
|
||||
.web
|
||||
.timeouts
|
||||
.carrier_negotiation_deadlines_secs,
|
||||
capability,
|
||||
key_fingerprint,
|
||||
max_sessions: profile
|
||||
|
||||
@@ -260,7 +260,15 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
||||
];
|
||||
|
||||
const WEB_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled", "carrier", "debug", "limits", "timeouts", "vhosts",
|
||||
"enabled",
|
||||
"carrier",
|
||||
"carriers",
|
||||
"carrier_learning",
|
||||
"carrier_negotiation_aggressiveness",
|
||||
"debug",
|
||||
"limits",
|
||||
"timeouts",
|
||||
"vhosts",
|
||||
];
|
||||
|
||||
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
@@ -271,10 +279,15 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"max_frames_per_body",
|
||||
"max_http_connections",
|
||||
"max_http_handlers",
|
||||
"max_lane_open_waits_per_session",
|
||||
"pending_bytes_per_lane",
|
||||
"pending_items_per_lane",
|
||||
"websocket_bytes_global",
|
||||
"websocket_admission_watermark_pct",
|
||||
"websocket_eviction_watermark_pct",
|
||||
"websocket_http_connection_reserve",
|
||||
"max_websocket_evictions_in_flight",
|
||||
"max_carrier_learning_entries",
|
||||
"max_body_readers",
|
||||
"max_body_bytes_global",
|
||||
"max_sessions_global",
|
||||
@@ -324,10 +337,17 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"header_secs",
|
||||
"body_secs",
|
||||
"stream_handshake_secs",
|
||||
"stream_first_byte_secs",
|
||||
"long_poll_secs",
|
||||
"lane_open_wait_secs",
|
||||
"carrier_health_secs",
|
||||
"websocket_upgrade_secs",
|
||||
"websocket_open_secs",
|
||||
"websocket_write_secs",
|
||||
"websocket_backpressure_secs",
|
||||
"websocket_eviction_secs",
|
||||
"carrier_negotiation_deadlines_secs",
|
||||
"carrier_learning_secs",
|
||||
"bootstrap_lifetime_secs",
|
||||
"reconnect_grace_secs",
|
||||
"http_idle_secs",
|
||||
|
||||
@@ -6,6 +6,10 @@ use super::*;
|
||||
mod debug;
|
||||
// Memory-envelope arithmetic remains isolated from protocol validation.
|
||||
mod memory;
|
||||
// Carrier ordering, cumulative deadlines, and fallback identity are validated together.
|
||||
mod negotiation;
|
||||
// Request and lifecycle timeout relationships are validated together.
|
||||
mod timeouts;
|
||||
// WebSocket transport policy is validated independently from HTTP body policy.
|
||||
mod websocket;
|
||||
|
||||
@@ -67,11 +71,17 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
||||
|
||||
validate_limits(&config.web.limits)?;
|
||||
debug::validate(&config.web.debug, &config.web.limits)?;
|
||||
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
|
||||
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
|
||||
let carriers = negotiation::validate(&config.web)?;
|
||||
if carriers.contains(&WebCarrier::Https) && config.web.limits.max_http_handlers < 2 {
|
||||
return config_error("WEB https candidates require web.limits.max_http_handlers >= 2");
|
||||
}
|
||||
validate_timeouts(&config.web.timeouts)?;
|
||||
websocket::validate(config.web.carrier, &config.web.limits, &config.web.timeouts)?;
|
||||
if carriers.contains(&WebCarrier::HttpsLanes) && config.web.limits.max_http_handlers < 4 {
|
||||
return config_error(
|
||||
"WEB https-lanes candidates require web.limits.max_http_handlers >= 4",
|
||||
);
|
||||
}
|
||||
timeouts::validate(&config.web.timeouts)?;
|
||||
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
|
||||
validate_vhosts(config)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -136,6 +146,11 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
if !(1..=MAX_WEB_TOMBSTONES_PER_SESSION).contains(&limits.max_tombstones_per_session) {
|
||||
return config_error("web.limits.max_tombstones_per_session must be within [1, 4096]");
|
||||
}
|
||||
if limits.pending_bytes_per_lane <= WEB_FRAME_HEADER_BYTES + WEB_QUEUE_ITEM_COST {
|
||||
return config_error(
|
||||
"web.limits.pending_bytes_per_lane must preserve one non-empty DATA frame",
|
||||
);
|
||||
}
|
||||
if limits.carrier_batch_bytes > limits.max_body_bytes
|
||||
|| limits.carrier_batch_bytes
|
||||
< limits
|
||||
@@ -155,6 +170,20 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
let positive = [
|
||||
("max_http_connections", limits.max_http_connections),
|
||||
("max_http_handlers", limits.max_http_handlers),
|
||||
(
|
||||
"max_lane_open_waits_per_session",
|
||||
limits.max_lane_open_waits_per_session,
|
||||
),
|
||||
("pending_bytes_per_lane", limits.pending_bytes_per_lane),
|
||||
("pending_items_per_lane", limits.pending_items_per_lane),
|
||||
(
|
||||
"max_websocket_evictions_in_flight",
|
||||
limits.max_websocket_evictions_in_flight,
|
||||
),
|
||||
(
|
||||
"max_carrier_learning_entries",
|
||||
limits.max_carrier_learning_entries,
|
||||
),
|
||||
("max_body_readers", limits.max_body_readers),
|
||||
("max_body_bytes_global", limits.max_body_bytes_global),
|
||||
("max_sessions_global", limits.max_sessions_global),
|
||||
@@ -227,8 +256,11 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
|| limits.max_bootstraps_per_ip > limits.max_bootstraps_global
|
||||
|| limits.max_http_handlers > limits.max_http_connections
|
||||
|| limits.max_body_readers > limits.max_http_handlers
|
||||
|| limits.max_lane_open_waits_per_session > limits.max_streams_per_session
|
||||
|| limits.pending_bytes_per_session > limits.pending_bytes_global
|
||||
|| limits.pending_items_per_session > limits.pending_items_global
|
||||
|| limits.pending_bytes_per_lane > limits.pending_bytes_per_session
|
||||
|| limits.pending_items_per_lane > limits.pending_items_per_session
|
||||
|| limits.control_bytes_per_session > limits.control_bytes_global
|
||||
|| limits.control_bytes_per_session > limits.pending_bytes_per_session
|
||||
|| limits.control_bytes_global > limits.pending_bytes_global
|
||||
@@ -324,41 +356,6 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_timeouts(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
let values = [
|
||||
("header_secs", timeouts.header_secs),
|
||||
("body_secs", timeouts.body_secs),
|
||||
("stream_handshake_secs", timeouts.stream_handshake_secs),
|
||||
("long_poll_secs", timeouts.long_poll_secs),
|
||||
("websocket_write_secs", timeouts.websocket_write_secs),
|
||||
(
|
||||
"websocket_backpressure_secs",
|
||||
timeouts.websocket_backpressure_secs,
|
||||
),
|
||||
("websocket_eviction_secs", timeouts.websocket_eviction_secs),
|
||||
("bootstrap_lifetime_secs", timeouts.bootstrap_lifetime_secs),
|
||||
("reconnect_grace_secs", timeouts.reconnect_grace_secs),
|
||||
("http_idle_secs", timeouts.http_idle_secs),
|
||||
("shutdown_secs", timeouts.shutdown_secs),
|
||||
("decoy_header_secs", timeouts.decoy_header_secs),
|
||||
];
|
||||
if let Some((field, _)) = values
|
||||
.into_iter()
|
||||
.find(|(_, value)| !(1..=3600).contains(value))
|
||||
{
|
||||
return config_error(&format!("web.timeouts.{field} must be within [1, 3600]"));
|
||||
}
|
||||
let request_deadline = timeouts
|
||||
.header_secs
|
||||
.max(timeouts.body_secs)
|
||||
.max(timeouts.long_poll_secs)
|
||||
.max(timeouts.decoy_header_secs);
|
||||
if request_deadline >= timeouts.http_idle_secs {
|
||||
return config_error("web.timeouts request deadlines must be lower than http_idle_secs");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
let limits = &config.web.limits;
|
||||
if config.web.vhosts.len() > limits.max_vhosts {
|
||||
|
||||
@@ -3,9 +3,14 @@ use super::*;
|
||||
const WEB_DEBUG_RENDERERS: usize = 2;
|
||||
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
||||
const WEB_LANE_STATE_BYTES: usize = 512;
|
||||
|
||||
/// Validates process-wide body, header, queue, static, and debug reservations.
|
||||
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
if limits.max_carrier_learning_entries == 0 {
|
||||
return config_error("web.limits.max_carrier_learning_entries must be > 0");
|
||||
}
|
||||
let body_reservation = limits
|
||||
.max_body_readers
|
||||
.checked_mul(limits.max_body_bytes)
|
||||
@@ -47,6 +52,21 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
.and_then(|scratch| value.checked_add(scratch))
|
||||
})
|
||||
.ok_or_else(|| ProxyError::Config("web.debug reservations overflowed usize".to_string()))?;
|
||||
let carrier_learning_reservation = limits
|
||||
.max_carrier_learning_entries
|
||||
.checked_mul(WEB_CARRIER_LEARNING_ENTRY_BYTES)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.carrier learning reservation overflowed usize".to_string())
|
||||
})?;
|
||||
let lane_state_reservation = limits
|
||||
.max_streams_per_session
|
||||
.checked_add(limits.max_tombstones_per_session)
|
||||
.and_then(|value| value.checked_add(1))
|
||||
.and_then(|value| value.checked_mul(limits.max_sessions_global))
|
||||
.and_then(|value| value.checked_mul(WEB_LANE_STATE_BYTES))
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits lane state reservation overflowed usize".to_string())
|
||||
})?;
|
||||
let reserved = limits
|
||||
.pending_bytes_global
|
||||
.checked_add(limits.max_body_bytes_global)
|
||||
@@ -54,6 +74,8 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
.and_then(|value| value.checked_add(debug_ring_index))
|
||||
.and_then(|value| value.checked_add(status_pages))
|
||||
.and_then(|value| value.checked_add(debug_reservation))
|
||||
.and_then(|value| value.checked_add(carrier_learning_reservation))
|
||||
.and_then(|value| value.checked_add(lane_state_reservation))
|
||||
.and_then(|value| value.checked_add(http_header_reservation))
|
||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||
if reserved > limits.memory_envelope_bytes
|
||||
@@ -65,3 +87,20 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_envelope_includes_bounded_lane_and_learning_metadata() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
assert!(validate(&limits).is_ok());
|
||||
|
||||
let previous_envelope = WebLimitsConfig {
|
||||
memory_envelope_bytes: 768 * 1024 * 1024,
|
||||
..limits
|
||||
};
|
||||
assert!(validate(&previous_envelope).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Validates bounded carrier selection and learning policy.
|
||||
pub(super) fn validate(config: &WebConfig) -> Result<Vec<WebCarrier>> {
|
||||
if let Some(carriers) = config.carriers.enabled() {
|
||||
if carriers.is_empty() {
|
||||
return config_error("web.carriers must contain at least one carrier");
|
||||
}
|
||||
let mut unique = HashSet::with_capacity(carriers.len());
|
||||
if carriers.iter().any(|carrier| !unique.insert(*carrier)) {
|
||||
return config_error("web.carriers must not contain duplicate carriers");
|
||||
}
|
||||
}
|
||||
let candidates = config.carrier_candidates();
|
||||
if config.carrier_negotiation_enabled()
|
||||
&& config.carrier_learning
|
||||
&& config.limits.max_carrier_learning_entries < 3
|
||||
{
|
||||
return config_error(
|
||||
"web.limits.max_carrier_learning_entries must be >= 3 when carrier learning is enabled",
|
||||
);
|
||||
}
|
||||
if candidates.len() > WebCarrier::ALL.len() {
|
||||
return config_error(
|
||||
"web.carriers and the web.carrier fallback must contain at most four carriers",
|
||||
);
|
||||
}
|
||||
let deadlines = config.timeouts.carrier_negotiation_deadlines_secs;
|
||||
if deadlines[0] == 0 || deadlines.windows(2).any(|pair| pair[0] >= pair[1]) {
|
||||
return config_error(
|
||||
"web.timeouts.carrier_negotiation_deadlines_secs must be non-zero and strictly increasing",
|
||||
);
|
||||
}
|
||||
let retained_chain_secs = deadlines[3]
|
||||
.checked_add(config.timeouts.carrier_health_secs)
|
||||
.and_then(|value| value.checked_add(1));
|
||||
if retained_chain_secs.is_none_or(|value| value >= config.timeouts.bootstrap_lifetime_secs) {
|
||||
return config_error(
|
||||
"web.timeouts final carrier deadline plus health and cleanup must be lower than bootstrap_lifetime_secs",
|
||||
);
|
||||
}
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_is_appended_once() {
|
||||
let config = WebConfig {
|
||||
carrier: WebCarrier::Https,
|
||||
carriers: WebCarriers::Enabled(vec![WebCarrier::Websocket, WebCarrier::Https]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
validate(&config).unwrap(),
|
||||
vec![WebCarrier::Websocket, WebCarrier::Https]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_carriers_are_rejected() {
|
||||
let config = WebConfig {
|
||||
carriers: WebCarriers::Enabled(vec![WebCarrier::Websocket, WebCarrier::Websocket]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(validate(&config).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_false_carriers_disable_negotiation() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Wrapper {
|
||||
value: WebCarriers,
|
||||
}
|
||||
|
||||
let config = WebConfig::default();
|
||||
assert!(!config.carrier_negotiation_enabled());
|
||||
assert_eq!(validate(&config).unwrap(), [WebCarrier::Https]);
|
||||
|
||||
let disabled: Wrapper = toml::from_str("value = false").unwrap();
|
||||
assert_eq!(disabled.value, WebCarriers::Disabled);
|
||||
assert!(toml::from_str::<Wrapper>("value = true").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_cannot_expand_the_candidate_set_beyond_four() {
|
||||
let mut config = WebConfig {
|
||||
carrier: WebCarrier::Https,
|
||||
carriers: WebCarriers::Enabled(vec![
|
||||
WebCarrier::HttpsLanes,
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::WebsocketLanes,
|
||||
WebCarrier::Https,
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(validate(&config).unwrap().len(), 4);
|
||||
|
||||
config.carriers = WebCarriers::Enabled(vec![
|
||||
WebCarrier::HttpsLanes,
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::WebsocketLanes,
|
||||
]);
|
||||
assert_eq!(validate(&config).unwrap().len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadlines_are_cumulative_and_bounded_by_bootstrap_lifetime() {
|
||||
let mut config = WebConfig::default();
|
||||
config.timeouts.carrier_negotiation_deadlines_secs = [3, 3, 8, 12];
|
||||
assert!(validate(&config).is_err());
|
||||
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 121];
|
||||
assert!(validate(&config).is_err());
|
||||
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 89];
|
||||
assert!(validate(&config).is_err());
|
||||
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 88];
|
||||
assert!(validate(&config).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use super::*;
|
||||
|
||||
/// Validates WEB request, learning, and lifecycle timeouts.
|
||||
pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
let values = [
|
||||
("header_secs", timeouts.header_secs),
|
||||
("body_secs", timeouts.body_secs),
|
||||
("stream_handshake_secs", timeouts.stream_handshake_secs),
|
||||
("stream_first_byte_secs", timeouts.stream_first_byte_secs),
|
||||
("long_poll_secs", timeouts.long_poll_secs),
|
||||
("lane_open_wait_secs", timeouts.lane_open_wait_secs),
|
||||
("carrier_health_secs", timeouts.carrier_health_secs),
|
||||
("websocket_upgrade_secs", timeouts.websocket_upgrade_secs),
|
||||
("websocket_open_secs", timeouts.websocket_open_secs),
|
||||
("websocket_write_secs", timeouts.websocket_write_secs),
|
||||
(
|
||||
"websocket_backpressure_secs",
|
||||
timeouts.websocket_backpressure_secs,
|
||||
),
|
||||
("websocket_eviction_secs", timeouts.websocket_eviction_secs),
|
||||
("bootstrap_lifetime_secs", timeouts.bootstrap_lifetime_secs),
|
||||
("reconnect_grace_secs", timeouts.reconnect_grace_secs),
|
||||
("http_idle_secs", timeouts.http_idle_secs),
|
||||
("shutdown_secs", timeouts.shutdown_secs),
|
||||
("decoy_header_secs", timeouts.decoy_header_secs),
|
||||
];
|
||||
if let Some((field, _)) = values
|
||||
.into_iter()
|
||||
.find(|(_, value)| !(1..=3600).contains(value))
|
||||
{
|
||||
return config_error(&format!("web.timeouts.{field} must be within [1, 3600]"));
|
||||
}
|
||||
if !(2..=86_400).contains(&timeouts.carrier_learning_secs) {
|
||||
return config_error("web.timeouts.carrier_learning_secs must be within [2, 86400]");
|
||||
}
|
||||
if timeouts.stream_first_byte_secs > 300 {
|
||||
return config_error("web.timeouts.stream_first_byte_secs must be within [1, 300]");
|
||||
}
|
||||
if timeouts.websocket_upgrade_secs > 60 {
|
||||
return config_error("web.timeouts.websocket_upgrade_secs must be within [1, 60]");
|
||||
}
|
||||
if timeouts.websocket_open_secs > 300 {
|
||||
return config_error("web.timeouts.websocket_open_secs must be within [1, 300]");
|
||||
}
|
||||
if timeouts.lane_open_wait_secs > timeouts.long_poll_secs {
|
||||
return config_error("web.timeouts.lane_open_wait_secs must not exceed long_poll_secs");
|
||||
}
|
||||
if timeouts.carrier_health_secs > timeouts.reconnect_grace_secs {
|
||||
return config_error(
|
||||
"web.timeouts.carrier_health_secs must not exceed reconnect_grace_secs",
|
||||
);
|
||||
}
|
||||
let request_deadline = timeouts
|
||||
.header_secs
|
||||
.max(timeouts.body_secs)
|
||||
.max(timeouts.long_poll_secs)
|
||||
.max(timeouts.decoy_header_secs);
|
||||
if request_deadline >= timeouts.http_idle_secs {
|
||||
return config_error("web.timeouts request deadlines must be lower than http_idle_secs");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -7,7 +7,7 @@ const WEBSOCKET_FRAME_OVERHEAD_BYTES: usize = 14;
|
||||
|
||||
/// Validates WebSocket admission, memory, and deadline invariants.
|
||||
pub(super) fn validate(
|
||||
carrier: WebCarrier,
|
||||
carriers: &[WebCarrier],
|
||||
limits: &WebLimitsConfig,
|
||||
timeouts: &WebTimeoutsConfig,
|
||||
) -> Result<()> {
|
||||
@@ -27,7 +27,7 @@ pub(super) fn validate(
|
||||
"web.timeouts.websocket_eviction_secs must not exceed websocket_write_secs",
|
||||
);
|
||||
}
|
||||
if !carrier.uses_websocket() {
|
||||
if !carriers.iter().any(|carrier| carrier.uses_websocket()) {
|
||||
return Ok(());
|
||||
}
|
||||
if limits.carrier_batch_bytes > MAX_WEBSOCKET_BATCH_BYTES {
|
||||
@@ -42,6 +42,14 @@ pub(super) fn validate(
|
||||
"WebSocket carriers require websocket_http_connection_reserve within [1, max_http_connections)",
|
||||
);
|
||||
}
|
||||
let websocket_capacity = limits
|
||||
.max_http_connections
|
||||
.saturating_sub(limits.websocket_http_connection_reserve);
|
||||
if limits.max_websocket_evictions_in_flight > websocket_capacity {
|
||||
return config_error(
|
||||
"web.limits.max_websocket_evictions_in_flight must not exceed WebSocket connection capacity",
|
||||
);
|
||||
}
|
||||
let socket_base = WEBSOCKET_IO_BUFFER_BYTES
|
||||
.checked_mul(2)
|
||||
.and_then(|value| value.checked_add(WEBSOCKET_DRIVER_OVERHEAD_BYTES))
|
||||
|
||||
@@ -49,6 +49,88 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
||||
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
||||
assert_eq!(vhost.profiles[0].key_fingerprint.len(), 16);
|
||||
assert_ne!(vhost.profiles[0].key_fingerprint, "0001020304050607");
|
||||
assert!(!vhost.profiles[0].carrier_negotiation_enabled);
|
||||
assert_eq!(
|
||||
vhost.profiles[0].carriers.as_ref(),
|
||||
[WebCarrier::HttpsLanes]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carriers_missing_or_false_disable_negotiation() {
|
||||
let missing = load_config_from_temp_toml(WEB_CONFIG);
|
||||
assert!(!missing.web.carrier_negotiation_enabled());
|
||||
assert!(!missing.web.runtime.unwrap().profiles[0].carrier_learning);
|
||||
|
||||
let disabled = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\ncarriers = false",
|
||||
);
|
||||
let disabled = load_config_from_temp_toml(&disabled);
|
||||
assert!(!disabled.web.carrier_negotiation_enabled());
|
||||
assert!(!disabled.web.runtime.as_ref().unwrap().profiles[0].carrier_learning);
|
||||
assert_eq!(
|
||||
disabled.web.runtime.unwrap().profiles[0].carriers.as_ref(),
|
||||
[WebCarrier::HttpsLanes]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carrier_array_enables_ordered_negotiation_and_appends_fallback() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\ncarriers = [\"websocket\", \"https\"]\ncarrier_learning = false",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert!(config.web.carrier_negotiation_enabled());
|
||||
assert!(!config.web.carrier_learning);
|
||||
let profile = &config.web.runtime.unwrap().profiles[0];
|
||||
assert_eq!(
|
||||
profile.carriers.as_ref(),
|
||||
[
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::Https,
|
||||
WebCarrier::HttpsLanes
|
||||
]
|
||||
);
|
||||
assert!(!profile.carrier_learning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carriers_reject_true_empty_and_duplicates() {
|
||||
for value in ["true", "[]", "[\"https\", \"https\"]"] {
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
&format!("carrier = \"https-lanes\"\ncarriers = {value}"),
|
||||
);
|
||||
assert!(load_config_error_from_temp_toml(&invalid).contains("web.carriers"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carrier_deadlines_and_learning_window_are_configurable() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert_eq!(
|
||||
config.web.timeouts.carrier_negotiation_deadlines_secs,
|
||||
[1, 2, 4, 9]
|
||||
);
|
||||
assert_eq!(config.web.timeouts.carrier_learning_secs, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carrier_learning_capacity_must_remain_nonzero() {
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.limits]\nmax_carrier_learning_entries = 0\n\n[[web.vhosts]]",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&invalid)
|
||||
.contains("web.limits.max_carrier_learning_entries")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -106,7 +188,7 @@ fn https_lanes_requires_separate_poll_and_control_handler_capacity() {
|
||||
"carrier = \"https-lanes\"\n\n[web.limits]\nmax_http_handlers = 1\nmax_body_readers = 1",
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&invalid);
|
||||
assert!(error.contains("web.carrier=https-lanes requires"));
|
||||
assert!(error.contains("WEB https-lanes candidates require"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+6
-2
@@ -24,6 +24,8 @@ mod network;
|
||||
mod policies;
|
||||
mod server;
|
||||
mod web;
|
||||
// WEB carrier tokens and fixed-slot policy helpers remain independent from bulky config types.
|
||||
mod web_carrier;
|
||||
// WEB debug capture policy is reusable by config reload and process storage.
|
||||
mod web_debug;
|
||||
|
||||
@@ -51,13 +53,15 @@ pub use server::{
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use web::{
|
||||
WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
||||
WebTimeoutsConfig, WebVhostConfig,
|
||||
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebLimitsConfig,
|
||||
WebProfileConfig, WebSecretMode, WebTimeoutsConfig, WebVhostConfig,
|
||||
};
|
||||
pub(crate) use web::{
|
||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||
WebStaticSite,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use web_carrier::{WebCarrier, WebCarriers};
|
||||
pub(crate) use web_debug::web_debug_fits_limits;
|
||||
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||
|
||||
|
||||
+125
-136
@@ -6,8 +6,13 @@ use std::sync::Arc;
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::web_carrier::{WebCarrier, WebCarriers};
|
||||
use super::web_debug::WebDebugConfig;
|
||||
|
||||
// Serialized WEB defaults remain separate from the runtime data model.
|
||||
mod defaults;
|
||||
use defaults::*;
|
||||
|
||||
/// Client-facing secret representation used to derive a WEB capability.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -18,48 +23,6 @@ pub enum WebSecretMode {
|
||||
Dd,
|
||||
}
|
||||
|
||||
/// Carrier selected for newly issued WEB bridge sessions.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum WebCarrier {
|
||||
/// Serialize all logical streams through one uplink and one downlink sequence.
|
||||
#[default]
|
||||
Https,
|
||||
/// Give every logical stream independent HTTPS sequencing and polling state.
|
||||
HttpsLanes,
|
||||
/// Multiplex all logical streams over one ordered WebSocket.
|
||||
Websocket,
|
||||
/// Give every logical stream an independently owned WebSocket lane.
|
||||
WebsocketLanes,
|
||||
}
|
||||
|
||||
impl WebCarrier {
|
||||
/// Returns the exact carrier token advertised to the browser bridge.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Https => "https",
|
||||
Self::HttpsLanes => "https-lanes",
|
||||
Self::Websocket => "websocket",
|
||||
Self::WebsocketLanes => "websocket-lanes",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether one carrier owns independent state per logical stream.
|
||||
pub(crate) const fn uses_lanes(self) -> bool {
|
||||
matches!(self, Self::HttpsLanes | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether carrier messages use RFC 6455 instead of HTTP bodies.
|
||||
pub(crate) const fn uses_websocket(self) -> bool {
|
||||
matches!(self, Self::Websocket | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether all logical streams share one carrier state machine.
|
||||
pub(crate) const fn is_multiplexed(self) -> bool {
|
||||
matches!(self, Self::Https | Self::Websocket)
|
||||
}
|
||||
}
|
||||
|
||||
/// One access user explicitly exposed through a WEB virtual host.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WebProfileConfig {
|
||||
@@ -135,6 +98,15 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide concurrently executing HTTP handler ceiling.
|
||||
#[serde(default = "default_web_max_http_handlers")]
|
||||
pub max_http_handlers: usize,
|
||||
/// Per-session ceiling for downlink polls waiting for a lane OPEN.
|
||||
#[serde(default = "default_web_max_lane_open_waits_per_session")]
|
||||
pub max_lane_open_waits_per_session: usize,
|
||||
/// Queued and resident DATA bytes allowed for one independent lane.
|
||||
#[serde(default = "default_web_pending_bytes_per_lane")]
|
||||
pub pending_bytes_per_lane: usize,
|
||||
/// Queued and resident DATA items allowed for one independent lane.
|
||||
#[serde(default = "default_web_pending_items_per_lane")]
|
||||
pub pending_items_per_lane: usize,
|
||||
/// Process-wide transient WebSocket byte sub-budget inside pending bytes.
|
||||
#[serde(default = "default_web_websocket_bytes_global")]
|
||||
pub websocket_bytes_global: usize,
|
||||
@@ -147,6 +119,12 @@ pub struct WebLimitsConfig {
|
||||
/// Accepted HTTP connections that WebSocket upgrades must leave available.
|
||||
#[serde(default = "default_web_websocket_http_connection_reserve")]
|
||||
pub websocket_http_connection_reserve: usize,
|
||||
/// Concurrent pressure-eviction claims allowed process-wide.
|
||||
#[serde(default = "default_web_max_websocket_evictions_in_flight")]
|
||||
pub max_websocket_evictions_in_flight: usize,
|
||||
/// Process-wide bounded carrier-learning evidence entry ceiling.
|
||||
#[serde(default = "default_web_max_carrier_learning_entries")]
|
||||
pub max_carrier_learning_entries: usize,
|
||||
/// Process-wide concurrently collected request body ceiling.
|
||||
#[serde(default = "default_web_max_body_readers")]
|
||||
pub max_body_readers: usize,
|
||||
@@ -216,7 +194,7 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide retained and in-flight WEB debug byte ceiling.
|
||||
#[serde(default = "default_web_debug_bytes_global")]
|
||||
pub debug_bytes_global: usize,
|
||||
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
|
||||
/// Declared process envelope for HTTP, queues, lane state, learning, and static snapshots.
|
||||
#[serde(default = "default_web_memory_envelope_bytes")]
|
||||
pub memory_envelope_bytes: usize,
|
||||
/// Sustained process-wide bootstrap issuance rate.
|
||||
@@ -249,10 +227,15 @@ impl Default for WebLimitsConfig {
|
||||
max_frames_per_body: default_web_max_frames_per_body(),
|
||||
max_http_connections: default_web_max_http_connections(),
|
||||
max_http_handlers: default_web_max_http_handlers(),
|
||||
max_lane_open_waits_per_session: default_web_max_lane_open_waits_per_session(),
|
||||
pending_bytes_per_lane: default_web_pending_bytes_per_lane(),
|
||||
pending_items_per_lane: default_web_pending_items_per_lane(),
|
||||
websocket_bytes_global: default_web_websocket_bytes_global(),
|
||||
websocket_admission_watermark_pct: default_web_websocket_admission_watermark_pct(),
|
||||
websocket_eviction_watermark_pct: default_web_websocket_eviction_watermark_pct(),
|
||||
websocket_http_connection_reserve: default_web_websocket_http_connection_reserve(),
|
||||
max_websocket_evictions_in_flight: default_web_max_websocket_evictions_in_flight(),
|
||||
max_carrier_learning_entries: default_web_max_carrier_learning_entries(),
|
||||
max_body_readers: default_web_max_body_readers(),
|
||||
max_body_bytes_global: default_web_max_body_bytes_global(),
|
||||
max_sessions_global: default_web_max_sessions_global(),
|
||||
@@ -299,9 +282,24 @@ pub struct WebTimeoutsConfig {
|
||||
/// Deadline from the first inner byte through MTProxy authentication.
|
||||
#[serde(default = "default_web_stream_handshake_timeout_secs")]
|
||||
pub stream_handshake_secs: u64,
|
||||
/// Absolute deadline for receiving the first inner MTProxy byte.
|
||||
#[serde(default = "default_web_stream_first_byte_secs")]
|
||||
pub stream_first_byte_secs: u64,
|
||||
/// Maximum wait for one empty downlink long poll.
|
||||
#[serde(default = "default_web_long_poll_timeout_secs")]
|
||||
pub long_poll_secs: u64,
|
||||
/// Grace for a canonical downlink poll that races its lane OPEN.
|
||||
#[serde(default = "default_web_lane_open_wait_secs")]
|
||||
pub lane_open_wait_secs: u64,
|
||||
/// Post-commit observation interval required before learning succeeds.
|
||||
#[serde(default = "default_web_carrier_health_secs")]
|
||||
pub carrier_health_secs: u64,
|
||||
/// Maximum wait for Hyper to transfer an accepted WebSocket upgrade.
|
||||
#[serde(default = "default_web_websocket_upgrade_secs")]
|
||||
pub websocket_upgrade_secs: u64,
|
||||
/// Absolute deadline for the first carrier binary message after upgrade.
|
||||
#[serde(default = "default_web_websocket_open_secs")]
|
||||
pub websocket_open_secs: u64,
|
||||
/// Maximum wait for one WebSocket write to complete.
|
||||
#[serde(default = "default_web_websocket_write_secs")]
|
||||
pub websocket_write_secs: u64,
|
||||
@@ -311,6 +309,12 @@ pub struct WebTimeoutsConfig {
|
||||
/// Maximum graceful close wait for an evicted WebSocket.
|
||||
#[serde(default = "default_web_websocket_eviction_secs")]
|
||||
pub websocket_eviction_secs: u64,
|
||||
/// Cumulative carrier-attempt deadlines for up to four unique candidates.
|
||||
#[serde(default = "default_web_carrier_negotiation_deadlines_secs")]
|
||||
pub carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// Fixed process-local carrier-learning evidence lifetime.
|
||||
#[serde(default = "default_web_carrier_learning_secs")]
|
||||
pub carrier_learning_secs: u64,
|
||||
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
|
||||
#[serde(default = "default_web_bootstrap_lifetime_secs")]
|
||||
pub bootstrap_lifetime_secs: u64,
|
||||
@@ -334,10 +338,17 @@ impl Default for WebTimeoutsConfig {
|
||||
header_secs: default_web_header_timeout_secs(),
|
||||
body_secs: default_web_body_timeout_secs(),
|
||||
stream_handshake_secs: default_web_stream_handshake_timeout_secs(),
|
||||
stream_first_byte_secs: default_web_stream_first_byte_secs(),
|
||||
long_poll_secs: default_web_long_poll_timeout_secs(),
|
||||
lane_open_wait_secs: default_web_lane_open_wait_secs(),
|
||||
carrier_health_secs: default_web_carrier_health_secs(),
|
||||
websocket_upgrade_secs: default_web_websocket_upgrade_secs(),
|
||||
websocket_open_secs: default_web_websocket_open_secs(),
|
||||
websocket_write_secs: default_web_websocket_write_secs(),
|
||||
websocket_backpressure_secs: default_web_websocket_backpressure_secs(),
|
||||
websocket_eviction_secs: default_web_websocket_eviction_secs(),
|
||||
carrier_negotiation_deadlines_secs: default_web_carrier_negotiation_deadlines_secs(),
|
||||
carrier_learning_secs: default_web_carrier_learning_secs(),
|
||||
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
|
||||
reconnect_grace_secs: default_web_reconnect_grace_secs(),
|
||||
http_idle_secs: default_web_http_idle_secs(),
|
||||
@@ -347,15 +358,37 @@ impl Default for WebTimeoutsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Sensitivity of process-local carrier-learning evidence.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum WebCarrierNegotiationAggressiveness {
|
||||
/// Require broad evidence and never rank by client IP.
|
||||
#[default]
|
||||
Conservative,
|
||||
/// Use moderate User-Agent, client-IP, and profile thresholds.
|
||||
Balanced,
|
||||
/// React to the first bounded evidence sample.
|
||||
Aggressive,
|
||||
}
|
||||
|
||||
/// WEB ingress, carrier, fallback, and lifecycle configuration.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WebConfig {
|
||||
/// Enables issuance of new WEB bridge and session credentials.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Carrier selected for newly issued WEB bridge sessions.
|
||||
/// Sole carrier when negotiation is disabled and final fallback when enabled.
|
||||
#[serde(default)]
|
||||
pub carrier: WebCarrier,
|
||||
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
||||
#[serde(default)]
|
||||
pub carriers: WebCarriers,
|
||||
/// Enables bounded process-local carrier learning for automatic sessions.
|
||||
#[serde(default = "default_web_carrier_learning")]
|
||||
pub carrier_learning: bool,
|
||||
/// Controls the evidence thresholds used by automatic carrier ranking.
|
||||
#[serde(default)]
|
||||
pub carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
/// Hard process and protocol limits.
|
||||
#[serde(default)]
|
||||
pub limits: WebLimitsConfig,
|
||||
@@ -373,6 +406,44 @@ pub struct WebConfig {
|
||||
pub(crate) runtime: Option<Arc<WebRuntimeConfig>>,
|
||||
}
|
||||
|
||||
impl WebConfig {
|
||||
/// Returns the configured negotiation order with the fallback appended once.
|
||||
pub(crate) fn carrier_candidates(&self) -> Vec<WebCarrier> {
|
||||
let Some(configured) = self.carriers.enabled() else {
|
||||
return vec![self.carrier];
|
||||
};
|
||||
let mut candidates = configured
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| *carrier != self.carrier)
|
||||
.collect::<Vec<_>>();
|
||||
candidates.push(self.carrier);
|
||||
candidates
|
||||
}
|
||||
|
||||
/// Returns whether the explicit candidate list enables auto-negotiation.
|
||||
pub(crate) fn carrier_negotiation_enabled(&self) -> bool {
|
||||
self.carriers.enabled().is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WebConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
carrier: WebCarrier::default(),
|
||||
carriers: WebCarriers::default(),
|
||||
carrier_learning: default_web_carrier_learning(),
|
||||
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||
limits: WebLimitsConfig::default(),
|
||||
debug: WebDebugConfig::default(),
|
||||
timeouts: WebTimeoutsConfig::default(),
|
||||
vhosts: Vec::new(),
|
||||
runtime: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeConfig {
|
||||
@@ -406,8 +477,16 @@ pub(crate) struct WebRuntimeProfile {
|
||||
pub(crate) user: String,
|
||||
/// Client secret representation and inner protocol policy.
|
||||
pub(crate) secret_mode: WebSecretMode,
|
||||
/// Carrier frozen into bridge and session state at issuance time.
|
||||
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||
pub(crate) carrier_negotiation_enabled: bool,
|
||||
/// Whether automatic outcomes consult and update process-local evidence.
|
||||
pub(crate) carrier_learning: bool,
|
||||
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
@@ -446,93 +525,3 @@ pub(crate) struct WebStaticAsset {
|
||||
/// Strong SHA-256 entity tag.
|
||||
pub(crate) etag: String,
|
||||
}
|
||||
|
||||
fn default_web_static_index() -> String {
|
||||
"index.html".to_string()
|
||||
}
|
||||
|
||||
macro_rules! usize_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> usize {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u32_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> u32 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u8_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> u8 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u64_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> u64 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
usize_default!(default_web_max_header_bytes, 16 * 1024);
|
||||
usize_default!(default_web_max_body_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
|
||||
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frames_per_body, 4096);
|
||||
usize_default!(default_web_max_http_connections, 1024);
|
||||
usize_default!(default_web_max_http_handlers, 512);
|
||||
usize_default!(default_web_websocket_bytes_global, 256 * 1024 * 1024);
|
||||
u8_default!(default_web_websocket_admission_watermark_pct, 75);
|
||||
u8_default!(default_web_websocket_eviction_watermark_pct, 90);
|
||||
usize_default!(default_web_websocket_http_connection_reserve, 64);
|
||||
usize_default!(default_web_max_body_readers, 32);
|
||||
usize_default!(default_web_max_body_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_max_sessions_global, 128);
|
||||
usize_default!(default_web_max_sessions_per_ip, 16);
|
||||
usize_default!(default_web_max_streams_per_session, 128);
|
||||
usize_default!(default_web_max_streams_global, 4096);
|
||||
usize_default!(default_web_max_stream_handshakes, 256);
|
||||
usize_default!(default_web_max_tombstones, 4096);
|
||||
usize_default!(default_web_pending_bytes_per_session, 32 * 1024 * 1024);
|
||||
usize_default!(default_web_pending_bytes_global, 512 * 1024 * 1024);
|
||||
usize_default!(default_web_pending_items_per_session, 16 * 1024);
|
||||
usize_default!(default_web_pending_items_global, 256 * 1024);
|
||||
usize_default!(default_web_control_bytes_per_session, 256 * 1024);
|
||||
usize_default!(default_web_control_bytes_global, 16 * 1024 * 1024);
|
||||
usize_default!(default_web_max_bootstraps_global, 512);
|
||||
usize_default!(default_web_max_bootstraps_per_ip, 64);
|
||||
usize_default!(default_web_max_vhosts, 8);
|
||||
usize_default!(default_web_max_profiles, 32);
|
||||
usize_default!(default_web_max_static_files, 4096);
|
||||
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
||||
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_debug_records_capacity, 65_536);
|
||||
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
|
||||
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
||||
u32_default!(default_web_new_bootstraps_burst, 256);
|
||||
u32_default!(default_web_new_sessions_per_minute, 600);
|
||||
u32_default!(default_web_new_sessions_burst, 128);
|
||||
u32_default!(default_web_new_streams_per_minute, 6000);
|
||||
u32_default!(default_web_new_streams_burst, 512);
|
||||
u64_default!(default_web_header_timeout_secs, 10);
|
||||
u64_default!(default_web_body_timeout_secs, 30);
|
||||
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||
u64_default!(default_web_websocket_write_secs, 30);
|
||||
u64_default!(default_web_websocket_backpressure_secs, 30);
|
||||
u64_default!(default_web_websocket_eviction_secs, 1);
|
||||
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
||||
u64_default!(default_web_reconnect_grace_secs, 120);
|
||||
u64_default!(default_web_http_idle_secs, 75);
|
||||
u64_default!(default_web_shutdown_secs, 15);
|
||||
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
pub(super) fn default_web_static_index() -> String {
|
||||
"index.html".to_string()
|
||||
}
|
||||
|
||||
macro_rules! usize_default {
|
||||
($name:ident, $value:expr) => {
|
||||
pub(super) fn $name() -> usize {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u32_default {
|
||||
($name:ident, $value:expr) => {
|
||||
pub(super) fn $name() -> u32 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u8_default {
|
||||
($name:ident, $value:expr) => {
|
||||
pub(super) fn $name() -> u8 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u64_default {
|
||||
($name:ident, $value:expr) => {
|
||||
pub(super) fn $name() -> u64 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
usize_default!(default_web_max_header_bytes, 16 * 1024);
|
||||
usize_default!(default_web_max_body_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
|
||||
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frames_per_body, 4096);
|
||||
usize_default!(default_web_max_http_connections, 1024);
|
||||
usize_default!(default_web_max_http_handlers, 512);
|
||||
usize_default!(default_web_max_lane_open_waits_per_session, 16);
|
||||
usize_default!(default_web_pending_bytes_per_lane, 8 * 1024 * 1024);
|
||||
usize_default!(default_web_pending_items_per_lane, 1024);
|
||||
usize_default!(default_web_websocket_bytes_global, 256 * 1024 * 1024);
|
||||
u8_default!(default_web_websocket_admission_watermark_pct, 75);
|
||||
u8_default!(default_web_websocket_eviction_watermark_pct, 90);
|
||||
usize_default!(default_web_websocket_http_connection_reserve, 64);
|
||||
usize_default!(default_web_max_websocket_evictions_in_flight, 8);
|
||||
usize_default!(default_web_max_carrier_learning_entries, 4096);
|
||||
usize_default!(default_web_max_body_readers, 32);
|
||||
usize_default!(default_web_max_body_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_max_sessions_global, 128);
|
||||
usize_default!(default_web_max_sessions_per_ip, 16);
|
||||
usize_default!(default_web_max_streams_per_session, 128);
|
||||
usize_default!(default_web_max_streams_global, 4096);
|
||||
usize_default!(default_web_max_stream_handshakes, 256);
|
||||
usize_default!(default_web_max_tombstones, 4096);
|
||||
usize_default!(default_web_pending_bytes_per_session, 32 * 1024 * 1024);
|
||||
usize_default!(default_web_pending_bytes_global, 512 * 1024 * 1024);
|
||||
usize_default!(default_web_pending_items_per_session, 16 * 1024);
|
||||
usize_default!(default_web_pending_items_global, 256 * 1024);
|
||||
usize_default!(default_web_control_bytes_per_session, 256 * 1024);
|
||||
usize_default!(default_web_control_bytes_global, 16 * 1024 * 1024);
|
||||
usize_default!(default_web_max_bootstraps_global, 512);
|
||||
usize_default!(default_web_max_bootstraps_per_ip, 64);
|
||||
usize_default!(default_web_max_vhosts, 8);
|
||||
usize_default!(default_web_max_profiles, 32);
|
||||
usize_default!(default_web_max_static_files, 4096);
|
||||
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
||||
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_debug_records_capacity, 65_536);
|
||||
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_memory_envelope_bytes, 1280 * 1024 * 1024);
|
||||
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
||||
u32_default!(default_web_new_bootstraps_burst, 256);
|
||||
u32_default!(default_web_new_sessions_per_minute, 600);
|
||||
u32_default!(default_web_new_sessions_burst, 128);
|
||||
u32_default!(default_web_new_streams_per_minute, 6000);
|
||||
u32_default!(default_web_new_streams_burst, 512);
|
||||
u64_default!(default_web_header_timeout_secs, 10);
|
||||
u64_default!(default_web_body_timeout_secs, 30);
|
||||
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
||||
u64_default!(default_web_stream_first_byte_secs, 30);
|
||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||
u64_default!(default_web_lane_open_wait_secs, 2);
|
||||
u64_default!(default_web_carrier_health_secs, 30);
|
||||
u64_default!(default_web_websocket_upgrade_secs, 5);
|
||||
u64_default!(default_web_websocket_open_secs, 15);
|
||||
u64_default!(default_web_websocket_write_secs, 30);
|
||||
u64_default!(default_web_websocket_backpressure_secs, 30);
|
||||
u64_default!(default_web_websocket_eviction_secs, 1);
|
||||
pub(super) fn default_web_carrier_negotiation_deadlines_secs() -> [u64; 4] {
|
||||
[3, 5, 8, 12]
|
||||
}
|
||||
u64_default!(default_web_carrier_learning_secs, 600);
|
||||
pub(super) fn default_web_carrier_learning() -> bool {
|
||||
true
|
||||
}
|
||||
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
||||
u64_default!(default_web_reconnect_grace_secs, 120);
|
||||
u64_default!(default_web_http_idle_secs, 75);
|
||||
u64_default!(default_web_shutdown_secs, 15);
|
||||
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
||||
@@ -0,0 +1,115 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Carrier selected for one newly issued WEB relay session.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum WebCarrier {
|
||||
/// Serialize all logical streams through one uplink and one downlink sequence.
|
||||
#[default]
|
||||
Https,
|
||||
/// Give every logical stream independent HTTPS sequencing and polling state.
|
||||
HttpsLanes,
|
||||
/// Multiplex all logical streams over one ordered WebSocket.
|
||||
Websocket,
|
||||
/// Give every logical stream an independently owned WebSocket lane.
|
||||
WebsocketLanes,
|
||||
}
|
||||
|
||||
impl WebCarrier {
|
||||
/// Every carrier supported by the WEB v1 bridge.
|
||||
pub(crate) const ALL: [Self; 4] = [
|
||||
Self::Https,
|
||||
Self::HttpsLanes,
|
||||
Self::Websocket,
|
||||
Self::WebsocketLanes,
|
||||
];
|
||||
|
||||
/// Returns the exact carrier token advertised to the browser bridge.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Https => "https",
|
||||
Self::HttpsLanes => "https-lanes",
|
||||
Self::Websocket => "websocket",
|
||||
Self::WebsocketLanes => "websocket-lanes",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the stable fixed-slot index used by bounded learning state.
|
||||
pub(crate) const fn index(self) -> usize {
|
||||
match self {
|
||||
Self::Https => 0,
|
||||
Self::HttpsLanes => 1,
|
||||
Self::Websocket => 2,
|
||||
Self::WebsocketLanes => 3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether one carrier owns independent state per logical stream.
|
||||
pub(crate) const fn uses_lanes(self) -> bool {
|
||||
matches!(self, Self::HttpsLanes | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether carrier messages use RFC 6455 instead of HTTP bodies.
|
||||
pub(crate) const fn uses_websocket(self) -> bool {
|
||||
matches!(self, Self::Websocket | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether all logical streams share one carrier state machine.
|
||||
pub(crate) const fn is_multiplexed(self) -> bool {
|
||||
matches!(self, Self::Https | Self::Websocket)
|
||||
}
|
||||
}
|
||||
|
||||
/// Optional ordered carrier list that enables server-side auto-negotiation.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub enum WebCarriers {
|
||||
/// Auto-negotiation is disabled and only `web.carrier` is used.
|
||||
#[default]
|
||||
Disabled,
|
||||
/// Auto-negotiation uses this ordered candidate list before the fallback.
|
||||
Enabled(Vec<WebCarrier>),
|
||||
}
|
||||
|
||||
impl WebCarriers {
|
||||
/// Returns the explicit candidate list when negotiation is enabled.
|
||||
pub fn enabled(&self) -> Option<&[WebCarrier]> {
|
||||
match self {
|
||||
Self::Disabled => None,
|
||||
Self::Enabled(carriers) => Some(carriers),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for WebCarriers {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::Disabled => false.serialize(serializer),
|
||||
Self::Enabled(carriers) => carriers.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for WebCarriers {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Repr {
|
||||
Flag(bool),
|
||||
List(Vec<WebCarrier>),
|
||||
}
|
||||
|
||||
match Repr::deserialize(deserializer)? {
|
||||
Repr::Flag(false) => Ok(Self::Disabled),
|
||||
Repr::Flag(true) => Err(serde::de::Error::custom(
|
||||
"web.carriers accepts false or a non-empty carrier array",
|
||||
)),
|
||||
Repr::List(carriers) => Ok(Self::Enabled(carriers)),
|
||||
}
|
||||
}
|
||||
}
|
||||
+205
-145
@@ -1,6 +1,5 @@
|
||||
use base64::Engine as _;
|
||||
|
||||
use crate::config::WebCarrier;
|
||||
use crate::crypto::SecureRandom;
|
||||
|
||||
/// Browser security policy for the transient Telegram Desktop bridge page.
|
||||
@@ -14,14 +13,17 @@ pub(crate) struct BridgePage {
|
||||
pub(crate) content_security_policy: String,
|
||||
}
|
||||
|
||||
/// Renders the selected HTTPS WEB carrier bridge with a fresh CSP nonce.
|
||||
/// Renders the bounded WEB carrier-negotiation bridge with a fresh CSP nonce.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn render(
|
||||
host: &str,
|
||||
bootstrap: &str,
|
||||
batch_limit: usize,
|
||||
queue_limit: usize,
|
||||
queue_items: usize,
|
||||
carrier: WebCarrier,
|
||||
negotiation_enabled: bool,
|
||||
candidate_count: usize,
|
||||
carrier_deadlines: [u64; 4],
|
||||
rng: &SecureRandom,
|
||||
) -> BridgePage {
|
||||
let mut nonce = [0u8; 18];
|
||||
@@ -34,7 +36,19 @@ pub(crate) fn render(
|
||||
.replace("__BATCH_LIMIT__", &batch_limit.to_string())
|
||||
.replace("__QUEUE_LIMIT__", &queue_limit.to_string())
|
||||
.replace("__QUEUE_ITEMS__", &queue_items.to_string())
|
||||
.replace("__CARRIER__", carrier.as_str());
|
||||
.replace(
|
||||
"__NEGOTIATION_ENABLED__",
|
||||
if negotiation_enabled { "true" } else { "false" },
|
||||
)
|
||||
.replace("__CANDIDATE_COUNT__", &candidate_count.to_string())
|
||||
.replace(
|
||||
"__CARRIER_DEADLINES__",
|
||||
&carrier_deadlines
|
||||
.iter()
|
||||
.map(u64::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
);
|
||||
BridgePage {
|
||||
body,
|
||||
content_security_policy: format!(
|
||||
@@ -54,21 +68,32 @@ const DOCUMENT: &str = r##"<!doctype html>
|
||||
<script nonce="__NONCE__">
|
||||
(()=>{
|
||||
'use strict';
|
||||
const relayOrigin='https://__HOST__',bootstrap="__BOOTSTRAP__",carrier='__CARRIER__';
|
||||
const bootstrap="__BOOTSTRAP__";
|
||||
const relayOrigin='https://__HOST__',carrierCapabilities='https,https-lanes,websocket,websocket-lanes';
|
||||
const negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
||||
let negotiatedCandidateCount=candidateCount,negotiatedFinalDeadline=candidateDeadlines[3],negotiatedFrozen=false;
|
||||
const batchLimit=__BATCH_LIMIT__,queueLimit=__QUEUE_LIMIT__,queueItemLimit=__QUEUE_ITEMS__;
|
||||
const laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItemLimit,1024),closedLaneLimit=4096;
|
||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'';
|
||||
history.replaceState(null,'',location.pathname);
|
||||
let initialized=false,closed=false,port=null,sessionToken='',createStarted=false,socket=null,socketReady=false;
|
||||
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
||||
let queuedBytes=0,queuedItems=0,upSequence=1,downCursor='0',upRunning=false,pollController=null;
|
||||
let helloFrame=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false;
|
||||
let negotiationStartedAt=0,carrierTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
||||
const pending=[],upPending=[],lanes=new Map(),closedLanes=new Set(),closedLaneOrder=[];
|
||||
const status=state=>{if(port&&!closed)port.postMessage({t:'status',state})};
|
||||
const pause=milliseconds=>new Promise(resolve=>setTimeout(resolve,milliseconds));
|
||||
const pause=(milliseconds,signal)=>new Promise((resolve,reject)=>{
|
||||
if(signal&&signal.aborted){reject(new Error('request aborted'));return}
|
||||
const timer=setTimeout(done,milliseconds);function done(){if(signal)signal.removeEventListener('abort',abort);resolve()}
|
||||
function abort(){clearTimeout(timer);signal.removeEventListener('abort',abort);reject(new Error('request aborted'))}
|
||||
if(signal)signal.addEventListener('abort',abort,{once:true});
|
||||
});
|
||||
const socketURL=()=>relayOrigin.replace(/^https:/,'wss:')+'/api/v1/ws';
|
||||
const options=(method,token,body,headers,signal,keepalive)=>({
|
||||
method,body,signal,keepalive:!!keepalive,mode:'same-origin',credentials:'omit',cache:'no-store',redirect:'error',referrerPolicy:'no-referrer',
|
||||
headers:Object.assign(token?{Authorization:'Bearer '+token}:{},body?{'Content-Type':'application/octet-stream'}:{},headers||{})
|
||||
});
|
||||
const attemptHeaders=(attempt,failure)=>negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{};
|
||||
function reserve(data,lane){
|
||||
let buffered=socket?socket.bufferedAmount:0;for(const value of lanes.values())if(value.socket)buffered+=value.socket.bufferedAmount;
|
||||
if(!data.byteLength||data.byteLength>queueLimit-queuedBytes-buffered||queuedItems>=queueItemLimit)return false;
|
||||
@@ -99,6 +124,27 @@ function splitFrames(value){
|
||||
}
|
||||
if(!result.length)throw new Error('empty frame batch');return result;
|
||||
}
|
||||
function findProbe(){
|
||||
for(let index=0;index<pending.length;index++){
|
||||
const value=pending[index],view=new DataView(value);let offset=0,frames=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8||frames++>=4096)throw new Error('invalid frame batch');
|
||||
const type=view.getUint8(offset),id=(view.getUint8(offset+1)<<16)|(view.getUint8(offset+2)<<8)|view.getUint8(offset+3);
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if((type===2&&!size)||size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
if(type===1||type===2)return {source:value,index,start:offset,end,id,data:value.slice(offset,end)};
|
||||
offset=end;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function consumeProbe(probe){
|
||||
if(pending[probe.index]!==probe.source)throw new Error('stale carrier probe');
|
||||
const before=probe.source.slice(0,probe.start),after=probe.source.slice(probe.end),remaining=before.byteLength+after.byteLength;
|
||||
if(!remaining){pending.splice(probe.index,1);release(probe.end-probe.start,1,null);return}
|
||||
const merged=new Uint8Array(remaining);merged.set(new Uint8Array(before),0);merged.set(new Uint8Array(after),before.byteLength);
|
||||
pending[probe.index]=merged.buffer;release(probe.end-probe.start,0,null);
|
||||
}
|
||||
function joinPending(values,lane){
|
||||
let total=0,count=0,frames=0;
|
||||
while(count<values.length){
|
||||
@@ -124,12 +170,13 @@ function retryAfterMs(response){
|
||||
if(Number.isFinite(when)){const delta=when-Date.now();return delta>0?Math.min(delta,30000):0}
|
||||
return 0;
|
||||
}
|
||||
async function request(path,makeOptions){
|
||||
async function request(path,frozenOptions){
|
||||
let delay=250,attempt=0;const deadline=Date.now()+90000;
|
||||
while(true){
|
||||
const requestOptions=makeOptions(),controller=new AbortController(),external=requestOptions.signal;
|
||||
const controller=new AbortController(),external=frozenOptions.signal;
|
||||
if(closed||(external&&external.aborted))throw new Error('request aborted');
|
||||
const abort=()=>controller.abort();if(external)external.addEventListener('abort',abort,{once:true});
|
||||
requestOptions.signal=controller.signal;const timer=setTimeout(abort,90000);
|
||||
const requestOptions=Object.assign({},frozenOptions,{signal:controller.signal});const timer=setTimeout(abort,90000);
|
||||
let serviceUnavailable=false,wait=0;
|
||||
try{
|
||||
const response=await fetch(relayOrigin+path,requestOptions);
|
||||
@@ -140,26 +187,129 @@ async function request(path,makeOptions){
|
||||
if(++attempt===9)throw new Error('carrier retry limit reached');
|
||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||
if(serviceUnavailable&&Date.now()>=deadline)throw new Error('carrier retry limit reached');
|
||||
status('reconnecting');await pause(wait||(delay+Math.floor(Math.random()*Math.max(1,delay/4))));
|
||||
status('reconnecting');await pause(wait||(delay+Math.floor(Math.random()*Math.max(1,delay/4))),external);
|
||||
if(closed||(external&&external.aborted))throw new Error('request aborted');
|
||||
if(!serviceUnavailable)delay=Math.min(delay*2,5000);
|
||||
}
|
||||
}
|
||||
function fail(){if(closed)return;status('failed');if(port)port.postMessage({t:'close'});close(true)}
|
||||
async function createSession(first){
|
||||
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
||||
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
||||
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'';
|
||||
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
||||
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:''}}
|
||||
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
||||
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
||||
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
||||
if(!Number.isInteger(echoedAttempt)||echoedAttempt<1||(exactAttempt?echoedAttempt!==expectedAttempt:echoedAttempt>expectedAttempt))throw new Error('invalid carrier attempt');
|
||||
if(parsedCount>candidateCount||parsedDeadline>candidateDeadlines[3])throw new Error('invalid carrier bounds');
|
||||
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
||||
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
||||
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
||||
return {selected,state};
|
||||
}
|
||||
function armCarrierDeadline(epoch){
|
||||
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
||||
if(carrierTimer)clearTimeout(carrierTimer);
|
||||
const deadline=carrierAttempt>=negotiatedCandidateCount?negotiatedFinalDeadline:candidateDeadlines[carrierAttempt-1];
|
||||
const remaining=negotiationStartedAt+deadline*1000-Date.now();
|
||||
carrierTimer=setTimeout(()=>advanceCarrier('timeout',epoch),Math.max(0,remaining));
|
||||
}
|
||||
function resetCandidate(){
|
||||
if(pollController)pollController.abort();pollController=null;
|
||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
||||
for(const lane of lanes.values()){if(lane.controller)lane.controller.abort();if(lane.socket)lane.socket.close()}
|
||||
lanes.clear();closedLanes.clear();closedLaneOrder.length=0;upPending.length=0;upSequence=1;downCursor='0';upRunning=false;
|
||||
sessionToken='';carrier='';candidateRunning=false;currentAttempt=null;
|
||||
}
|
||||
function advanceConfirmed(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
resetCandidate();
|
||||
if(carrierAttempt>=negotiatedCandidateCount||Date.now()>=negotiationStartedAt+negotiatedFinalDeadline*1000){switching=false;fail();return}
|
||||
carrierAttempt++;carrierFailure=reason;attemptEpoch++;const nextEpoch=attemptEpoch;switching=false;
|
||||
status('reconnecting');armCarrierDeadline(nextEpoch);createSession(nextEpoch);
|
||||
}
|
||||
function advanceCarrier(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch||switching)return;
|
||||
if(!negotiationEnabled){fail();return}
|
||||
switching=true;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||
const snapshot=currentAttempt;if(attemptController)attemptController.abort();attemptController=null;
|
||||
if(!snapshot||snapshot.epoch!==epoch){switching=false;fail();return}
|
||||
resolveAttempt(reason,epoch,snapshot);
|
||||
}
|
||||
async function resolveAttempt(reason,epoch,snapshot){
|
||||
const controller=new AbortController();attemptController=controller;
|
||||
const remaining=negotiationStartedAt+negotiatedFinalDeadline*1000-Date.now();
|
||||
if(remaining<=0){switching=false;fail();return}
|
||||
const timer=setTimeout(()=>controller.abort(),remaining);
|
||||
try{
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status===409){sessionEcho(response,snapshot.attempt,['committed','healthy'],false);await response.arrayBuffer();switching=false;fail();return}
|
||||
if(response.status!==200){await response.arrayBuffer();switching=false;fail();return}
|
||||
const echo=sessionEcho(response,snapshot.attempt,['provisional','committed','healthy'],true);
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'||(snapshot.selected&&echo.selected!==snapshot.selected))throw new Error('changed carrier replay');
|
||||
const welcome=await response.arrayBuffer();if(closed||epoch!==attemptEpoch)return;
|
||||
cleanupToken=token;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
if(echo.state!=='provisional'){switching=false;fail();return}
|
||||
advanceConfirmed(reason,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch){switching=false;fail()}}
|
||||
finally{clearTimeout(timer);if(attemptController===controller)attemptController=null}
|
||||
}
|
||||
function maybeStartCandidate(){
|
||||
let probe;try{probe=findProbe()}catch(error){fail();return}
|
||||
if(!probe||closed||carrierCommitted)return;
|
||||
if(!sessionToken||candidateRunning)return;
|
||||
candidateRunning=true;const epoch=attemptEpoch;
|
||||
if(carrier==='https')probeHttp(probe,null,epoch);
|
||||
else if(carrier==='https-lanes')probeHttp(probe,probe.id,epoch);
|
||||
else if(carrier==='websocket')openCandidateSocket(probe,null,epoch);
|
||||
else if(carrier==='websocket-lanes')openCandidateSocket(probe,probe.id,epoch);
|
||||
else advanceCarrier('protocol',epoch);
|
||||
}
|
||||
async function createSession(epoch){
|
||||
const controller=new AbortController(),attempt=carrierAttempt,failure=carrierFailure;
|
||||
const snapshot={epoch,attempt,failure,hello:helloFrame,selected:''};currentAttempt=snapshot;attemptController=controller;
|
||||
try{
|
||||
status('connecting');
|
||||
const response=await request('/api/v1/session',()=>options('POST',bootstrap,first));
|
||||
if(response.status!==200||response.headers.get('X-Carrier-Mode')!==carrier)throw new Error('session creation rejected');
|
||||
sessionToken=response.headers.get('X-Session-Token')||'';downCursor=response.headers.get('X-Down-Cursor')||'0';
|
||||
if(!sessionToken)throw new Error('missing session token');
|
||||
if(closed){deleteSession();return}
|
||||
const welcome=await response.arrayBuffer();
|
||||
port.postMessage(welcome,[welcome]);status('connected');
|
||||
if(carrier==='https-lanes')ensureLane(0);
|
||||
if(carrier==='websocket')openSocket();
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(attempt,failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status===409){sessionEcho(response,attempt,['committed','healthy'],false);await response.arrayBuffer();fail();return}
|
||||
if(response.status!==200){await response.arrayBuffer();advanceCarrier('http',epoch);return}
|
||||
const echo=sessionEcho(response,attempt,['provisional'],true),selected=echo.selected;snapshot.selected=selected;
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'){await response.arrayBuffer();advanceCarrier('protocol',epoch);return}
|
||||
const welcome=await response.arrayBuffer();if(closed||epoch!==attemptEpoch)return;
|
||||
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
maybeStartCandidate();
|
||||
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier('network',epoch)}
|
||||
}
|
||||
async function probeHttp(probe,laneID,epoch){
|
||||
try{
|
||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
||||
const response=await request('/api/v1/up',options('POST',token,body,headers,controller.signal));
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status!==204){await response.arrayBuffer();advanceCarrier('http',epoch);return}
|
||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
||||
if(laneID===null)upSequence=2;else ensureLane(laneID).sequence=2;
|
||||
commitCarrier(probe,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch)advanceCarrier('network',epoch)}
|
||||
}
|
||||
function commitCarrier(probe,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
if(switching){fail();return}
|
||||
try{consumeProbe(probe)}catch(error){fail();return}
|
||||
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||
attemptController=null;currentAttempt=null;
|
||||
status('connected');
|
||||
if(carrier==='https')poll();
|
||||
else if(carrier==='https-lanes'){const lane=lanes.get(probe.id);if(lane&&!lane.polling)pollLane(lane)}
|
||||
for(const data of pending.splice(0)){release(data.byteLength,1,null);queueCarrier(data)}
|
||||
if(carrier==='https')poll();else if(carrier==='https-lanes')pollLane(lanes.get(0));
|
||||
}catch(error){fail()}
|
||||
}
|
||||
function queueCarrier(data){
|
||||
try{
|
||||
@@ -174,22 +324,37 @@ async function runUp(){
|
||||
try{
|
||||
while(!closed&&sessionToken&&upPending.length){
|
||||
const batch=joinPending(upPending,null),sequence=String(upSequence);
|
||||
const response=await request('/api/v1/up',()=>options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence}));
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence}));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('uplink rejected');
|
||||
release(batch.total,batch.count,null);port.postMessage({t:'traffic',up:batch.total,down:0});upSequence++;
|
||||
}
|
||||
}catch(error){fail()}
|
||||
finally{upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}
|
||||
}
|
||||
function openSocket(){
|
||||
if(socket||closed)return;socket=new WebSocket(socketURL(),'tproxy-v1.'+sessionToken);socket.binaryType='arraybuffer';
|
||||
socket.onopen=()=>{if(closed)return;socketReady=true;status('connected');runSocketUp()};
|
||||
socket.onmessage=event=>{
|
||||
if(closed||!(event.data instanceof ArrayBuffer)){fail();return}
|
||||
try{const bound=frameBound(event.data,4096,batchLimit);if(bound.bytes!==event.data.byteLength)throw new Error('invalid frame batch')}catch(error){fail();return}
|
||||
function openCandidateSocket(probe,laneID,epoch){
|
||||
const token=sessionToken,protocol=laneID===null?(negotiationEnabled?'tproxy-auto-v1.':'tproxy-v1.')+token:(negotiationEnabled?'tproxy-auto-lane-v1.':'tproxy-lane-v1.')+token+'.'+String(laneID);
|
||||
const next=new WebSocket(socketURL(),protocol);next.binaryType='arraybuffer';let opened=false,lane=null;
|
||||
if(laneID===null)socket=next;else{lane=ensureLane(laneID);lane.socket=next}
|
||||
next.onopen=()=>{
|
||||
if(closed||epoch!==attemptEpoch)return;opened=true;
|
||||
if(lane){lane.ready=true}else socketReady=true;
|
||||
try{next.send(probe.data);if(!negotiationEnabled)commitCarrier(probe,epoch)}catch(error){advanceCarrier('upgrade',epoch)}
|
||||
};
|
||||
next.onmessage=event=>{
|
||||
if(closed||epoch!==attemptEpoch||!(event.data instanceof ArrayBuffer))return;
|
||||
if(!carrierCommitted){if(event.data.byteLength!==0){advanceCarrier('protocol',epoch);return}commitCarrier(probe,epoch);return}
|
||||
try{
|
||||
if(lane){const values=splitFrames(event.data);for(const value of values)if(value.id!==lane.id)throw new Error('cross-lane frame');if(values.some(value=>value.type===3))lane.remoteClosed=true}
|
||||
else{const bound=frameBound(event.data,4096,batchLimit);if(bound.bytes!==event.data.byteLength)throw new Error('invalid frame batch')}
|
||||
}catch(error){if(lane)finishLane(lane,true);else fail();return}
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
socket.onerror=()=>{};socket.onclose=()=>{socketReady=false;if(!closed)fail()};
|
||||
next.onerror=()=>{};
|
||||
next.onclose=()=>{
|
||||
if(epoch!==attemptEpoch||closed)return;
|
||||
if(!carrierCommitted){advanceCarrier(opened?'network':'upgrade',epoch);return}
|
||||
if(lane){lane.ready=false;lane.socket=null;finishLane(lane,true)}else{socketReady=false;fail()}
|
||||
};
|
||||
}
|
||||
function queueSocket(data){if(!reserve(data,null)){fail();return}upPending.push(data);runSocketUp()}
|
||||
async function waitSocket(next,size,limit){
|
||||
@@ -210,7 +375,7 @@ async function poll(){
|
||||
while(!closed&&sessionToken){
|
||||
try{
|
||||
pollController=new AbortController();
|
||||
const response=await request('/api/v1/down',()=>options('POST',sessionToken,null,{'X-Down-Cursor':downCursor},pollController.signal));
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':downCursor},pollController.signal));
|
||||
if(response.status===204){status('connected');continue}
|
||||
if(response.status!==200)throw new Error('downlink rejected');
|
||||
const next=response.headers.get('X-Down-Cursor')||'',data=await response.arrayBuffer();
|
||||
@@ -274,7 +439,7 @@ async function runLaneUp(lane){
|
||||
try{
|
||||
while(!closed&&sessionToken&&lane.pending.length){
|
||||
const batch=joinPending(lane.pending,lane),sequence=String(lane.sequence),laneID=String(lane.id);
|
||||
const response=await request('/api/v1/up',()=>options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID}));
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID}));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('lane uplink rejected');
|
||||
release(batch.total,batch.count,lane);port.postMessage({t:'traffic',up:batch.total,down:0});lane.sequence++;
|
||||
if(!lane.polling)pollLane(lane);
|
||||
@@ -287,7 +452,7 @@ async function pollLane(lane){
|
||||
try{
|
||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
||||
const controller=new AbortController(),laneID=String(lane.id);lane.controller=controller;
|
||||
const response=await request('/api/v1/down',()=>options('POST',sessionToken,null,{'X-Down-Cursor':lane.cursor,'X-Lane-ID':laneID},controller.signal));
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':lane.cursor,'X-Lane-ID':laneID},controller.signal));
|
||||
if(response.status===204){
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
||||
status('connected');continue;
|
||||
@@ -303,10 +468,10 @@ async function pollLane(lane){
|
||||
finally{lane.polling=false;lane.controller=null}
|
||||
}
|
||||
function deleteSession(){
|
||||
if(sessionToken)fetch(relayOrigin+'/api/v1/session',options('DELETE',sessionToken,null,null,undefined,true)).catch(()=>{});
|
||||
const token=cleanupToken||sessionToken;if(token)fetch(relayOrigin+'/api/v1/session',options('DELETE',token,null,null,undefined,true)).catch(()=>{});
|
||||
}
|
||||
function close(notifyServer){
|
||||
if(closed)return;closed=true;if(pollController)pollController.abort();
|
||||
if(closed)return;closed=true;if(carrierTimer)clearTimeout(carrierTimer);if(attemptController)attemptController.abort();if(pollController)pollController.abort();
|
||||
if(socket)socket.close();for(const lane of lanes.values()){if(lane.controller)lane.controller.abort();if(lane.socket)lane.socket.close()}
|
||||
if(notifyServer)deleteSession();pending.length=0;upPending.length=0;
|
||||
for(const lane of lanes.values())lane.pending.length=0;lanes.clear();queuedBytes=0;queuedItems=0;if(port)port.close();
|
||||
@@ -315,8 +480,8 @@ function activatePort(nextPort){
|
||||
initialized=true;port=nextPort;
|
||||
port.onmessage=message=>{
|
||||
if(message.data instanceof ArrayBuffer){
|
||||
if(!createStarted){createStarted=true;createSession(message.data)}
|
||||
else if(!sessionToken){if(!reserve(message.data,null)){fail();return}pending.push(message.data)}
|
||||
if(!createStarted){createStarted=true;helloFrame=message.data;if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}createSession(attemptEpoch)}
|
||||
else if(!carrierCommitted){if(!reserve(message.data,null)){fail();return}pending.push(message.data);maybeStartCandidate()}
|
||||
else queueCarrier(message.data);
|
||||
}else if(message.data&&message.data.t==='close')close(true);
|
||||
};
|
||||
@@ -348,111 +513,6 @@ addEventListener('pagehide',()=>close(true),{once:true});
|
||||
</html>
|
||||
"##;
|
||||
|
||||
// Rendered wire-contract tests remain separate from the embedded document.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rendered_page_contains_no_template_markers_or_capability() {
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::HttpsLanes,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(!page.body.contains("__"));
|
||||
assert!(!page.body.contains("bridge="));
|
||||
assert!(page.body.contains("X-Up-Seq"));
|
||||
assert!(page.body.contains("carrier='https-lanes'"));
|
||||
assert!(page.body.contains("X-Lane-ID"));
|
||||
assert!(page.body.contains("const when=Date.parse(header)"));
|
||||
assert!(page.body.contains("},{once:false});"));
|
||||
assert!(
|
||||
page.body
|
||||
.contains("if(!sessionToken)throw new Error('missing session token')")
|
||||
);
|
||||
assert!(!page.body.contains("welcomeBytes"));
|
||||
assert!(
|
||||
page.body
|
||||
.contains("for(const value of splitFrames(data))if(value.id!==lane.id)")
|
||||
);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("let frames;try{frames=splitFrames(value)}catch(error){fail();return}")
|
||||
);
|
||||
assert!(
|
||||
page.content_security_policy
|
||||
.contains("frame-ancestors http://127.0.0.1:*")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_is_parseable_by_ios_native_carrier() {
|
||||
let bootstrap = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
bootstrap,
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::Https,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
let accepted_shapes = [
|
||||
format!("const bootstrap=\"{bootstrap}\""),
|
||||
format!("const bootstrap='{bootstrap}'"),
|
||||
format!("bootstrap=\"{bootstrap}\""),
|
||||
];
|
||||
|
||||
assert!(
|
||||
accepted_shapes
|
||||
.iter()
|
||||
.any(|shape| page.body.contains(shape)),
|
||||
"the iOS native carrier cannot parse a comma-declared single-quoted bootstrap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_advertises_exact_websocket_carriers() {
|
||||
let websocket = render(
|
||||
"proxy.example.com",
|
||||
"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::Websocket,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(websocket.body.contains("carrier='websocket'"));
|
||||
assert!(
|
||||
websocket
|
||||
.body
|
||||
.contains("new WebSocket(socketURL(),'tproxy-v1.'+sessionToken)")
|
||||
);
|
||||
assert!(
|
||||
websocket
|
||||
.content_security_policy
|
||||
.contains("connect-src 'self' wss://proxy.example.com")
|
||||
);
|
||||
|
||||
let lanes = render(
|
||||
"proxy.example.com",
|
||||
"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::WebsocketLanes,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(lanes.body.contains("carrier='websocket-lanes'"));
|
||||
assert!(
|
||||
lanes
|
||||
.body
|
||||
.contains("'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id)")
|
||||
);
|
||||
assert!(!lanes.body.contains("__"));
|
||||
}
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
use super::*;
|
||||
|
||||
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||
render(
|
||||
"proxy.example.com",
|
||||
bootstrap,
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
true,
|
||||
candidate_count,
|
||||
[3, 5, 8, 12],
|
||||
&SecureRandom::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_contains_bounded_negotiation_contract() {
|
||||
let page = render_page("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 4);
|
||||
assert!(!page.body.contains("__"));
|
||||
assert!(!page.body.contains("bridge="));
|
||||
assert!(page.body.contains("X-Carrier-Capabilities"));
|
||||
assert!(page.body.contains("X-Carrier-Attempt"));
|
||||
assert!(page.body.contains("candidateCount=4"));
|
||||
assert!(page.body.contains("candidateDeadlines=[3,5,8,12]"));
|
||||
assert!(page.body.contains("X-Up-Seq"));
|
||||
assert!(page.body.contains("X-Lane-ID"));
|
||||
assert!(page.body.contains("tproxy-auto-v1."));
|
||||
assert!(page.body.contains("tproxy-auto-lane-v1."));
|
||||
assert!(
|
||||
page.content_security_policy
|
||||
.contains("frame-ancestors http://127.0.0.1:*")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_preserves_the_ios_bootstrap_literal() {
|
||||
let bootstrap = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
|
||||
let page = render_page(bootstrap, 2);
|
||||
assert!(
|
||||
page.body
|
||||
.contains(&format!("const bootstrap=\"{bootstrap}\""))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_deadline_formula_uses_the_final_checkpoint() {
|
||||
let page = render_page("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", 3);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("negotiatedFinalDeadline=candidateDeadlines[3]")
|
||||
);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("carrierAttempt>=negotiatedCandidateCount?negotiatedFinalDeadline")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
false,
|
||||
1,
|
||||
[3, 5, 8, 12],
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(page.body.contains(
|
||||
"if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}"
|
||||
));
|
||||
assert!(
|
||||
page.body
|
||||
.contains("negotiationEnabled?'tproxy-auto-v1.':'tproxy-v1.'")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_and_attempt_state_are_frozen_before_fetch() {
|
||||
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("async function request(path,frozenOptions)")
|
||||
);
|
||||
assert!(!page.body.contains("makeOptions"));
|
||||
assert!(
|
||||
page.body
|
||||
.contains("if(closed||(external&&external.aborted))throw new Error('request aborted')")
|
||||
);
|
||||
assert!(page.body.contains(
|
||||
"const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal)"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_commit_is_resolved_before_carrier_advance() {
|
||||
let page = render_page("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 4);
|
||||
assert!(page.body.contains("resolveAttempt(reason,epoch,snapshot)"));
|
||||
assert!(page.body.contains(
|
||||
"sessionEcho(response,snapshot.attempt,['provisional','committed','healthy'],true)"
|
||||
));
|
||||
assert!(
|
||||
page.body
|
||||
.contains("if(echo.state!=='provisional'){switching=false;fail();return}")
|
||||
);
|
||||
assert!(page.body.contains("const token=cleanupToken||sessionToken"));
|
||||
}
|
||||
+10
-149
@@ -1,6 +1,6 @@
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -19,7 +19,6 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
||||
use crate::web::bridge;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||
|
||||
// Response-body activity keeps connection idle accounting lifecycle-correct.
|
||||
@@ -34,6 +33,8 @@ mod down;
|
||||
mod request;
|
||||
// Carrier response construction and lane-header helpers are shared by handlers.
|
||||
mod response;
|
||||
// Session creation and replacement negotiation remain separate from request routing.
|
||||
mod session;
|
||||
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -56,12 +57,12 @@ use response::{
|
||||
bad_gateway, carrier_empty, carrier_headers, carrier_lane, full_response, generic_not_found,
|
||||
insert_header, service_unavailable,
|
||||
};
|
||||
use session::handle_session;
|
||||
|
||||
type BoxError = Box<dyn Error + Send + Sync>;
|
||||
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
||||
type HttpResponse = Response<HttpBody>;
|
||||
|
||||
const CREATE_BODY_LIMIT: usize = 64;
|
||||
const TRANSPORT_PATHS: [&str; 3] = ["/api/v1/session", "/api/v1/up", "/api/v1/down"];
|
||||
const WEBSOCKET_PATH: &str = "/api/v1/ws";
|
||||
|
||||
@@ -224,7 +225,6 @@ async fn handle_root(
|
||||
trace.set_route(TraceRoute::Bridge);
|
||||
trace.set_effective_ip(client_ip);
|
||||
}
|
||||
let carrier = profile.carrier;
|
||||
let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
|
||||
Ok(bootstrap) => bootstrap,
|
||||
Err(error) => {
|
||||
@@ -234,7 +234,7 @@ async fn handle_root(
|
||||
&profile,
|
||||
TraceLifecycleEvent::BootstrapRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
Some(error.as_str()),
|
||||
);
|
||||
strip_query(&mut request);
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
@@ -251,7 +251,9 @@ async fn handle_root(
|
||||
generation.config().web.limits.carrier_batch_bytes,
|
||||
generation.config().web.limits.pending_bytes_per_session,
|
||||
generation.config().web.limits.pending_items_per_session,
|
||||
carrier,
|
||||
profile.carrier_negotiation_enabled,
|
||||
profile.carriers.len(),
|
||||
profile.carrier_negotiation_deadlines_secs,
|
||||
&generation.rng,
|
||||
);
|
||||
let mut response = full_response(StatusCode::OK, Bytes::from(page.body));
|
||||
@@ -315,127 +317,6 @@ async fn handle_api(
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session(
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: crate::web::manager::TokenHash,
|
||||
client_ip: IpAddr,
|
||||
) -> HttpResponse {
|
||||
if request.headers().contains_key("x-lane-id") {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if request.method() == Method::DELETE {
|
||||
if request.headers().contains_key(header::CONTENT_TYPE) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(trace) = request_trace(&request)
|
||||
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
|
||||
{
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if !body.is_empty() || runtime.close_token(token_hash, &vhost.host).is_err() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
return carrier_empty(StatusCode::NO_CONTENT);
|
||||
}
|
||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some((trace_session_id, profile)) =
|
||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_profile(&profile, trace_session_id);
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, CREATE_BODY_LIMIT, false).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Request,
|
||||
&body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
match runtime.create_session(token_hash, &vhost.host, client_ip, &body) {
|
||||
Ok(result) => {
|
||||
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.register_redaction(result.token.as_bytes());
|
||||
trace.record_frames(
|
||||
TraceDirection::Response,
|
||||
&welcome,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
let mut response = full_response(StatusCode::OK, welcome);
|
||||
carrier_headers(&mut response);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-session-token"),
|
||||
&result.token,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-carrier-mode"),
|
||||
HeaderValue::from_static(result.carrier.as_str()),
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
HeaderValue::from_static("0"),
|
||||
);
|
||||
response
|
||||
}
|
||||
Err(
|
||||
error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent),
|
||||
) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
);
|
||||
service_unavailable()
|
||||
}
|
||||
Err(error) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
);
|
||||
serve_decoy(request, vhost, true, &runtime).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_up(
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
@@ -462,12 +343,7 @@ async fn handle_up(
|
||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let limit = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.limits
|
||||
.max_body_bytes;
|
||||
let limit = session.limits().max_body_bytes;
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
@@ -480,11 +356,7 @@ async fn handle_up(
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Request,
|
||||
&body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
trace.record_frames(TraceDirection::Request, &body, session.limits());
|
||||
}
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
||||
@@ -524,14 +396,3 @@ fn set_trace_route<B>(request: &Request<B>, route: TraceRoute) {
|
||||
trace.set_route(route);
|
||||
}
|
||||
}
|
||||
|
||||
fn manager_error_reason(error: ManagerError) -> &'static str {
|
||||
match error {
|
||||
ManagerError::Authentication => "authentication",
|
||||
ManagerError::Backpressure => "backpressure",
|
||||
ManagerError::Limit => "limit",
|
||||
ManagerError::Protocol => "protocol",
|
||||
ManagerError::Concurrent => "concurrent",
|
||||
ManagerError::Closed => "closed",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,8 +56,11 @@ pub(super) async fn handle_down(
|
||||
if !body.is_empty() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let _lane_poll = if lane_id.is_some() {
|
||||
let Some(permit) = runtime.try_lane_poll() else {
|
||||
let Some(_down_poll) = runtime.try_lane_poll(false) else {
|
||||
return service_unavailable();
|
||||
};
|
||||
let _control_lane_poll = if lane_id == Some(0) {
|
||||
let Some(permit) = runtime.try_lane_poll(true) else {
|
||||
return service_unavailable();
|
||||
};
|
||||
Some(permit)
|
||||
@@ -86,11 +89,7 @@ pub(super) async fn handle_down(
|
||||
}
|
||||
Ok(result) => {
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Response,
|
||||
&result.body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
trace.record_frames(TraceDirection::Response, &result.body, session.limits());
|
||||
}
|
||||
let mut response = full_response(StatusCode::OK, result.body);
|
||||
carrier_headers(&mut response);
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
||||
for (index, carrier) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let capability = [12 + index as u8; 32];
|
||||
let mut config = runtime_config(capability, carrier);
|
||||
config.web.timeouts.long_poll_secs = 0;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(active_runtime);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nCookie:\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (root_headers, root_body) = split_response(&root_response);
|
||||
assert!(root_headers.starts_with(b"HTTP/1.1 200"));
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let create = |cookie: &str| {
|
||||
let mut request = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\n{cookie}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
request.extend_from_slice(&hello);
|
||||
request
|
||||
};
|
||||
let nonempty_cookie =
|
||||
request(&listener, &runtime, create("Cookie: state=unexpected\r\n")).await;
|
||||
assert!(!nonempty_cookie.starts_with(b"HTTP/1.1 200"));
|
||||
let duplicate_cookie = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create("Cookie:\r\nCookie: state=unexpected\r\n"),
|
||||
)
|
||||
.await;
|
||||
assert!(!duplicate_cookie.starts_with(b"HTTP/1.1 200"));
|
||||
|
||||
let create_response = request(&listener, &runtime, create("Cookie:\r\n")).await;
|
||||
let (create_headers, create_body) = split_response(&create_response);
|
||||
assert!(create_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(create_headers, "x-carrier-mode"),
|
||||
carrier.as_str()
|
||||
);
|
||||
assert_eq!(create_body, frame::encode(FrameType::Welcome, 0, &[]));
|
||||
let session = response_header(create_headers, "x-session-token").to_string();
|
||||
let lane = if carrier == WebCarrier::HttpsLanes {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let mut uplink = format!(
|
||||
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nContent-Type: application/octet-stream\r\nCookie:\r\nX-Up-Seq: 1\r\n{lane}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
pong.len()
|
||||
)
|
||||
.into_bytes();
|
||||
uplink.extend_from_slice(&pong);
|
||||
let uplink_response = request(&listener, &runtime, uplink).await;
|
||||
let (uplink_headers, _) = split_response(&uplink_response);
|
||||
assert!(uplink_headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert_eq!(response_header(uplink_headers, "x-up-ack"), "1");
|
||||
|
||||
let downlink = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nCookie:\r\nX-Down-Cursor: 0\r\n{lane}Content-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let downlink_response = request(&listener, &runtime, downlink).await;
|
||||
let (downlink_headers, _) = split_response(&downlink_response);
|
||||
assert!(downlink_headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert_eq!(response_header(downlink_headers, "x-down-cursor"), "0");
|
||||
|
||||
let close = format!(
|
||||
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nCookie:\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let close_response = request(&listener, &runtime, close).await;
|
||||
assert!(close_response.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
use super::*;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const CAPABILITIES: &str = "https,https-lanes,websocket,websocket-lanes";
|
||||
const NATIVE_USER_AGENT_HEADER: &str =
|
||||
"User-Agent: Telegram/3951 CFNetwork/3896.100.1.2.1 Darwin/27.0.0\r\n";
|
||||
|
||||
fn issue_bootstrap(runtime: &Arc<WebProcessRuntime>, client_ip: &str) -> String {
|
||||
let profile = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.profiles[0]
|
||||
.clone();
|
||||
runtime
|
||||
.issue_bootstrap(profile, client_ip.parse().unwrap())
|
||||
.unwrap()
|
||||
.token
|
||||
}
|
||||
|
||||
fn create_request(
|
||||
bootstrap: &str,
|
||||
hello: &[u8],
|
||||
attempt: Option<u8>,
|
||||
failure: Option<&str>,
|
||||
) -> Vec<u8> {
|
||||
create_request_with_headers(bootstrap, hello, attempt, failure, "")
|
||||
}
|
||||
|
||||
fn create_request_with_headers(
|
||||
bootstrap: &str,
|
||||
hello: &[u8],
|
||||
attempt: Option<u8>,
|
||||
failure: Option<&str>,
|
||||
extra_headers: &str,
|
||||
) -> Vec<u8> {
|
||||
let negotiation = attempt.map_or_else(String::new, |attempt| {
|
||||
let failure = failure
|
||||
.map(|failure| format!("X-Carrier-Failure: {failure}\r\n"))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"X-Carrier-Capabilities: {CAPABILITIES}\r\nX-Carrier-Attempt: {attempt}\r\n{failure}"
|
||||
)
|
||||
});
|
||||
let mut request = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\n{negotiation}{extra_headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
request.extend_from_slice(hello);
|
||||
request
|
||||
}
|
||||
|
||||
fn optional_response_header<'a>(headers: &'a [u8], name: &str) -> Option<&'a str> {
|
||||
std::str::from_utf8(headers)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find_map(|(header, value)| header.eq_ignore_ascii_case(name).then_some(value.trim()))
|
||||
}
|
||||
|
||||
fn token_hash(token: &str) -> crate::web::manager::TokenHash {
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(token)
|
||||
.unwrap();
|
||||
Sha256::digest(raw).into()
|
||||
}
|
||||
|
||||
fn assert_no_negotiation_headers(headers: &[u8]) {
|
||||
for header in [
|
||||
"x-carrier-attempt",
|
||||
"x-carrier-candidate-count",
|
||||
"x-carrier-deadline",
|
||||
"x-carrier-state",
|
||||
] {
|
||||
assert!(optional_response_header(headers, header).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn absent_carriers_reject_negotiation_and_preserve_legacy_creation() {
|
||||
let capability = [41; 32];
|
||||
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
|
||||
let rejected = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, Some(1), None),
|
||||
)
|
||||
.await;
|
||||
let (rejected_headers, _) = split_response(&rejected);
|
||||
assert!(optional_response_header(rejected_headers, "x-session-token").is_none());
|
||||
|
||||
let legacy = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, None, None),
|
||||
)
|
||||
.await;
|
||||
let (legacy_headers, _) = split_response(&legacy);
|
||||
assert!(legacy_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(legacy_headers, "x-carrier-mode"), "https");
|
||||
assert!(optional_response_header(legacy_headers, "x-carrier-attempt").is_none());
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_free_native_client_can_use_each_fixed_carrier() {
|
||||
for (index, carrier) in WebCarrier::ALL.into_iter().enumerate() {
|
||||
let capability = [50 + index as u8; 32];
|
||||
let generation = test_runtime_generation(1, runtime_config(capability, carrier));
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
|
||||
let response = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request_with_headers(&bootstrap, &hello, None, None, NATIVE_USER_AGENT_HEADER),
|
||||
)
|
||||
.await;
|
||||
let (headers, _) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(headers, "x-carrier-mode"), carrier.as_str());
|
||||
assert_no_negotiation_headers(headers);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_free_native_client_uses_fallback_when_candidates_are_enabled() {
|
||||
let capability = [55; 32];
|
||||
let generation = test_runtime_generation(
|
||||
1,
|
||||
negotiation_runtime_config(
|
||||
capability,
|
||||
WebCarrier::HttpsLanes,
|
||||
false,
|
||||
Arc::from([WebCarrier::Websocket, WebCarrier::HttpsLanes]),
|
||||
),
|
||||
);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
|
||||
let response = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request_with_headers(&bootstrap, &hello, None, None, NATIVE_USER_AGENT_HEADER),
|
||||
)
|
||||
.await;
|
||||
let (headers, _) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(headers, "x-carrier-mode"), "https-lanes");
|
||||
assert_no_negotiation_headers(headers);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_native_capabilities_participate_in_automatic_selection() {
|
||||
let capability = [56; 32];
|
||||
let generation = test_runtime_generation(
|
||||
1,
|
||||
negotiation_runtime_config(
|
||||
capability,
|
||||
WebCarrier::Https,
|
||||
false,
|
||||
Arc::from([WebCarrier::WebsocketLanes, WebCarrier::Https]),
|
||||
),
|
||||
);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
|
||||
let response = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request_with_headers(&bootstrap, &hello, Some(1), None, NATIVE_USER_AGENT_HEADER),
|
||||
)
|
||||
.await;
|
||||
let (headers, _) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(headers, "x-carrier-mode"),
|
||||
"websocket-lanes"
|
||||
);
|
||||
assert_eq!(response_header(headers, "x-carrier-attempt"), "1");
|
||||
assert_eq!(response_header(headers, "x-carrier-candidate-count"), "2");
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
|
||||
let capability = [42; 32];
|
||||
let mut config = negotiation_runtime_config(
|
||||
capability,
|
||||
WebCarrier::Websocket,
|
||||
false,
|
||||
Arc::from([
|
||||
WebCarrier::Https,
|
||||
WebCarrier::HttpsLanes,
|
||||
WebCarrier::Websocket,
|
||||
]),
|
||||
);
|
||||
config.web.timeouts.long_poll_secs = 1;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
|
||||
let first_request = create_request(&bootstrap, &hello, Some(1), None);
|
||||
let first = request(&listener, &runtime, first_request.clone()).await;
|
||||
let (first_headers, _) = split_response(&first);
|
||||
assert_eq!(response_header(first_headers, "x-carrier-mode"), "https");
|
||||
assert_eq!(response_header(first_headers, "x-carrier-attempt"), "1");
|
||||
let first_token = response_header(first_headers, "x-session-token").to_string();
|
||||
|
||||
let replay = request(&listener, &runtime, first_request).await;
|
||||
let (replay_headers, _) = split_response(&replay);
|
||||
assert_eq!(
|
||||
response_header(replay_headers, "x-session-token"),
|
||||
first_token
|
||||
);
|
||||
assert_eq!(
|
||||
response_header(replay_headers, "x-carrier-candidate-count"),
|
||||
"3"
|
||||
);
|
||||
assert_eq!(
|
||||
response_header(replay_headers, "x-carrier-state"),
|
||||
"provisional"
|
||||
);
|
||||
|
||||
let second_request = create_request(&bootstrap, &hello, Some(2), Some("timeout"));
|
||||
let second = request(&listener, &runtime, second_request.clone()).await;
|
||||
let (second_headers, _) = split_response(&second);
|
||||
assert_eq!(
|
||||
response_header(second_headers, "x-carrier-mode"),
|
||||
"https-lanes"
|
||||
);
|
||||
assert_eq!(response_header(second_headers, "x-carrier-attempt"), "2");
|
||||
let second_token = response_header(second_headers, "x-session-token").to_string();
|
||||
assert_ne!(first_token, second_token);
|
||||
assert!(
|
||||
runtime
|
||||
.get_session(token_hash(&first_token), "proxy.example.com")
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let second_replay = request(&listener, &runtime, second_request).await;
|
||||
let (second_replay_headers, _) = split_response(&second_replay);
|
||||
for header in [
|
||||
"x-session-token",
|
||||
"x-carrier-mode",
|
||||
"x-carrier-attempt",
|
||||
"x-carrier-candidate-count",
|
||||
"x-carrier-deadline",
|
||||
"x-carrier-state",
|
||||
] {
|
||||
assert_eq!(
|
||||
response_header(second_replay_headers, header),
|
||||
response_header(second_headers, header),
|
||||
);
|
||||
}
|
||||
|
||||
let changed_failure = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, Some(2), Some("network")),
|
||||
)
|
||||
.await;
|
||||
let (changed_failure_headers, _) = split_response(&changed_failure);
|
||||
assert!(optional_response_header(changed_failure_headers, "x-session-token").is_none());
|
||||
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
let data = frame::encode(FrameType::Data, 7, &[0]);
|
||||
let mut body = Vec::with_capacity(open.len() + data.len());
|
||||
body.extend_from_slice(&open);
|
||||
body.extend_from_slice(&data);
|
||||
let mut uplink = format!(
|
||||
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nContent-Type: application/octet-stream\r\nX-Up-Seq: 1\r\nX-Lane-ID: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
)
|
||||
.into_bytes();
|
||||
uplink.extend_from_slice(&body);
|
||||
let accepted = request(&listener, &runtime, uplink).await;
|
||||
assert!(accepted.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
let first_down = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nX-Down-Cursor: 0\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let first_down = request(&listener, &runtime, first_down).await;
|
||||
let (first_down_headers, first_down_body) = split_response(&first_down);
|
||||
assert!(first_down_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert!(!first_down_body.is_empty());
|
||||
assert_eq!(response_header(first_down_headers, "x-down-cursor"), "1");
|
||||
|
||||
let acknowledgement = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nX-Down-Cursor: 1\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let acknowledgement = request(&listener, &runtime, acknowledgement).await;
|
||||
assert!(acknowledgement.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
let third = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, Some(3), Some("http")),
|
||||
)
|
||||
.await;
|
||||
let (third_headers, _) = split_response(&third);
|
||||
assert!(third_headers.starts_with(b"HTTP/1.1 409"));
|
||||
assert!(optional_response_header(third_headers, "x-session-token").is_none());
|
||||
assert_eq!(
|
||||
response_header(third_headers, "x-carrier-mode"),
|
||||
"https-lanes"
|
||||
);
|
||||
assert_eq!(response_header(third_headers, "x-carrier-attempt"), "2");
|
||||
assert_eq!(
|
||||
response_header(third_headers, "x-carrier-candidate-count"),
|
||||
"3"
|
||||
);
|
||||
assert_eq!(response_header(third_headers, "x-carrier-deadline"), "12");
|
||||
assert_eq!(
|
||||
response_header(third_headers, "x-carrier-state"),
|
||||
"committed"
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.get_session(token_hash(&second_token), "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_carrier_committed()
|
||||
);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timed_out_attempt_replays_before_successor_own_deadline() {
|
||||
let capability = [57; 32];
|
||||
let generation = test_runtime_generation(
|
||||
1,
|
||||
negotiation_runtime_config_with_deadlines(
|
||||
capability,
|
||||
WebCarrier::HttpsLanes,
|
||||
false,
|
||||
Arc::from([WebCarrier::Https, WebCarrier::HttpsLanes]),
|
||||
[1, 5, 8, 12],
|
||||
),
|
||||
);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let first_request = create_request(&bootstrap, &hello, Some(1), None);
|
||||
|
||||
let first = request(&listener, &runtime, first_request.clone()).await;
|
||||
let (first_headers, _) = split_response(&first);
|
||||
assert!(first_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(first_headers, "x-carrier-mode"), "https");
|
||||
let first_token = response_header(first_headers, "x-session-token").to_string();
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(1_100)).await;
|
||||
|
||||
let replay = request(&listener, &runtime, first_request).await;
|
||||
let (replay_headers, _) = split_response(&replay);
|
||||
assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(replay_headers, "x-session-token"),
|
||||
first_token
|
||||
);
|
||||
assert_eq!(
|
||||
response_header(replay_headers, "x-carrier-state"),
|
||||
"provisional"
|
||||
);
|
||||
|
||||
let second = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, Some(2), Some("timeout")),
|
||||
)
|
||||
.await;
|
||||
let (second_headers, _) = split_response(&second);
|
||||
assert!(second_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(second_headers, "x-carrier-mode"),
|
||||
"https-lanes"
|
||||
);
|
||||
assert_eq!(response_header(second_headers, "x-carrier-attempt"), "2");
|
||||
assert_ne!(
|
||||
response_header(second_headers, "x-session-token"),
|
||||
first_token
|
||||
);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn https_lane_downlink_can_arrive_before_its_uplink_open() {
|
||||
let capability = [43; 32];
|
||||
let mut config = negotiation_runtime_config(
|
||||
capability,
|
||||
WebCarrier::HttpsLanes,
|
||||
false,
|
||||
Arc::from([WebCarrier::HttpsLanes]),
|
||||
);
|
||||
config.web.timeouts.lane_open_wait_secs = 1;
|
||||
config.web.timeouts.long_poll_secs = 2;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap());
|
||||
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let created = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_request(&bootstrap, &hello, Some(1), None),
|
||||
)
|
||||
.await;
|
||||
let (created_headers, _) = split_response(&created);
|
||||
let token = response_header(created_headers, "x-session-token").to_string();
|
||||
|
||||
let down_request = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nX-Down-Cursor: 0\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let down_listener = Arc::clone(&listener);
|
||||
let down_runtime = Arc::clone(&runtime);
|
||||
let down =
|
||||
tokio::spawn(async move { request(&down_listener, &down_runtime, down_request).await });
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
let data = frame::encode(FrameType::Data, 7, &[1]);
|
||||
let mut body = Vec::with_capacity(open.len() + data.len());
|
||||
body.extend_from_slice(&open);
|
||||
body.extend_from_slice(&data);
|
||||
let mut uplink = format!(
|
||||
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nContent-Type: application/octet-stream\r\nX-Up-Seq: 1\r\nX-Lane-ID: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
)
|
||||
.into_bytes();
|
||||
uplink.extend_from_slice(&body);
|
||||
let accepted = request(&listener, &runtime, uplink).await;
|
||||
assert!(accepted.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
let down = tokio::time::timeout(Duration::from_secs(3), down)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (down_headers, _) = split_response(&down);
|
||||
assert!(down_headers.starts_with(b"HTTP/1.1 200") || down_headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert!(optional_response_header(down_headers, "x-down-cursor").is_some());
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
+272
-52
@@ -1,65 +1,21 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::Engine as _;
|
||||
use hyper::Request;
|
||||
use hyper::header;
|
||||
use ipnetwork::IpNetwork;
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use crate::config::{WebClientIpSource, WebRuntimeProfile, WebRuntimeVhost};
|
||||
use crate::web::manager::TokenHash;
|
||||
use crate::config::{WebRuntimeProfile, WebRuntimeVhost};
|
||||
use crate::web::manager::{
|
||||
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierRequest, TokenHash,
|
||||
};
|
||||
|
||||
/// Parses one lowercase canonical Host value restricted to the public HTTPS port.
|
||||
pub(super) fn canonical_request_host<B>(request: &Request<B>) -> Option<&str> {
|
||||
let values = request.headers().get_all(header::HOST);
|
||||
let mut values = values.iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let authority = value.parse::<hyper::http::uri::Authority>().ok()?;
|
||||
if authority.port_u16().is_some_and(|port| port != 443) {
|
||||
return None;
|
||||
}
|
||||
let host = value.strip_suffix(":443").unwrap_or(value);
|
||||
if authority.host() != host || host.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||
return None;
|
||||
}
|
||||
Some(host)
|
||||
}
|
||||
const USER_AGENT_CONTEXT: &[u8] = b"telemt-web-carrier-user-agent-v1\0";
|
||||
|
||||
/// Accepts one forwarded client address or the direct address of a trusted peer.
|
||||
pub(super) fn client_ip<B>(
|
||||
request: &Request<B>,
|
||||
peer: SocketAddr,
|
||||
source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
) -> Option<IpAddr> {
|
||||
if !trusted_proxy_cidrs
|
||||
.iter()
|
||||
.any(|network| network.contains(peer.ip()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let header_name = match source {
|
||||
WebClientIpSource::XForwardedFor => "x-forwarded-for",
|
||||
};
|
||||
let values = request.headers().get_all(header_name);
|
||||
let mut values = values.iter();
|
||||
let Some(value) = values.next() else {
|
||||
return Some(peer.ip());
|
||||
};
|
||||
let value = value.to_str().ok()?;
|
||||
if values.next().is_some() || value.trim() != value || value.contains(',') {
|
||||
return None;
|
||||
}
|
||||
if value.is_empty() {
|
||||
return Some(peer.ip());
|
||||
}
|
||||
value.parse::<IpAddr>().ok()
|
||||
}
|
||||
// Canonical host and forwarded-address provenance remain isolated from credentials.
|
||||
mod identity;
|
||||
pub(super) use identity::{canonical_request_host, carrier_ip_learning_eligible, client_ip};
|
||||
|
||||
/// Decodes an exact canonical bridge query without allocating credential strings.
|
||||
pub(super) fn bridge_candidate(query: Option<&str>) -> ([u8; 32], bool) {
|
||||
@@ -167,9 +123,148 @@ pub(super) fn canonical_u64_header<B>(request: &Request<B>, name: &'static str)
|
||||
(parsed.to_string() == value).then_some(parsed)
|
||||
}
|
||||
|
||||
/// Parses strict bridge negotiation metadata without trusting it for authentication.
|
||||
pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<CarrierRequest> {
|
||||
let user_agent_hash = normalized_user_agent_hash(request)?;
|
||||
let capabilities = single_header(request, "x-carrier-capabilities");
|
||||
let attempt = optional_canonical_u8_header(request, "x-carrier-attempt")?;
|
||||
let failure = optional_failure_header(request)?;
|
||||
let native_ios = native_ios_user_agent(request);
|
||||
match (capabilities, attempt) {
|
||||
(None, None) if failure.is_none() => {
|
||||
if native_ios {
|
||||
Some(CarrierRequest::ios(user_agent_hash))
|
||||
} else {
|
||||
Some(CarrierRequest::legacy(user_agent_hash))
|
||||
}
|
||||
}
|
||||
(Some(capabilities), Some(attempt)) => {
|
||||
let capabilities = parse_capabilities(capabilities)?;
|
||||
if (attempt == 1) != failure.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(CarrierRequest::automatic(
|
||||
if native_ios {
|
||||
CarrierClientClass::Ios
|
||||
} else {
|
||||
CarrierClientClass::Bridge
|
||||
},
|
||||
capabilities,
|
||||
attempt,
|
||||
failure,
|
||||
user_agent_hash,
|
||||
))
|
||||
}
|
||||
(None, Some(attempt)) if strict_browser_hint(request, host) => {
|
||||
if (attempt == 1) != failure.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(CarrierRequest::automatic(
|
||||
if native_ios {
|
||||
CarrierClientClass::Ios
|
||||
} else {
|
||||
CarrierClientClass::BrowserHint
|
||||
},
|
||||
CarrierCapabilities::all(),
|
||||
attempt,
|
||||
failure,
|
||||
user_agent_hash,
|
||||
))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn native_ios_user_agent<B>(request: &Request<B>) -> bool {
|
||||
single_header(request, header::USER_AGENT).is_some_and(|value| {
|
||||
let value = value.to_ascii_lowercase();
|
||||
value.contains("cfnetwork/") && value.contains("darwin/")
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_capabilities(value: &str) -> Option<CarrierCapabilities> {
|
||||
let mut bits = 0u8;
|
||||
let mut previous = None;
|
||||
for token in value.split(',') {
|
||||
let index = match token {
|
||||
"https" => 0,
|
||||
"https-lanes" => 1,
|
||||
"websocket" => 2,
|
||||
"websocket-lanes" => 3,
|
||||
_ => return None,
|
||||
};
|
||||
if previous.is_some_and(|previous| index <= previous) {
|
||||
return None;
|
||||
}
|
||||
previous = Some(index);
|
||||
bits |= 1 << index;
|
||||
}
|
||||
CarrierCapabilities::from_bits(bits)
|
||||
}
|
||||
|
||||
fn strict_browser_hint<B>(request: &Request<B>, host: &str) -> bool {
|
||||
single_header(request, header::ORIGIN).is_some_and(|value| value == format!("https://{host}"))
|
||||
&& single_header(request, "sec-fetch-site") == Some("same-origin")
|
||||
&& single_header(request, "sec-fetch-mode") == Some("cors")
|
||||
&& single_header(request, "sec-fetch-dest") == Some("empty")
|
||||
}
|
||||
|
||||
fn optional_canonical_u8_header<B>(request: &Request<B>, name: &'static str) -> Option<Option<u8>> {
|
||||
if !request.headers().contains_key(name) {
|
||||
return Some(None);
|
||||
}
|
||||
canonical_u64_header(request, name)
|
||||
.and_then(|value| u8::try_from(value).ok())
|
||||
.filter(|value| (1..=4).contains(value))
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
fn optional_failure_header<B>(request: &Request<B>) -> Option<Option<CarrierFailure>> {
|
||||
if !request.headers().contains_key("x-carrier-failure") {
|
||||
return Some(None);
|
||||
}
|
||||
single_header(request, "x-carrier-failure")
|
||||
.and_then(CarrierFailure::parse)
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
fn normalized_user_agent_hash<B>(request: &Request<B>) -> Option<[u8; 32]> {
|
||||
let user_agent = match single_header(request, header::USER_AGENT) {
|
||||
Some(value) => value.as_bytes(),
|
||||
None if !request.headers().contains_key(header::USER_AGENT) => &[],
|
||||
None => return None,
|
||||
};
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(USER_AGENT_CONTEXT);
|
||||
let mut emitted = false;
|
||||
let mut pending_whitespace = false;
|
||||
for &byte in user_agent {
|
||||
if byte.is_ascii_whitespace() {
|
||||
pending_whitespace = emitted;
|
||||
} else {
|
||||
if pending_whitespace {
|
||||
digest.update([b' ']);
|
||||
}
|
||||
digest.update([byte.to_ascii_lowercase()]);
|
||||
emitted = true;
|
||||
pending_whitespace = false;
|
||||
}
|
||||
}
|
||||
Some(digest.finalize().into())
|
||||
}
|
||||
|
||||
fn single_header<B>(request: &Request<B>, name: impl header::AsHeaderName) -> Option<&str> {
|
||||
let mut values = request.headers().get_all(name).iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
values.next().is_none().then_some(value)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::{WebCarrier, WebClientIpSource};
|
||||
|
||||
#[test]
|
||||
fn canonical_bridge_query_rejects_aliases() {
|
||||
@@ -319,4 +414,129 @@ mod tests {
|
||||
.append(header::COOKIE, "state=unexpected".parse().unwrap());
|
||||
assert!(!compatible_cookie_header(&duplicate_mixed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrier_metadata_is_canonical_and_legacy_safe() {
|
||||
let automatic = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.header("x-carrier-failure", "timeout")
|
||||
.header(header::USER_AGENT, "Example Browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert!(parsed.is_automatic());
|
||||
assert_eq!(parsed.attempt(), Some(2));
|
||||
assert_eq!(parsed.failure(), Some(CarrierFailure::Timeout));
|
||||
|
||||
let missing_failure = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&missing_failure, "proxy.example.com").is_none());
|
||||
|
||||
let legacy = Request::builder()
|
||||
.header(header::USER_AGENT, "Native")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(
|
||||
!carrier_request(&legacy, "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_automatic()
|
||||
);
|
||||
|
||||
let reordered = Request::builder()
|
||||
.header("x-carrier-capabilities", "websocket,https")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&reordered, "proxy.example.com").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_ios_user_agent_classifies_without_overriding_capabilities() {
|
||||
let metadata_free = Request::builder()
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&metadata_free, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(!parsed.is_automatic());
|
||||
assert!(!parsed.uses_capabilities());
|
||||
|
||||
let automatic = Request::builder()
|
||||
.header("x-carrier-capabilities", "https,https-lanes")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(parsed.is_automatic());
|
||||
assert!(parsed.supports(WebCarrier::Https));
|
||||
assert!(parsed.supports(WebCarrier::HttpsLanes));
|
||||
assert!(!parsed.supports(WebCarrier::Websocket));
|
||||
assert!(!parsed.supports(WebCarrier::WebsocketLanes));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_private_addresses_are_not_learning_evidence() {
|
||||
for address in ["::ffff:127.0.0.1", "::ffff:10.0.0.1"] {
|
||||
let effective_ip = address.parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", address)
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
let effective_ip = "::ffff:8.8.8.8".parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", "::ffff:8.8.8.8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_browser_metadata_recovers_a_stripped_capability_marker() {
|
||||
let request = Request::builder()
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(header::ORIGIN, "https://proxy.example.com")
|
||||
.header("sec-fetch-site", "same-origin")
|
||||
.header("sec-fetch-mode", "cors")
|
||||
.header("sec-fetch-dest", "empty")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&request, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::BrowserHint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_agent_learning_key_is_case_and_whitespace_normalized() {
|
||||
let first = Request::builder()
|
||||
.header(header::USER_AGENT, " Example\t Browser ")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let second = Request::builder()
|
||||
.header(header::USER_AGENT, "example browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
normalized_user_agent_hash(&first),
|
||||
normalized_user_agent_hash(&second)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use hyper::Request;
|
||||
use hyper::header;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::WebClientIpSource;
|
||||
|
||||
/// Parses one lowercase canonical Host value restricted to the public HTTPS port.
|
||||
pub(in crate::web::http) fn canonical_request_host<B>(request: &Request<B>) -> Option<&str> {
|
||||
let values = request.headers().get_all(header::HOST);
|
||||
let mut values = values.iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let authority = value.parse::<hyper::http::uri::Authority>().ok()?;
|
||||
if authority.port_u16().is_some_and(|port| port != 443) {
|
||||
return None;
|
||||
}
|
||||
let host = value.strip_suffix(":443").unwrap_or(value);
|
||||
if authority.host() != host || host.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||
return None;
|
||||
}
|
||||
Some(host)
|
||||
}
|
||||
|
||||
/// Accepts one forwarded client address or the direct address of a trusted peer.
|
||||
pub(in crate::web::http) fn client_ip<B>(
|
||||
request: &Request<B>,
|
||||
peer: SocketAddr,
|
||||
source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
) -> Option<IpAddr> {
|
||||
if !trusted_proxy_cidrs
|
||||
.iter()
|
||||
.any(|network| network.contains(peer.ip()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let header_name = match source {
|
||||
WebClientIpSource::XForwardedFor => "x-forwarded-for",
|
||||
};
|
||||
let values = request.headers().get_all(header_name);
|
||||
let mut values = values.iter();
|
||||
let Some(value) = values.next() else {
|
||||
return Some(peer.ip());
|
||||
};
|
||||
let value = value.to_str().ok()?;
|
||||
if values.next().is_some() || value.trim() != value || value.contains(',') {
|
||||
return None;
|
||||
}
|
||||
if value.is_empty() {
|
||||
return Some(peer.ip());
|
||||
}
|
||||
value.parse::<IpAddr>().ok()
|
||||
}
|
||||
|
||||
/// Allows IP learning only for one explicit globally routable forwarded address.
|
||||
pub(in crate::web::http) fn carrier_ip_learning_eligible<B>(
|
||||
request: &Request<B>,
|
||||
effective_ip: IpAddr,
|
||||
) -> bool {
|
||||
let mut values = request.headers().get_all("x-forwarded-for").iter();
|
||||
let Some(value) = values.next().and_then(|value| value.to_str().ok()) else {
|
||||
return false;
|
||||
};
|
||||
if values.next().is_some()
|
||||
|| value.trim() != value
|
||||
|| value.contains(',')
|
||||
|| value.parse::<IpAddr>().ok() != Some(effective_ip)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
globally_routable(effective_ip)
|
||||
}
|
||||
|
||||
fn globally_routable(address: IpAddr) -> bool {
|
||||
match address {
|
||||
IpAddr::V4(address) => globally_routable_v4(address),
|
||||
IpAddr::V6(address) => {
|
||||
if let Some(address) = address.to_ipv4_mapped() {
|
||||
return globally_routable_v4(address);
|
||||
}
|
||||
let segments = address.segments();
|
||||
!address.is_unspecified()
|
||||
&& !address.is_loopback()
|
||||
&& segments[0] & 0xfe00 != 0xfc00
|
||||
&& segments[0] & 0xffc0 != 0xfe80
|
||||
&& segments[0] & 0xff00 != 0xff00
|
||||
&& !(segments[0] == 0x2001 && segments[1] == 0x0db8)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn globally_routable_v4(address: std::net::Ipv4Addr) -> bool {
|
||||
let [a, b, c, _] = address.octets();
|
||||
!(a == 0
|
||||
|| a == 10
|
||||
|| a == 127
|
||||
|| (a == 100 && (64..=127).contains(&b))
|
||||
|| (a == 169 && b == 254)
|
||||
|| (a == 172 && (16..=31).contains(&b))
|
||||
|| (a == 192 && b == 0 && c == 0)
|
||||
|| (a == 192 && b == 0 && c == 2)
|
||||
|| (a == 192 && b == 168)
|
||||
|| (a == 198 && (b == 18 || b == 19))
|
||||
|| (a == 198 && b == 51 && c == 100)
|
||||
|| (a == 203 && b == 0 && c == 113)
|
||||
|| a >= 224)
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Method, Request, StatusCode};
|
||||
|
||||
use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||
use super::decoy::serve_decoy;
|
||||
use super::request::{binary_content_type, carrier_ip_learning_eligible, carrier_request};
|
||||
use super::response::{
|
||||
carrier_empty, carrier_headers, full_response, insert_header, service_unavailable,
|
||||
};
|
||||
use super::{HttpResponse, request_trace};
|
||||
use crate::config::WebRuntimeVhost;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
|
||||
use crate::web::trace::{TraceDirection, TraceLifecycleEvent, TraceRoute};
|
||||
|
||||
const CREATE_BODY_LIMIT: usize = 64;
|
||||
|
||||
/// Handles session creation, replacement replay, and authenticated closure.
|
||||
pub(super) async fn handle_session(
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
) -> HttpResponse {
|
||||
if request.headers().contains_key("x-lane-id") {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if request.method() == Method::DELETE {
|
||||
if request.headers().contains_key(header::CONTENT_TYPE) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(trace) = request_trace(&request)
|
||||
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
|
||||
{
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if !body.is_empty() || runtime.close_token(token_hash, &vhost.host).is_err() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
return carrier_empty(StatusCode::NO_CONTENT);
|
||||
}
|
||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
||||
let Some((trace_session_id, profile)) =
|
||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_profile(&profile, trace_session_id);
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, CREATE_BODY_LIMIT, false).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Request,
|
||||
&body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
match runtime.create_session(
|
||||
token_hash,
|
||||
&vhost.host,
|
||||
client_ip,
|
||||
&body,
|
||||
carrier_request,
|
||||
ip_learning_eligible,
|
||||
) {
|
||||
Ok(result) => {
|
||||
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.register_redaction(result.token.as_bytes());
|
||||
trace.record_frames(
|
||||
TraceDirection::Response,
|
||||
&welcome,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
let mut response = full_response(StatusCode::OK, welcome);
|
||||
carrier_headers(&mut response);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-session-token"),
|
||||
&result.token,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-carrier-mode"),
|
||||
HeaderValue::from_static(result.carrier.as_str()),
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
HeaderValue::from_static("0"),
|
||||
);
|
||||
if let Some(attempt) = result.attempt {
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-attempt"),
|
||||
&attempt.to_string(),
|
||||
);
|
||||
}
|
||||
if let Some(candidate_count) = result.candidate_count {
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-candidate-count"),
|
||||
&candidate_count.to_string(),
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-deadline"),
|
||||
&result.deadline_secs.unwrap_or_default().to_string(),
|
||||
);
|
||||
if let Some(state) = result.carrier_state {
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-state"),
|
||||
state,
|
||||
);
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
Err(ManagerError::Committed) => {
|
||||
let mut response = carrier_empty(StatusCode::CONFLICT);
|
||||
if let Some(echo) =
|
||||
runtime.carrier_echo(token_hash, &vhost.host, client_ip, carrier_request)
|
||||
{
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-carrier-mode"),
|
||||
HeaderValue::from_static(echo.carrier.as_str()),
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-attempt"),
|
||||
&echo.attempt.to_string(),
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-candidate-count"),
|
||||
&echo.candidate_count.to_string(),
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-deadline"),
|
||||
&echo.deadline_secs.to_string(),
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-carrier-state"),
|
||||
echo.state,
|
||||
);
|
||||
}
|
||||
response
|
||||
}
|
||||
Err(
|
||||
error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent),
|
||||
) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(error.as_str()),
|
||||
);
|
||||
service_unavailable()
|
||||
}
|
||||
Err(error) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(error.as_str()),
|
||||
);
|
||||
serve_decoy(request, vhost, true, &runtime).await
|
||||
}
|
||||
}
|
||||
}
|
||||
+77
-101
@@ -10,20 +10,85 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::serve_connection;
|
||||
use crate::config::{
|
||||
ProxyConfig, WebCarrier, WebClientIpSource, WebRuntimeConfig, WebRuntimeDecoy,
|
||||
ProxyConfig, WebCarrier, WebCarriers, WebClientIpSource, WebRuntimeConfig, WebRuntimeDecoy,
|
||||
WebRuntimeProfile, WebRuntimeVhost, WebSecretMode, WebStaticAsset, WebStaticSite,
|
||||
};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
#[path = "legacy_tests.rs"]
|
||||
mod legacy_tests;
|
||||
#[path = "negotiation_tests.rs"]
|
||||
mod negotiation_tests;
|
||||
|
||||
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
||||
|
||||
pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||
runtime_config_with_carriers(capability, carrier, false, true, Arc::from([carrier]))
|
||||
}
|
||||
|
||||
pub(super) fn negotiation_runtime_config(
|
||||
capability: [u8; 32],
|
||||
carrier: WebCarrier,
|
||||
carrier_learning: bool,
|
||||
carriers: Arc<[WebCarrier]>,
|
||||
) -> ProxyConfig {
|
||||
runtime_config_with_carriers(capability, carrier, true, carrier_learning, carriers)
|
||||
}
|
||||
|
||||
fn runtime_config_with_carriers(
|
||||
capability: [u8; 32],
|
||||
carrier: WebCarrier,
|
||||
carrier_negotiation_enabled: bool,
|
||||
carrier_learning: bool,
|
||||
carriers: Arc<[WebCarrier]>,
|
||||
) -> ProxyConfig {
|
||||
runtime_config_with_carriers_and_deadlines(
|
||||
capability,
|
||||
carrier,
|
||||
carrier_negotiation_enabled,
|
||||
carrier_learning,
|
||||
carriers,
|
||||
TEST_CARRIER_DEADLINES_SECS,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn negotiation_runtime_config_with_deadlines(
|
||||
capability: [u8; 32],
|
||||
carrier: WebCarrier,
|
||||
carrier_learning: bool,
|
||||
carriers: Arc<[WebCarrier]>,
|
||||
carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
) -> ProxyConfig {
|
||||
runtime_config_with_carriers_and_deadlines(
|
||||
capability,
|
||||
carrier,
|
||||
true,
|
||||
carrier_learning,
|
||||
carriers,
|
||||
carrier_negotiation_deadlines_secs,
|
||||
)
|
||||
}
|
||||
|
||||
fn runtime_config_with_carriers_and_deadlines(
|
||||
capability: [u8; 32],
|
||||
carrier: WebCarrier,
|
||||
carrier_negotiation_enabled: bool,
|
||||
carrier_learning: bool,
|
||||
carriers: Arc<[WebCarrier]>,
|
||||
carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
) -> ProxyConfig {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier,
|
||||
carrier_negotiation_enabled,
|
||||
carrier_learning,
|
||||
carriers: Arc::clone(&carriers),
|
||||
carrier_negotiation_deadlines_secs,
|
||||
capability,
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 4,
|
||||
@@ -63,6 +128,13 @@ pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> Proxy
|
||||
let mut config = ProxyConfig::default();
|
||||
config.web.enabled = true;
|
||||
config.web.carrier = carrier;
|
||||
config.web.carriers = if carrier_negotiation_enabled {
|
||||
WebCarriers::Enabled(carriers.to_vec())
|
||||
} else {
|
||||
WebCarriers::Disabled
|
||||
};
|
||||
config.web.carrier_learning = carrier_learning;
|
||||
config.web.timeouts.carrier_negotiation_deadlines_secs = carrier_negotiation_deadlines_secs;
|
||||
config.web.limits.max_bootstraps_per_ip = 1;
|
||||
config.web.timeouts.shutdown_secs = 1;
|
||||
config.web.runtime = Some(Arc::new(WebRuntimeConfig {
|
||||
@@ -106,7 +178,7 @@ pub(super) fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
|
||||
(&response[..separator], &response[separator + 4..])
|
||||
}
|
||||
|
||||
fn response_header<'a>(headers: &'a [u8], name: &str) -> &'a str {
|
||||
pub(super) fn response_header<'a>(headers: &'a [u8], name: &str) -> &'a str {
|
||||
std::str::from_utf8(headers)
|
||||
.unwrap()
|
||||
.lines()
|
||||
@@ -189,8 +261,8 @@ async fn https_carrier_bootstraps_and_closes_one_session() {
|
||||
);
|
||||
assert!(
|
||||
next_root_body
|
||||
.windows(21)
|
||||
.any(|value| value == b"carrier='https-lanes'")
|
||||
.windows(b"const negotiationEnabled=false".len())
|
||||
.any(|value| value == b"const negotiationEnabled=false")
|
||||
);
|
||||
|
||||
let close = format!(
|
||||
@@ -375,7 +447,7 @@ async fn https_lanes_is_advertised_and_requires_canonical_lane_headers() {
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
assert!(root_body.contains("carrier='https-lanes'"));
|
||||
assert!(root_body.contains("const negotiationEnabled=false"));
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
@@ -437,99 +509,3 @@ async fn https_lanes_is_advertised_and_requires_canonical_lane_headers() {
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
||||
for (index, carrier) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let capability = [12 + index as u8; 32];
|
||||
let mut config = runtime_config(capability, carrier);
|
||||
config.web.timeouts.long_poll_secs = 0;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(active_runtime);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nCookie:\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (root_headers, root_body) = split_response(&root_response);
|
||||
assert!(root_headers.starts_with(b"HTTP/1.1 200"));
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let create = |cookie: &str| {
|
||||
let mut request = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\n{cookie}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
request.extend_from_slice(&hello);
|
||||
request
|
||||
};
|
||||
let nonempty_cookie =
|
||||
request(&listener, &runtime, create("Cookie: state=unexpected\r\n")).await;
|
||||
assert!(!nonempty_cookie.starts_with(b"HTTP/1.1 200"));
|
||||
let duplicate_cookie = request(
|
||||
&listener,
|
||||
&runtime,
|
||||
create("Cookie:\r\nCookie: state=unexpected\r\n"),
|
||||
)
|
||||
.await;
|
||||
assert!(!duplicate_cookie.starts_with(b"HTTP/1.1 200"));
|
||||
|
||||
let create_response = request(&listener, &runtime, create("Cookie:\r\n")).await;
|
||||
let (create_headers, create_body) = split_response(&create_response);
|
||||
assert!(create_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(create_headers, "x-carrier-mode"),
|
||||
carrier.as_str()
|
||||
);
|
||||
assert_eq!(create_body, frame::encode(FrameType::Welcome, 0, &[]));
|
||||
let session = response_header(create_headers, "x-session-token").to_string();
|
||||
let lane = (carrier == WebCarrier::HttpsLanes)
|
||||
.then_some("X-Lane-ID: 0\r\n")
|
||||
.unwrap_or_default();
|
||||
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let mut uplink = format!(
|
||||
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nContent-Type: application/octet-stream\r\nCookie:\r\nX-Up-Seq: 1\r\n{lane}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
pong.len()
|
||||
)
|
||||
.into_bytes();
|
||||
uplink.extend_from_slice(&pong);
|
||||
let uplink_response = request(&listener, &runtime, uplink).await;
|
||||
let (uplink_headers, _) = split_response(&uplink_response);
|
||||
assert!(uplink_headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert_eq!(response_header(uplink_headers, "x-up-ack"), "1");
|
||||
|
||||
let downlink = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nCookie:\r\nX-Down-Cursor: 0\r\n{lane}Content-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let downlink_response = request(&listener, &runtime, downlink).await;
|
||||
let (downlink_headers, _) = split_response(&downlink_response);
|
||||
assert!(downlink_headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert_eq!(response_header(downlink_headers, "x-down-cursor"), "0");
|
||||
|
||||
let close = format!(
|
||||
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nCookie:\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let close_response = request(&listener, &runtime, close).await;
|
||||
assert!(close_response.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
+42
-20
@@ -200,7 +200,6 @@ impl AsyncRead for ConnectionIo {
|
||||
Poll::Ready(Ok(())) => {
|
||||
let filled = limited.filled().len();
|
||||
boundary.observe(limited.filled());
|
||||
drop(limited);
|
||||
buffer.advance(filled);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
@@ -244,6 +243,7 @@ struct ParsedUpgrade {
|
||||
protocol: String,
|
||||
accept: String,
|
||||
carrier: ParsedCarrier,
|
||||
acknowledge_commit: bool,
|
||||
}
|
||||
|
||||
pub(super) async fn handle(
|
||||
@@ -272,6 +272,10 @@ pub(super) async fn handle(
|
||||
(ParsedCarrier::Lane(lane_id), WebCarrier::WebsocketLanes) => WebSocketKind::Lane(lane_id),
|
||||
_ => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
let mut probe_reservation = match session.reserve_websocket_probe(parsed.acknowledge_commit) {
|
||||
Ok(reservation) => reservation,
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
let mut lane_reservation = match kind {
|
||||
WebSocketKind::Multiplex => None,
|
||||
WebSocketKind::Lane(lane_id) => match session.reserve_websocket_lane(lane_id) {
|
||||
@@ -279,22 +283,29 @@ pub(super) async fn handle(
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
},
|
||||
};
|
||||
let timeouts = runtime.active_generation().config().web.timeouts.clone();
|
||||
let timeouts = session.timeouts().clone();
|
||||
let connection = match runtime
|
||||
.admit_websocket(
|
||||
session.profile_key(),
|
||||
session.trace_session_id(),
|
||||
session.token_hash(),
|
||||
effective_ip,
|
||||
kind,
|
||||
BASE_BUDGET_BYTES,
|
||||
Duration::from_secs(timeouts.long_poll_secs),
|
||||
Duration::from_secs(timeouts.websocket_eviction_secs),
|
||||
session.carrier_cancellation(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(connection) => connection,
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
if let Some(reservation) = probe_reservation.as_mut()
|
||||
&& reservation.bind(connection.id()).is_err()
|
||||
{
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let trace_context = runtime.trace().websocket_context(
|
||||
&request,
|
||||
peer.ip(),
|
||||
@@ -324,7 +335,9 @@ pub(super) async fn handle(
|
||||
driver_session,
|
||||
connection,
|
||||
lane_reservation.take(),
|
||||
probe_reservation.take(),
|
||||
trace_context,
|
||||
parsed.acknowledge_commit,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -376,21 +389,17 @@ fn parse_upgrade<B>(request: &Request<B>) -> Option<ParsedUpgrade> {
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let (token, carrier) = if let Some(token) = protocol.strip_prefix("tproxy-v1.") {
|
||||
(token, ParsedCarrier::Multiplex)
|
||||
let (token, carrier, acknowledge_commit) =
|
||||
if let Some(token) = protocol.strip_prefix("tproxy-auto-v1.") {
|
||||
(token, ParsedCarrier::Multiplex, true)
|
||||
} else if let Some(lane) = protocol.strip_prefix("tproxy-auto-lane-v1.") {
|
||||
let (token, lane_id) = parse_lane_protocol(lane)?;
|
||||
(token, ParsedCarrier::Lane(lane_id), true)
|
||||
} else if let Some(token) = protocol.strip_prefix("tproxy-v1.") {
|
||||
(token, ParsedCarrier::Multiplex, false)
|
||||
} else if let Some(lane) = protocol.strip_prefix("tproxy-lane-v1.") {
|
||||
let (token, lane_id) = lane.split_once('.')?;
|
||||
if lane_id.is_empty()
|
||||
|| lane_id.starts_with('+')
|
||||
|| (lane_id.len() > 1 && lane_id.starts_with('0'))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let lane_id = lane_id
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.filter(|value| (1..=crate::web::frame::MAX_STREAM_ID).contains(value))?;
|
||||
(token, ParsedCarrier::Lane(lane_id))
|
||||
let (token, lane_id) = parse_lane_protocol(lane)?;
|
||||
(token, ParsedCarrier::Lane(lane_id), false)
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
@@ -412,13 +421,26 @@ fn parse_upgrade<B>(request: &Request<B>) -> Option<ParsedUpgrade> {
|
||||
protocol: protocol.to_string(),
|
||||
accept: base64::engine::general_purpose::STANDARD.encode(accept.finalize()),
|
||||
carrier,
|
||||
acknowledge_commit,
|
||||
})
|
||||
}
|
||||
|
||||
fn single_header<'a, B>(
|
||||
request: &'a Request<B>,
|
||||
name: impl hyper::header::AsHeaderName,
|
||||
) -> Option<&'a str> {
|
||||
fn parse_lane_protocol(value: &str) -> Option<(&str, u32)> {
|
||||
let (token, lane_id) = value.split_once('.')?;
|
||||
if lane_id.is_empty()
|
||||
|| lane_id.starts_with('+')
|
||||
|| (lane_id.len() > 1 && lane_id.starts_with('0'))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let lane_id = lane_id
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.filter(|value| (1..=crate::web::frame::MAX_STREAM_ID).contains(value))?;
|
||||
Some((token, lane_id))
|
||||
}
|
||||
|
||||
fn single_header<B>(request: &Request<B>, name: impl hyper::header::AsHeaderName) -> Option<&str> {
|
||||
let mut values = request.headers().get_all(name).iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
values.next().is_none().then_some(value)
|
||||
|
||||
+106
-396
@@ -2,31 +2,46 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio_tungstenite::WebSocketStream;
|
||||
use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::ConnectionIo;
|
||||
use crate::web::manager::{
|
||||
ManagerError, WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
|
||||
};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::manager::{WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation, WebSocketProbeReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
const READ_BUFFER_BYTES: usize = 64 * 1024;
|
||||
const WRITE_BUFFER_BYTES: usize = 64 * 1024;
|
||||
|
||||
// Cancellation-safe message I/O and budget retries remain separate from carrier loops.
|
||||
mod io;
|
||||
// Per-lane carrier state remains isolated from the multiplexed driver.
|
||||
mod lane;
|
||||
use io::{flush, process_multiplex, read_message, record_message, reserve_data, send};
|
||||
use lane::run_lane;
|
||||
|
||||
pub(super) async fn run_upgraded(
|
||||
on_upgrade: hyper::upgrade::OnUpgrade,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
session: Arc<WebSession>,
|
||||
connection: WebSocketConnection,
|
||||
mut lane_reservation: Option<WebSocketLaneReservation>,
|
||||
_probe_reservation: Option<WebSocketProbeReservation>,
|
||||
trace: Option<TraceWebSocketContext>,
|
||||
acknowledge_commit: bool,
|
||||
) {
|
||||
let Ok(upgraded) = on_upgrade.await else {
|
||||
let cancellation = connection.cancellation();
|
||||
let timeouts = session.timeouts().clone();
|
||||
let upgraded = tokio::select! {
|
||||
_ = cancellation.cancelled() => return,
|
||||
result = tokio::time::timeout(
|
||||
Duration::from_secs(timeouts.websocket_upgrade_secs),
|
||||
on_upgrade,
|
||||
) => result,
|
||||
};
|
||||
let Ok(Ok(upgraded)) = upgraded else {
|
||||
return;
|
||||
};
|
||||
let Ok(parts) = upgraded.downcast::<TokioIo<ConnectionIo>>() else {
|
||||
@@ -34,7 +49,7 @@ pub(super) async fn run_upgraded(
|
||||
};
|
||||
let mut io = parts.io.into_inner();
|
||||
io.enable_websocket(parts.read_buf);
|
||||
let limits = runtime.active_generation().config().web.limits.clone();
|
||||
let limits = session.limits().clone();
|
||||
let config = WebSocketConfig::default()
|
||||
.read_buffer_size(READ_BUFFER_BYTES)
|
||||
.write_buffer_size(WRITE_BUFFER_BYTES)
|
||||
@@ -46,8 +61,9 @@ pub(super) async fn run_upgraded(
|
||||
.max_message_size(Some(limits.carrier_batch_bytes))
|
||||
.max_frame_size(Some(limits.carrier_batch_bytes));
|
||||
let mut socket = WebSocketStream::from_raw_socket(io, Role::Server, Some(config)).await;
|
||||
connection.mark_opened();
|
||||
let cancellation = connection.cancellation();
|
||||
if !connection.mark_opened() {
|
||||
return;
|
||||
}
|
||||
if let Some(reservation) = lane_reservation.as_mut() {
|
||||
let _ = run_lane(
|
||||
&mut socket,
|
||||
@@ -57,6 +73,7 @@ pub(super) async fn run_upgraded(
|
||||
reservation,
|
||||
cancellation.clone(),
|
||||
trace.as_ref(),
|
||||
acknowledge_commit,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
@@ -67,22 +84,20 @@ pub(super) async fn run_upgraded(
|
||||
&connection,
|
||||
cancellation.clone(),
|
||||
trace.as_ref(),
|
||||
acknowledge_commit,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let eviction = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_eviction_secs,
|
||||
);
|
||||
let _ = tokio::time::timeout(eviction, socket.close(None)).await;
|
||||
let eviction = Duration::from_secs(timeouts.websocket_eviction_secs);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => {}
|
||||
_ = tokio::time::timeout(eviction, socket.close(None)) => {}
|
||||
}
|
||||
if let Some(reservation) = lane_reservation {
|
||||
session.close_websocket_lane(reservation.lane_id());
|
||||
drop(reservation);
|
||||
} else {
|
||||
} else if !acknowledge_commit || session.is_carrier_committed() {
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
@@ -96,6 +111,7 @@ async fn run_multiplex(
|
||||
connection: &WebSocketConnection,
|
||||
cancellation: CancellationToken,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
acknowledge_commit: bool,
|
||||
) -> Result<(), ()> {
|
||||
let mut sequence = 1u64;
|
||||
let mut cursor = 0u64;
|
||||
@@ -104,11 +120,18 @@ async fn run_multiplex(
|
||||
let mut read_budget = None;
|
||||
let liveness_interval = connection.liveness_interval();
|
||||
let mut next_ping = Instant::now() + liveness_interval;
|
||||
let open_deadline =
|
||||
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
|
||||
let backpressure_timeout = Duration::from_secs(session.timeouts().websocket_backpressure_secs);
|
||||
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
|
||||
let maximum_message = session.limits().carrier_batch_bytes;
|
||||
let mut active = false;
|
||||
loop {
|
||||
let down = session.poll_down(cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = tokio::time::sleep_until(open_deadline.into()), if !active => return Err(()),
|
||||
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
|
||||
incoming = read_message(
|
||||
socket,
|
||||
@@ -116,6 +139,8 @@ async fn run_multiplex(
|
||||
session.profile_key(),
|
||||
&cancellation,
|
||||
&mut read_budget,
|
||||
maximum_message,
|
||||
backpressure_timeout,
|
||||
) => {
|
||||
DriverEvent::Incoming(incoming?)
|
||||
}
|
||||
@@ -125,179 +150,13 @@ async fn run_multiplex(
|
||||
DriverEvent::Incoming((message, _budget)) => match message {
|
||||
Message::Binary(body) => {
|
||||
let started = Instant::now();
|
||||
let result =
|
||||
process_multiplex(runtime, session, sequence, &body, &cancellation).await;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
result?;
|
||||
sequence = sequence.checked_add(1).ok_or(())?;
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Pong(payload) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"pong",
|
||||
&payload,
|
||||
Instant::now(),
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
let started = Instant::now();
|
||||
flush(socket, runtime).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"ping",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"pong",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Close(_) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"close",
|
||||
&[],
|
||||
Instant::now(),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Message::Text(text) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"text",
|
||||
text.as_bytes(),
|
||||
Instant::now(),
|
||||
);
|
||||
return Err(());
|
||||
}
|
||||
Message::Frame(_) => return Err(()),
|
||||
},
|
||||
DriverEvent::Down(result) => {
|
||||
if result.body.is_empty() {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
} else {
|
||||
let _budget = reserve_data(
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
result.body.len(),
|
||||
&cancellation,
|
||||
)
|
||||
.await?;
|
||||
let body = result.body;
|
||||
let started = Instant::now();
|
||||
if trace.is_some() {
|
||||
send(socket, runtime, Message::Binary(body.clone())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
} else {
|
||||
send(socket, runtime, Message::Binary(body)).await?;
|
||||
}
|
||||
connection.mark_progress();
|
||||
}
|
||||
cursor = result.next_cursor;
|
||||
}
|
||||
DriverEvent::Liveness => {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_lane(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
connection: &WebSocketConnection,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
cancellation: CancellationToken,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
) -> Result<(), ()> {
|
||||
let mut sequence = 1u64;
|
||||
let mut cursor = 0u64;
|
||||
// Lane reads use the same cancellation-safe fragmented-message ownership.
|
||||
let mut read_budget = None;
|
||||
let liveness_interval = connection.liveness_interval();
|
||||
let mut next_ping = Instant::now() + liveness_interval;
|
||||
loop {
|
||||
let down = session.poll_down_lane(reservation.lane_id(), cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
|
||||
incoming = read_message(
|
||||
socket,
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
&cancellation,
|
||||
&mut read_budget,
|
||||
) => {
|
||||
DriverEvent::Incoming(incoming?)
|
||||
}
|
||||
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
|
||||
};
|
||||
match event {
|
||||
DriverEvent::Incoming((message, _budget)) => match message {
|
||||
Message::Binary(body) => {
|
||||
let started = Instant::now();
|
||||
let result = process_lane(
|
||||
let result = process_multiplex(
|
||||
runtime,
|
||||
session,
|
||||
reservation,
|
||||
sequence,
|
||||
&body,
|
||||
&cancellation,
|
||||
backpressure_timeout,
|
||||
)
|
||||
.await;
|
||||
record_message(
|
||||
@@ -308,7 +167,42 @@ async fn run_lane(
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
result?;
|
||||
let progressed = result?;
|
||||
if acknowledge_commit && sequence == 1 {
|
||||
if !session.needs_websocket_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
let started = Instant::now();
|
||||
send(
|
||||
socket,
|
||||
Message::Binary(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"carrier-ack",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
if !session.websocket_commit_ack_written(connection.id()) {
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
return Err(());
|
||||
}
|
||||
active = true;
|
||||
}
|
||||
sequence = sequence.checked_add(1).ok_or(())?;
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
@@ -327,7 +221,7 @@ async fn run_lane(
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
let started = Instant::now();
|
||||
flush(socket, runtime).await?;
|
||||
flush(socket, &cancellation, write_timeout).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
@@ -372,12 +266,15 @@ async fn run_lane(
|
||||
Message::Frame(_) => return Err(()),
|
||||
},
|
||||
DriverEvent::Down(result) => {
|
||||
if result.lane_closed {
|
||||
return Ok(());
|
||||
}
|
||||
if result.body.is_empty() {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
send(
|
||||
socket,
|
||||
Message::Ping(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
@@ -393,12 +290,19 @@ async fn run_lane(
|
||||
session.profile_key(),
|
||||
result.body.len(),
|
||||
&cancellation,
|
||||
backpressure_timeout,
|
||||
)
|
||||
.await?;
|
||||
let body = result.body;
|
||||
let started = Instant::now();
|
||||
if trace.is_some() {
|
||||
send(socket, runtime, Message::Binary(body.clone())).await?;
|
||||
send(
|
||||
socket,
|
||||
Message::Binary(body.clone()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
@@ -408,7 +312,7 @@ async fn run_lane(
|
||||
started,
|
||||
);
|
||||
} else {
|
||||
send(socket, runtime, Message::Binary(body)).await?;
|
||||
send(socket, Message::Binary(body), &cancellation, write_timeout).await?;
|
||||
}
|
||||
connection.mark_progress();
|
||||
}
|
||||
@@ -416,7 +320,13 @@ async fn run_lane(
|
||||
}
|
||||
DriverEvent::Liveness => {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
send(
|
||||
socket,
|
||||
Message::Ping(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
@@ -436,203 +346,3 @@ enum DriverEvent {
|
||||
Down(crate::web::session::PollResult),
|
||||
Liveness,
|
||||
}
|
||||
|
||||
async fn read_message(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
cancellation: &CancellationToken,
|
||||
retained_budget: &mut Option<WebSocketBudgetLease>,
|
||||
) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
|
||||
}
|
||||
if retained_budget.is_none() {
|
||||
let maximum = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.limits
|
||||
.carrier_batch_bytes;
|
||||
*retained_budget = Some(reserve_data(runtime, owner, maximum, cancellation).await?);
|
||||
}
|
||||
let message = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
message = socket.next() => message.ok_or(())?.map_err(|_| ())?,
|
||||
};
|
||||
if socket.get_ref().websocket_fragmented_message() {
|
||||
return Ok((message, None));
|
||||
}
|
||||
let mut budget = retained_budget.take().ok_or(())?;
|
||||
budget.shrink_to(message.len());
|
||||
Ok((message, Some(budget)))
|
||||
}
|
||||
|
||||
async fn reserve_data(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
bytes: usize,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<WebSocketBudgetLease, ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
|
||||
return Ok(budget);
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn process_multiplex(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), ()> {
|
||||
retry_backpressure(runtime, cancellation, || {
|
||||
session.process_up(sequence, body).map(|_| ())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn process_lane(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match session.process_websocket_lane(reservation, sequence, body) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn retry_backpressure<F>(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
cancellation: &CancellationToken,
|
||||
mut operation: F,
|
||||
) -> Result<(), ()>
|
||||
where
|
||||
F: FnMut() -> Result<(), ManagerError>,
|
||||
{
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match operation() {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn send(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &WebProcessRuntime,
|
||||
message: Message,
|
||||
) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_write_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, socket.send(message))
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
async fn flush(socket: &mut CarrierSocket, runtime: &WebProcessRuntime) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_write_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, socket.flush())
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
fn record_message(
|
||||
runtime: &WebProcessRuntime,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
direction: TraceDirection,
|
||||
message_type: &'static str,
|
||||
payload: &[u8],
|
||||
started: Instant,
|
||||
) {
|
||||
let Some(trace) = trace else {
|
||||
return;
|
||||
};
|
||||
runtime.trace().record_websocket_message(
|
||||
trace,
|
||||
direction,
|
||||
message_type,
|
||||
payload,
|
||||
started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use tokio_tungstenite::tungstenite::protocol::Message;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::CarrierSocket;
|
||||
use crate::web::manager::{ManagerError, WebProcessRuntime, WebSocketBudgetLease};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
pub(super) async fn read_message(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
cancellation: &CancellationToken,
|
||||
retained_budget: &mut Option<WebSocketBudgetLease>,
|
||||
maximum: usize,
|
||||
backpressure_timeout: Duration,
|
||||
) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
|
||||
}
|
||||
if retained_budget.is_none() {
|
||||
*retained_budget =
|
||||
Some(reserve_data(runtime, owner, maximum, cancellation, backpressure_timeout).await?);
|
||||
}
|
||||
let message = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
message = socket.next() => message.ok_or(())?.map_err(|_| ())?,
|
||||
};
|
||||
if socket.get_ref().websocket_fragmented_message() {
|
||||
return Ok((message, None));
|
||||
}
|
||||
let mut budget = retained_budget.take().ok_or(())?;
|
||||
budget.shrink_to(message.len());
|
||||
Ok((message, Some(budget)))
|
||||
}
|
||||
|
||||
pub(super) async fn reserve_data(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
bytes: usize,
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
) -> Result<WebSocketBudgetLease, ()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(());
|
||||
}
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
|
||||
return Ok(budget);
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
pub(super) async fn process_multiplex(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
) -> Result<bool, ()> {
|
||||
retry_backpressure(runtime, cancellation, timeout, || {
|
||||
session.process_websocket_multiplex(sequence, body)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn process_lane(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
) -> Result<bool, ()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(());
|
||||
}
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match session.process_websocket_lane(reservation, sequence, body) {
|
||||
Ok(progressed) => return Ok(progressed),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn retry_backpressure<F, T>(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
mut operation: F,
|
||||
) -> Result<T, ()>
|
||||
where
|
||||
F: FnMut() -> Result<T, ManagerError>,
|
||||
{
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(());
|
||||
}
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match operation() {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
pub(super) async fn send(
|
||||
socket: &mut CarrierSocket,
|
||||
message: Message,
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
) -> Result<(), ()> {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => Err(()),
|
||||
result = tokio::time::timeout(timeout, socket.send(message)) => {
|
||||
result.map_err(|_| ())?.map_err(|_| ())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn flush(
|
||||
socket: &mut CarrierSocket,
|
||||
cancellation: &CancellationToken,
|
||||
timeout: Duration,
|
||||
) -> Result<(), ()> {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => Err(()),
|
||||
result = tokio::time::timeout(timeout, socket.flush()) => {
|
||||
result.map_err(|_| ())?.map_err(|_| ())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_message(
|
||||
runtime: &WebProcessRuntime,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
direction: TraceDirection,
|
||||
message_type: &'static str,
|
||||
payload: &[u8],
|
||||
started: Instant,
|
||||
) {
|
||||
let Some(trace) = trace else {
|
||||
return;
|
||||
};
|
||||
runtime.trace().record_websocket_message(
|
||||
trace,
|
||||
direction,
|
||||
message_type,
|
||||
payload,
|
||||
started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use tokio_tungstenite::tungstenite::protocol::Message;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::CarrierSocket;
|
||||
use super::io::{flush, process_lane, read_message, record_message, reserve_data, send};
|
||||
use crate::web::manager::{WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_lane(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
connection: &WebSocketConnection,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
cancellation: CancellationToken,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
acknowledge_commit: bool,
|
||||
) -> Result<(), ()> {
|
||||
let mut sequence = 1u64;
|
||||
let mut cursor = 0u64;
|
||||
// Lane reads use the same cancellation-safe fragmented-message ownership.
|
||||
let mut read_budget = None;
|
||||
let liveness_interval = connection.liveness_interval();
|
||||
let mut next_ping = Instant::now() + liveness_interval;
|
||||
let open_deadline =
|
||||
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
|
||||
let backpressure_timeout = Duration::from_secs(session.timeouts().websocket_backpressure_secs);
|
||||
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
|
||||
let maximum_message = session.limits().carrier_batch_bytes;
|
||||
let mut active = false;
|
||||
loop {
|
||||
let down = session.poll_down_lane(reservation.lane_id(), cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = tokio::time::sleep_until(open_deadline.into()), if !active => return Err(()),
|
||||
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
|
||||
incoming = read_message(
|
||||
socket,
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
&cancellation,
|
||||
&mut read_budget,
|
||||
maximum_message,
|
||||
backpressure_timeout,
|
||||
) => {
|
||||
DriverEvent::Incoming(incoming?)
|
||||
}
|
||||
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
|
||||
};
|
||||
match event {
|
||||
DriverEvent::Incoming((message, _budget)) => match message {
|
||||
Message::Binary(body) => {
|
||||
let started = Instant::now();
|
||||
let result = process_lane(
|
||||
runtime,
|
||||
session,
|
||||
reservation,
|
||||
sequence,
|
||||
&body,
|
||||
&cancellation,
|
||||
backpressure_timeout,
|
||||
)
|
||||
.await;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
let progressed = result?;
|
||||
if acknowledge_commit && sequence == 1 {
|
||||
if !session.needs_websocket_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
let started = Instant::now();
|
||||
if send(
|
||||
socket,
|
||||
Message::Binary(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"carrier-ack",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
if !session.websocket_commit_ack_written(connection.id()) {
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
return Err(());
|
||||
}
|
||||
active = true;
|
||||
}
|
||||
sequence = sequence.checked_add(1).ok_or(())?;
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Pong(payload) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"pong",
|
||||
&payload,
|
||||
Instant::now(),
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
let started = Instant::now();
|
||||
flush(socket, &cancellation, write_timeout).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"ping",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"pong",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Close(_) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"close",
|
||||
&[],
|
||||
Instant::now(),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Message::Text(text) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"text",
|
||||
text.as_bytes(),
|
||||
Instant::now(),
|
||||
);
|
||||
return Err(());
|
||||
}
|
||||
Message::Frame(_) => return Err(()),
|
||||
},
|
||||
DriverEvent::Down(result) => {
|
||||
if result.lane_closed {
|
||||
return Ok(());
|
||||
}
|
||||
if result.body.is_empty() {
|
||||
let started = Instant::now();
|
||||
send(
|
||||
socket,
|
||||
Message::Ping(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
} else {
|
||||
let _budget = reserve_data(
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
result.body.len(),
|
||||
&cancellation,
|
||||
backpressure_timeout,
|
||||
)
|
||||
.await?;
|
||||
let body = result.body;
|
||||
let started = Instant::now();
|
||||
if trace.is_some() {
|
||||
send(
|
||||
socket,
|
||||
Message::Binary(body.clone()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
} else {
|
||||
send(socket, Message::Binary(body), &cancellation, write_timeout).await?;
|
||||
}
|
||||
connection.mark_progress();
|
||||
}
|
||||
cursor = result.next_cursor;
|
||||
}
|
||||
DriverEvent::Liveness => {
|
||||
let started = Instant::now();
|
||||
send(
|
||||
socket,
|
||||
Message::Ping(Bytes::new()),
|
||||
&cancellation,
|
||||
write_timeout,
|
||||
)
|
||||
.await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum DriverEvent {
|
||||
Incoming((Message, Option<WebSocketBudgetLease>)),
|
||||
Down(crate::web::session::PollResult),
|
||||
Liveness,
|
||||
}
|
||||
@@ -14,8 +14,10 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::maestro::generation::{RuntimeGeneration, test_runtime_generation};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::http::tests::runtime_config;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::http::tests::{negotiation_runtime_config, runtime_config};
|
||||
use crate::web::manager::{
|
||||
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierRequest, WebProcessRuntime,
|
||||
};
|
||||
|
||||
fn request(protocol: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
@@ -39,6 +41,14 @@ fn canonical_multiplex_and_lane_protocols_are_accepted() {
|
||||
|
||||
let lane = parse_upgrade(&request(&format!("tproxy-lane-v1.{token}.16777215"))).unwrap();
|
||||
assert!(matches!(lane.carrier, ParsedCarrier::Lane(16_777_215)));
|
||||
|
||||
let automatic = parse_upgrade(&request(&format!("tproxy-auto-v1.{token}"))).unwrap();
|
||||
assert!(automatic.acknowledge_commit);
|
||||
let automatic_lane =
|
||||
parse_upgrade(&request(&format!("tproxy-auto-lane-v1.{token}.7"))).unwrap();
|
||||
assert!(matches!(automatic_lane.carrier, ParsedCarrier::Lane(7)));
|
||||
assert!(automatic_lane.acknowledge_commit);
|
||||
assert!(!multiplex.acknowledge_commit);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -92,7 +102,20 @@ fn live_runtime(carrier: WebCarrier) -> LiveRuntime {
|
||||
}
|
||||
|
||||
fn live_runtime_with_long_poll(carrier: WebCarrier, long_poll_secs: u64) -> LiveRuntime {
|
||||
let mut config = runtime_config([31; 32], carrier);
|
||||
live_runtime_from_config(runtime_config([31; 32], carrier), long_poll_secs)
|
||||
}
|
||||
|
||||
fn live_negotiation_runtime(carrier: WebCarrier, carriers: Arc<[WebCarrier]>) -> LiveRuntime {
|
||||
live_runtime_from_config(
|
||||
negotiation_runtime_config([31; 32], carrier, false, carriers),
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
fn live_runtime_from_config(
|
||||
mut config: crate::config::ProxyConfig,
|
||||
long_poll_secs: u64,
|
||||
) -> LiveRuntime {
|
||||
config.web.timeouts.long_poll_secs = long_poll_secs;
|
||||
config.web.timeouts.websocket_write_secs = 2;
|
||||
config.web.timeouts.websocket_backpressure_secs = 2;
|
||||
@@ -105,6 +128,49 @@ fn live_runtime_with_long_poll(carrier: WebCarrier, long_poll_secs: u64) -> Live
|
||||
}
|
||||
}
|
||||
|
||||
fn create_automatic_session(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
) -> (TokenHash, Bytes, String, TokenHash) {
|
||||
let profile = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.profiles[0]
|
||||
.clone();
|
||||
let client_ip = "192.0.2.10".parse().unwrap();
|
||||
let bootstrap = runtime.issue_bootstrap(profile, client_ip).unwrap().token;
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(&bootstrap)
|
||||
.unwrap();
|
||||
let bootstrap_hash = Sha256::digest(raw).into();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let session = runtime
|
||||
.create_session(
|
||||
bootstrap_hash,
|
||||
"proxy.example.com",
|
||||
client_ip,
|
||||
&hello,
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
1,
|
||||
None,
|
||||
[9; 32],
|
||||
),
|
||||
false,
|
||||
)
|
||||
.unwrap()
|
||||
.token;
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(&session)
|
||||
.unwrap();
|
||||
let session_hash = Sha256::digest(raw).into();
|
||||
(bootstrap_hash, hello, session, session_hash)
|
||||
}
|
||||
|
||||
fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
|
||||
let profile = runtime
|
||||
.active_generation()
|
||||
@@ -123,7 +189,14 @@ fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
|
||||
let bootstrap_hash = Sha256::digest(raw).into();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let session = runtime
|
||||
.create_session(bootstrap_hash, "proxy.example.com", client_ip, &hello)
|
||||
.create_session(
|
||||
bootstrap_hash,
|
||||
"proxy.example.com",
|
||||
client_ip,
|
||||
&hello,
|
||||
CarrierRequest::legacy([0; 32]),
|
||||
false,
|
||||
)
|
||||
.unwrap()
|
||||
.token;
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
@@ -309,3 +382,106 @@ async fn malformed_websocket_lane_closes_only_that_lane() {
|
||||
let _ = second.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_websocket_carriers_commit_after_acknowledged_peer_progress() {
|
||||
for carrier in [WebCarrier::Websocket, WebCarrier::WebsocketLanes] {
|
||||
let live = live_negotiation_runtime(carrier, Arc::from([carrier]));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (_, _, session, session_hash) = create_automatic_session(&live.runtime);
|
||||
let protocol = match carrier {
|
||||
WebCarrier::Websocket => format!("tproxy-auto-v1.{session}"),
|
||||
WebCarrier::WebsocketLanes => format!("tproxy-auto-lane-v1.{session}.7"),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
|
||||
socket
|
||||
.send(Message::Binary(frame::encode(FrameType::Open, 7, &[])))
|
||||
.await
|
||||
.unwrap();
|
||||
let acknowledgement = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(acknowledgement, Message::Binary(Bytes::new()));
|
||||
assert!(
|
||||
live.runtime
|
||||
.get_session(session_hash, "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_carrier_committed()
|
||||
);
|
||||
socket
|
||||
.send(Message::Binary(frame::encode(
|
||||
FrameType::Window,
|
||||
7,
|
||||
&frame::window_payload(1),
|
||||
)))
|
||||
.await
|
||||
.unwrap();
|
||||
socket
|
||||
.send(Message::Ping(Bytes::from_static(b"commit")))
|
||||
.await
|
||||
.unwrap();
|
||||
loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if message == Message::Pong(Bytes::from_static(b"commit")) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
live.runtime
|
||||
.get_session(session_hash, "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_carrier_committed()
|
||||
);
|
||||
|
||||
let _ = socket.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_automatic_multiplex_socket_remains_supersedable() {
|
||||
let live = live_negotiation_runtime(
|
||||
WebCarrier::Https,
|
||||
Arc::from([WebCarrier::Websocket, WebCarrier::Https]),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (bootstrap_hash, hello, session, session_hash) = create_automatic_session(&live.runtime);
|
||||
let protocol = format!("tproxy-auto-v1.{session}");
|
||||
let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
|
||||
socket.close(None).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert!(
|
||||
live.runtime
|
||||
.get_session(session_hash, "proxy.example.com")
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
let replacement = live
|
||||
.runtime
|
||||
.create_session(
|
||||
bootstrap_hash,
|
||||
"proxy.example.com",
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
&hello,
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
2,
|
||||
Some(CarrierFailure::Upgrade),
|
||||
[9; 32],
|
||||
),
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(replacement.carrier, WebCarrier::Https);
|
||||
assert_eq!(replacement.attempt, Some(2));
|
||||
|
||||
live.shutdown().await;
|
||||
}
|
||||
|
||||
+97
-5
@@ -16,8 +16,19 @@ use crate::web::trace::WebTraceStore;
|
||||
|
||||
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
|
||||
mod state;
|
||||
// Carrier attempt metadata remains explicit and independent from HTTP parsing.
|
||||
mod negotiation;
|
||||
// Bounded process-local carrier evidence is isolated from session registries.
|
||||
#[path = "manager/carrier_learning.rs"]
|
||||
mod learning;
|
||||
// Bootstrap credentials and idempotent session creation are isolated from queue accounting.
|
||||
mod credentials;
|
||||
// First-session admission and bounded carrier replacement share one state machine.
|
||||
mod session_creation;
|
||||
// Session admission remains separate from stream tuple ownership.
|
||||
mod session_admission;
|
||||
// Carrier commit, health, and conflict echoes share one outcome publication path.
|
||||
mod carrier_outcome;
|
||||
// Stream admission and synthetic tuple ownership are process-scoped.
|
||||
mod admission;
|
||||
// Shutdown and expiry work remain outside request-path coordination.
|
||||
@@ -28,7 +39,10 @@ mod budget;
|
||||
mod websocket;
|
||||
pub(crate) use budget::WebSocketBudgetLease;
|
||||
use budget::{WebDataBudget, WebSocketBudgetClass};
|
||||
use state::ManagerState;
|
||||
pub(crate) use negotiation::{
|
||||
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierLearningContext, CarrierRequest,
|
||||
};
|
||||
use state::{ManagerState, StreamAdmissionState};
|
||||
pub(crate) use websocket::{WebSocketConnection, WebSocketKind};
|
||||
|
||||
const TOKEN_BYTES: usize = 32;
|
||||
@@ -52,16 +66,55 @@ pub(crate) enum ManagerError {
|
||||
Protocol,
|
||||
/// The operation conflicts with another in-flight operation.
|
||||
Concurrent,
|
||||
/// An authenticated attempt chain is already committed.
|
||||
Committed,
|
||||
/// The process or session has stopped accepting work.
|
||||
Closed,
|
||||
}
|
||||
|
||||
impl ManagerError {
|
||||
/// Returns the stable non-sensitive failure token used by WEB diagnostics.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Authentication => "authentication",
|
||||
Self::Backpressure => "backpressure",
|
||||
Self::Limit => "limit",
|
||||
Self::Protocol => "protocol",
|
||||
Self::Concurrent => "concurrent",
|
||||
Self::Committed => "committed",
|
||||
Self::Closed => "closed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Successful idempotent session creation result.
|
||||
pub(crate) struct CreateResult {
|
||||
/// Opaque bearer token for the created or replayed session.
|
||||
pub(crate) token: String,
|
||||
/// Carrier frozen into the created or replayed session.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// One-based carrier attempt echoed only for negotiated sessions.
|
||||
pub(crate) attempt: Option<u8>,
|
||||
/// Effective frozen candidate count on the first automatic response.
|
||||
pub(crate) candidate_count: Option<u8>,
|
||||
/// Cumulative final chain deadline on the first automatic response.
|
||||
pub(crate) deadline_secs: Option<u64>,
|
||||
/// Actual attempt-chain phase echoed for automatic sessions.
|
||||
pub(crate) carrier_state: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// Authenticated non-secret attempt-chain metadata returned with a conflict.
|
||||
pub(crate) struct CarrierEcho {
|
||||
/// Carrier frozen into the current committed attempt.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// One-based current attempt.
|
||||
pub(crate) attempt: u8,
|
||||
/// Frozen supported candidate count.
|
||||
pub(crate) candidate_count: u8,
|
||||
/// Frozen cumulative final deadline.
|
||||
pub(crate) deadline_secs: u64,
|
||||
/// Actual current chain phase.
|
||||
pub(crate) state: &'static str,
|
||||
}
|
||||
|
||||
/// Successful bridge bootstrap issuance result.
|
||||
@@ -78,9 +131,12 @@ pub(crate) struct WebProcessRuntime {
|
||||
trace: Arc<WebTraceStore>,
|
||||
limits: WebLimitsConfig,
|
||||
state: Mutex<ManagerState>,
|
||||
stream_admission: Mutex<StreamAdmissionState>,
|
||||
learning: Mutex<learning::CarrierLearning>,
|
||||
http_connections: Arc<Semaphore>,
|
||||
http_handlers: Arc<Semaphore>,
|
||||
lane_polls: Arc<Semaphore>,
|
||||
lane_aux_polls: Arc<Semaphore>,
|
||||
body_readers: Arc<Semaphore>,
|
||||
body_bytes: Arc<Semaphore>,
|
||||
stream_handshakes: Arc<Semaphore>,
|
||||
@@ -115,16 +171,28 @@ impl WebProcessRuntime {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
) -> Arc<Self> {
|
||||
let limits = active_runtime.load().config().web.limits.clone();
|
||||
let config = active_runtime.load().config();
|
||||
let limits = config.web.limits.clone();
|
||||
let learning_capacity = limits.max_carrier_learning_entries;
|
||||
let mut carrier_learning = learning::CarrierLearning::new(learning_capacity);
|
||||
let _ = carrier_learning.apply_policy(
|
||||
std::time::Instant::now(),
|
||||
config.web.carrier_negotiation_enabled() && config.web.carrier_learning,
|
||||
config.web.carrier_negotiation_aggressiveness,
|
||||
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
|
||||
);
|
||||
let websocket_connections = limits
|
||||
.max_http_connections
|
||||
.saturating_sub(limits.websocket_http_connection_reserve);
|
||||
let lane_poll_limit = limits.max_http_handlers / 2;
|
||||
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
|
||||
let runtime = Arc::new(Self {
|
||||
active_runtime,
|
||||
trace,
|
||||
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
||||
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
|
||||
lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))),
|
||||
lane_polls: Arc::new(Semaphore::new(lane_poll_limit)),
|
||||
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
|
||||
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
||||
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
||||
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
||||
@@ -136,6 +204,8 @@ impl WebProcessRuntime {
|
||||
data_budget: WebDataBudget::new(limits.clone()),
|
||||
limits,
|
||||
state: Mutex::new(ManagerState::default()),
|
||||
stream_admission: Mutex::new(StreamAdmissionState::default()),
|
||||
learning: Mutex::new(carrier_learning),
|
||||
shutdown: CancellationToken::new(),
|
||||
tasks: TaskTracker::new(),
|
||||
sessions_created: AtomicU64::new(0),
|
||||
@@ -197,8 +267,13 @@ impl WebProcessRuntime {
|
||||
}
|
||||
|
||||
/// Reserves one parked lane poll without exhausting all HTTP handlers.
|
||||
pub(crate) fn try_lane_poll(&self) -> Option<OwnedSemaphorePermit> {
|
||||
let permit = Arc::clone(&self.lane_polls).try_acquire_owned().ok();
|
||||
pub(crate) fn try_lane_poll(&self, auxiliary: bool) -> Option<OwnedSemaphorePermit> {
|
||||
let slots = if auxiliary {
|
||||
&self.lane_aux_polls
|
||||
} else {
|
||||
&self.lane_polls
|
||||
};
|
||||
let permit = Arc::clone(slots).try_acquire_owned().ok();
|
||||
if permit.is_none() {
|
||||
self.record_limit_hit();
|
||||
}
|
||||
@@ -251,6 +326,18 @@ impl WebProcessRuntime {
|
||||
Some((reader, body))
|
||||
}
|
||||
|
||||
/// Reserves transient bytes while one downlink batch replaces queued frames.
|
||||
pub(crate) fn try_downlink_staging_budget(&self, bytes: usize) -> Option<OwnedSemaphorePermit> {
|
||||
let bytes = u32::try_from(bytes).ok()?;
|
||||
let permit = Arc::clone(&self.body_bytes)
|
||||
.try_acquire_many_owned(bytes)
|
||||
.ok();
|
||||
if permit.is_none() {
|
||||
self.record_limit_hit();
|
||||
}
|
||||
permit
|
||||
}
|
||||
|
||||
/// Reserves bounded process-wide queue capacity for data or control traffic.
|
||||
pub(crate) fn try_reserve_pending(
|
||||
&self,
|
||||
@@ -307,25 +394,30 @@ impl WebProcessRuntime {
|
||||
}
|
||||
|
||||
/// Admits one WebSocket with owner-first bounded replacement.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn admit_websocket(
|
||||
self: &Arc<Self>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
session_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval: Duration,
|
||||
eviction_timeout: Duration,
|
||||
parent_cancellation: CancellationToken,
|
||||
) -> Result<WebSocketConnection, ManagerError> {
|
||||
websocket::admit(
|
||||
self,
|
||||
owner,
|
||||
session_id,
|
||||
session_hash,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval,
|
||||
eviction_timeout,
|
||||
parent_cancellation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ impl WebProcessRuntime {
|
||||
public_addr: SocketAddr,
|
||||
) -> Option<u16> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if state.closed
|
||||
|| state.streams_live >= self.limits.max_streams_global
|
||||
|| state
|
||||
@@ -54,7 +54,7 @@ impl WebProcessRuntime {
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if !release_stream_port(&mut state, client_ip, public_addr, peer_port) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -163,7 +163,6 @@ impl WebDataBudget {
|
||||
}
|
||||
remove_owner(&mut state.owner_bytes, owner, bytes);
|
||||
drop(state);
|
||||
self.pressured.store(false, Ordering::Release);
|
||||
self.notify.notify_waiters();
|
||||
}
|
||||
|
||||
@@ -217,6 +216,10 @@ impl WebDataBudget {
|
||||
self.pressured.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
pub(super) fn restore_pressure(&self) {
|
||||
self.pressured.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
@@ -259,7 +262,6 @@ impl WebDataBudget {
|
||||
state.websocket_bytes = state.websocket_bytes.saturating_sub(bytes);
|
||||
remove_owner(&mut state.owner_bytes, owner, bytes);
|
||||
drop(state);
|
||||
self.pressured.store(false, Ordering::Release);
|
||||
self.notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::net::IpAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::ProfileKey;
|
||||
use super::negotiation::{CarrierClientClass, CarrierLearningContext};
|
||||
use crate::config::{WebCarrier, WebCarrierNegotiationAggressiveness};
|
||||
|
||||
const PROFILE_WEIGHT: i16 = 32;
|
||||
const USER_AGENT_WEIGHT: i16 = 32;
|
||||
const IP_WEIGHT: i16 = 1;
|
||||
const SCORE_MIN: i8 = -8;
|
||||
const SCORE_MAX: i8 = 8;
|
||||
const MAX_COHORTS: usize = 4;
|
||||
const PRUNE_ENTRIES_PER_TICK: usize = 64;
|
||||
const COHORT_CONTEXT: &[u8] = b"telemt-web-carrier-cohort-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum EvidenceKey {
|
||||
Profile(ProfileKey),
|
||||
UserAgent(ProfileKey, CarrierClientClass, [u8; 32]),
|
||||
Ip(ProfileKey, IpAddr),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Bucket {
|
||||
slot: u64,
|
||||
valid: bool,
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; MAX_COHORTS],
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
fn reset(&mut self, slot: u64) {
|
||||
*self = Self {
|
||||
slot,
|
||||
valid: true,
|
||||
..Self::default()
|
||||
};
|
||||
}
|
||||
|
||||
fn update(&mut self, deltas: [i8; 4], cohort: Option<[u8; 32]>) {
|
||||
for (score, delta) in self.scores.iter_mut().zip(deltas) {
|
||||
*score = score.saturating_add(delta).clamp(SCORE_MIN, SCORE_MAX);
|
||||
}
|
||||
self.outcomes = self.outcomes.saturating_add(1);
|
||||
if let Some(cohort) = cohort
|
||||
&& !self.cohorts.contains(&Some(cohort))
|
||||
&& let Some(slot) = self.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*slot = Some(cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Evidence {
|
||||
insertion_sequence: u64,
|
||||
buckets: [Bucket; 2],
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
fn new(insertion_sequence: u64) -> Self {
|
||||
Self {
|
||||
insertion_sequence,
|
||||
buckets: [Bucket::default(), Bucket::default()],
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, slot: u64, deltas: [i8; 4], cohort: Option<[u8; 32]>) {
|
||||
let index = slot as usize % self.buckets.len();
|
||||
if !self.buckets[index].valid || self.buckets[index].slot != slot {
|
||||
self.buckets[index].reset(slot);
|
||||
}
|
||||
self.buckets[index].update(deltas, cohort);
|
||||
}
|
||||
|
||||
fn aggregate(&self, slot: u64) -> Aggregate {
|
||||
let mut aggregate = Aggregate::default();
|
||||
for bucket in &self.buckets {
|
||||
if !bucket.valid || (bucket.slot != slot && bucket.slot.saturating_add(1) != slot) {
|
||||
continue;
|
||||
}
|
||||
aggregate.outcomes = aggregate.outcomes.saturating_add(bucket.outcomes);
|
||||
for (score, value) in aggregate.scores.iter_mut().zip(bucket.scores) {
|
||||
*score = score.saturating_add(value).clamp(SCORE_MIN, SCORE_MAX);
|
||||
}
|
||||
for cohort in bucket.cohorts.iter().flatten() {
|
||||
if !aggregate.cohorts.contains(&Some(*cohort))
|
||||
&& let Some(target) = aggregate.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*target = Some(*cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
aggregate
|
||||
}
|
||||
|
||||
fn is_live(&self, slot: u64) -> bool {
|
||||
self.buckets.iter().any(|bucket| {
|
||||
bucket.valid && (bucket.slot == slot || bucket.slot.saturating_add(1) == slot)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Aggregate {
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; MAX_COHORTS],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct LearningPolicy {
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Thresholds {
|
||||
user_agent: u8,
|
||||
ip: Option<u8>,
|
||||
profile_outcomes: u8,
|
||||
profile_cohorts: usize,
|
||||
}
|
||||
|
||||
impl Thresholds {
|
||||
fn for_aggressiveness(value: WebCarrierNegotiationAggressiveness) -> Self {
|
||||
match value {
|
||||
WebCarrierNegotiationAggressiveness::Conservative => Self {
|
||||
user_agent: 3,
|
||||
ip: None,
|
||||
profile_outcomes: 8,
|
||||
profile_cohorts: 4,
|
||||
},
|
||||
WebCarrierNegotiationAggressiveness::Balanced => Self {
|
||||
user_agent: 2,
|
||||
ip: Some(3),
|
||||
profile_outcomes: 6,
|
||||
profile_cohorts: 3,
|
||||
},
|
||||
WebCarrierNegotiationAggressiveness::Aggressive => Self {
|
||||
user_agent: 1,
|
||||
ip: Some(1),
|
||||
profile_outcomes: 4,
|
||||
profile_cohorts: 2,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-local bounded two-bucket carrier evidence store.
|
||||
pub(super) struct CarrierLearning {
|
||||
entries: HashMap<EvidenceKey, Evidence>,
|
||||
insertion_order: VecDeque<(EvidenceKey, u64)>,
|
||||
capacity: usize,
|
||||
insertion_sequence: u64,
|
||||
epoch: Option<u64>,
|
||||
policy: Option<LearningPolicy>,
|
||||
policy_started_at: Instant,
|
||||
}
|
||||
|
||||
impl CarrierLearning {
|
||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||
pub(super) fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
insertion_order: VecDeque::new(),
|
||||
capacity,
|
||||
insertion_sequence: 1,
|
||||
epoch: Some(0),
|
||||
policy: None,
|
||||
policy_started_at: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies hot-reloaded learning policy and returns its outcome epoch.
|
||||
pub(super) fn apply_policy(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
) -> Option<u64> {
|
||||
let policy = LearningPolicy {
|
||||
enabled,
|
||||
aggressiveness,
|
||||
lifetime,
|
||||
};
|
||||
if self.policy != Some(policy) {
|
||||
self.entries.clear();
|
||||
self.insertion_order.clear();
|
||||
if !enabled {
|
||||
self.entries.shrink_to_fit();
|
||||
self.insertion_order.shrink_to_fit();
|
||||
}
|
||||
self.insertion_sequence = 1;
|
||||
self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1));
|
||||
self.policy = Some(policy);
|
||||
self.policy_started_at = now;
|
||||
}
|
||||
self.epoch
|
||||
}
|
||||
|
||||
/// Returns the current epoch only when the request snapshot matches owner policy.
|
||||
pub(super) fn epoch_for_policy(
|
||||
&self,
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
) -> Option<u64> {
|
||||
(self.policy
|
||||
== Some(LearningPolicy {
|
||||
enabled,
|
||||
aggressiveness,
|
||||
lifetime,
|
||||
}))
|
||||
.then_some(self.epoch)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Ranks supported configured candidates without scanning the evidence store.
|
||||
pub(super) fn rank(
|
||||
&self,
|
||||
now: Instant,
|
||||
configured: &[WebCarrier],
|
||||
request: super::CarrierRequest,
|
||||
profile_key: ProfileKey,
|
||||
client_ip: IpAddr,
|
||||
ip_learning_eligible: bool,
|
||||
) -> (Vec<WebCarrier>, [i16; 4]) {
|
||||
let Some(policy) = self.policy.filter(|policy| policy.enabled) else {
|
||||
return (supported(configured, request), [0; 4]);
|
||||
};
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let thresholds = Thresholds::for_aggressiveness(policy.aggressiveness);
|
||||
let profile = self
|
||||
.entries
|
||||
.get(&EvidenceKey::Profile(profile_key))
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let user_agent = self
|
||||
.entries
|
||||
.get(&EvidenceKey::UserAgent(
|
||||
profile_key,
|
||||
request.class(),
|
||||
request.user_agent_hash(),
|
||||
))
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let ip = (ip_learning_eligible && thresholds.ip.is_some())
|
||||
.then(|| self.entries.get(&EvidenceKey::Ip(profile_key, client_ip)))
|
||||
.flatten()
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let profile_ready = profile.as_ref().is_some_and(|entry| {
|
||||
entry.outcomes >= thresholds.profile_outcomes
|
||||
&& entry.cohorts.iter().flatten().count() >= thresholds.profile_cohorts
|
||||
});
|
||||
let user_agent_ready = user_agent
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.outcomes >= thresholds.user_agent);
|
||||
let ip_ready = thresholds
|
||||
.ip
|
||||
.is_some_and(|minimum| ip.as_ref().is_some_and(|entry| entry.outcomes >= minimum));
|
||||
let mut scores = [0i16; 4];
|
||||
for carrier in WebCarrier::ALL {
|
||||
let index = carrier.index();
|
||||
if profile_ready {
|
||||
scores[index] += i16::from(profile.as_ref().map_or(0, |value| value.scores[index]))
|
||||
* PROFILE_WEIGHT;
|
||||
}
|
||||
if user_agent_ready {
|
||||
scores[index] +=
|
||||
i16::from(user_agent.as_ref().map_or(0, |value| value.scores[index]))
|
||||
* USER_AGENT_WEIGHT;
|
||||
}
|
||||
if ip_ready {
|
||||
scores[index] +=
|
||||
i16::from(ip.as_ref().map_or(0, |value| value.scores[index])) * IP_WEIGHT;
|
||||
}
|
||||
}
|
||||
let mut ranked = supported(configured, request);
|
||||
let fallback = configured
|
||||
.last()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier));
|
||||
if let Some(fallback) = fallback {
|
||||
ranked.retain(|carrier| *carrier != fallback);
|
||||
}
|
||||
ranked.sort_by_key(|carrier| std::cmp::Reverse(scores[carrier.index()]));
|
||||
if let Some(fallback) = fallback {
|
||||
ranked.push(fallback);
|
||||
}
|
||||
(ranked, scores)
|
||||
}
|
||||
|
||||
/// Applies one complete attempt chain as one atomic evidence sample.
|
||||
pub(super) fn record_chain(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
epoch: u64,
|
||||
context: CarrierLearningContext,
|
||||
failures: &[WebCarrier],
|
||||
winner: WebCarrier,
|
||||
) {
|
||||
let Some(policy) = self.policy.filter(|policy| policy.enabled) else {
|
||||
return;
|
||||
};
|
||||
if Some(epoch) != self.epoch {
|
||||
return;
|
||||
}
|
||||
let mut deltas = [0i8; 4];
|
||||
let _ = failures;
|
||||
deltas[winner.index()] = deltas[winner.index()].saturating_add(1);
|
||||
let thresholds = Thresholds::for_aggressiveness(policy.aggressiveness);
|
||||
let keys = [
|
||||
Some(EvidenceKey::Profile(context.profile_key)),
|
||||
Some(EvidenceKey::UserAgent(
|
||||
context.profile_key,
|
||||
context.class,
|
||||
context.user_agent_hash,
|
||||
)),
|
||||
(context.ip_learning_eligible && thresholds.ip.is_some())
|
||||
.then_some(EvidenceKey::Ip(context.profile_key, context.client_ip)),
|
||||
];
|
||||
self.make_room(&keys);
|
||||
let missing = keys
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|key| !self.entries.contains_key(key))
|
||||
.count();
|
||||
if self.entries.len().saturating_add(missing) > self.capacity {
|
||||
return;
|
||||
}
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let cohort = cohort_hash(context);
|
||||
for (index, key) in keys.into_iter().enumerate() {
|
||||
let Some(key) = key else { continue };
|
||||
self.update_key(key, slot, deltas, (index == 0).then_some(cohort));
|
||||
}
|
||||
}
|
||||
|
||||
/// Reclaims a fixed number of entries outside both half-window buckets.
|
||||
pub(super) fn prune(&mut self, now: Instant) {
|
||||
let Some(policy) = self.policy else { return };
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let budget = self.insertion_order.len().min(PRUNE_ENTRIES_PER_TICK);
|
||||
for _ in 0..budget {
|
||||
let Some((key, sequence)) = self.insertion_order.pop_front() else {
|
||||
break;
|
||||
};
|
||||
let current = self
|
||||
.entries
|
||||
.get(&key)
|
||||
.is_some_and(|entry| entry.insertion_sequence == sequence);
|
||||
if !current {
|
||||
continue;
|
||||
}
|
||||
if self
|
||||
.entries
|
||||
.get(&key)
|
||||
.is_some_and(|entry| entry.is_live(slot))
|
||||
{
|
||||
self.insertion_order.push_back((key, sequence));
|
||||
} else {
|
||||
self.entries.remove(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_room(&mut self, keys: &[Option<EvidenceKey>; 3]) {
|
||||
let missing = keys
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|key| !self.entries.contains_key(key))
|
||||
.count();
|
||||
let mut remaining = self.insertion_order.len();
|
||||
while self.entries.len().saturating_add(missing) > self.capacity && remaining > 0 {
|
||||
remaining -= 1;
|
||||
let Some((oldest, sequence)) = self.insertion_order.pop_front() else {
|
||||
break;
|
||||
};
|
||||
if self
|
||||
.entries
|
||||
.get(&oldest)
|
||||
.is_none_or(|entry| entry.insertion_sequence != sequence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if keys.contains(&Some(oldest)) {
|
||||
self.insertion_order.push_back((oldest, sequence));
|
||||
continue;
|
||||
}
|
||||
self.entries.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_key(
|
||||
&mut self,
|
||||
key: EvidenceKey,
|
||||
slot: u64,
|
||||
deltas: [i8; 4],
|
||||
cohort: Option<[u8; 32]>,
|
||||
) {
|
||||
if let Some(entry) = self.entries.get_mut(&key) {
|
||||
entry.update(slot, deltas, cohort);
|
||||
return;
|
||||
}
|
||||
let Some(insertion_sequence) = self.next_insertion_sequence() else {
|
||||
return;
|
||||
};
|
||||
self.entries.insert(key, Evidence::new(insertion_sequence));
|
||||
self.insertion_order.push_back((key, insertion_sequence));
|
||||
if let Some(entry) = self.entries.get_mut(&key) {
|
||||
entry.update(slot, deltas, cohort);
|
||||
}
|
||||
}
|
||||
|
||||
fn next_insertion_sequence(&mut self) -> Option<u64> {
|
||||
let sequence = self.insertion_sequence;
|
||||
self.insertion_sequence = sequence.checked_add(1)?;
|
||||
Some(sequence)
|
||||
}
|
||||
}
|
||||
|
||||
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
|
||||
configured
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bucket_slot(start: Instant, now: Instant, lifetime: Duration) -> u64 {
|
||||
let half = (lifetime / 2).max(Duration::from_nanos(1));
|
||||
let quotient = now.saturating_duration_since(start).as_nanos() / half.as_nanos();
|
||||
quotient.min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
fn cohort_hash(context: CarrierLearningContext) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(COHORT_CONTEXT);
|
||||
digest.update(context.profile_key);
|
||||
digest.update([match context.class {
|
||||
CarrierClientClass::Legacy => 0,
|
||||
CarrierClientClass::Bridge => 1,
|
||||
CarrierClientClass::BrowserHint => 2,
|
||||
CarrierClientClass::Ios => 3,
|
||||
}]);
|
||||
digest.update(context.user_agent_hash);
|
||||
match context.client_ip {
|
||||
IpAddr::V4(address) => {
|
||||
digest.update([4]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
IpAddr::V6(address) => {
|
||||
digest.update([6]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "carrier_learning/tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,222 @@
|
||||
use super::*;
|
||||
use crate::web::manager::{CarrierCapabilities, CarrierRequest};
|
||||
|
||||
fn request(hash: u8) -> CarrierRequest {
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
1,
|
||||
None,
|
||||
[hash; 32],
|
||||
)
|
||||
}
|
||||
|
||||
fn context(hash: u8) -> CarrierLearningContext {
|
||||
CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, hash)),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [hash; 32],
|
||||
epoch: 1,
|
||||
ip_learning_eligible: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_epoch_rejects_late_outcomes_and_clears_state() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Websocket);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
let next = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
false,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
assert_ne!(epoch, next);
|
||||
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggressive_policy_ranks_one_atomic_chain_sample() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(now, epoch, context(2), &[], WebCarrier::Websocket);
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&[
|
||||
WebCarrier::Https,
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::HttpsLanes,
|
||||
],
|
||||
request(2),
|
||||
[1; 32],
|
||||
context(2).client_ip,
|
||||
true,
|
||||
);
|
||||
assert_eq!(
|
||||
ranked,
|
||||
[
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::Https,
|
||||
WebCarrier::HttpsLanes,
|
||||
]
|
||||
);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 33);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_half_windows_expire_without_sliding_updates() {
|
||||
let start = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
start,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(start, epoch, context(3), &[], WebCarrier::Websocket);
|
||||
learning.record_chain(
|
||||
start + Duration::from_secs(6),
|
||||
epoch,
|
||||
context(3),
|
||||
&[],
|
||||
WebCarrier::Websocket,
|
||||
);
|
||||
learning.prune(start + Duration::from_secs(11));
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
learning.prune(start + Duration::from_secs(16));
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_epoch_and_insertion_identifiers_fail_closed() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
learning.epoch = Some(u64::MAX);
|
||||
assert_eq!(
|
||||
learning.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
),
|
||||
None
|
||||
);
|
||||
learning.record_chain(now, u64::MAX, context(4), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
|
||||
learning.epoch = Some(1);
|
||||
learning.insertion_sequence = u64::MAX;
|
||||
learning.record_chain(now, 1, context(4), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_reported_failures_do_not_create_negative_evidence() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(
|
||||
now,
|
||||
epoch,
|
||||
context(5),
|
||||
&[WebCarrier::Websocket],
|
||||
WebCarrier::Https,
|
||||
);
|
||||
let (_, scores) = learning.rank(
|
||||
now,
|
||||
&[WebCarrier::Websocket, WebCarrier::HttpsLanes],
|
||||
request(5),
|
||||
[1; 32],
|
||||
context(5).client_ip,
|
||||
true,
|
||||
);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_new_old_policy_rejects_both_stale_epochs() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let old = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
let middle = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Balanced,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
let current = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
assert_ne!(old, middle);
|
||||
assert_ne!(middle, current);
|
||||
assert_ne!(old, current);
|
||||
learning.record_chain(now, old, context(6), &[], WebCarrier::Https);
|
||||
learning.record_chain(now, middle, context(6), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
learning.record_chain(now, current, context(6), &[], WebCarrier::Https);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_metadata_stays_within_the_entry_capacity() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
for hash in 1..=32 {
|
||||
learning.record_chain(now, epoch, context(hash), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.len() <= 3);
|
||||
assert!(learning.insertion_order.len() <= 3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::state::CarrierChainPhase;
|
||||
use super::{
|
||||
CarrierClientClass, CarrierEcho, CarrierLearningContext, CarrierRequest, TokenHash,
|
||||
WebProcessRuntime,
|
||||
};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Returns authenticated current chain metadata after a committed retry conflict.
|
||||
pub(crate) fn carrier_echo(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
request: CarrierRequest,
|
||||
) -> Option<CarrierEcho> {
|
||||
let state = self.state.lock();
|
||||
let entry = state.bootstraps.get(&bootstrap_hash)?;
|
||||
let session = entry.session.as_ref()?;
|
||||
if !entry.used
|
||||
|| entry.profile.host != host
|
||||
|| entry.session_client_ip != Some(client_ip)
|
||||
|| entry
|
||||
.carrier_request
|
||||
.is_none_or(|current| !current.matches_client(request))
|
||||
|| !(matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
) || session.is_carrier_committed())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(CarrierEcho {
|
||||
carrier: session.carrier(),
|
||||
attempt: entry.carrier_attempt,
|
||||
candidate_count: u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
|
||||
deadline_secs: entry.profile.carrier_negotiation_deadlines_secs[3],
|
||||
state: if entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& session.is_carrier_committed()
|
||||
{
|
||||
CarrierChainPhase::CommittedPendingHealth.as_str()
|
||||
} else {
|
||||
entry.carrier_phase.as_str()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Restores the exact old attempt after successor admission fails.
|
||||
pub(super) fn cancel_replacement(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
old_session: &Arc<WebSession>,
|
||||
) {
|
||||
old_session.cancel_carrier_supersede();
|
||||
let mut state = self.state.lock();
|
||||
if let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash)
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, old_session))
|
||||
{
|
||||
entry.carrier_transitioning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Freezes replacement immediately after accepted carrier state mutation.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn carrier_committed(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
session_hash: TokenHash,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
class: CarrierClientClass,
|
||||
client_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
let scores = state.bootstraps.get_mut(&bootstrap_hash).and_then(|entry| {
|
||||
if entry.carrier_attempt == attempt
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.token_hash() == session_hash)
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
{
|
||||
entry.carrier_phase = CarrierChainPhase::CommittedPendingHealth;
|
||||
Some(entry.carrier_scores)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
drop(state);
|
||||
let Some(scores) = scores else { return false };
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierCommitted,
|
||||
class.as_str(),
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Promotes one exact committed attempt after transport-specific health evidence.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn carrier_became_healthy(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
session_hash: TokenHash,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
class: CarrierClientClass,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
client_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
) {
|
||||
let (scores, failures) = {
|
||||
let mut state = self.state.lock();
|
||||
let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash) else {
|
||||
return;
|
||||
};
|
||||
if entry.carrier_attempt != attempt
|
||||
|| entry.carrier_phase != CarrierChainPhase::CommittedPendingHealth
|
||||
|| entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_none_or(|session| session.token_hash() != session_hash)
|
||||
{
|
||||
return;
|
||||
}
|
||||
entry.carrier_phase = CarrierChainPhase::Healthy;
|
||||
(entry.carrier_scores, entry.carrier_failures)
|
||||
};
|
||||
if let Some(context) = learning_context {
|
||||
let now = Instant::now();
|
||||
let mut learning = self.learning.lock();
|
||||
let failures = failures.into_iter().flatten().collect::<Vec<_>>();
|
||||
learning.record_chain(now, context.epoch, context, &failures, carrier);
|
||||
}
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity,
|
||||
TraceLifecycleEvent::CarrierHealthy,
|
||||
class.as_str(),
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
+29
-131
@@ -3,19 +3,14 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::state::{
|
||||
Bootstrap, allow_rate, decrement_map, evict_oldest_unused_bootstrap, matching_profile,
|
||||
new_unique_token, profile_key, remove_expired_locked,
|
||||
};
|
||||
use super::{
|
||||
BootstrapResult, CreateResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime,
|
||||
Bootstrap, CarrierChainPhase, allow_rate, evict_oldest_unused_bootstrap, matching_profile,
|
||||
new_unique_token, remove_expired_locked,
|
||||
};
|
||||
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
use crate::web::frame;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
impl WebProcessRuntime {
|
||||
@@ -78,6 +73,19 @@ impl WebProcessRuntime {
|
||||
body_digest: [0; TOKEN_BYTES],
|
||||
session_token: Zeroizing::new(String::new()),
|
||||
session: None,
|
||||
carrier_request: None,
|
||||
carrier_candidates: Arc::from([]),
|
||||
carrier_scores: [0; 4],
|
||||
carrier_attempt: 0,
|
||||
carrier_transitioning: false,
|
||||
carrier_phase: CarrierChainPhase::Provisional,
|
||||
carrier_started_at: None,
|
||||
carrier_deadline_at: None,
|
||||
carrier_failures: [None; 3],
|
||||
carrier_learning_epoch: 0,
|
||||
close_requested: false,
|
||||
session_client_ip: None,
|
||||
session_ip_learning_eligible: false,
|
||||
used: false,
|
||||
},
|
||||
);
|
||||
@@ -117,128 +125,6 @@ impl WebProcessRuntime {
|
||||
.map(|entry| (entry.trace_session_id, Arc::clone(&entry.profile)))
|
||||
}
|
||||
|
||||
/// Creates a session exactly once or replays the original successful result.
|
||||
pub(crate) fn create_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
body: &[u8],
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !frame::validate_hello(body, &self.limits) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let body_digest: TokenHash = Sha256::digest(body).into();
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||
return Err(ManagerError::Authentication);
|
||||
};
|
||||
if entry.profile.host != host || now > entry.expires_at {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.used {
|
||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||
if !digest_matches {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||
let result = CreateResult {
|
||||
token: entry.session_token.as_str().to_owned(),
|
||||
carrier: session.carrier(),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
drop(state);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
crate::web::trace::TraceLifecycleEvent::SessionReplayed,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
return Ok(result);
|
||||
}
|
||||
let trace_session_id = entry.trace_session_id;
|
||||
let issued_profile = Arc::clone(&entry.profile);
|
||||
if state.closed || !config.web.enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &issued_profile))
|
||||
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
let profile_key = profile_key(&profile);
|
||||
if state.sessions.len() >= self.limits.max_sessions_global
|
||||
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
>= self.limits.max_sessions_per_ip
|
||||
|| state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= profile.max_sessions
|
||||
|| !allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
self.limits.new_sessions_per_minute,
|
||||
self.limits.new_sessions_burst,
|
||||
)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
trace_session_id,
|
||||
profile,
|
||||
profile_key,
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
let identity = session.trace_identity();
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: session.carrier(),
|
||||
};
|
||||
drop(state);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
crate::web::trace::TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Resolves an authenticated session token.
|
||||
pub(crate) fn get_session(
|
||||
&self,
|
||||
@@ -260,12 +146,24 @@ impl WebProcessRuntime {
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<(), ManagerError> {
|
||||
let state = self.state.lock();
|
||||
let mut state = self.state.lock();
|
||||
let session = state
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.filter(|session| session.matches_host(host))
|
||||
.cloned();
|
||||
if session.is_some() {
|
||||
for bootstrap in state.bootstraps.values_mut() {
|
||||
if bootstrap
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|current| current.token_hash() == hash)
|
||||
{
|
||||
bootstrap.close_requested = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let closed = state
|
||||
.closed_tokens
|
||||
.get(&hash)
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
/// Telemt Carrier Selection and Failure Dampening - Copyright 2077
|
||||
/// anhand des Kundenverhaltens Rückschlüsse gegen DSGVO ziehen...?!
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::negotiation::{CarrierClientClass, CarrierLearningContext};
|
||||
use super::ProfileKey;
|
||||
use crate::config::WebCarrier;
|
||||
|
||||
const PROFILE_WEIGHT: i16 = 4;
|
||||
const USER_AGENT_WEIGHT: i16 = 4;
|
||||
const IP_WEIGHT: i16 = 1;
|
||||
const SCORE_MIN: i8 = -8;
|
||||
const SCORE_MAX: i8 = 8;
|
||||
const PROFILE_MIN_OUTCOMES: u8 = 8;
|
||||
const PROFILE_MIN_COHORTS: usize = 4;
|
||||
const COHORT_CONTEXT: &[u8] = b"telemt-web-carrier-cohort-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum EvidenceKey {
|
||||
Profile(ProfileKey),
|
||||
UserAgent(ProfileKey, CarrierClientClass, [u8; 32]),
|
||||
Ip(ProfileKey, IpAddr),
|
||||
}
|
||||
|
||||
struct Evidence {
|
||||
created_at: Instant,
|
||||
lifetime: Duration,
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; PROFILE_MIN_COHORTS],
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
fn new(created_at: Instant, lifetime: Duration) -> Self {
|
||||
Self {
|
||||
created_at,
|
||||
lifetime,
|
||||
scores: [0; 4],
|
||||
outcomes: 0,
|
||||
cohorts: [None; PROFILE_MIN_COHORTS],
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, carrier: WebCarrier, delta: i8, cohort: Option<[u8; 32]>) {
|
||||
let score = &mut self.scores[carrier.index()];
|
||||
*score = score.saturating_add(delta).clamp(SCORE_MIN, SCORE_MAX);
|
||||
self.outcomes = self.outcomes.saturating_add(1).min(PROFILE_MIN_OUTCOMES);
|
||||
if let Some(cohort) = cohort
|
||||
&& !self.cohorts.contains(&Some(cohort))
|
||||
&& let Some(slot) = self.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*slot = Some(cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-local bounded fixed-window carrier evidence store.
|
||||
pub(super) struct CarrierLearning {
|
||||
entries: HashMap<EvidenceKey, Evidence>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl CarrierLearning {
|
||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||
pub(super) fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: HashMap::with_capacity(capacity),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ranks supported configured candidates using only unexpired evidence.
|
||||
pub(super) fn rank(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
configured: &[WebCarrier],
|
||||
request: super::CarrierRequest,
|
||||
profile_key: ProfileKey,
|
||||
client_ip: IpAddr,
|
||||
) -> (Vec<WebCarrier>, [i16; 4]) {
|
||||
self.prune(now);
|
||||
let mut scores = [0i16; 4];
|
||||
let profile = self.entries.get(&EvidenceKey::Profile(profile_key));
|
||||
let profile_ready = profile.is_some_and(|entry| {
|
||||
entry.outcomes >= PROFILE_MIN_OUTCOMES
|
||||
&& entry.cohorts.iter().flatten().count() >= PROFILE_MIN_COHORTS
|
||||
});
|
||||
let user_agent = self.entries.get(&EvidenceKey::UserAgent(
|
||||
profile_key,
|
||||
request.class(),
|
||||
request.user_agent_hash(),
|
||||
));
|
||||
let ip = self.entries.get(&EvidenceKey::Ip(profile_key, client_ip));
|
||||
for carrier in WebCarrier::ALL {
|
||||
let index = carrier.index();
|
||||
if profile_ready {
|
||||
scores[index] += i16::from(profile.map_or(0, |entry| entry.scores[index]))
|
||||
* PROFILE_WEIGHT;
|
||||
}
|
||||
scores[index] += i16::from(user_agent.map_or(0, |entry| entry.scores[index]))
|
||||
* USER_AGENT_WEIGHT;
|
||||
scores[index] +=
|
||||
i16::from(ip.map_or(0, |entry| entry.scores[index])) * IP_WEIGHT;
|
||||
}
|
||||
let mut ranked = configured
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier))
|
||||
.collect::<Vec<_>>();
|
||||
ranked.sort_by_key(|carrier| std::cmp::Reverse(scores[carrier.index()]));
|
||||
(ranked, scores)
|
||||
}
|
||||
|
||||
/// Records one committed success or one server-accepted supersession failure.
|
||||
pub(super) fn record(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
lifetime: Duration,
|
||||
context: CarrierLearningContext,
|
||||
carrier: WebCarrier,
|
||||
success: bool,
|
||||
) {
|
||||
self.prune(now);
|
||||
let delta = if success { 1 } else { -1 };
|
||||
let cohort = cohort_hash(context);
|
||||
self.update(
|
||||
EvidenceKey::Profile(context.profile_key),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
Some(cohort),
|
||||
);
|
||||
self.update(
|
||||
EvidenceKey::UserAgent(
|
||||
context.profile_key,
|
||||
context.class,
|
||||
context.user_agent_hash,
|
||||
),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
None,
|
||||
);
|
||||
self.update(
|
||||
EvidenceKey::Ip(context.profile_key, context.client_ip),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
/// Removes fixed-window entries after their creation-time expiry.
|
||||
pub(super) fn prune(&mut self, now: Instant) {
|
||||
self.entries.retain(|_, entry| {
|
||||
now.saturating_duration_since(entry.created_at) <= entry.lifetime
|
||||
});
|
||||
}
|
||||
|
||||
fn update(
|
||||
&mut self,
|
||||
key: EvidenceKey,
|
||||
now: Instant,
|
||||
lifetime: Duration,
|
||||
carrier: WebCarrier,
|
||||
delta: i8,
|
||||
cohort: Option<[u8; 32]>,
|
||||
) {
|
||||
if !self.entries.contains_key(&key) && self.entries.len() >= self.capacity {
|
||||
let oldest = self
|
||||
.entries
|
||||
.iter()
|
||||
.min_by_key(|(_, entry)| entry.created_at)
|
||||
.map(|(key, _)| *key);
|
||||
if let Some(oldest) = oldest {
|
||||
self.entries.remove(&oldest);
|
||||
}
|
||||
}
|
||||
self.entries
|
||||
.entry(key)
|
||||
.or_insert_with(|| Evidence::new(now, lifetime))
|
||||
.update(carrier, delta, cohort);
|
||||
}
|
||||
}
|
||||
|
||||
fn cohort_hash(context: CarrierLearningContext) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(COHORT_CONTEXT);
|
||||
digest.update(context.profile_key);
|
||||
digest.update([match context.class {
|
||||
CarrierClientClass::Legacy => 0,
|
||||
CarrierClientClass::Bridge => 1,
|
||||
CarrierClientClass::BrowserHint => 2,
|
||||
}]);
|
||||
digest.update(context.user_agent_hash);
|
||||
match context.client_ip {
|
||||
IpAddr::V4(address) => {
|
||||
digest.update([4]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
IpAddr::V6(address) => {
|
||||
digest.update([6]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::web::manager::{CarrierCapabilities, CarrierRequest};
|
||||
|
||||
fn request(hash: u8) -> CarrierRequest {
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
1,
|
||||
None,
|
||||
[hash; 32],
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evidence_is_bounded_and_expires_without_sliding() {
|
||||
let start = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let context = CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: "192.0.2.1".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [2; 32],
|
||||
};
|
||||
learning.record(
|
||||
start,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
learning.record(
|
||||
start + Duration::from_secs(5),
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
learning.prune(start + Duration::from_secs(11));
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_agent_and_ip_evidence_rank_stably() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(16);
|
||||
let context = CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: "192.0.2.1".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [2; 32],
|
||||
};
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&[WebCarrier::Https, WebCarrier::Websocket],
|
||||
request(2),
|
||||
context.profile_key,
|
||||
context.client_ip,
|
||||
);
|
||||
assert_eq!(ranked, [WebCarrier::Websocket, WebCarrier::Https]);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_evidence_requires_outcome_and_cohort_thresholds() {
|
||||
let now = Instant::now();
|
||||
let profile_key = [1; 32];
|
||||
let configured = [WebCarrier::Https, WebCarrier::Websocket];
|
||||
let unrelated_ip = "198.51.100.10".parse().unwrap();
|
||||
let mut learning = CarrierLearning::new(64);
|
||||
for cohort in 1..=3u8 {
|
||||
let context = CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip: IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, cohort)),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [cohort; 32],
|
||||
};
|
||||
for _ in 0..2 {
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
let (ranked, _) = learning.rank(
|
||||
now,
|
||||
&configured,
|
||||
request(99),
|
||||
profile_key,
|
||||
unrelated_ip,
|
||||
);
|
||||
assert_eq!(ranked, configured);
|
||||
|
||||
let fourth = CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip: "192.0.2.4".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [4; 32],
|
||||
};
|
||||
for _ in 0..2 {
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
fourth,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
}
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&configured,
|
||||
request(99),
|
||||
profile_key,
|
||||
unrelated_ip,
|
||||
);
|
||||
assert_eq!(ranked, [WebCarrier::Websocket, WebCarrier::Https]);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 32);
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ use std::time::{Duration, Instant};
|
||||
|
||||
use tracing::info;
|
||||
|
||||
use super::state::{ClosedToken, decrement_map, remove_bootstrap_locked, remove_expired_locked};
|
||||
use super::state::{
|
||||
decrement_map, remember_closed_token_locked, remove_bootstrap_locked, remove_expired_locked,
|
||||
};
|
||||
use super::{ProfileKey, TokenHash, WebProcessRuntime};
|
||||
|
||||
impl WebProcessRuntime {
|
||||
@@ -22,33 +24,20 @@ impl WebProcessRuntime {
|
||||
}
|
||||
decrement_map(&mut state.sessions_per_ip, &client_ip);
|
||||
decrement_map(&mut state.sessions_per_profile, &profile_key);
|
||||
let expiry = Instant::now()
|
||||
+ Duration::from_secs(
|
||||
remember_closed_token_locked(
|
||||
&mut state,
|
||||
hash,
|
||||
profile_host,
|
||||
Duration::from_secs(
|
||||
self.active_runtime
|
||||
.load()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.bootstrap_lifetime_secs,
|
||||
),
|
||||
self.limits.max_sessions_global.saturating_mul(16),
|
||||
);
|
||||
state.closed_tokens.insert(
|
||||
hash,
|
||||
ClosedToken {
|
||||
expires_at: expiry,
|
||||
host: profile_host.to_string(),
|
||||
},
|
||||
);
|
||||
while state.closed_tokens.len() > self.limits.max_sessions_global.saturating_mul(16) {
|
||||
let Some(oldest) = state
|
||||
.closed_tokens
|
||||
.iter()
|
||||
.min_by_key(|(_, closed)| closed.expires_at)
|
||||
.map(|(hash, _)| *hash)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
state.closed_tokens.remove(&oldest);
|
||||
}
|
||||
let bootstrap_hashes = state
|
||||
.bootstraps
|
||||
.iter()
|
||||
@@ -78,6 +67,7 @@ impl WebProcessRuntime {
|
||||
state.bootstraps_per_ip.clear();
|
||||
state.sessions.values().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
self.stream_admission.lock().closed = true;
|
||||
for session in &sessions {
|
||||
session.close();
|
||||
}
|
||||
@@ -96,10 +86,8 @@ impl WebProcessRuntime {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), waits).await;
|
||||
self.tasks.close();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), self.tasks.wait()).await;
|
||||
let (sessions_live, streams_live) = {
|
||||
let state = self.state.lock();
|
||||
(state.sessions.len(), state.streams_live)
|
||||
};
|
||||
let sessions_live = self.state.lock().sessions.len();
|
||||
let streams_live = self.stream_admission.lock().streams_live;
|
||||
let budget = self.data_budget.snapshot();
|
||||
info!(
|
||||
target: "telemt::web",
|
||||
@@ -124,13 +112,53 @@ impl WebProcessRuntime {
|
||||
pub(super) fn cleanup(&self) {
|
||||
self.cleanup_websockets();
|
||||
let now = Instant::now();
|
||||
let sessions = {
|
||||
let generation = self.active_generation();
|
||||
let config = &generation.config().web;
|
||||
let learning_enabled = config.carrier_negotiation_enabled() && config.carrier_learning;
|
||||
let mut learning = self.learning.lock();
|
||||
let _ = learning.apply_policy(
|
||||
now,
|
||||
learning_enabled,
|
||||
config.carrier_negotiation_aggressiveness,
|
||||
Duration::from_secs(config.timeouts.carrier_learning_secs),
|
||||
);
|
||||
learning.prune(now);
|
||||
drop(learning);
|
||||
let (sessions, expired_chains) = {
|
||||
let mut state = self.state.lock();
|
||||
let expired = state
|
||||
.bootstraps
|
||||
.iter()
|
||||
.filter_map(|(hash, bootstrap)| {
|
||||
(bootstrap.carrier_phase == super::state::CarrierChainPhase::Provisional
|
||||
&& bootstrap
|
||||
.carrier_deadline_at
|
||||
.is_some_and(|deadline| now >= deadline)
|
||||
&& bootstrap
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| !session.is_carrier_committed()))
|
||||
.then_some((*hash, bootstrap.session.clone()))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let expired_chains = expired
|
||||
.iter()
|
||||
.filter_map(|(_, session)| session.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for (hash, _) in expired {
|
||||
remove_bootstrap_locked(&mut state, hash);
|
||||
}
|
||||
remove_expired_locked(&mut state, now);
|
||||
state.sessions.values().cloned().collect::<Vec<_>>()
|
||||
(
|
||||
state.sessions.values().cloned().collect::<Vec<_>>(),
|
||||
expired_chains,
|
||||
)
|
||||
};
|
||||
for session in sessions.into_iter().filter(|session| session.is_idle(now)) {
|
||||
for session in expired_chains {
|
||||
session.close();
|
||||
}
|
||||
for session in sessions {
|
||||
session.close_if_due(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
use std::net::IpAddr;
|
||||
|
||||
use crate::config::WebCarrier;
|
||||
|
||||
/// Stable client classification used only for carrier negotiation and learning.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub(crate) enum CarrierClientClass {
|
||||
/// A client that did not present server-bridge negotiation metadata.
|
||||
Legacy,
|
||||
/// The generated bridge presented the explicit capability marker.
|
||||
Bridge,
|
||||
/// Strict same-origin browser metadata survived while the marker did not.
|
||||
BrowserHint,
|
||||
/// A native iOS client classified for diagnostics and learning only.
|
||||
Ios,
|
||||
}
|
||||
|
||||
impl CarrierClientClass {
|
||||
/// Returns the non-sensitive token exposed by WEB debug lifecycle records.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Legacy => "legacy",
|
||||
Self::Bridge => "bridge",
|
||||
Self::BrowserHint => "browser-hint",
|
||||
Self::Ios => "ios",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical carrier failure category reported by the generated bridge.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum CarrierFailure {
|
||||
/// The cumulative attempt deadline elapsed.
|
||||
Timeout,
|
||||
/// The browser observed a network failure.
|
||||
Network,
|
||||
/// A WebSocket upgrade or post-upgrade acknowledgement failed.
|
||||
Upgrade,
|
||||
/// The carrier returned an unexpected HTTP result.
|
||||
Http,
|
||||
/// The carrier violated its response or framing contract.
|
||||
Protocol,
|
||||
}
|
||||
|
||||
impl CarrierFailure {
|
||||
/// Parses one canonical bridge failure token.
|
||||
pub(crate) const fn parse(value: &str) -> Option<Self> {
|
||||
match value.as_bytes() {
|
||||
b"timeout" => Some(Self::Timeout),
|
||||
b"network" => Some(Self::Network),
|
||||
b"upgrade" => Some(Self::Upgrade),
|
||||
b"http" => Some(Self::Http),
|
||||
b"protocol" => Some(Self::Protocol),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the canonical non-sensitive failure token.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Timeout => "timeout",
|
||||
Self::Network => "network",
|
||||
Self::Upgrade => "upgrade",
|
||||
Self::Http => "http",
|
||||
Self::Protocol => "protocol",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validated carrier capability set sent by a negotiation-capable client.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CarrierCapabilities(u8);
|
||||
|
||||
impl CarrierCapabilities {
|
||||
/// Returns a set containing every carrier implemented by the generated bridge.
|
||||
pub(crate) const fn all() -> Self {
|
||||
Self(0b1111)
|
||||
}
|
||||
|
||||
/// Builds a set from a validated bit representation.
|
||||
pub(crate) const fn from_bits(bits: u8) -> Option<Self> {
|
||||
if bits != 0 && bits & !0b1111 == 0 {
|
||||
Some(Self(bits))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the bridge can run one carrier.
|
||||
pub(crate) const fn contains(self, carrier: WebCarrier) -> bool {
|
||||
self.0 & (1 << carrier.index()) != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable metadata attached to one session-creation attempt.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CarrierRequest {
|
||||
class: CarrierClientClass,
|
||||
capabilities: Option<CarrierCapabilities>,
|
||||
attempt: Option<u8>,
|
||||
failure: Option<CarrierFailure>,
|
||||
user_agent_hash: [u8; 32],
|
||||
}
|
||||
|
||||
impl CarrierRequest {
|
||||
/// Constructs legacy metadata without enabling negotiation or learning.
|
||||
pub(crate) const fn legacy(user_agent_hash: [u8; 32]) -> Self {
|
||||
Self {
|
||||
class: CarrierClientClass::Legacy,
|
||||
capabilities: None,
|
||||
attempt: None,
|
||||
failure: None,
|
||||
user_agent_hash,
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a metadata-free native client without inferring capabilities.
|
||||
pub(crate) const fn ios(user_agent_hash: [u8; 32]) -> Self {
|
||||
Self {
|
||||
class: CarrierClientClass::Ios,
|
||||
capabilities: None,
|
||||
attempt: None,
|
||||
failure: None,
|
||||
user_agent_hash,
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs validated automatic-negotiation metadata.
|
||||
pub(crate) const fn automatic(
|
||||
class: CarrierClientClass,
|
||||
capabilities: CarrierCapabilities,
|
||||
attempt: u8,
|
||||
failure: Option<CarrierFailure>,
|
||||
user_agent_hash: [u8; 32],
|
||||
) -> Self {
|
||||
Self {
|
||||
class,
|
||||
capabilities: Some(capabilities),
|
||||
attempt: Some(attempt),
|
||||
failure,
|
||||
user_agent_hash,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this request participates in server-side negotiation.
|
||||
pub(crate) const fn is_automatic(self) -> bool {
|
||||
self.capabilities.is_some()
|
||||
}
|
||||
|
||||
/// Returns whether server capability filtering applies to this request.
|
||||
pub(crate) const fn uses_capabilities(self) -> bool {
|
||||
self.capabilities.is_some()
|
||||
}
|
||||
|
||||
/// Returns the canonical attempt number when negotiation is active.
|
||||
pub(crate) const fn attempt(self) -> Option<u8> {
|
||||
self.attempt
|
||||
}
|
||||
|
||||
/// Returns the reported reason for advancing from the previous candidate.
|
||||
pub(crate) const fn failure(self) -> Option<CarrierFailure> {
|
||||
self.failure
|
||||
}
|
||||
|
||||
/// Returns whether a carrier is supported by this request.
|
||||
pub(crate) const fn supports(self, carrier: WebCarrier) -> bool {
|
||||
match self.capabilities {
|
||||
Some(capabilities) => capabilities.contains(carrier),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the stable non-sensitive client class.
|
||||
pub(crate) const fn class(self) -> CarrierClientClass {
|
||||
self.class
|
||||
}
|
||||
|
||||
/// Returns the normalized User-Agent digest.
|
||||
pub(crate) const fn user_agent_hash(self) -> [u8; 32] {
|
||||
self.user_agent_hash
|
||||
}
|
||||
|
||||
/// Checks the capability identity frozen across sequential attempts.
|
||||
pub(crate) fn matches_client(self, other: Self) -> bool {
|
||||
self.class == other.class
|
||||
&& self.capabilities_bits() == other.capabilities_bits()
|
||||
&& self.user_agent_hash == other.user_agent_hash
|
||||
}
|
||||
|
||||
/// Checks the complete idempotent identity of one exact attempt request.
|
||||
pub(crate) fn matches_attempt(self, other: Self) -> bool {
|
||||
self.matches_client(other) && self.attempt == other.attempt && self.failure == other.failure
|
||||
}
|
||||
|
||||
fn capabilities_bits(self) -> Option<u8> {
|
||||
self.capabilities.map(|capabilities| capabilities.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the cumulative deadline slot assigned to one carrier attempt.
|
||||
pub(super) const fn carrier_attempt_deadline_index(
|
||||
candidate_count: u8,
|
||||
attempt: u8,
|
||||
) -> Option<usize> {
|
||||
if candidate_count == 0 || candidate_count > 4 || attempt == 0 || attempt > candidate_count {
|
||||
return None;
|
||||
}
|
||||
if attempt == candidate_count {
|
||||
Some(3)
|
||||
} else {
|
||||
Some((attempt - 1) as usize)
|
||||
}
|
||||
}
|
||||
|
||||
/// Secret-independent evidence owner frozen into an automatic session.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CarrierLearningContext {
|
||||
/// Stable profile namespace.
|
||||
pub(crate) profile_key: super::ProfileKey,
|
||||
/// Effective client address from the trusted L7 boundary.
|
||||
pub(crate) client_ip: IpAddr,
|
||||
/// Client-class namespace for normalized User-Agent evidence.
|
||||
pub(crate) class: CarrierClientClass,
|
||||
/// Domain-separated normalized User-Agent digest.
|
||||
pub(crate) user_agent_hash: [u8; 32],
|
||||
/// Hot-reload epoch that rejects late outcomes from an older policy.
|
||||
pub(crate) epoch: u64,
|
||||
/// Whether the authoritative client address is safe to use as learning evidence.
|
||||
pub(crate) ip_learning_eligible: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::carrier_attempt_deadline_index;
|
||||
|
||||
#[test]
|
||||
fn final_candidate_uses_the_final_cumulative_deadline_slot() {
|
||||
assert_eq!(carrier_attempt_deadline_index(1, 1), Some(3));
|
||||
assert_eq!(carrier_attempt_deadline_index(2, 1), Some(0));
|
||||
assert_eq!(carrier_attempt_deadline_index(2, 2), Some(3));
|
||||
assert_eq!(carrier_attempt_deadline_index(3, 1), Some(0));
|
||||
assert_eq!(carrier_attempt_deadline_index(3, 2), Some(1));
|
||||
assert_eq!(carrier_attempt_deadline_index(3, 3), Some(3));
|
||||
assert_eq!(carrier_attempt_deadline_index(4, 1), Some(0));
|
||||
assert_eq!(carrier_attempt_deadline_index(4, 2), Some(1));
|
||||
assert_eq!(carrier_attempt_deadline_index(4, 3), Some(2));
|
||||
assert_eq!(carrier_attempt_deadline_index(4, 4), Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_candidate_or_attempt_counts_have_no_deadline_slot() {
|
||||
for (candidate_count, attempt) in [(0, 1), (5, 1), (1, 0), (1, 2), (3, 4)] {
|
||||
assert_eq!(
|
||||
carrier_attempt_deadline_index(candidate_count, attempt),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::state::{ManagerState, allow_rate};
|
||||
use super::{ProfileKey, WebProcessRuntime};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
|
||||
/// Applies process, address, profile, and rate ceilings to one initial session.
|
||||
pub(super) fn admit_initial(
|
||||
runtime: &WebProcessRuntime,
|
||||
state: &mut ManagerState,
|
||||
now: Instant,
|
||||
client_ip: IpAddr,
|
||||
profile_key: ProfileKey,
|
||||
profile: &WebRuntimeProfile,
|
||||
) -> bool {
|
||||
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
|
||||
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
< runtime.limits.max_sessions_per_ip
|
||||
&& state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
< profile.max_sessions
|
||||
&& allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
runtime.limits.new_sessions_per_minute,
|
||||
runtime.limits.new_sessions_burst,
|
||||
);
|
||||
if !admitted {
|
||||
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
admitted
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::negotiation::carrier_attempt_deadline_index;
|
||||
use super::session_admission::admit_initial;
|
||||
use super::state::{
|
||||
CarrierChainPhase, decrement_map, matching_profile, new_unique_token, profile_key,
|
||||
remember_closed_token_locked, remove_expired_locked,
|
||||
};
|
||||
use super::{
|
||||
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
|
||||
WebProcessRuntime,
|
||||
};
|
||||
use crate::config::{WebCarrier, WebRuntimeProfile};
|
||||
use crate::web::frame;
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::trace::TraceLifecycleEvent;
|
||||
|
||||
struct Replacement {
|
||||
old_session: Arc<WebSession>,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
profile_key: super::ProfileKey,
|
||||
trace_session_id: u64,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
request: CarrierRequest,
|
||||
scores: [i16; 4],
|
||||
learning_epoch: u64,
|
||||
ip_learning_eligible: bool,
|
||||
carrier_deadline_at: Instant,
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Creates, replays, or atomically supersedes one pre-commit carrier session.
|
||||
pub(crate) fn create_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
body: &[u8],
|
||||
carrier_request: CarrierRequest,
|
||||
ip_learning_eligible: bool,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !frame::validate_hello(body, &self.limits) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let body_digest: TokenHash = Sha256::digest(body).into();
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||
return Err(ManagerError::Authentication);
|
||||
};
|
||||
if entry.profile.host != host || now > entry.expires_at {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.used {
|
||||
if entry
|
||||
.carrier_deadline_at
|
||||
.is_some_and(|deadline| now >= deadline)
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| !session.is_carrier_committed())
|
||||
{
|
||||
let session = entry.session.clone();
|
||||
drop(state);
|
||||
if let Some(session) = session {
|
||||
session.close();
|
||||
}
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if entry.close_requested {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||
let client_matches = entry.session_client_ip == Some(client_ip)
|
||||
&& entry.session_ip_learning_eligible == ip_learning_eligible;
|
||||
let request_matches = entry
|
||||
.carrier_request
|
||||
.is_some_and(|current| current.matches_client(carrier_request));
|
||||
if !digest_matches || !client_matches || !request_matches {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.carrier_transitioning {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
if carrier_request.attempt() == Some(entry.carrier_attempt)
|
||||
|| (!carrier_request.is_automatic() && entry.carrier_attempt == 1)
|
||||
{
|
||||
if entry
|
||||
.carrier_request
|
||||
.is_none_or(|current| !current.matches_attempt(carrier_request))
|
||||
{
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||
let automatic = carrier_request.is_automatic();
|
||||
let carrier_state = if entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& session.is_carrier_committed()
|
||||
{
|
||||
CarrierChainPhase::CommittedPendingHealth.as_str()
|
||||
} else {
|
||||
entry.carrier_phase.as_str()
|
||||
};
|
||||
let result = CreateResult {
|
||||
token: entry.session_token.as_str().to_owned(),
|
||||
carrier: session.carrier(),
|
||||
attempt: carrier_request.attempt(),
|
||||
candidate_count: automatic
|
||||
.then(|| u8::try_from(entry.carrier_candidates.len()).unwrap_or(4)),
|
||||
deadline_secs: automatic
|
||||
.then_some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: automatic.then_some(carrier_state),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
drop(state);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
TraceLifecycleEvent::SessionReplayed,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
return Ok(result);
|
||||
}
|
||||
let next_attempt = entry.carrier_attempt.saturating_add(1);
|
||||
if !carrier_request.is_automatic()
|
||||
|| carrier_request.attempt() != Some(next_attempt)
|
||||
|| matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
)
|
||||
{
|
||||
return Err(
|
||||
if matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
) {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Protocol
|
||||
},
|
||||
);
|
||||
}
|
||||
let Some(carrier) = entry
|
||||
.carrier_candidates
|
||||
.get(usize::from(next_attempt - 1))
|
||||
.copied()
|
||||
else {
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
let candidate_count =
|
||||
u8::try_from(entry.carrier_candidates.len()).map_err(|_| ManagerError::Protocol)?;
|
||||
let deadline_index = carrier_attempt_deadline_index(candidate_count, next_attempt)
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
if entry.carrier_started_at.is_some_and(|started| {
|
||||
now.saturating_duration_since(started)
|
||||
>= Duration::from_secs(
|
||||
entry.profile.carrier_negotiation_deadlines_secs[deadline_index],
|
||||
)
|
||||
}) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let old_session = entry.session.clone().ok_or(ManagerError::Authentication)?;
|
||||
let replacement = Replacement {
|
||||
profile: Arc::clone(&entry.profile),
|
||||
profile_key: old_session.profile_key(),
|
||||
trace_session_id: entry.trace_session_id,
|
||||
old_session,
|
||||
attempt: next_attempt,
|
||||
carrier,
|
||||
request: carrier_request,
|
||||
scores: entry.carrier_scores,
|
||||
learning_epoch: entry.carrier_learning_epoch,
|
||||
ip_learning_eligible,
|
||||
carrier_deadline_at: entry.carrier_deadline_at.ok_or(ManagerError::Protocol)?,
|
||||
};
|
||||
state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?
|
||||
.carrier_transitioning = true;
|
||||
drop(state);
|
||||
return self.replace_session(bootstrap_hash, client_ip, replacement);
|
||||
}
|
||||
|
||||
if (carrier_request.is_automatic() && carrier_request.attempt() != Some(1))
|
||||
|| (!carrier_request.is_automatic() && carrier_request.attempt().is_some())
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let trace_session_id = entry.trace_session_id;
|
||||
let issued_profile = Arc::clone(&entry.profile);
|
||||
if state.closed || !config.web.enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &issued_profile))
|
||||
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
let profile_key = profile_key(&profile);
|
||||
if carrier_request.is_automatic() && !profile.carrier_negotiation_enabled {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let capability_selection =
|
||||
carrier_request.uses_capabilities() && profile.carrier_negotiation_enabled;
|
||||
let learning_policy = (
|
||||
config.web.carrier_negotiation_enabled() && config.web.carrier_learning,
|
||||
config.web.carrier_negotiation_aggressiveness,
|
||||
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
|
||||
);
|
||||
let (candidates, scores, learning_epoch) = if capability_selection
|
||||
&& profile.carrier_learning
|
||||
{
|
||||
let learning = self.learning.lock();
|
||||
if let Some(epoch) =
|
||||
learning.epoch_for_policy(learning_policy.0, learning_policy.1, learning_policy.2)
|
||||
{
|
||||
let (candidates, scores) = learning.rank(
|
||||
now,
|
||||
&profile.carriers,
|
||||
carrier_request,
|
||||
profile_key,
|
||||
client_ip,
|
||||
ip_learning_eligible,
|
||||
);
|
||||
(candidates, scores, Some(epoch))
|
||||
} else {
|
||||
(
|
||||
profile
|
||||
.carriers
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| carrier_request.supports(*carrier))
|
||||
.collect(),
|
||||
[0; 4],
|
||||
None,
|
||||
)
|
||||
}
|
||||
} else if capability_selection {
|
||||
(
|
||||
profile
|
||||
.carriers
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| carrier_request.supports(*carrier))
|
||||
.collect(),
|
||||
[0; 4],
|
||||
None,
|
||||
)
|
||||
} else if carrier_request.uses_capabilities() && !carrier_request.supports(profile.carrier)
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
} else {
|
||||
(vec![profile.carrier], [0; 4], None)
|
||||
};
|
||||
let Some(carrier) = candidates.first().copied() else {
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
if !admit_initial(self, &mut state, now, client_ip, profile_key, &profile) {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let carrier_deadline_at = carrier_request
|
||||
.is_automatic()
|
||||
.then_some(now + Duration::from_secs(profile.carrier_negotiation_deadlines_secs[3]));
|
||||
let learning_context = learning_epoch.map(|epoch| CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip,
|
||||
class: carrier_request.class(),
|
||||
user_agent_hash: carrier_request.user_agent_hash(),
|
||||
epoch,
|
||||
ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
trace_session_id,
|
||||
Arc::clone(&profile),
|
||||
profile_key,
|
||||
carrier,
|
||||
1,
|
||||
bootstrap_hash,
|
||||
carrier_deadline_at,
|
||||
carrier_request.class(),
|
||||
learning_context,
|
||||
carrier_request.is_automatic(),
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(carrier_request);
|
||||
entry.carrier_candidates = candidates.into();
|
||||
entry.carrier_scores = scores;
|
||||
entry.carrier_attempt = 1;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
||||
entry.carrier_deadline_at = carrier_deadline_at;
|
||||
entry.carrier_failures = [None; 3];
|
||||
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
||||
entry.expires_at = now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs);
|
||||
entry.session_client_ip = Some(client_ip);
|
||||
entry.session_ip_learning_eligible = ip_learning_eligible;
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
let candidate_count = u8::try_from(entry.carrier_candidates.len()).unwrap_or(4);
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
let identity = session.trace_identity();
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier,
|
||||
attempt: carrier_request.attempt(),
|
||||
candidate_count: carrier_request.is_automatic().then_some(candidate_count),
|
||||
deadline_secs: carrier_request
|
||||
.is_automatic()
|
||||
.then_some(profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: carrier_request
|
||||
.is_automatic()
|
||||
.then_some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
drop(state);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierClassified,
|
||||
carrier_request.class().as_str(),
|
||||
carrier,
|
||||
1,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierSelected,
|
||||
carrier_request.class().as_str(),
|
||||
carrier,
|
||||
1,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn replace_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
replacement: Replacement,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !replacement.old_session.begin_carrier_supersede() {
|
||||
let committed = replacement.old_session.is_carrier_committed();
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(if committed {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Closed
|
||||
});
|
||||
}
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
||||
entry.carrier_transitioning
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& !entry.close_requested
|
||||
&& entry.carrier_attempt.saturating_add(1) == replacement.attempt
|
||||
&& now < replacement.carrier_deadline_at
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session))
|
||||
}) && state
|
||||
.sessions
|
||||
.get(&replacement.old_session.token_hash())
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
|
||||
if !valid
|
||||
|| state.closed
|
||||
|| !config.web.enabled
|
||||
|| !generation
|
||||
.proxy_shared
|
||||
.is_user_enabled(&replacement.profile.user)
|
||||
{
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let learning_context = (replacement.profile.carrier_learning
|
||||
&& replacement.learning_epoch != 0)
|
||||
.then_some(CarrierLearningContext {
|
||||
profile_key: replacement.profile_key,
|
||||
client_ip,
|
||||
class: replacement.request.class(),
|
||||
user_agent_hash: replacement.request.user_agent_hash(),
|
||||
epoch: replacement.learning_epoch,
|
||||
ip_learning_eligible: replacement.ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
replacement.trace_session_id,
|
||||
Arc::clone(&replacement.profile),
|
||||
replacement.profile_key,
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
bootstrap_hash,
|
||||
Some(replacement.carrier_deadline_at),
|
||||
replacement.request.class(),
|
||||
learning_context,
|
||||
true,
|
||||
self.limits.clone(),
|
||||
replacement.old_session.timeouts().clone(),
|
||||
);
|
||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
session.close();
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let old_hash = replacement.old_session.token_hash();
|
||||
state.sessions.remove(&old_hash);
|
||||
remember_closed_token_locked(
|
||||
&mut state,
|
||||
old_hash,
|
||||
&replacement.profile.host,
|
||||
Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
||||
self.limits.max_sessions_global.saturating_mul(16),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(replacement.request);
|
||||
entry.carrier_attempt = replacement.attempt;
|
||||
entry.carrier_transitioning = false;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
if let Some(slot) = entry
|
||||
.carrier_failures
|
||||
.get_mut(usize::from(replacement.attempt.saturating_sub(2)))
|
||||
{
|
||||
*slot = Some(replacement.old_session.carrier());
|
||||
}
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: replacement.carrier,
|
||||
attempt: Some(replacement.attempt),
|
||||
candidate_count: Some(u8::try_from(entry.carrier_candidates.len()).unwrap_or(4)),
|
||||
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
let old_identity = replacement.old_session.trace_identity();
|
||||
drop(state);
|
||||
supersede.finish();
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity.clone(),
|
||||
TraceLifecycleEvent::CarrierFailed,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity,
|
||||
TraceLifecycleEvent::CarrierSuperseded,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierSelected,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
replacement.scores,
|
||||
None,
|
||||
);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
+96
-13
@@ -1,19 +1,36 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::Engine as _;
|
||||
use sha2::{Digest, Sha256};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{ProfileKey, TOKEN_BYTES, TokenHash};
|
||||
use crate::config::{WebRuntimeConfig, WebRuntimeProfile};
|
||||
use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
|
||||
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile};
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
const WEB_PROFILE_OWNER_CONTEXT: &[u8] = b"telemt-web-profile-owner-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum CarrierChainPhase {
|
||||
Provisional,
|
||||
CommittedPendingHealth,
|
||||
Healthy,
|
||||
}
|
||||
|
||||
impl CarrierChainPhase {
|
||||
pub(super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Provisional => "provisional",
|
||||
Self::CommittedPendingHealth => "committed",
|
||||
Self::Healthy => "healthy",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One issued bootstrap and optional idempotent session-creation replay state.
|
||||
pub(super) struct Bootstrap {
|
||||
/// Credential and replay-state expiry deadline.
|
||||
@@ -32,6 +49,32 @@ pub(super) struct Bootstrap {
|
||||
pub(super) session_token: Zeroizing<String>,
|
||||
/// Created session retained while retry replay remains valid.
|
||||
pub(super) session: Option<Arc<WebSession>>,
|
||||
/// Metadata that defines exact attempt replay and candidate advancement.
|
||||
pub(super) carrier_request: Option<CarrierRequest>,
|
||||
/// Learning-ranked carrier order frozen by the first automatic attempt.
|
||||
pub(super) carrier_candidates: Arc<[WebCarrier]>,
|
||||
/// Weighted learning scores captured when the candidate order was frozen.
|
||||
pub(super) carrier_scores: [i16; 4],
|
||||
/// Current one-based carrier attempt, or zero before session creation.
|
||||
pub(super) carrier_attempt: u8,
|
||||
/// Prevents concurrent retries from replacing the same attempt twice.
|
||||
pub(super) carrier_transitioning: bool,
|
||||
/// Manager-owned attempt-chain phase used for replacement linearization.
|
||||
pub(super) carrier_phase: CarrierChainPhase,
|
||||
/// Monotonic start of the first automatic session attempt.
|
||||
pub(super) carrier_started_at: Option<Instant>,
|
||||
/// Absolute server-side end of the automatic attempt chain.
|
||||
pub(super) carrier_deadline_at: Option<Instant>,
|
||||
/// Failed candidates staged until one winner becomes healthy.
|
||||
pub(super) carrier_failures: [Option<WebCarrier>; 3],
|
||||
/// Learning-policy epoch frozen by the first automatic attempt.
|
||||
pub(super) carrier_learning_epoch: u64,
|
||||
/// DELETE observed before an in-flight replacement committed its swap.
|
||||
pub(super) close_requested: bool,
|
||||
/// Effective address frozen by the first session-creation request.
|
||||
pub(super) session_client_ip: Option<IpAddr>,
|
||||
/// Whether the first request carried an authoritative public forwarded address.
|
||||
pub(super) session_ip_learning_eligible: bool,
|
||||
/// Distinguishes unused issuance quota from completed creation replay state.
|
||||
pub(super) used: bool,
|
||||
}
|
||||
@@ -56,6 +99,20 @@ struct StreamPortState {
|
||||
next: u16,
|
||||
}
|
||||
|
||||
/// Stream admission and KDF tuple ownership isolated from credential transitions.
|
||||
#[derive(Default)]
|
||||
pub(super) struct StreamAdmissionState {
|
||||
/// Process shutdown admission latch.
|
||||
pub(super) closed: bool,
|
||||
/// Live stream counts by stable profile key.
|
||||
pub(super) streams_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Process-wide live relay-task count.
|
||||
pub(super) streams_live: usize,
|
||||
stream_ports: HashMap<(IpAddr, SocketAddr), StreamPortState>,
|
||||
/// Logical-stream creation rate limiter.
|
||||
pub(super) stream_rate: RateState,
|
||||
}
|
||||
|
||||
/// Process-wide WEB registries and quota accounting protected by one short lock.
|
||||
#[derive(Default)]
|
||||
pub(super) struct ManagerState {
|
||||
@@ -71,17 +128,10 @@ pub(super) struct ManagerState {
|
||||
pub(super) sessions_per_ip: HashMap<IpAddr, usize>,
|
||||
/// Live session counts by stable profile key.
|
||||
pub(super) sessions_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Live relay-task counts by stable profile key.
|
||||
pub(super) streams_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Process-wide live relay-task count.
|
||||
pub(super) streams_live: usize,
|
||||
stream_ports: HashMap<(IpAddr, SocketAddr), StreamPortState>,
|
||||
/// Bootstrap issuance rate limiter.
|
||||
pub(super) bootstrap_rate: RateState,
|
||||
/// Session creation rate limiter.
|
||||
pub(super) session_rate: RateState,
|
||||
/// Logical-stream creation rate limiter.
|
||||
pub(super) stream_rate: RateState,
|
||||
/// Process shutdown admission latch.
|
||||
pub(super) closed: bool,
|
||||
}
|
||||
@@ -135,6 +185,11 @@ pub(super) fn matching_profile(
|
||||
&& profile.user == expected.user
|
||||
&& profile.secret_mode == expected.secret_mode
|
||||
&& profile.carrier == expected.carrier
|
||||
&& profile.carrier_negotiation_enabled == expected.carrier_negotiation_enabled
|
||||
&& profile.carrier_learning == expected.carrier_learning
|
||||
&& profile.carriers == expected.carriers
|
||||
&& profile.carrier_negotiation_deadlines_secs
|
||||
== expected.carrier_negotiation_deadlines_secs
|
||||
&& profile.capability == expected.capability
|
||||
&& profile.key_fingerprint == expected.key_fingerprint
|
||||
})
|
||||
@@ -198,6 +253,34 @@ pub(super) fn remove_bootstrap_locked(state: &mut ManagerState, hash: TokenHash)
|
||||
}
|
||||
}
|
||||
|
||||
/// Retains one bounded host-bound marker for an invalidated session credential.
|
||||
pub(super) fn remember_closed_token_locked(
|
||||
state: &mut ManagerState,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
lifetime: Duration,
|
||||
capacity: usize,
|
||||
) {
|
||||
state.closed_tokens.insert(
|
||||
hash,
|
||||
ClosedToken {
|
||||
expires_at: Instant::now() + lifetime,
|
||||
host: host.to_string(),
|
||||
},
|
||||
);
|
||||
while state.closed_tokens.len() > capacity {
|
||||
let Some(oldest) = state
|
||||
.closed_tokens
|
||||
.iter()
|
||||
.min_by_key(|(_, closed)| closed.expires_at)
|
||||
.map(|(hash, _)| *hash)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
state.closed_tokens.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrements one counted owner and removes its map entry at zero.
|
||||
pub(super) fn decrement_map<K, Q>(values: &mut HashMap<K, usize>, key: &Q)
|
||||
where
|
||||
@@ -217,7 +300,7 @@ where
|
||||
|
||||
/// Allocates a non-zero source port unique among live streams for one KDF route.
|
||||
pub(super) fn allocate_stream_port(
|
||||
state: &mut ManagerState,
|
||||
state: &mut StreamAdmissionState,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
) -> Option<u16> {
|
||||
@@ -240,7 +323,7 @@ pub(super) fn allocate_stream_port(
|
||||
|
||||
/// Releases one source port and reclaims empty per-route allocator state.
|
||||
pub(super) fn release_stream_port(
|
||||
state: &mut ManagerState,
|
||||
state: &mut StreamAdmissionState,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
@@ -262,7 +345,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn synthetic_ports_are_unique_per_live_route_and_state_is_reclaimed() {
|
||||
let mut state = ManagerState::default();
|
||||
let mut state = StreamAdmissionState::default();
|
||||
let client_ip = "192.0.2.10".parse().unwrap();
|
||||
let public_addr = "203.0.113.10:443".parse().unwrap();
|
||||
let first = allocate_stream_port(&mut state, client_ip, public_addr).unwrap();
|
||||
|
||||
+231
-50
@@ -1,7 +1,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
@@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
|
||||
|
||||
/// One process-owned WebSocket carrier class used for eviction priority.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub(crate) enum WebSocketKind {
|
||||
/// One connection multiplexes every logical stream in a session.
|
||||
Multiplex,
|
||||
@@ -18,23 +18,42 @@ pub(crate) enum WebSocketKind {
|
||||
Lane(u32),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
struct WebSocketClaimKey {
|
||||
session_hash: super::TokenHash,
|
||||
kind: WebSocketKind,
|
||||
}
|
||||
|
||||
#[repr(u8)]
|
||||
enum WebSocketPhase {
|
||||
Claimed,
|
||||
Upgraded,
|
||||
Active,
|
||||
Closing,
|
||||
}
|
||||
|
||||
pub(super) struct WebSocketEntry {
|
||||
id: u64,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
claim: WebSocketClaimKey,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
liveness_interval_ms: u64,
|
||||
created_tick: u64,
|
||||
last_peer_tick: AtomicU64,
|
||||
last_progress_tick: AtomicU64,
|
||||
opened: AtomicBool,
|
||||
phase: AtomicU8,
|
||||
closing: AtomicBool,
|
||||
cancel: CancellationToken,
|
||||
released: CancellationToken,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct WebSocketRegistry {
|
||||
entries: HashMap<u64, Arc<WebSocketEntry>>,
|
||||
claims: HashMap<WebSocketClaimKey, u64>,
|
||||
evictions_in_flight: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
@@ -63,9 +82,45 @@ impl WebSocketConnection {
|
||||
}
|
||||
|
||||
/// Marks successful ownership transfer from HTTP to the WebSocket codec.
|
||||
pub(crate) fn mark_opened(&self) {
|
||||
self.entry.opened.store(true, Ordering::Release);
|
||||
pub(crate) fn mark_opened(&self) -> bool {
|
||||
if self.entry.closing.load(Ordering::Acquire)
|
||||
|| self
|
||||
.entry
|
||||
.phase
|
||||
.compare_exchange(
|
||||
WebSocketPhase::Claimed as u8,
|
||||
WebSocketPhase::Upgraded as u8,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_err()
|
||||
|| self.entry.closing.load(Ordering::Acquire)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.mark_progress();
|
||||
true
|
||||
}
|
||||
|
||||
/// Marks the first validated carrier binary message as active progress.
|
||||
pub(crate) fn mark_active(&self) -> bool {
|
||||
if self.entry.closing.load(Ordering::Acquire)
|
||||
|| self
|
||||
.entry
|
||||
.phase
|
||||
.compare_exchange(
|
||||
WebSocketPhase::Upgraded as u8,
|
||||
WebSocketPhase::Active as u8,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_err()
|
||||
|| self.entry.closing.load(Ordering::Acquire)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.mark_peer_activity();
|
||||
true
|
||||
}
|
||||
|
||||
/// Refreshes the peer-liveness deadline after any received WebSocket message.
|
||||
@@ -90,93 +145,167 @@ impl WebSocketConnection {
|
||||
impl Drop for WebSocketConnection {
|
||||
fn drop(&mut self) {
|
||||
if let Some(runtime) = self.runtime.upgrade() {
|
||||
runtime.websockets.lock().entries.remove(&self.entry.id);
|
||||
let mut registry = runtime.websockets.lock();
|
||||
registry.entries.remove(&self.entry.id);
|
||||
if registry.claims.get(&self.entry.claim) == Some(&self.entry.id) {
|
||||
registry.claims.remove(&self.entry.claim);
|
||||
}
|
||||
if self.entry.closing.load(Ordering::Acquire) {
|
||||
if registry.evictions_in_flight == 0 {
|
||||
registry.closed = true;
|
||||
} else {
|
||||
registry.evictions_in_flight -= 1;
|
||||
}
|
||||
}
|
||||
drop(registry);
|
||||
drop(self.base_budget.take());
|
||||
drop(self.slot.take());
|
||||
self.entry.released.cancel();
|
||||
runtime.websocket_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn admit(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
session_hash: super::TokenHash,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval: Duration,
|
||||
eviction_timeout: Duration,
|
||||
parent_cancellation: CancellationToken,
|
||||
) -> Result<WebSocketConnection, ManagerError> {
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
|
||||
if let Some(connection) = try_admit(
|
||||
match try_admit(
|
||||
runtime,
|
||||
owner,
|
||||
session_id,
|
||||
session_hash,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval_ms,
|
||||
&parent_cancellation,
|
||||
) {
|
||||
return Ok(connection);
|
||||
Ok(connection) => return Ok(connection),
|
||||
Err(TryAdmitError::Conflict) => return Err(ManagerError::Concurrent),
|
||||
Err(TryAdmitError::Closed) => return Err(ManagerError::Closed),
|
||||
Err(TryAdmitError::Capacity) => {}
|
||||
}
|
||||
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None) else {
|
||||
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None, true) else {
|
||||
runtime.record_limit_hit();
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let released = runtime.websocket_notify.notified();
|
||||
let released = victim.released.cancelled();
|
||||
victim.cancel.cancel();
|
||||
let _ = tokio::time::timeout(eviction_timeout, released).await;
|
||||
try_admit(
|
||||
tokio::select! {
|
||||
_ = parent_cancellation.cancelled() => return Err(ManagerError::Closed),
|
||||
_ = tokio::time::timeout(eviction_timeout, released) => {}
|
||||
}
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
match try_admit(
|
||||
runtime,
|
||||
owner,
|
||||
session_id,
|
||||
session_hash,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval_ms,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
&parent_cancellation,
|
||||
) {
|
||||
Ok(connection) => Ok(connection),
|
||||
Err(TryAdmitError::Conflict) => Err(ManagerError::Concurrent),
|
||||
Err(TryAdmitError::Closed) => Err(ManagerError::Closed),
|
||||
Err(TryAdmitError::Capacity) => {
|
||||
runtime.record_limit_hit();
|
||||
ManagerError::Limit
|
||||
})
|
||||
Err(ManagerError::Limit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum TryAdmitError {
|
||||
Capacity,
|
||||
Conflict,
|
||||
Closed,
|
||||
}
|
||||
|
||||
fn try_admit(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
session_hash: super::TokenHash,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval_ms: u64,
|
||||
) -> Option<WebSocketConnection> {
|
||||
parent_cancellation: &CancellationToken,
|
||||
) -> Result<WebSocketConnection, TryAdmitError> {
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
let claim = WebSocketClaimKey { session_hash, kind };
|
||||
{
|
||||
let registry = runtime.websockets.lock();
|
||||
if registry.closed {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
if registry.claims.contains_key(&claim) {
|
||||
return Err(TryAdmitError::Conflict);
|
||||
}
|
||||
}
|
||||
let slot = Arc::clone(&runtime.websocket_connections)
|
||||
.try_acquire_owned()
|
||||
.ok()?;
|
||||
let base_budget = runtime.try_websocket_base_budget(owner, base_bytes)?;
|
||||
let id = runtime.websocket_next_id.fetch_add(1, Ordering::Relaxed);
|
||||
.map_err(|_| TryAdmitError::Capacity)?;
|
||||
let base_budget = runtime
|
||||
.try_websocket_base_budget(owner, base_bytes)
|
||||
.ok_or(TryAdmitError::Capacity)?;
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
let id = runtime
|
||||
.websocket_next_id
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
|
||||
value.checked_add(1)
|
||||
})
|
||||
.map_err(|_| TryAdmitError::Capacity)?;
|
||||
let now = runtime.websocket_tick();
|
||||
let entry = Arc::new(WebSocketEntry {
|
||||
id,
|
||||
owner,
|
||||
session_id,
|
||||
claim,
|
||||
client_ip,
|
||||
kind,
|
||||
liveness_interval_ms,
|
||||
created_tick: now,
|
||||
last_peer_tick: AtomicU64::new(now),
|
||||
last_progress_tick: AtomicU64::new(now),
|
||||
opened: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
phase: AtomicU8::new(WebSocketPhase::Claimed as u8),
|
||||
closing: AtomicBool::new(false),
|
||||
cancel: parent_cancellation.child_token(),
|
||||
released: CancellationToken::new(),
|
||||
});
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if registry.closed {
|
||||
return None;
|
||||
if registry.closed || parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
if registry.claims.contains_key(&claim) {
|
||||
return Err(TryAdmitError::Conflict);
|
||||
}
|
||||
registry.claims.insert(claim, id);
|
||||
registry.entries.insert(id, Arc::clone(&entry));
|
||||
drop(registry);
|
||||
Some(WebSocketConnection {
|
||||
Ok(WebSocketConnection {
|
||||
runtime: Arc::downgrade(runtime),
|
||||
entry,
|
||||
slot: Some(slot),
|
||||
@@ -191,22 +320,13 @@ impl WebProcessRuntime {
|
||||
|
||||
pub(super) fn cleanup_websockets(&self) {
|
||||
let now = self.websocket_tick();
|
||||
let mut victims = self
|
||||
.websockets
|
||||
.lock()
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| {
|
||||
now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire))
|
||||
>= dead_after(entry)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if victims.is_empty()
|
||||
&& self.data_budget.take_pressure()
|
||||
&& let Some(victim) = select_pressure_victim(self, now)
|
||||
{
|
||||
let mut victims = claim_stale_victims(self, now);
|
||||
if victims.is_empty() && self.data_budget.take_pressure() {
|
||||
if let Some(victim) = select_pressure_victim(self, now, true) {
|
||||
victims.push(victim);
|
||||
} else {
|
||||
self.data_budget.restore_pressure();
|
||||
}
|
||||
}
|
||||
for victim in victims {
|
||||
victim.cancel.cancel();
|
||||
@@ -231,16 +351,20 @@ fn select_victim(
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
excluded_id: Option<u64>,
|
||||
claim: bool,
|
||||
) -> Option<Arc<WebSocketEntry>> {
|
||||
let fair_share = runtime.data_budget.fair_share(Some(owner));
|
||||
let requester_usage = runtime.data_budget.owner_usage(owner);
|
||||
let now = runtime.websocket_tick();
|
||||
runtime
|
||||
.websockets
|
||||
.lock()
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
|
||||
return None;
|
||||
}
|
||||
let selected = registry
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| Some(entry.id) != excluded_id)
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.filter_map(|entry| {
|
||||
let owner_rank = if entry.session_id == session_id {
|
||||
0
|
||||
@@ -269,15 +393,26 @@ fn select_victim(
|
||||
))
|
||||
})
|
||||
.min_by_key(|(key, _)| *key)
|
||||
.map(|(_, entry)| entry)
|
||||
.map(|(_, entry)| entry)?;
|
||||
if claim && !claim_entry(&mut registry, &selected, runtime) {
|
||||
return None;
|
||||
}
|
||||
Some(selected)
|
||||
}
|
||||
|
||||
fn select_pressure_victim(runtime: &WebProcessRuntime, now: u64) -> Option<Arc<WebSocketEntry>> {
|
||||
runtime
|
||||
.websockets
|
||||
.lock()
|
||||
fn select_pressure_victim(
|
||||
runtime: &WebProcessRuntime,
|
||||
now: u64,
|
||||
claim: bool,
|
||||
) -> Option<Arc<WebSocketEntry>> {
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
|
||||
return None;
|
||||
}
|
||||
let selected = registry
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.map(|entry| {
|
||||
(
|
||||
(
|
||||
@@ -290,11 +425,57 @@ fn select_pressure_victim(runtime: &WebProcessRuntime, now: u64) -> Option<Arc<W
|
||||
)
|
||||
})
|
||||
.min_by_key(|(key, _)| *key)
|
||||
.map(|(_, entry)| entry)
|
||||
.map(|(_, entry)| entry)?;
|
||||
if claim && !claim_entry(&mut registry, &selected, runtime) {
|
||||
return None;
|
||||
}
|
||||
Some(selected)
|
||||
}
|
||||
|
||||
fn claim_stale_victims(runtime: &WebProcessRuntime, now: u64) -> Vec<Arc<WebSocketEntry>> {
|
||||
let mut registry = runtime.websockets.lock();
|
||||
let available = runtime
|
||||
.limits
|
||||
.max_websocket_evictions_in_flight
|
||||
.saturating_sub(registry.evictions_in_flight);
|
||||
let candidates = registry
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.filter(|entry| {
|
||||
now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
})
|
||||
.take(available)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|entry| claim_entry(&mut registry, entry, runtime))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn claim_entry(
|
||||
registry: &mut WebSocketRegistry,
|
||||
entry: &Arc<WebSocketEntry>,
|
||||
runtime: &WebProcessRuntime,
|
||||
) -> bool {
|
||||
if registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight
|
||||
|| entry
|
||||
.closing
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
entry
|
||||
.phase
|
||||
.store(WebSocketPhase::Closing as u8, Ordering::Release);
|
||||
registry.evictions_in_flight += 1;
|
||||
true
|
||||
}
|
||||
|
||||
fn entry_priority(entry: &WebSocketEntry, now: u64) -> u8 {
|
||||
if !entry.opened.load(Ordering::Acquire)
|
||||
if entry.phase.load(Ordering::Acquire) < WebSocketPhase::Active as u8
|
||||
|| now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
{
|
||||
0
|
||||
|
||||
@@ -1,18 +1,29 @@
|
||||
use super::*;
|
||||
|
||||
fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
|
||||
let phase = if opened {
|
||||
WebSocketPhase::Active
|
||||
} else {
|
||||
WebSocketPhase::Claimed
|
||||
};
|
||||
WebSocketEntry {
|
||||
id: 1,
|
||||
owner: [0; 32],
|
||||
session_id: 1,
|
||||
claim: WebSocketClaimKey {
|
||||
session_hash: [0; 32],
|
||||
kind,
|
||||
},
|
||||
client_ip: "192.0.2.10".parse().unwrap(),
|
||||
kind,
|
||||
liveness_interval_ms: 10,
|
||||
created_tick: 1,
|
||||
last_peer_tick: AtomicU64::new(peer_tick),
|
||||
last_progress_tick: AtomicU64::new(peer_tick),
|
||||
opened: AtomicBool::new(opened),
|
||||
phase: AtomicU8::new(phase as u8),
|
||||
closing: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
released: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+165
-107
@@ -2,9 +2,9 @@ use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::io;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize};
|
||||
use std::task::{Context, Poll, Waker};
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use parking_lot::Mutex;
|
||||
@@ -14,17 +14,30 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebTimeoutsConfig};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::{ProfileKey, TokenHash, WebProcessRuntime};
|
||||
use crate::web::manager::{
|
||||
CarrierClientClass, CarrierLearningContext, ProfileKey, TokenHash, WebProcessRuntime,
|
||||
};
|
||||
|
||||
// Backend tasks own generation admission and authenticated MTProxy relay lifetimes.
|
||||
mod backend;
|
||||
// Downlink queues own cursor replay, flow control, and memory reservations.
|
||||
mod downlink;
|
||||
// Response ownership keeps detached batches charged until the last body clone drops.
|
||||
mod resident;
|
||||
// Lane carrier state isolates request sequencing and downlink replay per logical stream.
|
||||
mod lanes;
|
||||
// Lane batch staging transfers queue ownership without escaping process budgets.
|
||||
mod lane_downlink;
|
||||
// Lane uplink creation remains transactional across validation and queue reservations.
|
||||
mod lane_uplink;
|
||||
// WebSocket carrier state owns pre-OPEN lane reservations and failure isolation.
|
||||
mod websocket;
|
||||
pub(crate) use websocket::WebSocketLaneReservation;
|
||||
pub(crate) use websocket::WebSocketProbeReservation;
|
||||
// Carrier commit and health evidence share one session-locked state machine.
|
||||
mod negotiation;
|
||||
// Session closure and carrier-attempt transitions share one cancellation boundary.
|
||||
mod lifecycle;
|
||||
// Uplink batches own exactly-once sequencing and client-frame validation.
|
||||
mod uplink;
|
||||
|
||||
@@ -43,7 +56,14 @@ struct InboundChunk {
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StreamIdentity {
|
||||
pub(crate) id: u32,
|
||||
pub(crate) instance: u64,
|
||||
}
|
||||
|
||||
struct StreamState {
|
||||
instance: u64,
|
||||
inbound: VecDeque<InboundChunk>,
|
||||
receive_window: u32,
|
||||
send_credit: u64,
|
||||
@@ -61,15 +81,21 @@ struct QueuedFrame {
|
||||
|
||||
struct DownBatch {
|
||||
body: Bytes,
|
||||
lease: Arc<resident::PendingResponseLease>,
|
||||
base_cursor: u64,
|
||||
next_cursor: u64,
|
||||
data_bytes: usize,
|
||||
data_items: usize,
|
||||
control_bytes: usize,
|
||||
control_items: usize,
|
||||
carrier_health_eligible: bool,
|
||||
}
|
||||
|
||||
struct CarrierLane {
|
||||
instance: u64,
|
||||
pending_bytes: usize,
|
||||
pending_items: usize,
|
||||
resident: Arc<resident::ResidentCounters>,
|
||||
pending_frames: VecDeque<QueuedFrame>,
|
||||
pending_windows: HashMap<u32, usize>,
|
||||
unacked: Option<DownBatch>,
|
||||
@@ -82,8 +108,12 @@ struct CarrierLane {
|
||||
}
|
||||
|
||||
impl CarrierLane {
|
||||
fn new() -> Self {
|
||||
fn new(instance: u64) -> Self {
|
||||
Self {
|
||||
instance,
|
||||
pending_bytes: 0,
|
||||
pending_items: 0,
|
||||
resident: Arc::new(resident::ResidentCounters::default()),
|
||||
pending_frames: VecDeque::new(),
|
||||
pending_windows: HashMap::new(),
|
||||
unacked: None,
|
||||
@@ -99,6 +129,8 @@ impl CarrierLane {
|
||||
|
||||
struct SessionState {
|
||||
streams: HashMap<u32, StreamState>,
|
||||
closing_streams: HashMap<u32, u64>,
|
||||
next_stream_instance: u64,
|
||||
active_peer_ports: HashSet<u16>,
|
||||
closed_streams: HashSet<u32>,
|
||||
closed_order: VecDeque<u32>,
|
||||
@@ -110,15 +142,38 @@ struct SessionState {
|
||||
last_up_sequence: u64,
|
||||
last_up_digest: TokenHash,
|
||||
carrier_lanes: HashMap<u32, CarrierLane>,
|
||||
lane_open_waits: usize,
|
||||
next_lane_instance: u64,
|
||||
websocket_lane_reservations: HashMap<u32, u16>,
|
||||
pending_bytes: usize,
|
||||
pending_items: usize,
|
||||
pending_control_bytes: usize,
|
||||
pending_control_items: usize,
|
||||
last_activity: Instant,
|
||||
negotiation_phase: SessionNegotiationPhase,
|
||||
carrier_health_due_at: Option<Instant>,
|
||||
carrier_health_activity_at: Option<Instant>,
|
||||
carrier_health_uplink: bool,
|
||||
carrier_health_downlink: bool,
|
||||
carrier_commit_published: bool,
|
||||
carrier_health_reported: bool,
|
||||
websocket_carrier_active: bool,
|
||||
websocket_commit_ack_pending: bool,
|
||||
websocket_commit_ack_owner: Option<u64>,
|
||||
websocket_commit_ack_written: bool,
|
||||
websocket_probe_claimed: bool,
|
||||
close_requested: bool,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum SessionNegotiationPhase {
|
||||
Uncommitted,
|
||||
Replacing,
|
||||
Committed,
|
||||
Superseded,
|
||||
}
|
||||
|
||||
/// One bounded WEB carrier session containing logical MTProxy streams.
|
||||
pub(crate) struct WebSession {
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
@@ -127,13 +182,22 @@ pub(crate) struct WebSession {
|
||||
trace_session_id: u64,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
profile_key: ProfileKey,
|
||||
selected_carrier: WebCarrier,
|
||||
carrier_attempt: u8,
|
||||
bootstrap_hash: TokenHash,
|
||||
carrier_deadline_at: Option<Instant>,
|
||||
carrier_class: CarrierClientClass,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
automatic_carrier: bool,
|
||||
limits: WebLimitsConfig,
|
||||
timeouts: WebTimeoutsConfig,
|
||||
state: Mutex<SessionState>,
|
||||
down_notify: Arc<Notify>,
|
||||
lane_open_notify: Arc<Notify>,
|
||||
cancel: CancellationToken,
|
||||
tasks_live: AtomicUsize,
|
||||
tasks_done: Arc<Notify>,
|
||||
resident: Arc<resident::ResidentCounters>,
|
||||
finished: AtomicBool,
|
||||
up_active: AtomicBool,
|
||||
}
|
||||
@@ -158,12 +222,21 @@ impl WebSession {
|
||||
trace_session_id: u64,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
profile_key: ProfileKey,
|
||||
selected_carrier: WebCarrier,
|
||||
carrier_attempt: u8,
|
||||
bootstrap_hash: TokenHash,
|
||||
carrier_deadline_at: Option<Instant>,
|
||||
carrier_class: CarrierClientClass,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
automatic_carrier: bool,
|
||||
limits: WebLimitsConfig,
|
||||
timeouts: WebTimeoutsConfig,
|
||||
) -> Arc<Self> {
|
||||
let mut carrier_lanes = HashMap::new();
|
||||
if profile.carrier == WebCarrier::HttpsLanes {
|
||||
carrier_lanes.insert(0, CarrierLane::new());
|
||||
let mut next_lane_instance = 1;
|
||||
if selected_carrier == WebCarrier::HttpsLanes {
|
||||
carrier_lanes.insert(0, CarrierLane::new(next_lane_instance));
|
||||
next_lane_instance += 1;
|
||||
}
|
||||
Arc::new(Self {
|
||||
manager,
|
||||
@@ -172,10 +245,19 @@ impl WebSession {
|
||||
trace_session_id,
|
||||
profile,
|
||||
profile_key,
|
||||
selected_carrier,
|
||||
carrier_attempt,
|
||||
bootstrap_hash,
|
||||
carrier_deadline_at,
|
||||
carrier_class,
|
||||
learning_context,
|
||||
automatic_carrier,
|
||||
limits,
|
||||
timeouts,
|
||||
state: Mutex::new(SessionState {
|
||||
streams: HashMap::new(),
|
||||
closing_streams: HashMap::new(),
|
||||
next_stream_instance: 1,
|
||||
active_peer_ports: HashSet::new(),
|
||||
closed_streams: HashSet::new(),
|
||||
closed_order: VecDeque::new(),
|
||||
@@ -187,18 +269,35 @@ impl WebSession {
|
||||
last_up_sequence: 0,
|
||||
last_up_digest: [0; 32],
|
||||
carrier_lanes,
|
||||
lane_open_waits: 0,
|
||||
next_lane_instance,
|
||||
websocket_lane_reservations: HashMap::new(),
|
||||
pending_bytes: 0,
|
||||
pending_items: 0,
|
||||
pending_control_bytes: 0,
|
||||
pending_control_items: 0,
|
||||
last_activity: Instant::now(),
|
||||
negotiation_phase: SessionNegotiationPhase::Uncommitted,
|
||||
carrier_health_due_at: None,
|
||||
carrier_health_activity_at: None,
|
||||
carrier_health_uplink: false,
|
||||
carrier_health_downlink: false,
|
||||
carrier_commit_published: false,
|
||||
carrier_health_reported: false,
|
||||
websocket_carrier_active: false,
|
||||
websocket_commit_ack_pending: false,
|
||||
websocket_commit_ack_owner: None,
|
||||
websocket_commit_ack_written: false,
|
||||
websocket_probe_claimed: false,
|
||||
close_requested: false,
|
||||
closed: false,
|
||||
}),
|
||||
down_notify: Arc::new(Notify::new()),
|
||||
lane_open_notify: Arc::new(Notify::new()),
|
||||
cancel: CancellationToken::new(),
|
||||
tasks_live: AtomicUsize::new(0),
|
||||
tasks_done: Arc::new(Notify::new()),
|
||||
resident: Arc::new(resident::ResidentCounters::default()),
|
||||
finished: AtomicBool::new(false),
|
||||
up_active: AtomicBool::new(false),
|
||||
})
|
||||
@@ -216,7 +315,7 @@ impl WebSession {
|
||||
|
||||
/// Returns the immutable carrier selected when this session was created.
|
||||
pub(crate) fn carrier(&self) -> WebCarrier {
|
||||
self.profile.carrier
|
||||
self.selected_carrier
|
||||
}
|
||||
|
||||
/// Returns the stable quota owner without exposing profile credentials.
|
||||
@@ -229,6 +328,11 @@ impl WebSession {
|
||||
self.trace_session_id
|
||||
}
|
||||
|
||||
/// Creates a child cancellation boundary for one owned carrier task.
|
||||
pub(crate) fn carrier_cancellation(&self) -> CancellationToken {
|
||||
self.cancel.child_token()
|
||||
}
|
||||
|
||||
/// Returns a cloned non-secret identity only for enabled debug capture.
|
||||
pub(crate) fn trace_identity(&self) -> crate::web::trace::TraceIdentity {
|
||||
crate::web::trace::TraceIdentity::from_profile(self.trace_session_id, &self.profile)
|
||||
@@ -253,96 +357,34 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||
pub(crate) fn close(&self) {
|
||||
let (data_bytes, data_items, control_bytes, control_items) = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return;
|
||||
}
|
||||
state.closed = true;
|
||||
for stream in state.streams.values_mut() {
|
||||
if let Some(waker) = stream.read_waker.take() {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(waker) = stream.write_waker.take() {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
state.streams.clear();
|
||||
state.pending_frames.clear();
|
||||
state.pending_windows.clear();
|
||||
state.unacked = None;
|
||||
for lane in state.carrier_lanes.values() {
|
||||
lane.notify.notify_waiters();
|
||||
}
|
||||
state.carrier_lanes.clear();
|
||||
let control_bytes = state.pending_control_bytes;
|
||||
let control_items = state.pending_control_items;
|
||||
let data_bytes = state.pending_bytes.saturating_sub(control_bytes);
|
||||
let data_items = state.pending_items.saturating_sub(control_items);
|
||||
state.pending_bytes = 0;
|
||||
state.pending_items = 0;
|
||||
state.pending_control_bytes = 0;
|
||||
state.pending_control_items = 0;
|
||||
(data_bytes, data_items, control_bytes, control_items)
|
||||
};
|
||||
self.cancel.cancel();
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_pending(self.profile_key, data_bytes, data_items, false);
|
||||
manager.release_pending(self.profile_key, control_bytes, control_items, true);
|
||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||
None,
|
||||
Some("closed"),
|
||||
);
|
||||
manager.session_finished(
|
||||
self.token_hash,
|
||||
self.client_ip,
|
||||
self.profile_key,
|
||||
&self.profile.host,
|
||||
);
|
||||
}
|
||||
}
|
||||
/// Returns the immutable limits frozen when this carrier chain was created.
|
||||
pub(crate) fn limits(&self) -> &WebLimitsConfig {
|
||||
&self.limits
|
||||
}
|
||||
|
||||
/// Waits for all logical-stream tasks after admission has closed.
|
||||
pub(crate) async fn wait(&self) {
|
||||
loop {
|
||||
let notified = self.tasks_done.notified();
|
||||
if self.tasks_live.load(Ordering::Acquire) == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether reconnect grace elapsed without activity.
|
||||
pub(crate) fn is_idle(&self, now: Instant) -> bool {
|
||||
let state = self.state.lock();
|
||||
!state.closed
|
||||
&& now.saturating_duration_since(state.last_activity)
|
||||
>= Duration::from_secs(self.timeouts.reconnect_grace_secs)
|
||||
/// Returns the immutable timeouts frozen when this carrier chain was created.
|
||||
pub(crate) fn timeouts(&self) -> &WebTimeoutsConfig {
|
||||
&self.timeouts
|
||||
}
|
||||
|
||||
/// Polls client-to-server bytes and returns consumed flow-control credit.
|
||||
pub(super) fn poll_read(
|
||||
&self,
|
||||
stream_id: u32,
|
||||
stream: StreamIdentity,
|
||||
cx: &mut Context<'_>,
|
||||
output: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let mut state = self.state.lock();
|
||||
let (count, finished) = {
|
||||
let Some(stream) = state.streams.get_mut(&stream_id) else {
|
||||
let Some(stream_state) = state
|
||||
.streams
|
||||
.get_mut(&stream.id)
|
||||
.filter(|state| state.instance == stream.instance)
|
||||
else {
|
||||
return Poll::Ready(Ok(()));
|
||||
};
|
||||
let Some(chunk) = stream.inbound.front_mut() else {
|
||||
stream.read_waker = Some(cx.waker().clone());
|
||||
let Some(chunk) = stream_state.inbound.front_mut() else {
|
||||
stream_state.read_waker = Some(cx.waker().clone());
|
||||
return Poll::Pending;
|
||||
};
|
||||
let available = &chunk.bytes[chunk.offset..];
|
||||
@@ -351,14 +393,14 @@ impl WebSession {
|
||||
chunk.offset += count;
|
||||
let finished = chunk.offset == chunk.bytes.len();
|
||||
if finished {
|
||||
stream.inbound.pop_front();
|
||||
stream_state.inbound.pop_front();
|
||||
}
|
||||
stream.receive_window = stream.receive_window.saturating_add(count as u32);
|
||||
stream_state.receive_window = stream_state.receive_window.saturating_add(count as u32);
|
||||
(count, finished)
|
||||
};
|
||||
let overhead = if finished { QUEUE_ITEM_COST } else { 0 };
|
||||
self.release_locked(&mut state, count + overhead, usize::from(finished), false);
|
||||
if !self.queue_window_locked(&mut state, stream_id, count as u32) {
|
||||
if !self.queue_window_locked(&mut state, stream.id, count as u32) {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Poll::Ready(Err(io::Error::other(
|
||||
@@ -371,7 +413,7 @@ impl WebSession {
|
||||
/// Polls server-to-client writes against stream credit and bounded queues.
|
||||
pub(super) fn poll_write(
|
||||
&self,
|
||||
stream_id: u32,
|
||||
stream: StreamIdentity,
|
||||
cx: &mut Context<'_>,
|
||||
input: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
@@ -379,7 +421,11 @@ impl WebSession {
|
||||
return Poll::Ready(Ok(0));
|
||||
}
|
||||
let mut state = self.state.lock();
|
||||
let Some(stream) = state.streams.get_mut(&stream_id) else {
|
||||
let Some(stream_state) = state
|
||||
.streams
|
||||
.get_mut(&stream.id)
|
||||
.filter(|state| state.instance == stream.instance)
|
||||
else {
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::BrokenPipe,
|
||||
"WEB logical stream is closed",
|
||||
@@ -389,24 +435,39 @@ impl WebSession {
|
||||
.len()
|
||||
.min(frame::DATA_CHUNK_BYTES)
|
||||
.min(self.limits.max_frame_payload_bytes)
|
||||
.min(stream.send_credit as usize);
|
||||
.min(if self.carrier().uses_lanes() {
|
||||
self.limits
|
||||
.pending_bytes_per_lane
|
||||
.saturating_sub(frame::HEADER_BYTES + QUEUE_ITEM_COST)
|
||||
} else {
|
||||
usize::MAX
|
||||
})
|
||||
.min(stream_state.send_credit as usize);
|
||||
if count == 0 {
|
||||
stream.write_waker = Some(cx.waker().clone());
|
||||
stream_state.write_waker = Some(cx.waker().clone());
|
||||
return Poll::Pending;
|
||||
}
|
||||
if !self.queue_data_locked(&mut state, stream_id, &input[..count]) {
|
||||
if let Some(stream) = state.streams.get_mut(&stream_id) {
|
||||
stream.write_waker = Some(cx.waker().clone());
|
||||
if !self.queue_data_locked(&mut state, stream.id, &input[..count]) {
|
||||
if let Some(stream_state) = state
|
||||
.streams
|
||||
.get_mut(&stream.id)
|
||||
.filter(|state| state.instance == stream.instance)
|
||||
{
|
||||
stream_state.write_waker = Some(cx.waker().clone());
|
||||
}
|
||||
return Poll::Pending;
|
||||
}
|
||||
let Some(stream) = state.streams.get_mut(&stream_id) else {
|
||||
let Some(stream_state) = state
|
||||
.streams
|
||||
.get_mut(&stream.id)
|
||||
.filter(|state| state.instance == stream.instance)
|
||||
else {
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::BrokenPipe,
|
||||
"WEB logical stream is closed",
|
||||
)));
|
||||
};
|
||||
stream.send_credit -= count as u64;
|
||||
stream_state.send_credit -= count as u64;
|
||||
state.last_activity = Instant::now();
|
||||
drop(state);
|
||||
if self.carrier().is_multiplexed() {
|
||||
@@ -421,18 +482,6 @@ impl WebSession {
|
||||
.upgrade()
|
||||
.map(|manager| manager.budget_notify())
|
||||
}
|
||||
|
||||
fn release_stream_reservation(&self, peer_port: u16) {
|
||||
let removed = self.state.lock().active_peer_ports.remove(&peer_port);
|
||||
if removed && let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn inbound_queue_cost(queue: &VecDeque<InboundChunk>) -> (usize, usize) {
|
||||
@@ -456,3 +505,12 @@ fn remember_closed(state: &mut SessionState, stream_id: u32, limit: usize) -> Op
|
||||
}
|
||||
evicted
|
||||
}
|
||||
|
||||
fn insert_carrier_lane(state: &mut SessionState, lane_id: u32) -> Option<u64> {
|
||||
let instance = state.next_lane_instance;
|
||||
state.next_lane_instance = instance.checked_add(1)?;
|
||||
state
|
||||
.carrier_lanes
|
||||
.insert(lane_id, CarrierLane::new(instance));
|
||||
Some(instance)
|
||||
}
|
||||
|
||||
+106
-51
@@ -7,7 +7,7 @@ use crate::proxy::shared_state::ConntrackClosePolicy;
|
||||
use crate::web::frame::FrameType;
|
||||
use crate::web::stream::WebLogicalStream;
|
||||
|
||||
use super::{WebSession, inbound_queue_cost};
|
||||
use super::{StreamIdentity, WebSession, inbound_queue_cost};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
@@ -17,61 +17,64 @@ impl WebSession {
|
||||
/// Starts one owned inner handshake and relay task for an admitted stream.
|
||||
pub(super) fn spawn_stream(
|
||||
self: &Arc<Self>,
|
||||
stream_id: u32,
|
||||
peer_port: u16,
|
||||
completion: StreamCompletion,
|
||||
retain_reservation_on_reject: bool,
|
||||
) -> bool {
|
||||
let stream = completion.stream;
|
||||
let peer_port = completion.peer_port;
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
completion
|
||||
.retain_rejected
|
||||
.store(retain_reservation_on_reject, Ordering::Release);
|
||||
drop(completion);
|
||||
return false;
|
||||
};
|
||||
let generation = manager.active_generation();
|
||||
if !*generation.admission_rx.borrow() {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some(stream.id),
|
||||
Some("admission_closed"),
|
||||
);
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
completion
|
||||
.retain_rejected
|
||||
.store(retain_reservation_on_reject, Ordering::Release);
|
||||
drop(completion);
|
||||
return false;
|
||||
}
|
||||
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
|
||||
manager.record_stream_rejected();
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some(stream.id),
|
||||
Some("connection_limit"),
|
||||
);
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
completion
|
||||
.retain_rejected
|
||||
.store(retain_reservation_on_reject, Ordering::Release);
|
||||
drop(completion);
|
||||
return false;
|
||||
};
|
||||
let deps = generation.client_runtime_deps();
|
||||
let replay_checker = Arc::clone(&generation.replay_checker);
|
||||
let session = Arc::clone(self);
|
||||
let cancel = self.cancel.clone();
|
||||
let retain_rejected = Arc::new(AtomicBool::new(false));
|
||||
self.tasks_live.fetch_add(1, Ordering::AcqRel);
|
||||
let completion = StreamCompletion {
|
||||
session: Arc::clone(&session),
|
||||
stream_id,
|
||||
peer_port,
|
||||
retain_rejected: Arc::clone(&retain_rejected),
|
||||
};
|
||||
let retain_rejected = Arc::clone(&completion.retain_rejected);
|
||||
let future = async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let _completion = completion;
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamAdmitted,
|
||||
Some(stream_id),
|
||||
Some(stream.id),
|
||||
None,
|
||||
);
|
||||
let stream = WebLogicalStream::new(Arc::clone(&session), stream_id);
|
||||
let logical_stream = WebLogicalStream::new(Arc::clone(&session), stream);
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = run_stream(
|
||||
Arc::clone(&session),
|
||||
stream_id,
|
||||
stream,
|
||||
logical_stream,
|
||||
deps,
|
||||
replay_checker,
|
||||
peer_port,
|
||||
@@ -82,7 +85,7 @@ impl WebSession {
|
||||
retain_rejected.store(retain_reservation_on_reject, Ordering::Release);
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some(stream.id),
|
||||
Some("generation_closed"),
|
||||
);
|
||||
drop(future);
|
||||
@@ -93,21 +96,32 @@ impl WebSession {
|
||||
|
||||
fn stream_rejected_before_spawn(
|
||||
&self,
|
||||
stream_id: u32,
|
||||
stream: StreamIdentity,
|
||||
peer_port: u16,
|
||||
retain_reservation: bool,
|
||||
) {
|
||||
if !retain_reservation {
|
||||
self.stream_finished(stream_id, peer_port);
|
||||
self.stream_finished(stream, peer_port);
|
||||
return;
|
||||
}
|
||||
let queued = {
|
||||
let mut state = self.state.lock();
|
||||
state.streams.remove(&stream_id).map(|stream| {
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
if state.closing_streams.get(&stream.id) == Some(&stream.instance) {
|
||||
state.closing_streams.remove(&stream.id);
|
||||
self.remember_closed_locked(&mut state, stream.id);
|
||||
}
|
||||
state
|
||||
.streams
|
||||
.get(&stream.id)
|
||||
.filter(|state| state.instance == stream.instance)
|
||||
.is_some()
|
||||
.then(|| state.streams.remove(&stream.id))
|
||||
.flatten()
|
||||
.map(|stream_state| {
|
||||
let (bytes, items) = inbound_queue_cost(&stream_state.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
self.remember_closed_locked(&mut state, stream_id);
|
||||
self.queue_control_locked(&mut state, FrameType::Close, stream_id, &[])
|
||||
self.remember_closed_locked(&mut state, stream.id);
|
||||
self.queue_control_locked(&mut state, FrameType::Close, stream.id, &[])
|
||||
})
|
||||
};
|
||||
if queued.is_some_and(|queued| !queued) {
|
||||
@@ -115,16 +129,27 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_finished(&self, stream_id: u32, peer_port: u16) {
|
||||
fn stream_finished(&self, stream: StreamIdentity, peer_port: u16) {
|
||||
let (queued, reserved) = {
|
||||
let mut state = self.state.lock();
|
||||
let reserved = state.active_peer_ports.remove(&peer_port);
|
||||
let queued = state.streams.remove(&stream_id).map(|stream| {
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
let current = state
|
||||
.streams
|
||||
.get(&stream.id)
|
||||
.is_some_and(|state| state.instance == stream.instance);
|
||||
let queued = current
|
||||
.then(|| state.streams.remove(&stream.id))
|
||||
.flatten()
|
||||
.map(|stream_state| {
|
||||
let (bytes, items) = inbound_queue_cost(&stream_state.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
self.remember_closed_locked(&mut state, stream_id);
|
||||
self.queue_control_locked(&mut state, FrameType::Close, stream_id, &[])
|
||||
self.remember_closed_locked(&mut state, stream.id);
|
||||
self.queue_control_locked(&mut state, FrameType::Close, stream.id, &[])
|
||||
});
|
||||
if state.closing_streams.get(&stream.id) == Some(&stream.instance) {
|
||||
state.closing_streams.remove(&stream.id);
|
||||
self.remember_closed_locked(&mut state, stream.id);
|
||||
}
|
||||
(queued, reserved)
|
||||
};
|
||||
if reserved && let Some(manager) = self.manager.upgrade() {
|
||||
@@ -146,25 +171,41 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
struct StreamCompletion {
|
||||
pub(super) struct StreamCompletion {
|
||||
session: Arc<WebSession>,
|
||||
stream_id: u32,
|
||||
peer_port: u16,
|
||||
pub(super) stream: StreamIdentity,
|
||||
pub(super) peer_port: u16,
|
||||
retain_rejected: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
pub(super) fn own_stream_task(
|
||||
self: &Arc<Self>,
|
||||
stream: StreamIdentity,
|
||||
peer_port: u16,
|
||||
) -> StreamCompletion {
|
||||
self.tasks_live.fetch_add(1, Ordering::AcqRel);
|
||||
StreamCompletion {
|
||||
session: Arc::clone(self),
|
||||
stream,
|
||||
peer_port,
|
||||
retain_rejected: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StreamCompletion {
|
||||
fn drop(&mut self) {
|
||||
self.session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamClosed,
|
||||
Some(self.stream_id),
|
||||
Some(self.stream.id),
|
||||
None,
|
||||
);
|
||||
if self.retain_rejected.load(Ordering::Acquire) {
|
||||
self.session
|
||||
.stream_rejected_before_spawn(self.stream_id, self.peer_port, true);
|
||||
.stream_rejected_before_spawn(self.stream, self.peer_port, true);
|
||||
} else {
|
||||
self.session.stream_finished(self.stream_id, self.peer_port);
|
||||
self.session.stream_finished(self.stream, self.peer_port);
|
||||
}
|
||||
if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 {
|
||||
self.session.tasks_done.notify_waiters();
|
||||
@@ -174,7 +215,7 @@ impl Drop for StreamCompletion {
|
||||
|
||||
async fn run_stream(
|
||||
session: Arc<WebSession>,
|
||||
stream_id: u32,
|
||||
stream_identity: StreamIdentity,
|
||||
stream: WebLogicalStream,
|
||||
deps: crate::proxy::authenticated::ClientRuntimeDeps,
|
||||
replay_checker: Arc<crate::stats::ReplayChecker>,
|
||||
@@ -191,13 +232,27 @@ async fn run_stream(
|
||||
let peer = std::net::SocketAddr::new(session.client_ip, peer_port);
|
||||
deps.stats.increment_connects_all();
|
||||
|
||||
// A carrier may publish OPEN before the local MTProto socket writes its
|
||||
// first byte. Session and stream quotas bound this idle phase without
|
||||
// consuming the process-wide active-handshake budget.
|
||||
if reader.read_exact(&mut handshake[..1]).await.is_err() {
|
||||
// Silent OPEN ownership has a separate absolute deadline so it cannot
|
||||
// consume stream and tuple quotas indefinitely before handshake admission.
|
||||
let first_byte = tokio::time::timeout(
|
||||
Duration::from_secs(session.timeouts.stream_first_byte_secs),
|
||||
reader.read_exact(&mut handshake[..1]),
|
||||
)
|
||||
.await;
|
||||
if first_byte.is_err() {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeTimeout,
|
||||
Some(stream_identity.id),
|
||||
Some("first_byte_timeout"),
|
||||
);
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_first_byte_timeout");
|
||||
return;
|
||||
}
|
||||
if first_byte.is_ok_and(|result| result.is_err()) {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some("first_byte_io"),
|
||||
);
|
||||
deps.stats
|
||||
@@ -206,7 +261,7 @@ async fn run_stream(
|
||||
}
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamFirstByte,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
None,
|
||||
);
|
||||
let Some(manager) = session.manager.upgrade() else {
|
||||
@@ -215,7 +270,7 @@ async fn run_stream(
|
||||
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some("handshake_limit"),
|
||||
);
|
||||
return;
|
||||
@@ -246,7 +301,7 @@ async fn run_stream(
|
||||
Err(_) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeTimeout,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some("timeout"),
|
||||
);
|
||||
deps.stats
|
||||
@@ -258,7 +313,7 @@ async fn run_stream(
|
||||
Ok(Err(_)) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some("io"),
|
||||
);
|
||||
deps.stats
|
||||
@@ -268,7 +323,7 @@ async fn run_stream(
|
||||
Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeSucceeded,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
None,
|
||||
);
|
||||
(reader, writer, success)
|
||||
@@ -276,7 +331,7 @@ async fn run_stream(
|
||||
Ok(Ok(_)) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeRejected,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some("bad_client"),
|
||||
);
|
||||
deps.stats
|
||||
@@ -286,7 +341,7 @@ async fn run_stream(
|
||||
};
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::RelayStarted,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
None,
|
||||
);
|
||||
let relay_result = run_authenticated(
|
||||
@@ -301,7 +356,7 @@ async fn run_stream(
|
||||
.await;
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::RelayEnded,
|
||||
Some(stream_id),
|
||||
Some(stream_identity.id),
|
||||
Some(if relay_result.is_ok() {
|
||||
"completed"
|
||||
} else {
|
||||
|
||||
@@ -70,6 +70,10 @@ fn test_runtime_with_dc(
|
||||
user: "default".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier,
|
||||
carrier_negotiation_enabled: false,
|
||||
carrier_learning: true,
|
||||
carriers: Arc::from([carrier]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [7; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 4,
|
||||
@@ -120,6 +124,13 @@ fn test_runtime_with_dc(
|
||||
1,
|
||||
profile,
|
||||
[7; 32],
|
||||
carrier,
|
||||
1,
|
||||
[9; 32],
|
||||
None,
|
||||
crate::web::manager::CarrierClientClass::Legacy,
|
||||
None,
|
||||
false,
|
||||
limits,
|
||||
timeouts,
|
||||
);
|
||||
@@ -323,10 +334,11 @@ async fn silent_streams_do_not_consume_active_handshake_capacity() {
|
||||
assert_eq!(runtime.process_frame(2, 4, FrameType::Data, &[2]), Ok(4));
|
||||
settle_tasks().await;
|
||||
|
||||
{
|
||||
let state = runtime.session.state.lock();
|
||||
assert!(state.streams.contains_key(&1));
|
||||
assert!(!state.streams.contains_key(&2));
|
||||
drop(state);
|
||||
}
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -361,3 +373,17 @@ async fn cancellation_while_waiting_for_data_releases_stream_ownership() {
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausted_stream_identity_does_not_acquire_synthetic_port_ownership() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
runtime.session.state.lock().next_stream_instance = u64::MAX;
|
||||
|
||||
assert_eq!(
|
||||
runtime.process_frame(1, 1, FrameType::Open, &[]),
|
||||
Err(ManagerError::Closed)
|
||||
);
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
+75
-121
@@ -1,7 +1,9 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
|
||||
use super::resident::{OwnedBatchBody, PendingCounts, PendingResponseLease};
|
||||
use super::{
|
||||
DownBatch, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState, WebSession,
|
||||
};
|
||||
@@ -14,7 +16,7 @@ impl WebSession {
|
||||
if !self.carrier().is_multiplexed() {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let epoch = {
|
||||
let (epoch, healthy) = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
@@ -33,15 +35,29 @@ impl WebSession {
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let carrier_health_eligible = unacked.carrier_health_eligible;
|
||||
self.release_unacked_locked(&mut state);
|
||||
state.carrier_health_downlink |= carrier_health_eligible;
|
||||
if carrier_health_eligible {
|
||||
state.carrier_health_activity_at = Some(Instant::now());
|
||||
}
|
||||
} else if cursor != state.down_cursor {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
state.down_epoch = state.down_epoch.wrapping_add(1).max(1);
|
||||
state.down_epoch
|
||||
let Some(epoch) = state.down_epoch.checked_add(1) else {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
state.down_epoch = epoch;
|
||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||
(state.down_epoch, healthy)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
self.down_notify.notify_waiters();
|
||||
|
||||
let deadline = Duration::from_secs(self.timeouts.long_poll_secs);
|
||||
@@ -60,6 +76,9 @@ impl WebSession {
|
||||
if !state.pending_frames.is_empty() {
|
||||
let batch = match self.take_down_batch_locked(&mut state, cursor) {
|
||||
Ok(batch) => batch,
|
||||
Err(ManagerError::Backpressure) => {
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
Err(error) => {
|
||||
drop(state);
|
||||
self.close();
|
||||
@@ -121,6 +140,15 @@ impl WebSession {
|
||||
.limits
|
||||
.pending_items_per_session
|
||||
.saturating_sub(item_reserve);
|
||||
let resident = self.resident.snapshot();
|
||||
let pending_bytes = state.pending_bytes.saturating_add(resident.bytes());
|
||||
let pending_items = state.pending_items.saturating_add(resident.items());
|
||||
let pending_control_bytes = state
|
||||
.pending_control_bytes
|
||||
.saturating_add(resident.control_bytes);
|
||||
let pending_control_items = state
|
||||
.pending_control_items
|
||||
.saturating_add(resident.control_items);
|
||||
if state.closed {
|
||||
return false;
|
||||
}
|
||||
@@ -128,20 +156,14 @@ impl WebSession {
|
||||
let fits = if control {
|
||||
bytes <= self.limits.control_bytes_per_session
|
||||
&& items <= item_reserve
|
||||
&& state.pending_bytes
|
||||
<= self.limits.pending_bytes_per_session.saturating_sub(bytes)
|
||||
&& state.pending_items
|
||||
<= self.limits.pending_items_per_session.saturating_sub(items)
|
||||
&& state.pending_control_bytes
|
||||
&& pending_bytes <= self.limits.pending_bytes_per_session.saturating_sub(bytes)
|
||||
&& pending_items <= self.limits.pending_items_per_session.saturating_sub(items)
|
||||
&& pending_control_bytes
|
||||
<= self.limits.control_bytes_per_session.saturating_sub(bytes)
|
||||
&& state.pending_control_items <= item_reserve.saturating_sub(items)
|
||||
&& pending_control_items <= item_reserve.saturating_sub(items)
|
||||
} else {
|
||||
let data_bytes = state
|
||||
.pending_bytes
|
||||
.saturating_sub(state.pending_control_bytes);
|
||||
let data_items = state
|
||||
.pending_items
|
||||
.saturating_sub(state.pending_control_items);
|
||||
let data_bytes = pending_bytes.saturating_sub(pending_control_bytes);
|
||||
let data_items = pending_items.saturating_sub(pending_control_items);
|
||||
let (byte_limit, item_limit) = if class == PendingClass::Downlink {
|
||||
let uplink_bytes = self.limits.max_body_bytes.saturating_add(
|
||||
self.limits
|
||||
@@ -203,6 +225,21 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn release_local_locked(
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) {
|
||||
state.pending_bytes = state.pending_bytes.saturating_sub(bytes);
|
||||
state.pending_items = state.pending_items.saturating_sub(items);
|
||||
if control {
|
||||
state.pending_control_bytes = state.pending_control_bytes.saturating_sub(bytes);
|
||||
state.pending_control_items = state.pending_control_items.saturating_sub(items);
|
||||
}
|
||||
}
|
||||
|
||||
/// Coalesces one flow-control update into the bounded control queue.
|
||||
pub(super) fn queue_window_locked(
|
||||
&self,
|
||||
@@ -351,6 +388,12 @@ impl WebSession {
|
||||
body_len += queued.encoded.len();
|
||||
count += 1;
|
||||
}
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let Some(_staging) = manager.try_downlink_staging_budget(body_len) else {
|
||||
return Err(ManagerError::Backpressure);
|
||||
};
|
||||
let mut body = BytesMut::with_capacity(body_len);
|
||||
let mut data_bytes = 0usize;
|
||||
let mut data_items = 0usize;
|
||||
@@ -383,14 +426,25 @@ impl WebSession {
|
||||
*index = index.saturating_sub(count);
|
||||
}
|
||||
state.down_cursor = next_cursor;
|
||||
let counts = PendingCounts {
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
};
|
||||
let lease = PendingResponseLease::new(self, counts, None);
|
||||
let body = Bytes::from_owner(OwnedBatchBody::new(body.freeze(), Arc::clone(&lease)));
|
||||
Ok(DownBatch {
|
||||
body: body.freeze(),
|
||||
body,
|
||||
lease,
|
||||
base_cursor: cursor,
|
||||
next_cursor,
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
carrier_health_eligible: state.negotiation_phase
|
||||
== super::SessionNegotiationPhase::Committed,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -398,8 +452,9 @@ impl WebSession {
|
||||
let Some(batch) = state.unacked.take() else {
|
||||
return;
|
||||
};
|
||||
self.release_locked(state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_locked(state, batch.control_bytes, batch.control_items, true);
|
||||
batch.lease.detach();
|
||||
self.release_local_locked(state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(state, batch.control_bytes, batch.control_items, true);
|
||||
for stream in state.streams.values_mut() {
|
||||
if let Some(waker) = stream.write_waker.take() {
|
||||
waker.wake();
|
||||
@@ -409,106 +464,5 @@ impl WebSession {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::config::{
|
||||
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn session() -> Arc<WebSession> {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: SocketAddr::from(([203, 0, 113, 10], 443)),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::Https,
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
});
|
||||
WebSession::new(
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebLimitsConfig::default(),
|
||||
WebTimeoutsConfig::default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn queue_close(session: &WebSession) {
|
||||
let encoded = frame::encode(FrameType::Close, 1, &[]);
|
||||
session.state.lock().pending_frames.push_back(QueuedFrame {
|
||||
encoded: BytesMut::from(encoded.as_ref()),
|
||||
frame_type: FrameType::Close,
|
||||
stream_id: 1,
|
||||
control: true,
|
||||
cost: frame::HEADER_BYTES + QUEUE_ITEM_COST,
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downlink_replays_unacknowledged_batch_byte_for_byte() {
|
||||
let session = session();
|
||||
queue_close(&session);
|
||||
let first = session.poll_down(0).await.unwrap();
|
||||
let replay = session.poll_down(0).await.unwrap();
|
||||
assert_eq!(first.next_cursor, 1);
|
||||
assert_eq!(replay.next_cursor, 1);
|
||||
assert_eq!(first.body, replay.body);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_or_overflowing_cursor_closes_session() {
|
||||
let invalid = session();
|
||||
assert!(matches!(
|
||||
invalid.poll_down(1).await,
|
||||
Err(ManagerError::Protocol)
|
||||
));
|
||||
assert!(invalid.state.lock().closed);
|
||||
|
||||
let overflow = session();
|
||||
{
|
||||
let mut state = overflow.state.lock();
|
||||
state.down_cursor = u64::MAX;
|
||||
}
|
||||
queue_close(&overflow);
|
||||
assert!(matches!(
|
||||
overflow.poll_down(u64::MAX).await,
|
||||
Err(ManagerError::Protocol)
|
||||
));
|
||||
assert!(overflow.state.lock().closed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn newer_poll_supersedes_older_poll_without_closing_session() {
|
||||
let session = session();
|
||||
let first_session = Arc::clone(&session);
|
||||
let first = tokio::spawn(async move { first_session.poll_down(0).await });
|
||||
while session.state.lock().down_epoch < 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let second_session = Arc::clone(&session);
|
||||
let second = tokio::spawn(async move { second_session.poll_down(0).await });
|
||||
while session.state.lock().down_epoch < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let superseded = tokio::time::timeout(Duration::from_secs(1), first)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(superseded.body.is_empty());
|
||||
assert_eq!(superseded.next_cursor, 0);
|
||||
assert!(!session.state.lock().closed);
|
||||
second.abort();
|
||||
}
|
||||
}
|
||||
#[path = "downlink_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
use super::*;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
|
||||
use crate::config::{
|
||||
ProxyConfig, WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn session() -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||
let generation = test_runtime_generation(1, ProxyConfig::default());
|
||||
let manager = WebProcessRuntime::start(Arc::new(ArcSwap::from(generation)));
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: SocketAddr::from(([203, 0, 113, 10], 443)),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::Https,
|
||||
carrier_negotiation_enabled: false,
|
||||
carrier_learning: false,
|
||||
carriers: Arc::from([WebCarrier::Https]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
});
|
||||
let mut timeouts = WebTimeoutsConfig::default();
|
||||
timeouts.long_poll_secs = 1;
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(&manager),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebCarrier::Https,
|
||||
1,
|
||||
[3; 32],
|
||||
None,
|
||||
crate::web::manager::CarrierClientClass::Legacy,
|
||||
None,
|
||||
false,
|
||||
WebLimitsConfig::default(),
|
||||
timeouts,
|
||||
);
|
||||
(session, manager)
|
||||
}
|
||||
|
||||
fn queue_close(session: &WebSession) {
|
||||
let mut state = session.state.lock();
|
||||
assert!(session.queue_control_locked(&mut state, FrameType::Close, 1, &[]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downlink_replays_unacknowledged_batch_byte_for_byte() {
|
||||
let (session, manager) = session();
|
||||
queue_close(&session);
|
||||
let first = session.poll_down(0).await.unwrap();
|
||||
let replay = session.poll_down(0).await.unwrap();
|
||||
assert_eq!(first.next_cursor, 1);
|
||||
assert_eq!(replay.next_cursor, 1);
|
||||
assert_eq!(first.body, replay.body);
|
||||
drop(first);
|
||||
drop(replay);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acknowledged_response_stays_resident_until_the_last_body_clone_drops() {
|
||||
let (session, manager) = session();
|
||||
queue_close(&session);
|
||||
let response = session.poll_down(0).await.unwrap();
|
||||
let retained = response.body.clone();
|
||||
{
|
||||
let mut state = session.state.lock();
|
||||
session.release_unacked_locked(&mut state);
|
||||
assert_eq!(state.pending_bytes, 0);
|
||||
}
|
||||
assert!(session.resident.snapshot().bytes() > 0);
|
||||
drop(response);
|
||||
assert!(session.resident.snapshot().bytes() > 0);
|
||||
drop(retained);
|
||||
assert_eq!(session.resident.snapshot().bytes(), 0);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_or_overflowing_cursor_closes_session() {
|
||||
let (invalid, invalid_manager) = session();
|
||||
assert!(matches!(
|
||||
invalid.poll_down(1).await,
|
||||
Err(ManagerError::Protocol)
|
||||
));
|
||||
assert!(invalid.state.lock().closed);
|
||||
invalid_manager.shutdown().await;
|
||||
|
||||
let (overflow, overflow_manager) = session();
|
||||
{
|
||||
let mut state = overflow.state.lock();
|
||||
state.down_cursor = u64::MAX;
|
||||
}
|
||||
queue_close(&overflow);
|
||||
assert!(matches!(
|
||||
overflow.poll_down(u64::MAX).await,
|
||||
Err(ManagerError::Protocol)
|
||||
));
|
||||
assert!(overflow.state.lock().closed);
|
||||
overflow_manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn newer_poll_supersedes_older_poll_without_closing_session() {
|
||||
let (session, manager) = session();
|
||||
let first_session = Arc::clone(&session);
|
||||
let first = tokio::spawn(async move { first_session.poll_down(0).await });
|
||||
while session.state.lock().down_epoch < 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let second_session = Arc::clone(&session);
|
||||
let second = tokio::spawn(async move { second_session.poll_down(0).await });
|
||||
while session.state.lock().down_epoch < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let superseded = tokio::time::timeout(Duration::from_secs(1), first)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(superseded.body.is_empty());
|
||||
assert_eq!(superseded.next_cursor, 0);
|
||||
assert!(!session.state.lock().closed);
|
||||
second.abort();
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
|
||||
use super::resident::{OwnedBatchBody, PendingCounts, PendingResponseLease};
|
||||
use super::{CarrierLane, DownBatch, WebSession};
|
||||
use crate::config::WebLimitsConfig;
|
||||
use crate::web::frame::FrameType;
|
||||
use crate::web::manager::ManagerError;
|
||||
|
||||
/// Stages one bounded lane batch and transfers its accounting into response ownership.
|
||||
pub(super) fn take_lane_down_batch(
|
||||
session: &WebSession,
|
||||
limits: &WebLimitsConfig,
|
||||
lane: &mut CarrierLane,
|
||||
cursor: u64,
|
||||
carrier_health_eligible: bool,
|
||||
) -> Result<DownBatch, ManagerError> {
|
||||
let next_cursor = lane
|
||||
.down_cursor
|
||||
.checked_add(1)
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
let mut count = 0usize;
|
||||
let mut body_len = 0usize;
|
||||
for queued in &lane.pending_frames {
|
||||
if count >= limits.max_frames_per_body
|
||||
|| (count != 0
|
||||
&& body_len.saturating_add(queued.encoded.len()) > limits.carrier_batch_bytes)
|
||||
{
|
||||
break;
|
||||
}
|
||||
body_len += queued.encoded.len();
|
||||
count += 1;
|
||||
}
|
||||
let Some(manager) = session.manager.upgrade() else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let Some(_staging) = manager.try_downlink_staging_budget(body_len) else {
|
||||
return Err(ManagerError::Backpressure);
|
||||
};
|
||||
let mut body = BytesMut::with_capacity(body_len);
|
||||
let mut data_bytes = 0usize;
|
||||
let mut data_items = 0usize;
|
||||
let mut control_bytes = 0usize;
|
||||
let mut control_items = 0usize;
|
||||
for index in 0..count {
|
||||
let Some(queued) = lane.pending_frames.get(index) else {
|
||||
break;
|
||||
};
|
||||
if queued.frame_type == FrameType::Window
|
||||
&& lane.pending_windows.get(&queued.stream_id) == Some(&index)
|
||||
{
|
||||
lane.pending_windows.remove(&queued.stream_id);
|
||||
}
|
||||
}
|
||||
for _ in 0..count {
|
||||
let Some(queued) = lane.pending_frames.pop_front() else {
|
||||
break;
|
||||
};
|
||||
body.extend_from_slice(&queued.encoded);
|
||||
if queued.control {
|
||||
control_bytes += queued.cost;
|
||||
control_items += 1;
|
||||
} else {
|
||||
data_bytes += queued.cost;
|
||||
data_items += 1;
|
||||
}
|
||||
}
|
||||
for index in lane.pending_windows.values_mut() {
|
||||
*index = index.saturating_sub(count);
|
||||
}
|
||||
lane.down_cursor = next_cursor;
|
||||
let counts = PendingCounts {
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
};
|
||||
let lease = PendingResponseLease::new(session, counts, Some(Arc::clone(&lane.resident)));
|
||||
let body = Bytes::from_owner(OwnedBatchBody::new(body.freeze(), Arc::clone(&lease)));
|
||||
Ok(DownBatch {
|
||||
body,
|
||||
lease,
|
||||
base_cursor: cursor,
|
||||
next_cursor,
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
carrier_health_eligible,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use super::uplink::{AppliedProgress, inbound_reservation, validate_batch};
|
||||
use super::{PendingClass, WebSession, insert_carrier_lane};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
|
||||
impl WebSession {
|
||||
/// Applies one exactly-once uplink batch to an independent HTTPS lane.
|
||||
pub(crate) fn process_up_lane(
|
||||
self: &Arc<Self>,
|
||||
lane_id: u32,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<u64, ManagerError> {
|
||||
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let frames = match frame::parse_all(body, &self.limits) {
|
||||
Ok(frames) => frames,
|
||||
Err(_) => {
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
};
|
||||
if frames
|
||||
.iter()
|
||||
.copied()
|
||||
.any(|value| value.stream_id != lane_id || frame::validate_client_shape(value).is_err())
|
||||
{
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
state.last_activity = Instant::now();
|
||||
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
||||
if new_lane {
|
||||
if lane_id != 0
|
||||
&& frames
|
||||
.first()
|
||||
.is_some_and(|value| value.frame_type != FrameType::Open)
|
||||
&& only_late_frames(&frames)
|
||||
{
|
||||
return if self.automatic_carrier
|
||||
&& state.negotiation_phase != super::SessionNegotiationPhase::Committed
|
||||
{
|
||||
Err(ManagerError::Backpressure)
|
||||
} else {
|
||||
Ok(sequence)
|
||||
};
|
||||
}
|
||||
if lane_id == 0
|
||||
|| frames
|
||||
.first()
|
||||
.is_none_or(|value| value.frame_type != FrameType::Open)
|
||||
{
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let lane_limit = self
|
||||
.profile
|
||||
.max_streams_per_session
|
||||
.saturating_add(self.limits.max_tombstones_per_session)
|
||||
.saturating_add(1);
|
||||
if state.carrier_lanes.len() >= lane_limit {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
}
|
||||
let (last_sequence, last_digest, up_active) = state
|
||||
.carrier_lanes
|
||||
.get(&lane_id)
|
||||
.map_or((0, [0; 32], false), |lane| {
|
||||
(lane.last_up_sequence, lane.last_up_digest, lane.up_active)
|
||||
});
|
||||
if sequence == last_sequence && sequence != 0 {
|
||||
return if bool::from(last_digest.ct_eq(&digest)) {
|
||||
Ok(sequence)
|
||||
} else {
|
||||
drop(state);
|
||||
self.close();
|
||||
Err(ManagerError::Protocol)
|
||||
};
|
||||
}
|
||||
if sequence == 0 || sequence != last_sequence.saturating_add(1) {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if up_active {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
if !validate_batch(&state, &frames) {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||
if !self.reserve_locked(
|
||||
&mut state,
|
||||
reserve_bytes,
|
||||
reserve_items,
|
||||
PendingClass::Uplink,
|
||||
) {
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
}
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
if new_lane && insert_carrier_lane(&mut state, lane_id).is_none() {
|
||||
self.release_locked(&mut state, reserve_bytes, reserve_items, false);
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
self.release_locked(&mut state, reserve_bytes, reserve_items, false);
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
lane.up_active = true;
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let mut progress = AppliedProgress::default();
|
||||
let applied = self.apply_batch_locked(
|
||||
&mut state,
|
||||
&frames,
|
||||
&mut opened,
|
||||
&mut None,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
&mut progress,
|
||||
);
|
||||
self.release_locked(&mut state, unused_bytes, unused_items, false);
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
if applied {
|
||||
lane.last_up_sequence = sequence;
|
||||
lane.last_up_digest = digest;
|
||||
}
|
||||
}
|
||||
if applied {
|
||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||
}
|
||||
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
||||
};
|
||||
if matches!(result, Err(ManagerError::Backpressure)) {
|
||||
return result;
|
||||
}
|
||||
if result.is_err() {
|
||||
self.close();
|
||||
drop(opened);
|
||||
return result;
|
||||
}
|
||||
if self.automatic_carrier && !self.is_carrier_committed() {
|
||||
self.lane_open_notify.notify_waiters();
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
self.lane_open_notify.notify_waiters();
|
||||
for completion in opened {
|
||||
self.spawn_stream(completion, false);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
fn only_late_frames(frames: &[Frame<'_>]) -> bool {
|
||||
frames.iter().all(|value| {
|
||||
matches!(
|
||||
value.frame_type,
|
||||
FrameType::Data | FrameType::Window | FrameType::Close
|
||||
)
|
||||
})
|
||||
}
|
||||
+187
-224
@@ -2,150 +2,17 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
use super::uplink::{inbound_reservation, validate_batch};
|
||||
use super::lane_downlink::take_lane_down_batch;
|
||||
use super::{
|
||||
CarrierLane, DownBatch, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState,
|
||||
WebSession, remember_closed,
|
||||
PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState, WebSession,
|
||||
remember_closed,
|
||||
};
|
||||
use crate::config::{WebCarrier, WebLimitsConfig};
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::ManagerError;
|
||||
|
||||
impl WebSession {
|
||||
/// Applies one exactly-once uplink batch to an independent HTTPS lane.
|
||||
pub(crate) fn process_up_lane(
|
||||
self: &Arc<Self>,
|
||||
lane_id: u32,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<u64, ManagerError> {
|
||||
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let frames = match frame::parse_all(body, &self.limits) {
|
||||
Ok(frames) => frames,
|
||||
Err(_) => {
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
};
|
||||
if frames
|
||||
.iter()
|
||||
.copied()
|
||||
.any(|value| value.stream_id != lane_id || frame::validate_client_shape(value).is_err())
|
||||
{
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
if !state.carrier_lanes.contains_key(&lane_id) {
|
||||
if lane_id != 0
|
||||
&& frames
|
||||
.first()
|
||||
.is_some_and(|value| value.frame_type != FrameType::Open)
|
||||
&& only_late_frames(&frames)
|
||||
{
|
||||
return Ok(sequence);
|
||||
}
|
||||
if lane_id == 0
|
||||
|| frames
|
||||
.first()
|
||||
.is_none_or(|value| value.frame_type != FrameType::Open)
|
||||
{
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
state.carrier_lanes.insert(lane_id, CarrierLane::new());
|
||||
}
|
||||
let lane = state
|
||||
.carrier_lanes
|
||||
.get_mut(&lane_id)
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
if sequence == lane.last_up_sequence && sequence != 0 {
|
||||
return if bool::from(lane.last_up_digest.ct_eq(&digest)) {
|
||||
Ok(sequence)
|
||||
} else {
|
||||
drop(state);
|
||||
self.close();
|
||||
Err(ManagerError::Protocol)
|
||||
};
|
||||
}
|
||||
if sequence == 0 || sequence != lane.last_up_sequence.saturating_add(1) {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if lane.up_active {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
lane.up_active = true;
|
||||
if !validate_batch(&state, &frames) {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||
if !self.reserve_locked(
|
||||
&mut state,
|
||||
reserve_bytes,
|
||||
reserve_items,
|
||||
PendingClass::Uplink,
|
||||
) {
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
}
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let applied = self.apply_batch_locked(
|
||||
&mut state,
|
||||
&frames,
|
||||
&mut opened,
|
||||
&mut None,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
);
|
||||
self.release_locked(&mut state, unused_bytes, unused_items, false);
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
if applied {
|
||||
lane.last_up_sequence = sequence;
|
||||
lane.last_up_digest = digest;
|
||||
}
|
||||
}
|
||||
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
||||
};
|
||||
if matches!(result, Err(ManagerError::Backpressure)) {
|
||||
return result;
|
||||
}
|
||||
if result.is_err() {
|
||||
self.close();
|
||||
for (_, peer_port) in opened {
|
||||
self.release_stream_reservation(peer_port);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
for (stream_id, peer_port) in opened {
|
||||
self.spawn_stream(stream_id, peer_port, false);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Polls one lane with independent cursor replay and newest-poll-wins semantics.
|
||||
pub(crate) async fn poll_down_lane(
|
||||
&self,
|
||||
@@ -155,14 +22,26 @@ impl WebSession {
|
||||
if !self.carrier().uses_lanes() || lane_id > frame::MAX_STREAM_ID {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (epoch, notify) = {
|
||||
if !self.wait_for_lane_open(lane_id, cursor).await? {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: false,
|
||||
});
|
||||
}
|
||||
let (instance, epoch, notify, healthy) = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
let acknowledged = {
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
return Err(ManagerError::Protocol);
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: true,
|
||||
});
|
||||
};
|
||||
if let Some(unacked) = &lane.unacked {
|
||||
if cursor == unacked.base_cursor {
|
||||
@@ -188,22 +67,46 @@ impl WebSession {
|
||||
}
|
||||
};
|
||||
if let Some(batch) = acknowledged {
|
||||
self.release_locked(&mut state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_locked(&mut state, batch.control_bytes, batch.control_items, true);
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.pending_bytes = lane.pending_bytes.saturating_sub(batch.data_bytes);
|
||||
lane.pending_items = lane.pending_items.saturating_sub(batch.data_items);
|
||||
}
|
||||
batch.lease.detach();
|
||||
self.release_local_locked(&mut state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(
|
||||
&mut state,
|
||||
batch.control_bytes,
|
||||
batch.control_items,
|
||||
true,
|
||||
);
|
||||
state.carrier_health_downlink |= batch.carrier_health_eligible;
|
||||
if batch.carrier_health_eligible {
|
||||
state.carrier_health_activity_at = Some(Instant::now());
|
||||
}
|
||||
if let Some(stream) = state.streams.get_mut(&lane_id)
|
||||
&& let Some(waker) = stream.write_waker.take()
|
||||
{
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
let lane = state
|
||||
.carrier_lanes
|
||||
.get_mut(&lane_id)
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
lane.down_epoch = lane.down_epoch.wrapping_add(1).max(1);
|
||||
(lane.down_epoch, Arc::clone(&lane.notify))
|
||||
let Some(epoch) = lane.down_epoch.checked_add(1) else {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
lane.down_epoch = epoch;
|
||||
let instance = lane.instance;
|
||||
let notify = Arc::clone(&lane.notify);
|
||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||
(instance, epoch, notify, healthy)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
notify.notify_waiters();
|
||||
|
||||
let deadline = Duration::from_secs(self.timeouts.long_poll_secs);
|
||||
@@ -215,6 +118,8 @@ impl WebSession {
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let carrier_health_eligible = lane_id != 0
|
||||
&& state.negotiation_phase == super::SessionNegotiationPhase::Committed;
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
@@ -222,6 +127,13 @@ impl WebSession {
|
||||
lane_closed: true,
|
||||
});
|
||||
};
|
||||
if lane.instance != instance {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: true,
|
||||
});
|
||||
}
|
||||
if lane.down_epoch != epoch {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
@@ -230,8 +142,17 @@ impl WebSession {
|
||||
});
|
||||
}
|
||||
if !lane.pending_frames.is_empty() {
|
||||
let batch = match take_lane_down_batch(&self.limits, lane, cursor) {
|
||||
let batch = match take_lane_down_batch(
|
||||
self,
|
||||
&self.limits,
|
||||
lane,
|
||||
cursor,
|
||||
carrier_health_eligible,
|
||||
) {
|
||||
Ok(batch) => batch,
|
||||
Err(ManagerError::Backpressure) => {
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
Err(error) => {
|
||||
drop(state);
|
||||
self.close();
|
||||
@@ -288,13 +209,18 @@ impl WebSession {
|
||||
lane_closed: true,
|
||||
});
|
||||
}
|
||||
if state
|
||||
.carrier_lanes
|
||||
.get(&lane_id)
|
||||
.is_some_and(|lane| lane.down_epoch == epoch)
|
||||
{
|
||||
if let Some(lane) = state.carrier_lanes.get(&lane_id) {
|
||||
if lane.instance != instance {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: true,
|
||||
});
|
||||
}
|
||||
if lane.down_epoch == epoch {
|
||||
state.last_activity = Instant::now();
|
||||
}
|
||||
}
|
||||
Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
@@ -304,6 +230,76 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_lane_open(&self, lane_id: u32, cursor: u64) -> Result<bool, ManagerError> {
|
||||
let wait = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if state.carrier_lanes.contains_key(&lane_id) {
|
||||
return Ok(true);
|
||||
}
|
||||
if cursor != 0 || lane_id == 0 {
|
||||
drop(state);
|
||||
self.close();
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if state.closed_streams.contains(&lane_id)
|
||||
|| state.closing_streams.contains_key(&lane_id)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
if state.lane_open_waits >= self.limits.max_lane_open_waits_per_session {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let Some(auxiliary) = manager.try_lane_poll(true) else {
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
state.lane_open_waits += 1;
|
||||
LaneOpenWaitGuard {
|
||||
session: self,
|
||||
_auxiliary: auxiliary,
|
||||
}
|
||||
};
|
||||
let deadline = Duration::from_secs(self.timeouts.lane_open_wait_secs);
|
||||
let opened = tokio::time::timeout(deadline, async {
|
||||
loop {
|
||||
let notified = self.lane_open_notify.notified();
|
||||
{
|
||||
let state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if state.carrier_lanes.contains_key(&lane_id)
|
||||
|| state.closed_streams.contains(&lane_id)
|
||||
|| state.closing_streams.contains_key(&lane_id)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
drop(wait);
|
||||
match opened {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
let state = self.state.lock();
|
||||
if state.closed {
|
||||
Err(ManagerError::Closed)
|
||||
} else {
|
||||
Ok(state.carrier_lanes.contains_key(&lane_id)
|
||||
|| state.closed_streams.contains(&lane_id)
|
||||
|| state.closing_streams.contains_key(&lane_id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn queue_lane_frame_locked(
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
@@ -350,6 +346,17 @@ impl WebSession {
|
||||
<= self.limits.max_frame_payload_bytes
|
||||
});
|
||||
if can_coalesce {
|
||||
if state.carrier_lanes.get(&stream_id).is_none_or(|lane| {
|
||||
let resident = lane.resident.snapshot();
|
||||
payload.len() > self.limits.pending_bytes_per_lane
|
||||
|| lane.pending_bytes.saturating_add(resident.data_bytes)
|
||||
> self
|
||||
.limits
|
||||
.pending_bytes_per_lane
|
||||
.saturating_sub(payload.len())
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
if !self.reserve_locked(state, payload.len(), 0, PendingClass::Downlink) {
|
||||
return false;
|
||||
}
|
||||
@@ -365,6 +372,7 @@ impl WebSession {
|
||||
last.cost += payload.len();
|
||||
let payload_len = (last.encoded.len() - frame::HEADER_BYTES) as u32;
|
||||
last.encoded[4..8].copy_from_slice(&payload_len.to_be_bytes());
|
||||
lane.pending_bytes += payload.len();
|
||||
lane.notify.notify_waiters();
|
||||
return true;
|
||||
}
|
||||
@@ -374,6 +382,18 @@ impl WebSession {
|
||||
} else {
|
||||
PendingClass::Downlink
|
||||
};
|
||||
if !control
|
||||
&& state.carrier_lanes.get(&stream_id).is_none_or(|lane| {
|
||||
let resident = lane.resident.snapshot();
|
||||
cost > self.limits.pending_bytes_per_lane
|
||||
|| lane.pending_bytes.saturating_add(resident.data_bytes)
|
||||
> self.limits.pending_bytes_per_lane.saturating_sub(cost)
|
||||
|| lane.pending_items.saturating_add(resident.data_items)
|
||||
>= self.limits.pending_items_per_lane
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.reserve_locked(state, cost, 1, class) {
|
||||
return false;
|
||||
}
|
||||
@@ -396,6 +416,10 @@ impl WebSession {
|
||||
control,
|
||||
cost,
|
||||
});
|
||||
if !control {
|
||||
lane.pending_bytes += cost;
|
||||
lane.pending_items += 1;
|
||||
}
|
||||
if frame_type == FrameType::Window {
|
||||
lane.pending_windows.insert(stream_id, index);
|
||||
}
|
||||
@@ -435,87 +459,26 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
if let Some(batch) = lane.unacked.take() {
|
||||
data_bytes = data_bytes.saturating_add(batch.data_bytes);
|
||||
data_items = data_items.saturating_add(batch.data_items);
|
||||
control_bytes = control_bytes.saturating_add(batch.control_bytes);
|
||||
control_items = control_items.saturating_add(batch.control_items);
|
||||
batch.lease.detach();
|
||||
self.release_local_locked(state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(state, batch.control_bytes, batch.control_items, true);
|
||||
}
|
||||
self.release_locked(state, data_bytes, data_items, false);
|
||||
self.release_locked(state, control_bytes, control_items, true);
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
fn only_late_frames(frames: &[Frame<'_>]) -> bool {
|
||||
frames.iter().all(|value| {
|
||||
matches!(
|
||||
value.frame_type,
|
||||
FrameType::Data | FrameType::Window | FrameType::Close
|
||||
)
|
||||
})
|
||||
struct LaneOpenWaitGuard<'a> {
|
||||
session: &'a WebSession,
|
||||
_auxiliary: OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
fn take_lane_down_batch(
|
||||
limits: &WebLimitsConfig,
|
||||
lane: &mut CarrierLane,
|
||||
cursor: u64,
|
||||
) -> Result<DownBatch, ManagerError> {
|
||||
let next_cursor = lane
|
||||
.down_cursor
|
||||
.checked_add(1)
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
let mut count = 0usize;
|
||||
let mut body_len = 0usize;
|
||||
for queued in &lane.pending_frames {
|
||||
if count >= limits.max_frames_per_body
|
||||
|| (count != 0
|
||||
&& body_len.saturating_add(queued.encoded.len()) > limits.carrier_batch_bytes)
|
||||
{
|
||||
break;
|
||||
impl Drop for LaneOpenWaitGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.session.state.lock();
|
||||
state.lane_open_waits = state.lane_open_waits.saturating_sub(1);
|
||||
}
|
||||
body_len += queued.encoded.len();
|
||||
count += 1;
|
||||
}
|
||||
let mut body = BytesMut::with_capacity(body_len);
|
||||
let mut data_bytes = 0usize;
|
||||
let mut data_items = 0usize;
|
||||
let mut control_bytes = 0usize;
|
||||
let mut control_items = 0usize;
|
||||
for index in 0..count {
|
||||
let Some(queued) = lane.pending_frames.get(index) else {
|
||||
break;
|
||||
};
|
||||
if queued.frame_type == FrameType::Window
|
||||
&& lane.pending_windows.get(&queued.stream_id) == Some(&index)
|
||||
{
|
||||
lane.pending_windows.remove(&queued.stream_id);
|
||||
}
|
||||
}
|
||||
for _ in 0..count {
|
||||
let Some(queued) = lane.pending_frames.pop_front() else {
|
||||
break;
|
||||
};
|
||||
body.extend_from_slice(&queued.encoded);
|
||||
if queued.control {
|
||||
control_bytes += queued.cost;
|
||||
control_items += 1;
|
||||
} else {
|
||||
data_bytes += queued.cost;
|
||||
data_items += 1;
|
||||
}
|
||||
}
|
||||
for index in lane.pending_windows.values_mut() {
|
||||
*index = index.saturating_sub(count);
|
||||
}
|
||||
lane.down_cursor = next_cursor;
|
||||
Ok(DownBatch {
|
||||
body: body.freeze(),
|
||||
base_cursor: cursor,
|
||||
next_cursor,
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
})
|
||||
}
|
||||
|
||||
// Lane-specific protocol, replay, and lifecycle tests.
|
||||
|
||||
+172
-16
@@ -1,19 +1,43 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use bytes::BytesMut;
|
||||
|
||||
use super::*;
|
||||
use crate::config::{WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig};
|
||||
use crate::config::{
|
||||
ProxyConfig, WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::session::{CarrierLane, insert_carrier_lane};
|
||||
|
||||
fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
||||
new_session(limits, std::sync::Weak::new())
|
||||
}
|
||||
|
||||
fn new_session(
|
||||
limits: WebLimitsConfig,
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
) -> Arc<WebSession> {
|
||||
new_session_with_automatic(limits, manager, false)
|
||||
}
|
||||
|
||||
fn new_session_with_automatic(
|
||||
limits: WebLimitsConfig,
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
automatic: bool,
|
||||
) -> Arc<WebSession> {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: SocketAddr::from(([203, 0, 113, 10], 443)),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::HttpsLanes,
|
||||
carrier_negotiation_enabled: false,
|
||||
carrier_learning: true,
|
||||
carriers: Arc::from([WebCarrier::HttpsLanes]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
@@ -21,28 +45,150 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
||||
max_streams_per_session: 2,
|
||||
});
|
||||
WebSession::new(
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
manager,
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebCarrier::HttpsLanes,
|
||||
1,
|
||||
[3; 32],
|
||||
None,
|
||||
if automatic {
|
||||
crate::web::manager::CarrierClientClass::Bridge
|
||||
} else {
|
||||
crate::web::manager::CarrierClientClass::Legacy
|
||||
},
|
||||
None,
|
||||
automatic,
|
||||
limits,
|
||||
WebTimeoutsConfig::default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn session_with_manager() -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||
session_with_manager_limits(WebLimitsConfig::default())
|
||||
}
|
||||
|
||||
fn session_with_manager_limits(
|
||||
limits: WebLimitsConfig,
|
||||
) -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||
let generation = test_runtime_generation(1, ProxyConfig::default());
|
||||
let manager = WebProcessRuntime::start(Arc::new(ArcSwap::from(generation)));
|
||||
let session = new_session(limits, Arc::downgrade(&manager));
|
||||
(session, manager)
|
||||
}
|
||||
|
||||
fn session() -> Arc<WebSession> {
|
||||
session_with_limits(WebLimitsConfig::default())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn early_down_waits_without_creating_a_provisional_lane() {
|
||||
let (session, manager) = session_with_manager();
|
||||
let polling = Arc::clone(&session);
|
||||
let poll = tokio::spawn(async move { polling.poll_down_lane(7, 0).await });
|
||||
while session.state.lock().lane_open_waits == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(!session.state.lock().carrier_lanes.contains_key(&7));
|
||||
{
|
||||
let mut state = session.state.lock();
|
||||
assert!(insert_carrier_lane(&mut state, 7).is_some());
|
||||
state.closed_streams.insert(7);
|
||||
assert!(session.queue_control_locked(&mut state, FrameType::Close, 7, &[]));
|
||||
}
|
||||
session.lane_open_notify.notify_waiters();
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), poll)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!result.body.is_empty());
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
drop(result);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn early_down_timeout_is_empty_and_releases_its_session_slot() {
|
||||
let (session, manager) = session_with_manager();
|
||||
let polling = Arc::clone(&session);
|
||||
let poll = tokio::spawn(async move { polling.poll_down_lane(7, 0).await });
|
||||
while session.state.lock().lane_open_waits == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
let result = poll.await.unwrap().unwrap();
|
||||
assert!(result.body.is_empty());
|
||||
assert_eq!(result.next_cursor, 0);
|
||||
assert!(!result.lane_closed);
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn early_down_admission_is_bounded_and_cancellation_safe() {
|
||||
let limits = WebLimitsConfig {
|
||||
max_lane_open_waits_per_session: 2,
|
||||
..WebLimitsConfig::default()
|
||||
};
|
||||
let (session, manager) = session_with_manager_limits(limits);
|
||||
let mut waits = Vec::new();
|
||||
for lane_id in [7, 8] {
|
||||
let polling = Arc::clone(&session);
|
||||
waits.push(tokio::spawn(async move {
|
||||
polling.poll_down_lane(lane_id, 0).await
|
||||
}));
|
||||
}
|
||||
while session.state.lock().lane_open_waits < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(matches!(
|
||||
session.poll_down_lane(9, 0).await,
|
||||
Err(ManagerError::Limit)
|
||||
));
|
||||
for wait in waits {
|
||||
wait.abort();
|
||||
let _ = wait.await;
|
||||
}
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_close_wakes_early_down_with_closed_state() {
|
||||
let (session, manager) = session_with_manager();
|
||||
let polling = Arc::clone(&session);
|
||||
let poll = tokio::spawn(async move { polling.poll_down_lane(7, 0).await });
|
||||
while session.state.lock().lane_open_waits == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
session.close();
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(Duration::from_secs(1), poll)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Err(ManagerError::Closed)
|
||||
));
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lane_uplink_sequences_are_independent_and_exactly_once() {
|
||||
let session = session();
|
||||
{
|
||||
let mut state = session.state.lock();
|
||||
for lane_id in [51, 52] {
|
||||
state.carrier_lanes.insert(lane_id, CarrierLane::new());
|
||||
state
|
||||
.carrier_lanes
|
||||
.insert(lane_id, CarrierLane::new(u64::from(lane_id)));
|
||||
state.closed_streams.insert(lane_id);
|
||||
}
|
||||
}
|
||||
@@ -67,20 +213,12 @@ fn cross_lane_frame_is_fatal_to_https_lane_session() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn drained_closed_lane_replays_then_signals_completion() {
|
||||
let session = session();
|
||||
let (session, manager) = session_with_manager();
|
||||
{
|
||||
let mut state = session.state.lock();
|
||||
state.carrier_lanes.insert(7, CarrierLane::new());
|
||||
state.carrier_lanes.insert(7, CarrierLane::new(7));
|
||||
state.closed_streams.insert(7);
|
||||
let lane = state.carrier_lanes.get_mut(&7).unwrap();
|
||||
let encoded = frame::encode(FrameType::Close, 7, &[]);
|
||||
lane.pending_frames.push_back(QueuedFrame {
|
||||
encoded: BytesMut::from(encoded.as_ref()),
|
||||
frame_type: FrameType::Close,
|
||||
stream_id: 7,
|
||||
control: true,
|
||||
cost: frame::HEADER_BYTES + QUEUE_ITEM_COST,
|
||||
});
|
||||
assert!(session.queue_control_locked(&mut state, FrameType::Close, 7, &[]));
|
||||
}
|
||||
let first = session.poll_down_lane(7, 0).await.unwrap();
|
||||
let replay = session.poll_down_lane(7, 0).await.unwrap();
|
||||
@@ -89,6 +227,10 @@ async fn drained_closed_lane_replays_then_signals_completion() {
|
||||
let finished = session.poll_down_lane(7, 1).await.unwrap();
|
||||
assert!(finished.body.is_empty());
|
||||
assert!(finished.lane_closed);
|
||||
drop(first);
|
||||
drop(replay);
|
||||
session.close();
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -100,7 +242,7 @@ fn tombstone_eviction_releases_lane_budget_and_accepts_late_frames() {
|
||||
let session = session_with_limits(limits);
|
||||
{
|
||||
let mut state = session.state.lock();
|
||||
state.carrier_lanes.insert(7, CarrierLane::new());
|
||||
state.carrier_lanes.insert(7, CarrierLane::new(7));
|
||||
let encoded = frame::encode(FrameType::Close, 7, &[]);
|
||||
let cost = encoded.len() + QUEUE_ITEM_COST;
|
||||
state
|
||||
@@ -120,7 +262,7 @@ fn tombstone_eviction_releases_lane_budget_and_accepts_late_frames() {
|
||||
state.pending_control_bytes = cost;
|
||||
state.pending_control_items = 1;
|
||||
session.remember_closed_locked(&mut state, 7);
|
||||
state.carrier_lanes.insert(8, CarrierLane::new());
|
||||
state.carrier_lanes.insert(8, CarrierLane::new(8));
|
||||
session.remember_closed_locked(&mut state, 8);
|
||||
assert!(!state.carrier_lanes.contains_key(&7));
|
||||
assert_eq!(state.pending_bytes, 0);
|
||||
@@ -130,3 +272,17 @@ fn tombstone_eviction_releases_lane_budget_and_accepts_late_frames() {
|
||||
assert_eq!(session.process_up_lane(7, 7, &late), Ok(7));
|
||||
assert!(!session.state.lock().closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_lane_does_not_ack_a_missing_lane_without_real_progress() {
|
||||
let session =
|
||||
new_session_with_automatic(WebLimitsConfig::default(), std::sync::Weak::new(), true);
|
||||
let late = frame::encode(FrameType::Data, 7, b"late");
|
||||
|
||||
assert_eq!(
|
||||
session.process_up_lane(7, 1, &late),
|
||||
Err(ManagerError::Backpressure)
|
||||
);
|
||||
assert!(!session.is_carrier_committed());
|
||||
assert!(!session.state.lock().closed);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::{SessionNegotiationPhase, WebSession};
|
||||
|
||||
struct ReleasedQueues {
|
||||
data_bytes: usize,
|
||||
data_items: usize,
|
||||
control_bytes: usize,
|
||||
control_items: usize,
|
||||
}
|
||||
|
||||
/// Deferred queue release after manager publication linearizes a supersede.
|
||||
#[must_use]
|
||||
pub(crate) struct CarrierSupersedeCompletion<'a> {
|
||||
session: &'a WebSession,
|
||||
released: ReleasedQueues,
|
||||
}
|
||||
|
||||
impl CarrierSupersedeCompletion<'_> {
|
||||
/// Releases process budgets and signals cancellation after manager locks are dropped.
|
||||
pub(crate) fn finish(self) {
|
||||
self.session.finish_close(self.released, true);
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||
pub(crate) fn close(&self) {
|
||||
let Some(released) = self.begin_close(false, None) else {
|
||||
return;
|
||||
};
|
||||
self.finish_close(released, false);
|
||||
}
|
||||
|
||||
/// Atomically prevents first-frame commit while one successor is prepared.
|
||||
pub(crate) fn begin_carrier_supersede(&self) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || state.close_requested {
|
||||
return false;
|
||||
}
|
||||
match state.negotiation_phase {
|
||||
SessionNegotiationPhase::Uncommitted => {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Replacing;
|
||||
true
|
||||
}
|
||||
SessionNegotiationPhase::Replacing
|
||||
| SessionNegotiationPhase::Committed
|
||||
| SessionNegotiationPhase::Superseded => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Restores an uncommitted attempt after successor admission failed.
|
||||
pub(crate) fn cancel_carrier_supersede(&self) {
|
||||
let close_requested = {
|
||||
let mut state = self.state.lock();
|
||||
if !state.closed && state.negotiation_phase == SessionNegotiationPhase::Replacing {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Uncommitted;
|
||||
}
|
||||
state.close_requested
|
||||
};
|
||||
if close_requested {
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
/// Linearizes manager publication against close requests on the old token.
|
||||
pub(crate) fn prepare_carrier_supersede(&self) -> Option<CarrierSupersedeCompletion<'_>> {
|
||||
let released = self.begin_close(true, None)?;
|
||||
Some(CarrierSupersedeCompletion {
|
||||
session: self,
|
||||
released,
|
||||
})
|
||||
}
|
||||
|
||||
/// Waits for all logical-stream tasks after admission has closed.
|
||||
pub(crate) async fn wait(&self) {
|
||||
loop {
|
||||
let notified = self.tasks_done.notified();
|
||||
if self.tasks_live.load(Ordering::Acquire) == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically closes a session only when reconnect grace is still due.
|
||||
pub(crate) fn close_if_due(&self, now: Instant) -> bool {
|
||||
let healthy = {
|
||||
let mut state = self.state.lock();
|
||||
self.carrier_health_ready_locked(&mut state, now)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
let Some(released) = self.begin_close(false, Some(now)) else {
|
||||
return false;
|
||||
};
|
||||
self.finish_close(released, false);
|
||||
true
|
||||
}
|
||||
|
||||
fn begin_close(&self, superseded: bool, idle_now: Option<Instant>) -> Option<ReleasedQueues> {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed
|
||||
|| (superseded
|
||||
&& (state.negotiation_phase != SessionNegotiationPhase::Replacing
|
||||
|| state.close_requested))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(now) = idle_now
|
||||
&& (state.negotiation_phase == SessionNegotiationPhase::Replacing
|
||||
|| now.saturating_duration_since(state.last_activity)
|
||||
< Duration::from_secs(self.timeouts.reconnect_grace_secs))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if !superseded && state.negotiation_phase == SessionNegotiationPhase::Replacing {
|
||||
state.close_requested = true;
|
||||
return None;
|
||||
}
|
||||
state.closed = true;
|
||||
if superseded {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Superseded;
|
||||
}
|
||||
for stream in state.streams.values_mut() {
|
||||
if let Some(waker) = stream.read_waker.take() {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(waker) = stream.write_waker.take() {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
state.streams.clear();
|
||||
state.pending_frames.clear();
|
||||
state.pending_windows.clear();
|
||||
if let Some(batch) = state.unacked.take() {
|
||||
batch.lease.detach();
|
||||
self.release_local_locked(&mut state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(&mut state, batch.control_bytes, batch.control_items, true);
|
||||
}
|
||||
let mut lane_data_bytes = 0usize;
|
||||
let mut lane_data_items = 0usize;
|
||||
let mut lane_control_bytes = 0usize;
|
||||
let mut lane_control_items = 0usize;
|
||||
for lane in state.carrier_lanes.values_mut() {
|
||||
lane.notify.notify_waiters();
|
||||
if let Some(batch) = lane.unacked.take() {
|
||||
batch.lease.detach();
|
||||
lane_data_bytes = lane_data_bytes.saturating_add(batch.data_bytes);
|
||||
lane_data_items = lane_data_items.saturating_add(batch.data_items);
|
||||
lane_control_bytes = lane_control_bytes.saturating_add(batch.control_bytes);
|
||||
lane_control_items = lane_control_items.saturating_add(batch.control_items);
|
||||
}
|
||||
}
|
||||
self.release_local_locked(&mut state, lane_data_bytes, lane_data_items, false);
|
||||
self.release_local_locked(&mut state, lane_control_bytes, lane_control_items, true);
|
||||
state.carrier_lanes.clear();
|
||||
let control_bytes = state.pending_control_bytes;
|
||||
let control_items = state.pending_control_items;
|
||||
let data_bytes = state.pending_bytes.saturating_sub(control_bytes);
|
||||
let data_items = state.pending_items.saturating_sub(control_items);
|
||||
state.pending_bytes = 0;
|
||||
state.pending_items = 0;
|
||||
state.pending_control_bytes = 0;
|
||||
state.pending_control_items = 0;
|
||||
Some(ReleasedQueues {
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
})
|
||||
}
|
||||
|
||||
fn finish_close(&self, released: ReleasedQueues, superseded: bool) {
|
||||
self.cancel.cancel();
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
}
|
||||
if self.carrier().uses_lanes() {
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_pending(
|
||||
self.profile_key,
|
||||
released.data_bytes,
|
||||
released.data_items,
|
||||
false,
|
||||
);
|
||||
manager.release_pending(
|
||||
self.profile_key,
|
||||
released.control_bytes,
|
||||
released.control_items,
|
||||
true,
|
||||
);
|
||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||
None,
|
||||
Some(if superseded { "superseded" } else { "closed" }),
|
||||
);
|
||||
if !superseded {
|
||||
manager.session_finished(
|
||||
self.token_hash,
|
||||
self.client_ip,
|
||||
self.profile_key,
|
||||
&self.profile.host,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::uplink::AppliedProgress;
|
||||
use super::{SessionNegotiationPhase, SessionState, WebSession};
|
||||
|
||||
impl WebSession {
|
||||
/// Returns whether accepted carrier progress made this attempt immutable.
|
||||
pub(crate) fn is_carrier_committed(&self) -> bool {
|
||||
self.state.lock().negotiation_phase == SessionNegotiationPhase::Committed
|
||||
}
|
||||
|
||||
/// Rejects mutation after replacement, supersede, or the final chain deadline.
|
||||
pub(super) fn ensure_carrier_active_locked(
|
||||
&self,
|
||||
state: &SessionState,
|
||||
) -> Result<(), crate::web::manager::ManagerError> {
|
||||
if state.negotiation_phase == SessionNegotiationPhase::Uncommitted
|
||||
&& self
|
||||
.carrier_deadline_at
|
||||
.is_some_and(|deadline| Instant::now() >= deadline)
|
||||
{
|
||||
return Err(crate::web::manager::ManagerError::Closed);
|
||||
}
|
||||
match state.negotiation_phase {
|
||||
SessionNegotiationPhase::Uncommitted | SessionNegotiationPhase::Committed => Ok(()),
|
||||
SessionNegotiationPhase::Replacing | SessionNegotiationPhase::Superseded => {
|
||||
Err(crate::web::manager::ManagerError::Closed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes the already-linearized session commit to process state.
|
||||
pub(super) fn finish_carrier_commit(&self) {
|
||||
let published = self.manager.upgrade().is_some_and(|manager| {
|
||||
manager.carrier_committed(
|
||||
self.bootstrap_hash,
|
||||
self.token_hash,
|
||||
self.carrier_attempt,
|
||||
self.selected_carrier,
|
||||
self.carrier_class,
|
||||
self.client_ip,
|
||||
self.trace_identity(),
|
||||
)
|
||||
});
|
||||
if !published {
|
||||
return;
|
||||
}
|
||||
let healthy = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || state.negotiation_phase != SessionNegotiationPhase::Committed {
|
||||
false
|
||||
} else {
|
||||
state.carrier_commit_published = true;
|
||||
self.carrier_health_ready_locked(&mut state, Instant::now())
|
||||
}
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes complete transport-specific health evidence to process state.
|
||||
pub(super) fn finish_carrier_health(&self) {
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.carrier_became_healthy(
|
||||
self.bootstrap_hash,
|
||||
self.token_hash,
|
||||
self.carrier_attempt,
|
||||
self.selected_carrier,
|
||||
self.carrier_class,
|
||||
self.learning_context,
|
||||
self.client_ip,
|
||||
self.trace_identity(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Records accepted OPEN or DATA progress and returns commit and health transitions.
|
||||
pub(super) fn record_uplink_progress_locked(
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
progress: AppliedProgress,
|
||||
) -> (bool, bool) {
|
||||
if !self.automatic_carrier || !progress.any() {
|
||||
return (false, false);
|
||||
}
|
||||
if self.selected_carrier.uses_websocket() {
|
||||
state.websocket_carrier_active = true;
|
||||
state.websocket_commit_ack_pending = true;
|
||||
} else if progress.accepted_data {
|
||||
state.carrier_health_uplink = true;
|
||||
}
|
||||
let now = Instant::now();
|
||||
let committed = if state.negotiation_phase == SessionNegotiationPhase::Uncommitted {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Committed;
|
||||
state.carrier_health_due_at =
|
||||
Some(now + Duration::from_secs(self.timeouts.carrier_health_secs));
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
let healthy = self.carrier_health_ready_locked(state, now);
|
||||
(committed, healthy)
|
||||
}
|
||||
|
||||
/// Consumes complete post-commit health evidence at most once.
|
||||
pub(super) fn carrier_health_ready_locked(
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
if !self.automatic_carrier
|
||||
|| state.closed
|
||||
|| state.negotiation_phase != SessionNegotiationPhase::Committed
|
||||
|| !state.carrier_commit_published
|
||||
|| state.carrier_health_reported
|
||||
|| state.carrier_health_due_at.is_none_or(|due| now < due)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let evidence = if state.websocket_carrier_active {
|
||||
state.websocket_probe_claimed
|
||||
&& state.websocket_commit_ack_owner.is_some()
|
||||
&& state.websocket_commit_ack_written
|
||||
&& state.carrier_health_uplink
|
||||
} else {
|
||||
state.carrier_health_uplink
|
||||
&& state.carrier_health_downlink
|
||||
&& state
|
||||
.carrier_health_activity_at
|
||||
.zip(state.carrier_health_due_at)
|
||||
.is_some_and(|(activity, due)| activity >= due)
|
||||
};
|
||||
if evidence {
|
||||
state.carrier_health_reported = true;
|
||||
}
|
||||
evidence
|
||||
}
|
||||
|
||||
/// Returns whether the exact automatic WebSocket owner must receive a commit acknowledgement.
|
||||
pub(crate) fn needs_websocket_commit_ack(&self, owner: u64) -> bool {
|
||||
let state = self.state.lock();
|
||||
!state.closed
|
||||
&& state.websocket_commit_ack_owner == Some(owner)
|
||||
&& state.websocket_commit_ack_pending
|
||||
&& !state.websocket_commit_ack_written
|
||||
}
|
||||
|
||||
/// Records that the commit acknowledgement reached its exact WebSocket owner.
|
||||
pub(crate) fn websocket_commit_ack_written(&self, owner: u64) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed
|
||||
|| state.websocket_commit_ack_owner != Some(owner)
|
||||
|| !state.websocket_commit_ack_pending
|
||||
{
|
||||
return false;
|
||||
}
|
||||
state.websocket_commit_ack_written = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Records validated binary peer progress after the exact WebSocket acknowledgement.
|
||||
pub(crate) fn websocket_peer_after_commit_ack(&self, owner: u64) -> bool {
|
||||
let healthy = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed
|
||||
|| state.websocket_commit_ack_owner != Some(owner)
|
||||
|| !state.websocket_commit_ack_written
|
||||
{
|
||||
return false;
|
||||
}
|
||||
state.carrier_health_uplink = true;
|
||||
let now = Instant::now();
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
self.carrier_health_ready_locked(&mut state, now)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
use crate::config::{
|
||||
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::web::manager::{CarrierClientClass, WebProcessRuntime};
|
||||
|
||||
fn session(carrier: WebCarrier, deadline: Instant) -> Arc<WebSession> {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: SocketAddr::from(([203, 0, 113, 10], 443)),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier,
|
||||
carrier_negotiation_enabled: true,
|
||||
carrier_learning: false,
|
||||
carriers: Arc::from([carrier]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
});
|
||||
WebSession::new(
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
carrier,
|
||||
1,
|
||||
[3; 32],
|
||||
Some(deadline),
|
||||
CarrierClientClass::Bridge,
|
||||
None,
|
||||
true,
|
||||
WebLimitsConfig::default(),
|
||||
WebTimeoutsConfig::default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_deadline_refuses_uncommitted_progress() {
|
||||
let session = session(WebCarrier::Https, Instant::now() - Duration::from_secs(1));
|
||||
let state = session.state.lock();
|
||||
assert_eq!(
|
||||
session.ensure_carrier_active_locked(&state),
|
||||
Err(crate::web::manager::ManagerError::Closed)
|
||||
);
|
||||
assert!(matches!(
|
||||
state.negotiation_phase,
|
||||
SessionNegotiationPhase::Uncommitted
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_health_requires_authenticated_activity_after_the_window() {
|
||||
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
let now = Instant::now();
|
||||
let mut state = session.state.lock();
|
||||
state.negotiation_phase = SessionNegotiationPhase::Committed;
|
||||
state.carrier_commit_published = true;
|
||||
state.carrier_health_due_at = Some(now - Duration::from_secs(1));
|
||||
state.carrier_health_uplink = true;
|
||||
state.carrier_health_downlink = true;
|
||||
state.carrier_health_activity_at = Some(now - Duration::from_secs(2));
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_health_requires_the_exact_live_probe_owner() {
|
||||
let session = session(
|
||||
WebCarrier::Websocket,
|
||||
Instant::now() + Duration::from_secs(60),
|
||||
);
|
||||
let now = Instant::now();
|
||||
let mut state = session.state.lock();
|
||||
state.negotiation_phase = SessionNegotiationPhase::Committed;
|
||||
state.carrier_commit_published = true;
|
||||
state.carrier_health_due_at = Some(now - Duration::from_secs(1));
|
||||
state.websocket_carrier_active = true;
|
||||
state.websocket_commit_ack_owner = Some(7);
|
||||
state.websocket_commit_ack_written = true;
|
||||
state.carrier_health_uplink = true;
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
state.websocket_probe_claimed = true;
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_waits_for_manager_commit_publication() {
|
||||
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
let now = Instant::now();
|
||||
let mut state = session.state.lock();
|
||||
state.negotiation_phase = SessionNegotiationPhase::Committed;
|
||||
state.carrier_health_due_at = Some(now - Duration::from_secs(1));
|
||||
state.carrier_health_uplink = true;
|
||||
state.carrier_health_downlink = true;
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(!state.carrier_health_reported);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commit_and_supersede_have_one_session_lock_winner() {
|
||||
let committed = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
{
|
||||
let mut state = committed.state.lock();
|
||||
assert!(
|
||||
committed
|
||||
.record_uplink_progress_locked(
|
||||
&mut state,
|
||||
AppliedProgress {
|
||||
accepted_open: true,
|
||||
accepted_data: true,
|
||||
},
|
||||
)
|
||||
.0
|
||||
);
|
||||
}
|
||||
assert!(!committed.begin_carrier_supersede());
|
||||
|
||||
let replacing = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
assert!(replacing.begin_carrier_supersede());
|
||||
assert_eq!(
|
||||
replacing.ensure_carrier_active_locked(&replacing.state.lock()),
|
||||
Err(crate::web::manager::ManagerError::Closed)
|
||||
);
|
||||
replacing.cancel_carrier_supersede();
|
||||
assert!(
|
||||
replacing
|
||||
.ensure_carrier_active_locked(&replacing.state.lock())
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
|
||||
use super::WebSession;
|
||||
use crate::web::manager::{ProfileKey, WebProcessRuntime};
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(super) struct PendingCounts {
|
||||
pub(super) data_bytes: usize,
|
||||
pub(super) data_items: usize,
|
||||
pub(super) control_bytes: usize,
|
||||
pub(super) control_items: usize,
|
||||
}
|
||||
|
||||
impl PendingCounts {
|
||||
pub(super) fn bytes(self) -> usize {
|
||||
self.data_bytes.saturating_add(self.control_bytes)
|
||||
}
|
||||
|
||||
pub(super) fn items(self) -> usize {
|
||||
self.data_items.saturating_add(self.control_items)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ResidentCounters {
|
||||
data_bytes: AtomicUsize,
|
||||
data_items: AtomicUsize,
|
||||
control_bytes: AtomicUsize,
|
||||
control_items: AtomicUsize,
|
||||
}
|
||||
|
||||
impl ResidentCounters {
|
||||
pub(super) fn snapshot(&self) -> PendingCounts {
|
||||
PendingCounts {
|
||||
data_bytes: self.data_bytes.load(Ordering::Acquire),
|
||||
data_items: self.data_items.load(Ordering::Acquire),
|
||||
control_bytes: self.control_bytes.load(Ordering::Acquire),
|
||||
control_items: self.control_items.load(Ordering::Acquire),
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&self, counts: PendingCounts) {
|
||||
self.data_bytes
|
||||
.fetch_add(counts.data_bytes, Ordering::AcqRel);
|
||||
self.data_items
|
||||
.fetch_add(counts.data_items, Ordering::AcqRel);
|
||||
self.control_bytes
|
||||
.fetch_add(counts.control_bytes, Ordering::AcqRel);
|
||||
self.control_items
|
||||
.fetch_add(counts.control_items, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn remove(&self, counts: PendingCounts) {
|
||||
self.data_bytes
|
||||
.fetch_sub(counts.data_bytes, Ordering::AcqRel);
|
||||
self.data_items
|
||||
.fetch_sub(counts.data_items, Ordering::AcqRel);
|
||||
self.control_bytes
|
||||
.fetch_sub(counts.control_bytes, Ordering::AcqRel);
|
||||
self.control_items
|
||||
.fetch_sub(counts.control_items, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PendingResponseLease {
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
counts: PendingCounts,
|
||||
session: Arc<ResidentCounters>,
|
||||
lane: Option<Arc<ResidentCounters>>,
|
||||
detached: AtomicBool,
|
||||
}
|
||||
|
||||
impl PendingResponseLease {
|
||||
pub(super) fn new(
|
||||
session: &WebSession,
|
||||
counts: PendingCounts,
|
||||
lane: Option<Arc<ResidentCounters>>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
manager: session.manager.clone(),
|
||||
owner: session.profile_key,
|
||||
counts,
|
||||
session: Arc::clone(&session.resident),
|
||||
lane,
|
||||
detached: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn detach(&self) {
|
||||
if self
|
||||
.detached
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.session.add(self.counts);
|
||||
if let Some(lane) = &self.lane {
|
||||
lane.add(self.counts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingResponseLease {
|
||||
fn drop(&mut self) {
|
||||
if self.detached.load(Ordering::Acquire) {
|
||||
self.session.remove(self.counts);
|
||||
if let Some(lane) = &self.lane {
|
||||
lane.remove(self.counts);
|
||||
}
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_pending(
|
||||
self.owner,
|
||||
self.counts.data_bytes,
|
||||
self.counts.data_items,
|
||||
false,
|
||||
);
|
||||
manager.release_pending(
|
||||
self.owner,
|
||||
self.counts.control_bytes,
|
||||
self.counts.control_items,
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct OwnedBatchBody {
|
||||
bytes: Bytes,
|
||||
_lease: Arc<PendingResponseLease>,
|
||||
}
|
||||
|
||||
impl OwnedBatchBody {
|
||||
pub(super) fn new(bytes: Bytes, lease: Arc<PendingResponseLease>) -> Self {
|
||||
Self {
|
||||
bytes,
|
||||
_lease: lease,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for OwnedBatchBody {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
+114
-15
@@ -7,13 +7,26 @@ use bytes::Bytes;
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use super::backend::StreamCompletion;
|
||||
use super::{
|
||||
InboundChunk, PendingClass, QUEUE_ITEM_COST, SessionState, StreamState, WebSession,
|
||||
inbound_queue_cost,
|
||||
InboundChunk, PendingClass, QUEUE_ITEM_COST, SessionState, StreamIdentity, StreamState,
|
||||
WebSession, inbound_queue_cost,
|
||||
};
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(super) struct AppliedProgress {
|
||||
pub(super) accepted_open: bool,
|
||||
pub(super) accepted_data: bool,
|
||||
}
|
||||
|
||||
impl AppliedProgress {
|
||||
pub(super) fn any(self) -> bool {
|
||||
self.accepted_open || self.accepted_data
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Applies one exactly-once uplink batch.
|
||||
pub(crate) fn process_up(
|
||||
@@ -21,6 +34,28 @@ impl WebSession {
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<u64, ManagerError> {
|
||||
let (acknowledged, progressed) = self.process_up_inner(sequence, body)?;
|
||||
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
Ok(acknowledged)
|
||||
}
|
||||
|
||||
/// Applies one WebSocket uplink batch and reports actual carrier progress.
|
||||
pub(crate) fn process_websocket_multiplex(
|
||||
self: &Arc<Self>,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<bool, ManagerError> {
|
||||
self.process_up_inner(sequence, body)
|
||||
.map(|(_, progress)| progress)
|
||||
}
|
||||
|
||||
fn process_up_inner(
|
||||
self: &Arc<Self>,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<(u64, bool), ManagerError> {
|
||||
if !self.carrier().is_multiplexed() {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
@@ -49,15 +84,18 @@ impl WebSession {
|
||||
}
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
state.last_activity = Instant::now();
|
||||
if sequence == state.last_up_sequence && sequence != 0 {
|
||||
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
||||
Ok(sequence)
|
||||
Ok((sequence, false))
|
||||
} else {
|
||||
drop(state);
|
||||
self.close();
|
||||
@@ -85,6 +123,7 @@ impl WebSession {
|
||||
}
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let mut progress = AppliedProgress::default();
|
||||
let applied = self.apply_batch_locked(
|
||||
&mut state,
|
||||
&frames,
|
||||
@@ -92,6 +131,7 @@ impl WebSession {
|
||||
&mut None,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
&mut progress,
|
||||
);
|
||||
self.release_locked(&mut state, unused_bytes, unused_items, false);
|
||||
if !applied {
|
||||
@@ -99,7 +139,8 @@ impl WebSession {
|
||||
} else {
|
||||
state.last_up_sequence = sequence;
|
||||
state.last_up_digest = digest;
|
||||
Ok(sequence)
|
||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||
Ok((sequence, progress.any()))
|
||||
}
|
||||
};
|
||||
if matches!(result, Err(ManagerError::Backpressure)) {
|
||||
@@ -107,13 +148,17 @@ impl WebSession {
|
||||
}
|
||||
if result.is_err() {
|
||||
self.close();
|
||||
for (_, peer_port) in opened {
|
||||
self.release_stream_reservation(peer_port);
|
||||
}
|
||||
drop(opened);
|
||||
return result;
|
||||
}
|
||||
for (stream_id, peer_port) in opened {
|
||||
self.spawn_stream(stream_id, peer_port, false);
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
for completion in opened {
|
||||
self.spawn_stream(completion, false);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
@@ -122,21 +167,26 @@ impl WebSession {
|
||||
}
|
||||
|
||||
pub(super) fn apply_batch_locked(
|
||||
&self,
|
||||
self: &Arc<Self>,
|
||||
state: &mut SessionState,
|
||||
frames: &[Frame<'_>],
|
||||
opened: &mut Vec<(u32, u16)>,
|
||||
opened: &mut Vec<StreamCompletion>,
|
||||
reserved_open: &mut Option<(u32, u16)>,
|
||||
unused_bytes: &mut usize,
|
||||
unused_items: &mut usize,
|
||||
progress: &mut AppliedProgress,
|
||||
) -> bool {
|
||||
for value in frames {
|
||||
if value.stream_id == 0 {
|
||||
continue;
|
||||
}
|
||||
let was_closed = state.closed_streams.contains(&value.stream_id);
|
||||
let was_closed = state.closed_streams.contains(&value.stream_id)
|
||||
|| state.closing_streams.contains_key(&value.stream_id);
|
||||
match value.frame_type {
|
||||
FrameType::Open => {
|
||||
let Some(stream) = next_stream_identity(state, value.stream_id) else {
|
||||
return false;
|
||||
};
|
||||
let peer_port = match reserved_open.take() {
|
||||
Some((reserved_stream_id, peer_port))
|
||||
if reserved_stream_id == value.stream_id =>
|
||||
@@ -166,6 +216,7 @@ impl WebSession {
|
||||
state.streams.insert(
|
||||
value.stream_id,
|
||||
StreamState {
|
||||
instance: stream.instance,
|
||||
inbound: VecDeque::new(),
|
||||
receive_window: frame::INITIAL_STREAM_WINDOW,
|
||||
send_credit: u64::from(frame::INITIAL_STREAM_WINDOW),
|
||||
@@ -173,7 +224,8 @@ impl WebSession {
|
||||
write_waker: None,
|
||||
},
|
||||
);
|
||||
opened.push((value.stream_id, peer_port));
|
||||
progress.accepted_open = true;
|
||||
opened.push(self.own_stream_task(stream, peer_port));
|
||||
}
|
||||
FrameType::Data if !was_closed => {
|
||||
let Some(stream) = state.streams.get_mut(&value.stream_id) else {
|
||||
@@ -184,6 +236,7 @@ impl WebSession {
|
||||
bytes: Bytes::copy_from_slice(value.payload),
|
||||
offset: 0,
|
||||
});
|
||||
progress.accepted_data = true;
|
||||
*unused_bytes =
|
||||
unused_bytes.saturating_sub(value.payload.len() + QUEUE_ITEM_COST);
|
||||
*unused_items = unused_items.saturating_sub(1);
|
||||
@@ -208,6 +261,9 @@ impl WebSession {
|
||||
let Some(stream) = state.streams.remove(&value.stream_id) else {
|
||||
return false;
|
||||
};
|
||||
state
|
||||
.closing_streams
|
||||
.insert(value.stream_id, stream.instance);
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
self.release_locked(state, bytes, items, false);
|
||||
self.remember_closed_locked(state, value.stream_id);
|
||||
@@ -249,6 +305,15 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
fn next_stream_identity(state: &mut SessionState, stream_id: u32) -> Option<StreamIdentity> {
|
||||
let instance = state.next_stream_instance;
|
||||
state.next_stream_instance = instance.checked_add(1)?;
|
||||
Some(StreamIdentity {
|
||||
id: stream_id,
|
||||
instance,
|
||||
})
|
||||
}
|
||||
|
||||
struct UplinkGuard<'a>(&'a AtomicBool);
|
||||
|
||||
impl Drop for UplinkGuard<'_> {
|
||||
@@ -271,8 +336,9 @@ pub(super) fn validate_batch(state: &SessionState, frames: &[Frame<'_>]) -> bool
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let was_closed =
|
||||
state.closed_streams.contains(&value.stream_id) || closed.contains(&value.stream_id);
|
||||
let was_closed = state.closed_streams.contains(&value.stream_id)
|
||||
|| state.closing_streams.contains_key(&value.stream_id)
|
||||
|| closed.contains(&value.stream_id);
|
||||
match value.frame_type {
|
||||
FrameType::Open => {
|
||||
if live.contains_key(&value.stream_id) || was_closed {
|
||||
@@ -356,12 +422,20 @@ mod tests {
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn session() -> Arc<WebSession> {
|
||||
session_with_automatic(false)
|
||||
}
|
||||
|
||||
fn session_with_automatic(automatic: bool) -> Arc<WebSession> {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: SocketAddr::from(([203, 0, 113, 10], 443)),
|
||||
user: "alice".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::Https,
|
||||
carrier_negotiation_enabled: false,
|
||||
carrier_learning: true,
|
||||
carriers: Arc::from([WebCarrier::Https]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
@@ -375,6 +449,17 @@ mod tests {
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebCarrier::Https,
|
||||
1,
|
||||
[3; 32],
|
||||
None,
|
||||
if automatic {
|
||||
crate::web::manager::CarrierClientClass::Bridge
|
||||
} else {
|
||||
crate::web::manager::CarrierClientClass::Legacy
|
||||
},
|
||||
None,
|
||||
automatic,
|
||||
WebLimitsConfig::default(),
|
||||
WebTimeoutsConfig::default(),
|
||||
)
|
||||
@@ -411,6 +496,7 @@ mod tests {
|
||||
state.streams.insert(
|
||||
1,
|
||||
StreamState {
|
||||
instance: 1,
|
||||
inbound: VecDeque::new(),
|
||||
receive_window: frame::INITIAL_STREAM_WINDOW,
|
||||
send_credit: u64::from(frame::INITIAL_STREAM_WINDOW),
|
||||
@@ -439,4 +525,17 @@ mod tests {
|
||||
assert_eq!(session.process_up(2, &body), Err(ManagerError::Protocol));
|
||||
assert!(session.state.lock().closed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_uplink_does_not_ack_a_batch_without_real_progress() {
|
||||
let session = session_with_automatic(true);
|
||||
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||
|
||||
assert_eq!(
|
||||
session.process_up(1, &body),
|
||||
Err(ManagerError::Backpressure)
|
||||
);
|
||||
assert!(!session.is_carrier_committed());
|
||||
assert!(!session.state.lock().closed);
|
||||
}
|
||||
}
|
||||
|
||||
+114
-10
@@ -3,8 +3,8 @@ use std::time::Instant;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::uplink::{inbound_reservation, validate_batch};
|
||||
use super::{CarrierLane, PendingClass, WebSession, inbound_queue_cost};
|
||||
use super::uplink::{AppliedProgress, inbound_reservation, validate_batch};
|
||||
use super::{PendingClass, WebSession, inbound_queue_cost, insert_carrier_lane};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame;
|
||||
use crate::web::manager::ManagerError;
|
||||
@@ -17,6 +17,43 @@ pub(crate) struct WebSocketLaneReservation {
|
||||
transferred: bool,
|
||||
}
|
||||
|
||||
/// Session-wide ownership of the only automatic WebSocket carrier probe.
|
||||
pub(crate) struct WebSocketProbeReservation {
|
||||
session: Arc<WebSession>,
|
||||
owner: Option<u64>,
|
||||
}
|
||||
|
||||
impl WebSocketProbeReservation {
|
||||
/// Binds the admitted process connection to the future commit acknowledgement.
|
||||
pub(crate) fn bind(&mut self, owner: u64) -> Result<(), ManagerError> {
|
||||
let mut state = self.session.state.lock();
|
||||
if state.closed
|
||||
|| !state.websocket_probe_claimed
|
||||
|| state.websocket_commit_ack_owner.is_some()
|
||||
{
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state.websocket_commit_ack_owner = Some(owner);
|
||||
self.owner = Some(owner);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketProbeReservation {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.session.state.lock();
|
||||
state.websocket_probe_claimed = false;
|
||||
if state.websocket_commit_ack_owner == self.owner {
|
||||
state.websocket_commit_ack_owner = None;
|
||||
if !state.carrier_health_reported {
|
||||
state.websocket_commit_ack_written = false;
|
||||
state.carrier_health_uplink = false;
|
||||
state.carrier_health_activity_at = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSocketLaneReservation {
|
||||
/// Returns the logical stream owned by this connection.
|
||||
pub(crate) fn lane_id(&self) -> u32 {
|
||||
@@ -46,6 +83,42 @@ impl Drop for WebSocketLaneReservation {
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Reserves the only automatic WebSocket probe before any HTTP 101 response.
|
||||
pub(crate) fn reserve_websocket_probe(
|
||||
self: &Arc<Self>,
|
||||
acknowledge_commit: bool,
|
||||
) -> Result<Option<WebSocketProbeReservation>, ManagerError> {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
if !self.automatic_carrier {
|
||||
return if acknowledge_commit {
|
||||
Err(ManagerError::Protocol)
|
||||
} else {
|
||||
Ok(None)
|
||||
};
|
||||
}
|
||||
match state.negotiation_phase {
|
||||
super::SessionNegotiationPhase::Uncommitted if acknowledge_commit => {
|
||||
if state.websocket_probe_claimed || state.websocket_commit_ack_owner.is_some() {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
state.websocket_probe_claimed = true;
|
||||
Ok(Some(WebSocketProbeReservation {
|
||||
session: Arc::clone(self),
|
||||
owner: None,
|
||||
}))
|
||||
}
|
||||
super::SessionNegotiationPhase::Committed if !acknowledge_commit => Ok(None),
|
||||
super::SessionNegotiationPhase::Committed => Err(ManagerError::Committed),
|
||||
super::SessionNegotiationPhase::Uncommitted => Err(ManagerError::Protocol),
|
||||
super::SessionNegotiationPhase::Replacing
|
||||
| super::SessionNegotiationPhase::Superseded => Err(ManagerError::Closed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquires stream quota and tuple ownership before a lane returns HTTP 101.
|
||||
pub(crate) fn reserve_websocket_lane(
|
||||
self: &Arc<Self>,
|
||||
@@ -63,6 +136,7 @@ impl WebSession {
|
||||
}
|
||||
if state.active_peer_ports.len() >= self.profile.max_streams_per_session
|
||||
|| state.streams.contains_key(&lane_id)
|
||||
|| state.closing_streams.contains_key(&lane_id)
|
||||
|| state.closed_streams.contains(&lane_id)
|
||||
|| state.websocket_lane_reservations.contains_key(&lane_id)
|
||||
{
|
||||
@@ -89,7 +163,19 @@ impl WebSession {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
state.websocket_lane_reservations.insert(lane_id, peer_port);
|
||||
state.carrier_lanes.insert(lane_id, CarrierLane::new());
|
||||
if insert_carrier_lane(&mut state, lane_id).is_none() {
|
||||
state.websocket_lane_reservations.remove(&lane_id);
|
||||
state.active_peer_ports.remove(&peer_port);
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
drop(state);
|
||||
self.lane_open_notify.notify_waiters();
|
||||
Ok(WebSocketLaneReservation {
|
||||
session: Arc::clone(self),
|
||||
lane_id,
|
||||
@@ -104,7 +190,7 @@ impl WebSession {
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<(), ManagerError> {
|
||||
) -> Result<bool, ManagerError> {
|
||||
if !Arc::ptr_eq(self, &reservation.session)
|
||||
|| reservation.lane_id == 0
|
||||
|| reservation.lane_id > frame::MAX_STREAM_ID
|
||||
@@ -122,11 +208,14 @@ impl WebSession {
|
||||
}
|
||||
let digest = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
if !reservation.transferred
|
||||
&& state.websocket_lane_reservations.get(&lane_id) != Some(&reservation.peer_port)
|
||||
{
|
||||
@@ -162,6 +251,7 @@ impl WebSession {
|
||||
}
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let mut progress = AppliedProgress::default();
|
||||
let mut reserved_open =
|
||||
(!reservation.transferred).then_some((lane_id, reservation.peer_port));
|
||||
let applied = self.apply_batch_locked(
|
||||
@@ -171,6 +261,7 @@ impl WebSession {
|
||||
&mut reserved_open,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
&mut progress,
|
||||
);
|
||||
self.release_locked(&mut state, unused_bytes, unused_items, false);
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
@@ -181,14 +272,25 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
applied.then_some(()).ok_or(ManagerError::Protocol)
|
||||
if applied {
|
||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||
}
|
||||
applied
|
||||
.then_some(progress.any())
|
||||
.ok_or(ManagerError::Protocol)
|
||||
};
|
||||
result?;
|
||||
for (stream_id, peer_port) in opened {
|
||||
if stream_id != lane_id || peer_port != reservation.peer_port {
|
||||
let progressed = result?;
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
for completion in opened {
|
||||
if completion.stream.id != lane_id || completion.peer_port != reservation.peer_port {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if !self.spawn_stream(stream_id, peer_port, true) {
|
||||
if !self.spawn_stream(completion, true) {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
reservation.transfer_to_stream();
|
||||
@@ -199,7 +301,7 @@ impl WebSession {
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
}
|
||||
Ok(())
|
||||
Ok(progressed)
|
||||
}
|
||||
|
||||
/// Ends one failed or disconnected lane without closing its parent session.
|
||||
@@ -208,6 +310,7 @@ impl WebSession {
|
||||
let mut state = self.state.lock();
|
||||
let reserved = state.websocket_lane_reservations.remove(&lane_id);
|
||||
if let Some(stream) = state.streams.remove(&lane_id) {
|
||||
state.closing_streams.insert(lane_id, stream.instance);
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
if let Some(waker) = stream.read_waker {
|
||||
@@ -224,6 +327,7 @@ impl WebSession {
|
||||
if let Some(peer_port) = reserved {
|
||||
self.release_websocket_lane_reservation(lane_id, peer_port);
|
||||
}
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
|
||||
fn release_websocket_lane_reservation(&self, lane_id: u32, peer_port: u16) {
|
||||
|
||||
@@ -33,6 +33,10 @@ fn runtime(admission: bool) -> TestRuntime {
|
||||
user: "default".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::WebsocketLanes,
|
||||
carrier_negotiation_enabled: false,
|
||||
carrier_learning: true,
|
||||
carriers: Arc::from([WebCarrier::WebsocketLanes]),
|
||||
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||
capability: [7; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 2,
|
||||
@@ -60,6 +64,13 @@ fn runtime(admission: bool) -> TestRuntime {
|
||||
1,
|
||||
profile,
|
||||
[7; 32],
|
||||
WebCarrier::WebsocketLanes,
|
||||
1,
|
||||
[9; 32],
|
||||
None,
|
||||
crate::web::manager::CarrierClientClass::Legacy,
|
||||
None,
|
||||
false,
|
||||
limits,
|
||||
timeouts,
|
||||
);
|
||||
|
||||
+7
-7
@@ -7,21 +7,21 @@ use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
use tokio::sync::futures::OwnedNotified;
|
||||
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::session::{StreamIdentity, WebSession};
|
||||
|
||||
/// Async byte stream that maps one WEB stream identifier onto carrier frames.
|
||||
pub(crate) struct WebLogicalStream {
|
||||
session: Arc<WebSession>,
|
||||
stream_id: u32,
|
||||
stream: StreamIdentity,
|
||||
budget_wait: Option<Pin<Box<OwnedNotified>>>,
|
||||
}
|
||||
|
||||
impl WebLogicalStream {
|
||||
/// Binds a virtual byte stream to one live carrier stream identifier.
|
||||
pub(crate) fn new(session: Arc<WebSession>, stream_id: u32) -> Self {
|
||||
pub(crate) fn new(session: Arc<WebSession>, stream: StreamIdentity) -> Self {
|
||||
Self {
|
||||
session,
|
||||
stream_id,
|
||||
stream,
|
||||
budget_wait: None,
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,7 @@ impl AsyncRead for WebLogicalStream {
|
||||
cx: &mut Context<'_>,
|
||||
output: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
self.session.poll_read(self.stream_id, cx, output)
|
||||
self.session.poll_read(self.stream, cx, output)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ impl AsyncWrite for WebLogicalStream {
|
||||
cx: &mut Context<'_>,
|
||||
input: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
let result = self.session.poll_write(self.stream_id, cx, input);
|
||||
let result = self.session.poll_write(self.stream, cx, input);
|
||||
if !result.is_pending() {
|
||||
self.budget_wait = None;
|
||||
return result;
|
||||
@@ -64,7 +64,7 @@ impl AsyncWrite for WebLogicalStream {
|
||||
}
|
||||
self.budget_wait = None;
|
||||
}
|
||||
match self.session.poll_write(self.stream_id, cx, input) {
|
||||
match self.session.poll_write(self.stream, cx, input) {
|
||||
Poll::Ready(result) => {
|
||||
self.budget_wait = None;
|
||||
Poll::Ready(result)
|
||||
|
||||
+16
-11
@@ -81,10 +81,11 @@ impl HttpTraceExchange {
|
||||
.headers()
|
||||
.get(hyper::header::USER_AGENT)
|
||||
.map(|value| bounded_text(value.as_bytes(), USER_AGENT_MAX_BYTES)),
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_headers(request.headers()))
|
||||
.unwrap_or_default(),
|
||||
if policy.capture_headers {
|
||||
sanitized_headers(request.headers())
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
sensitive_values(request.headers(), request.uri().query()),
|
||||
)
|
||||
} else {
|
||||
@@ -423,13 +424,17 @@ mod tests {
|
||||
.header("authorization", format!("Bearer {request_token}"))
|
||||
.body(())
|
||||
.unwrap();
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
policy.body_capture = WebDebugBodyCapture::Prefix;
|
||||
policy.body_prefix_bytes = 256;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 4;
|
||||
limits.debug_bytes_global = 16 * 1024;
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
body_capture: WebDebugBodyCapture::Prefix,
|
||||
body_prefix_bytes: 256,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = WebLimitsConfig {
|
||||
debug_records_capacity: 4,
|
||||
debug_bytes_global: 16 * 1024,
|
||||
..Default::default()
|
||||
};
|
||||
let store = WebTraceStore::new(policy, &limits);
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.30".parse().unwrap())
|
||||
|
||||
@@ -13,5 +13,6 @@ pub(crate) use exchange::HttpTraceExchange;
|
||||
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
|
||||
pub(crate) use types::{
|
||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
||||
TraceLifecycleEvent, TraceRecord, TraceRecordKind, TraceRoute, TraceWebSocketContext,
|
||||
TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind, TraceRoute,
|
||||
TraceWebSocketContext,
|
||||
};
|
||||
|
||||
+22
-18
@@ -28,12 +28,11 @@ pub(super) fn request_dynamic_bytes<B>(
|
||||
lossy_text_reservation(value.as_bytes(), USER_AGENT_MAX_BYTES)
|
||||
}),
|
||||
)
|
||||
.saturating_add(
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_header_bytes(request.headers()))
|
||||
.unwrap_or(0),
|
||||
)
|
||||
.saturating_add(if policy.capture_headers {
|
||||
sanitized_header_bytes(request.headers())
|
||||
} else {
|
||||
0
|
||||
})
|
||||
.saturating_add(sensitive_value_bytes(
|
||||
request.headers(),
|
||||
request.uri().query(),
|
||||
@@ -45,10 +44,11 @@ pub(super) fn response_dynamic_bytes<B>(
|
||||
response: &hyper::Response<B>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> usize {
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_header_bytes(response.headers()))
|
||||
.unwrap_or(0)
|
||||
if policy.capture_headers {
|
||||
sanitized_header_bytes(response.headers())
|
||||
} else {
|
||||
0
|
||||
}
|
||||
.saturating_add(sensitive_value_bytes(response.headers(), None))
|
||||
}
|
||||
|
||||
@@ -173,11 +173,11 @@ fn sanitized_header_bytes(headers: &hyper::HeaderMap) -> usize {
|
||||
total
|
||||
.saturating_add(std::mem::size_of::<TraceHeader>())
|
||||
.saturating_add(name.as_str().len())
|
||||
.saturating_add(
|
||||
header_value_allowed(name)
|
||||
.then(|| lossy_text_reservation(value.as_bytes(), 4096))
|
||||
.unwrap_or(0),
|
||||
)
|
||||
.saturating_add(if header_value_allowed(name) {
|
||||
lossy_text_reservation(value.as_bytes(), 4096)
|
||||
} else {
|
||||
0
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -195,6 +195,8 @@ fn header_value_allowed(name: &hyper::header::HeaderName) -> bool {
|
||||
| "x-down-cursor"
|
||||
| "x-lane-id"
|
||||
| "x-lane-closed"
|
||||
| "x-carrier-attempt"
|
||||
| "x-carrier-failure"
|
||||
| "x-carrier-mode"
|
||||
| "retry-after"
|
||||
| "cache-control"
|
||||
@@ -273,9 +275,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn full_capture_keeps_decoy_bodies_prefix_bounded() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.body_capture = WebDebugBodyCapture::Full;
|
||||
policy.decoy_body_prefix_bytes = 123;
|
||||
let policy = WebDebugConfig {
|
||||
body_capture: WebDebugBodyCapture::Full,
|
||||
decoy_body_prefix_bytes: 123,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(capture_limit(&policy, TraceRoute::Decoy, 4096), Some(123));
|
||||
assert_eq!(capture_limit(&policy, TraceRoute::Uplink, 4096), Some(4096));
|
||||
}
|
||||
|
||||
+68
-9
@@ -9,7 +9,8 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
use super::exchange::HttpTraceExchange;
|
||||
use super::types::{
|
||||
TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind,
|
||||
TraceCarrierDetail, TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord,
|
||||
TraceRecordKind,
|
||||
};
|
||||
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
||||
|
||||
@@ -166,6 +167,57 @@ impl WebTraceStore {
|
||||
event: TraceLifecycleEvent,
|
||||
stream_id: Option<u32>,
|
||||
reason: Option<&'static str>,
|
||||
) {
|
||||
self.record_lifecycle_detail(
|
||||
peer_ip,
|
||||
effective_ip,
|
||||
identity,
|
||||
event,
|
||||
stream_id,
|
||||
reason,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
/// Records one carrier transition with structured, non-secret negotiation fields.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn record_carrier_lifecycle(
|
||||
&self,
|
||||
effective_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
event: TraceLifecycleEvent,
|
||||
client_class: &'static str,
|
||||
carrier: crate::config::WebCarrier,
|
||||
attempt: u8,
|
||||
scores: [i16; 4],
|
||||
reason: Option<&'static str>,
|
||||
) {
|
||||
self.record_lifecycle_detail(
|
||||
None,
|
||||
Some(effective_ip),
|
||||
identity,
|
||||
event,
|
||||
None,
|
||||
reason,
|
||||
Some(TraceCarrierDetail {
|
||||
client_class,
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn record_lifecycle_detail(
|
||||
&self,
|
||||
peer_ip: Option<IpAddr>,
|
||||
effective_ip: Option<IpAddr>,
|
||||
identity: TraceIdentity,
|
||||
event: TraceLifecycleEvent,
|
||||
stream_id: Option<u32>,
|
||||
reason: Option<&'static str>,
|
||||
carrier: Option<TraceCarrierDetail>,
|
||||
) {
|
||||
if !self.enabled.load(Ordering::Acquire) {
|
||||
return;
|
||||
@@ -197,6 +249,7 @@ impl WebTraceStore {
|
||||
event,
|
||||
stream_id,
|
||||
reason,
|
||||
carrier,
|
||||
}),
|
||||
};
|
||||
if !self.try_commit(record, reservation, epoch) {
|
||||
@@ -378,11 +431,15 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = records_capacity;
|
||||
limits.debug_bytes_global = bytes_capacity;
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = WebLimitsConfig {
|
||||
debug_records_capacity: records_capacity,
|
||||
debug_bytes_global: bytes_capacity,
|
||||
..Default::default()
|
||||
};
|
||||
WebTraceStore::new(policy, &limits)
|
||||
}
|
||||
|
||||
@@ -427,9 +484,11 @@ mod tests {
|
||||
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let mut changed = WebDebugConfig::default();
|
||||
changed.enabled = true;
|
||||
changed.capture_headers = false;
|
||||
let changed = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_headers: false,
|
||||
..Default::default()
|
||||
};
|
||||
store.apply_policy(&changed);
|
||||
exchange.commit();
|
||||
|
||||
|
||||
@@ -65,16 +65,15 @@ impl WebTraceStore {
|
||||
)
|
||||
.unwrap_or(0);
|
||||
let capture_bytes = payload.len().min(capture_limit);
|
||||
let frame_reservation = policy
|
||||
.capture_frames
|
||||
.then(|| {
|
||||
let frame_reservation = if policy.capture_frames {
|
||||
payload
|
||||
.len()
|
||||
.div_ceil(frame::HEADER_BYTES)
|
||||
.clamp(1, self.frame_limits.max_frames_per_body)
|
||||
.saturating_mul(std::mem::size_of::<TraceFrame>())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let identity_bytes = context
|
||||
.identity
|
||||
.user
|
||||
@@ -168,14 +167,18 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn websocket_message_capture_retains_bounded_identity_body_timing_and_frames() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
policy.capture_frames = true;
|
||||
policy.capture_timings = true;
|
||||
policy.body_capture = WebDebugBodyCapture::Full;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 4;
|
||||
limits.debug_bytes_global = 64 * 1024;
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_frames: true,
|
||||
capture_timings: true,
|
||||
body_capture: WebDebugBodyCapture::Full,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = WebLimitsConfig {
|
||||
debug_records_capacity: 4,
|
||||
debug_bytes_global: 64 * 1024,
|
||||
..Default::default()
|
||||
};
|
||||
let store = WebTraceStore::new(policy, &limits);
|
||||
let request = hyper::Request::builder()
|
||||
.header(hyper::header::USER_AGENT, "trace-client")
|
||||
|
||||
@@ -228,6 +228,18 @@ pub(crate) enum TraceLifecycleEvent {
|
||||
BridgeIssued,
|
||||
/// Bootstrap or bridge admission was rejected.
|
||||
BootstrapRejected,
|
||||
/// A session request was classified for legacy or automatic negotiation.
|
||||
CarrierClassified,
|
||||
/// One carrier candidate was selected for an attempt.
|
||||
CarrierSelected,
|
||||
/// A provisional carrier was reported failed by the bridge.
|
||||
CarrierFailed,
|
||||
/// An uncommitted carrier session was atomically superseded.
|
||||
CarrierSuperseded,
|
||||
/// Bidirectional carrier evidence made replacement unsafe.
|
||||
CarrierCommitted,
|
||||
/// A committed carrier survived its configured health interval.
|
||||
CarrierHealthy,
|
||||
/// A new session was created.
|
||||
SessionCreated,
|
||||
/// An idempotent session creation was replayed.
|
||||
@@ -264,6 +276,12 @@ impl TraceLifecycleEvent {
|
||||
match self {
|
||||
Self::BridgeIssued => "bridge_issued",
|
||||
Self::BootstrapRejected => "bootstrap_rejected",
|
||||
Self::CarrierClassified => "carrier_classified",
|
||||
Self::CarrierSelected => "carrier_selected",
|
||||
Self::CarrierFailed => "carrier_failed",
|
||||
Self::CarrierSuperseded => "carrier_superseded",
|
||||
Self::CarrierCommitted => "carrier_committed",
|
||||
Self::CarrierHealthy => "carrier_healthy",
|
||||
Self::SessionCreated => "session_created",
|
||||
Self::SessionReplayed => "session_replayed",
|
||||
Self::SessionRejected => "session_rejected",
|
||||
@@ -282,6 +300,19 @@ impl TraceLifecycleEvent {
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-sensitive carrier-negotiation fields attached to lifecycle records.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceCarrierDetail {
|
||||
/// Stable client classification; never a raw User-Agent.
|
||||
pub(crate) client_class: &'static str,
|
||||
/// Candidate associated with the lifecycle transition.
|
||||
pub(crate) carrier: crate::config::WebCarrier,
|
||||
/// One-based candidate attempt.
|
||||
pub(crate) attempt: u8,
|
||||
/// Weighted learning scores indexed by the canonical carrier order.
|
||||
pub(crate) scores: [i16; 4],
|
||||
}
|
||||
|
||||
/// One typed WEB lifecycle observation.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceLifecycleRecord {
|
||||
@@ -291,6 +322,8 @@ pub(crate) struct TraceLifecycleRecord {
|
||||
pub(crate) stream_id: Option<u32>,
|
||||
/// Closed outcome or rejection reason.
|
||||
pub(crate) reason: Option<&'static str>,
|
||||
/// Carrier negotiation detail when this is a carrier lifecycle event.
|
||||
pub(crate) carrier: Option<TraceCarrierDetail>,
|
||||
}
|
||||
|
||||
/// Trace record payload variant.
|
||||
|
||||
Reference in New Issue
Block a user