WEB Carriers Safe-matrix Refactored

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-26 20:33:33 +03:00
parent 8b2b88f30c
commit 34eeb2d856
49 changed files with 3406 additions and 936 deletions
+6 -4
View File
@@ -56,9 +56,11 @@ pub(super) async fn handle_down(
if !body.is_empty() {
return serve_decoy(request, vhost, true, &runtime).await;
}
let _lane_poll = if lane_id.is_some() {
let auxiliary = lane_id.is_some_and(|lane_id| session.lane_poll_is_auxiliary(lane_id));
let Some(permit) = runtime.try_lane_poll(auxiliary) else {
let Some(_down_poll) = runtime.try_lane_poll(false) else {
return service_unavailable();
};
let _control_lane_poll = if lane_id == Some(0) {
let Some(permit) = runtime.try_lane_poll(true) else {
return service_unavailable();
};
Some(permit)
@@ -90,7 +92,7 @@ pub(super) async fn handle_down(
trace.record_frames(
TraceDirection::Response,
&result.body,
&runtime.active_generation().config().web.limits,
session.limits(),
);
}
let mut response = full_response(StatusCode::OK, result.body);
+125 -9
View File
@@ -1,5 +1,7 @@
use super::*;
use std::time::Duration;
use sha2::{Digest, Sha256};
const CAPABILITIES: &str = "https,https-lanes,websocket,websocket-lanes";
@@ -95,7 +97,7 @@ async fn absent_carriers_reject_negotiation_and_preserve_legacy_creation() {
#[tokio::test]
async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
let capability = [42; 32];
let config = negotiation_runtime_config(
let mut config = negotiation_runtime_config(
capability,
WebCarrier::Websocket,
false,
@@ -105,6 +107,7 @@ async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
WebCarrier::Websocket,
]),
);
config.web.timeouts.long_poll_secs = 1;
let generation = test_runtime_generation(1, config);
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -121,6 +124,8 @@ async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
let replay = request(&listener, &runtime, first_request).await;
let (replay_headers, _) = split_response(&replay);
assert_eq!(response_header(replay_headers, "x-session-token"), first_token);
assert_eq!(response_header(replay_headers, "x-carrier-candidate-count"), "3");
assert_eq!(response_header(replay_headers, "x-carrier-state"), "provisional");
let second_request = create_request(&bootstrap, &hello, Some(2), Some("timeout"));
let second = request(&listener, &runtime, second_request.clone()).await;
@@ -140,20 +145,59 @@ async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
let second_replay = request(&listener, &runtime, second_request).await;
let (second_replay_headers, _) = split_response(&second_replay);
assert_eq!(
response_header(second_replay_headers, "x-session-token"),
second_token
);
for header in [
"x-session-token",
"x-carrier-mode",
"x-carrier-attempt",
"x-carrier-candidate-count",
"x-carrier-deadline",
"x-carrier-state",
] {
assert_eq!(
response_header(second_replay_headers, header),
response_header(second_headers, header),
);
}
let changed_failure = request(
&listener,
&runtime,
create_request(&bootstrap, &hello, Some(2), Some("network")),
)
.await;
let (changed_failure_headers, _) = split_response(&changed_failure);
assert!(optional_response_header(changed_failure_headers, "x-session-token").is_none());
let open = frame::encode(FrameType::Open, 7, &[]);
let data = frame::encode(FrameType::Data, 7, &[0]);
let mut body = Vec::with_capacity(open.len() + data.len());
body.extend_from_slice(&open);
body.extend_from_slice(&data);
let mut uplink = format!(
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nContent-Type: application/octet-stream\r\nX-Up-Seq: 1\r\nX-Lane-ID: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
open.len()
body.len()
)
.into_bytes();
uplink.extend_from_slice(&open);
let committed = request(&listener, &runtime, uplink).await;
assert!(committed.starts_with(b"HTTP/1.1 204"));
uplink.extend_from_slice(&body);
let accepted = request(&listener, &runtime, uplink).await;
assert!(accepted.starts_with(b"HTTP/1.1 204"));
let first_down = format!(
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nX-Down-Cursor: 0\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
.into_bytes();
let first_down = request(&listener, &runtime, first_down).await;
let (first_down_headers, first_down_body) = split_response(&first_down);
assert!(first_down_headers.starts_with(b"HTTP/1.1 200"));
assert!(!first_down_body.is_empty());
assert_eq!(response_header(first_down_headers, "x-down-cursor"), "1");
let acknowledgement = format!(
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {second_token}\r\nX-Down-Cursor: 1\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
.into_bytes();
let acknowledgement = request(&listener, &runtime, acknowledgement).await;
assert!(acknowledgement.starts_with(b"HTTP/1.1 204"));
let third = request(
&listener,
@@ -162,7 +206,13 @@ async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
)
.await;
let (third_headers, _) = split_response(&third);
assert!(third_headers.starts_with(b"HTTP/1.1 409"));
assert!(optional_response_header(third_headers, "x-session-token").is_none());
assert_eq!(response_header(third_headers, "x-carrier-mode"), "https-lanes");
assert_eq!(response_header(third_headers, "x-carrier-attempt"), "2");
assert_eq!(response_header(third_headers, "x-carrier-candidate-count"), "3");
assert_eq!(response_header(third_headers, "x-carrier-deadline"), "12");
assert_eq!(response_header(third_headers, "x-carrier-state"), "committed");
assert!(
runtime
.get_session(token_hash(&second_token), "proxy.example.com")
@@ -174,3 +224,69 @@ async fn negotiation_replays_replaces_and_freezes_after_carrier_commit() {
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
#[tokio::test]
async fn https_lane_downlink_can_arrive_before_its_uplink_open() {
let capability = [43; 32];
let mut config = negotiation_runtime_config(
capability,
WebCarrier::HttpsLanes,
false,
Arc::from([WebCarrier::HttpsLanes]),
);
config.web.timeouts.lane_open_wait_secs = 1;
config.web.timeouts.long_poll_secs = 2;
let generation = test_runtime_generation(1, config);
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap());
let bootstrap = issue_bootstrap(&runtime, "192.0.2.10");
let hello = frame::encode(FrameType::Hello, 0, &[1]);
let created = request(
&listener,
&runtime,
create_request(&bootstrap, &hello, Some(1), None),
)
.await;
let (created_headers, _) = split_response(&created);
let token = response_header(created_headers, "x-session-token").to_string();
let down_request = format!(
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nX-Down-Cursor: 0\r\nX-Lane-ID: 7\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
.into_bytes();
let down_listener = Arc::clone(&listener);
let down_runtime = Arc::clone(&runtime);
let down = tokio::spawn(async move {
request(&down_listener, &down_runtime, down_request).await
});
tokio::task::yield_now().await;
let open = frame::encode(FrameType::Open, 7, &[]);
let data = frame::encode(FrameType::Data, 7, &[1]);
let mut body = Vec::with_capacity(open.len() + data.len());
body.extend_from_slice(&open);
body.extend_from_slice(&data);
let mut uplink = format!(
"POST /api/v1/up HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nContent-Type: application/octet-stream\r\nX-Up-Seq: 1\r\nX-Lane-ID: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.into_bytes();
uplink.extend_from_slice(&body);
let accepted = request(&listener, &runtime, uplink).await;
assert!(accepted.starts_with(b"HTTP/1.1 204"));
let down = tokio::time::timeout(Duration::from_secs(3), down)
.await
.unwrap()
.unwrap();
let (down_headers, _) = split_response(&down);
assert!(
down_headers.starts_with(b"HTTP/1.1 200")
|| down_headers.starts_with(b"HTTP/1.1 204")
);
assert!(optional_response_header(down_headers, "x-down-cursor").is_some());
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
+84 -57
View File
@@ -1,69 +1,23 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use base64::Engine as _;
use hyper::Request;
use hyper::header;
use ipnetwork::IpNetwork;
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use crate::config::{WebClientIpSource, WebRuntimeProfile, WebRuntimeVhost};
use crate::config::{WebRuntimeProfile, WebRuntimeVhost};
use crate::web::manager::{
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierRequest, TokenHash,
};
const USER_AGENT_CONTEXT: &[u8] = b"telemt-web-carrier-user-agent-v1\0";
/// Parses one lowercase canonical Host value restricted to the public HTTPS port.
pub(super) fn canonical_request_host<B>(request: &Request<B>) -> Option<&str> {
let values = request.headers().get_all(header::HOST);
let mut values = values.iter();
let value = values.next()?.to_str().ok()?;
if values.next().is_some() {
return None;
}
let authority = value.parse::<hyper::http::uri::Authority>().ok()?;
if authority.port_u16().is_some_and(|port| port != 443) {
return None;
}
let host = value.strip_suffix(":443").unwrap_or(value);
if authority.host() != host || host.bytes().any(|byte| byte.is_ascii_uppercase()) {
return None;
}
Some(host)
}
/// Accepts one forwarded client address or the direct address of a trusted peer.
pub(super) fn client_ip<B>(
request: &Request<B>,
peer: SocketAddr,
source: WebClientIpSource,
trusted_proxy_cidrs: &[IpNetwork],
) -> Option<IpAddr> {
if !trusted_proxy_cidrs
.iter()
.any(|network| network.contains(peer.ip()))
{
return None;
}
let header_name = match source {
WebClientIpSource::XForwardedFor => "x-forwarded-for",
};
let values = request.headers().get_all(header_name);
let mut values = values.iter();
let Some(value) = values.next() else {
return Some(peer.ip());
};
let value = value.to_str().ok()?;
if values.next().is_some() || value.trim() != value || value.contains(',') {
return None;
}
if value.is_empty() {
return Some(peer.ip());
}
value.parse::<IpAddr>().ok()
}
// Canonical host and forwarded-address provenance remain isolated from credentials.
mod identity;
pub(super) use identity::{
canonical_request_host, carrier_ip_learning_eligible, client_ip,
};
/// Decodes an exact canonical bridge query without allocating credential strings.
pub(super) fn bridge_candidate(query: Option<&str>) -> ([u8; 32], bool) {
@@ -177,16 +131,31 @@ pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<Car
let capabilities = single_header(request, "x-carrier-capabilities");
let attempt = optional_canonical_u8_header(request, "x-carrier-attempt")?;
let failure = optional_failure_header(request)?;
let native_ios = native_ios_user_agent(request);
match (capabilities, attempt) {
(None, None) if failure.is_none() => Some(CarrierRequest::legacy(user_agent_hash)),
(None, None) if failure.is_none() => {
if native_ios {
Some(CarrierRequest::ios(user_agent_hash))
} else {
Some(CarrierRequest::legacy(user_agent_hash))
}
}
(Some(capabilities), Some(attempt)) => {
let capabilities = parse_capabilities(capabilities)?;
if (attempt == 1) != failure.is_none() {
return None;
}
Some(CarrierRequest::automatic(
CarrierClientClass::Bridge,
capabilities,
if native_ios {
CarrierClientClass::Ios
} else {
CarrierClientClass::Bridge
},
if native_ios {
CarrierCapabilities::ios()
} else {
capabilities
},
attempt,
failure,
user_agent_hash,
@@ -197,8 +166,16 @@ pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<Car
return None;
}
Some(CarrierRequest::automatic(
CarrierClientClass::BrowserHint,
CarrierCapabilities::all(),
if native_ios {
CarrierClientClass::Ios
} else {
CarrierClientClass::BrowserHint
},
if native_ios {
CarrierCapabilities::ios()
} else {
CarrierCapabilities::all()
},
attempt,
failure,
user_agent_hash,
@@ -208,6 +185,13 @@ pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<Car
}
}
fn native_ios_user_agent<B>(request: &Request<B>) -> bool {
single_header(request, header::USER_AGENT).is_some_and(|value| {
let value = value.to_ascii_lowercase();
value.contains("cfnetwork/") && value.contains("darwin/")
})
}
fn parse_capabilities(value: &str) -> Option<CarrierCapabilities> {
let mut bits = 0u8;
let mut previous = None;
@@ -292,6 +276,9 @@ fn single_header<B>(request: &Request<B>, name: impl header::AsHeaderName) -> Op
#[cfg(test)]
mod tests {
use super::*;
use ipnetwork::IpNetwork;
use crate::config::{WebCarrier, WebClientIpSource};
#[test]
fn canonical_bridge_query_rejects_aliases() {
@@ -485,6 +472,46 @@ mod tests {
assert!(carrier_request(&reordered, "proxy.example.com").is_none());
}
#[test]
fn native_ios_capability_claims_cannot_enable_parallel_carriers() {
let request = Request::builder()
.header(
"x-carrier-capabilities",
"https,https-lanes,websocket,websocket-lanes",
)
.header("x-carrier-attempt", "1")
.header(
header::USER_AGENT,
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
)
.body(())
.unwrap();
let parsed = carrier_request(&request, "proxy.example.com").unwrap();
assert_eq!(parsed.class(), CarrierClientClass::Ios);
assert!(parsed.supports(WebCarrier::Https));
assert!(!parsed.supports(WebCarrier::HttpsLanes));
assert!(!parsed.supports(WebCarrier::Websocket));
assert!(!parsed.supports(WebCarrier::WebsocketLanes));
}
#[test]
fn mapped_private_addresses_are_not_learning_evidence() {
for address in ["::ffff:127.0.0.1", "::ffff:10.0.0.1"] {
let effective_ip = address.parse().unwrap();
let request = Request::builder()
.header("x-forwarded-for", address)
.body(())
.unwrap();
assert!(!carrier_ip_learning_eligible(&request, effective_ip));
}
let effective_ip = "::ffff:8.8.8.8".parse().unwrap();
let request = Request::builder()
.header("x-forwarded-for", "::ffff:8.8.8.8")
.body(())
.unwrap();
assert!(carrier_ip_learning_eligible(&request, effective_ip));
}
#[test]
fn strict_browser_metadata_recovers_a_stripped_capability_marker() {
let request = Request::builder()
+111
View File
@@ -0,0 +1,111 @@
use std::net::{IpAddr, SocketAddr};
use hyper::Request;
use hyper::header;
use ipnetwork::IpNetwork;
use crate::config::WebClientIpSource;
/// Parses one lowercase canonical Host value restricted to the public HTTPS port.
pub(in crate::web::http) fn canonical_request_host<B>(request: &Request<B>) -> Option<&str> {
let values = request.headers().get_all(header::HOST);
let mut values = values.iter();
let value = values.next()?.to_str().ok()?;
if values.next().is_some() {
return None;
}
let authority = value.parse::<hyper::http::uri::Authority>().ok()?;
if authority.port_u16().is_some_and(|port| port != 443) {
return None;
}
let host = value.strip_suffix(":443").unwrap_or(value);
if authority.host() != host || host.bytes().any(|byte| byte.is_ascii_uppercase()) {
return None;
}
Some(host)
}
/// Accepts one forwarded client address or the direct address of a trusted peer.
pub(in crate::web::http) fn client_ip<B>(
request: &Request<B>,
peer: SocketAddr,
source: WebClientIpSource,
trusted_proxy_cidrs: &[IpNetwork],
) -> Option<IpAddr> {
if !trusted_proxy_cidrs
.iter()
.any(|network| network.contains(peer.ip()))
{
return None;
}
let header_name = match source {
WebClientIpSource::XForwardedFor => "x-forwarded-for",
};
let values = request.headers().get_all(header_name);
let mut values = values.iter();
let Some(value) = values.next() else {
return Some(peer.ip());
};
let value = value.to_str().ok()?;
if values.next().is_some() || value.trim() != value || value.contains(',') {
return None;
}
if value.is_empty() {
return Some(peer.ip());
}
value.parse::<IpAddr>().ok()
}
/// Allows IP learning only for one explicit globally routable forwarded address.
pub(in crate::web::http) fn carrier_ip_learning_eligible<B>(
request: &Request<B>,
effective_ip: IpAddr,
) -> bool {
let mut values = request.headers().get_all("x-forwarded-for").iter();
let Some(value) = values.next().and_then(|value| value.to_str().ok()) else {
return false;
};
if values.next().is_some()
|| value.trim() != value
|| value.contains(',')
|| value.parse::<IpAddr>().ok() != Some(effective_ip)
{
return false;
}
globally_routable(effective_ip)
}
fn globally_routable(address: IpAddr) -> bool {
match address {
IpAddr::V4(address) => globally_routable_v4(address),
IpAddr::V6(address) => {
if let Some(address) = address.to_ipv4_mapped() {
return globally_routable_v4(address);
}
let segments = address.segments();
!address.is_unspecified()
&& !address.is_loopback()
&& segments[0] & 0xfe00 != 0xfc00
&& segments[0] & 0xffc0 != 0xfe80
&& segments[0] & 0xff00 != 0xff00
&& !(segments[0] == 0x2001 && segments[1] == 0x0db8)
}
}
}
fn globally_routable_v4(address: std::net::Ipv4Addr) -> bool {
let [a, b, c, _] = address.octets();
!(a == 0
|| a == 10
|| a == 127
|| (a == 100 && (64..=127).contains(&b))
|| (a == 169 && b == 254)
|| (a == 172 && (16..=31).contains(&b))
|| (a == 192 && b == 0 && c == 0)
|| (a == 192 && b == 0 && c == 2)
|| (a == 192 && b == 168)
|| (a == 198 && (b == 18 || b == 19))
|| (a == 198 && b == 51 && c == 100)
|| (a == 203 && b == 0 && c == 113)
|| a >= 224)
}
+11 -1
View File
@@ -272,6 +272,10 @@ pub(super) async fn handle(
(ParsedCarrier::Lane(lane_id), WebCarrier::WebsocketLanes) => WebSocketKind::Lane(lane_id),
_ => return serve_decoy(request, vhost, true, &runtime).await,
};
let mut probe_reservation = match session.reserve_websocket_probe(parsed.acknowledge_commit) {
Ok(reservation) => reservation,
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
};
let mut lane_reservation = match kind {
WebSocketKind::Multiplex => None,
WebSocketKind::Lane(lane_id) => match session.reserve_websocket_lane(lane_id) {
@@ -279,7 +283,7 @@ pub(super) async fn handle(
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
},
};
let timeouts = runtime.active_generation().config().web.timeouts.clone();
let timeouts = session.timeouts().clone();
let connection = match runtime
.admit_websocket(
session.profile_key(),
@@ -297,6 +301,11 @@ pub(super) async fn handle(
Ok(connection) => connection,
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
};
if let Some(reservation) = probe_reservation.as_mut()
&& reservation.bind(connection.id()).is_err()
{
return serve_decoy(request, vhost, true, &runtime).await;
}
let trace_context = runtime.trace().websocket_context(
&request,
peer.ip(),
@@ -326,6 +335,7 @@ pub(super) async fn handle(
driver_session,
connection,
lane_reservation.take(),
probe_reservation.take(),
trace_context,
parsed.acknowledge_commit,
)
+136 -54
View File
@@ -11,7 +11,9 @@ use super::ConnectionIo;
use crate::web::manager::{
WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
};
use crate::web::session::{WebSession, WebSocketLaneReservation};
use crate::web::session::{
WebSession, WebSocketLaneReservation, WebSocketProbeReservation,
};
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
const READ_BUFFER_BYTES: usize = 64 * 1024;
@@ -27,11 +29,12 @@ pub(super) async fn run_upgraded(
session: Arc<WebSession>,
connection: WebSocketConnection,
mut lane_reservation: Option<WebSocketLaneReservation>,
_probe_reservation: Option<WebSocketProbeReservation>,
trace: Option<TraceWebSocketContext>,
acknowledge_commit: bool,
) {
let cancellation = connection.cancellation();
let timeouts = runtime.active_generation().config().web.timeouts.clone();
let timeouts = session.timeouts().clone();
let upgraded = tokio::select! {
_ = cancellation.cancelled() => return,
result = tokio::time::timeout(
@@ -47,7 +50,7 @@ pub(super) async fn run_upgraded(
};
let mut io = parts.io.into_inner();
io.enable_websocket(parts.read_buf);
let limits = runtime.active_generation().config().web.limits.clone();
let limits = session.limits().clone();
let config = WebSocketConfig::default()
.read_buffer_size(READ_BUFFER_BYTES)
.write_buffer_size(WRITE_BUFFER_BYTES)
@@ -59,7 +62,9 @@ pub(super) async fn run_upgraded(
.max_message_size(Some(limits.carrier_batch_bytes))
.max_frame_size(Some(limits.carrier_batch_bytes));
let mut socket = WebSocketStream::from_raw_socket(io, Role::Server, Some(config)).await;
connection.mark_opened();
if !connection.mark_opened() {
return;
}
if let Some(reservation) = lane_reservation.as_mut() {
let _ = run_lane(
&mut socket,
@@ -84,16 +89,11 @@ pub(super) async fn run_upgraded(
)
.await;
}
let eviction = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_eviction_secs,
);
if !cancellation.is_cancelled() {
let _ = tokio::time::timeout(eviction, socket.close(None)).await;
let eviction = Duration::from_secs(timeouts.websocket_eviction_secs);
tokio::select! {
biased;
_ = cancellation.cancelled() => {}
_ = tokio::time::timeout(eviction, socket.close(None)) => {}
}
if let Some(reservation) = lane_reservation {
session.close_websocket_lane(reservation.lane_id());
@@ -121,15 +121,12 @@ async fn run_multiplex(
let mut read_budget = None;
let liveness_interval = connection.liveness_interval();
let mut next_ping = Instant::now() + liveness_interval;
let open_deadline = Instant::now()
+ Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_open_secs,
);
let open_deadline =
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
let backpressure_timeout =
Duration::from_secs(session.timeouts().websocket_backpressure_secs);
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
let maximum_message = session.limits().carrier_batch_bytes;
let mut active = false;
loop {
let down = session.poll_down(cursor);
@@ -144,6 +141,8 @@ async fn run_multiplex(
session.profile_key(),
&cancellation,
&mut read_budget,
maximum_message,
backpressure_timeout,
) => {
DriverEvent::Incoming(incoming?)
}
@@ -153,8 +152,15 @@ async fn run_multiplex(
DriverEvent::Incoming((message, _budget)) => match message {
Message::Binary(body) => {
let started = Instant::now();
let result =
process_multiplex(runtime, session, sequence, &body, &cancellation).await;
let result = process_multiplex(
runtime,
session,
sequence,
&body,
&cancellation,
backpressure_timeout,
)
.await;
record_message(
runtime,
trace,
@@ -163,14 +169,17 @@ async fn run_multiplex(
&body,
started,
);
result?;
if acknowledge_commit && sequence == 1 && session.is_carrier_committed() {
let progressed = result?;
if acknowledge_commit && sequence == 1 {
if !session.needs_websocket_commit_ack(connection.id()) {
return Err(());
}
let started = Instant::now();
send(
socket,
runtime,
Message::Binary(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
@@ -181,9 +190,19 @@ async fn run_multiplex(
&[],
started,
);
if !session.websocket_commit_ack_written(connection.id()) {
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
}
if !active {
connection.mark_active();
if !active && progressed {
if !connection.mark_active() {
return Err(());
}
active = true;
}
sequence = sequence.checked_add(1).ok_or(())?;
@@ -204,7 +223,7 @@ async fn run_multiplex(
}
Message::Ping(payload) => {
let started = Instant::now();
flush(socket, runtime, &cancellation).await?;
flush(socket, &cancellation, write_timeout).await?;
record_message(
runtime,
trace,
@@ -251,7 +270,13 @@ async fn run_multiplex(
DriverEvent::Down(result) => {
if result.body.is_empty() {
let started = Instant::now();
send(socket, runtime, Message::Ping(Bytes::new()), &cancellation).await?;
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
@@ -267,12 +292,19 @@ async fn run_multiplex(
session.profile_key(),
result.body.len(),
&cancellation,
backpressure_timeout,
)
.await?;
let body = result.body;
let started = Instant::now();
if trace.is_some() {
send(socket, runtime, Message::Binary(body.clone()), &cancellation).await?;
send(
socket,
Message::Binary(body.clone()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
@@ -282,7 +314,13 @@ async fn run_multiplex(
started,
);
} else {
send(socket, runtime, Message::Binary(body), &cancellation).await?;
send(
socket,
Message::Binary(body),
&cancellation,
write_timeout,
)
.await?;
}
connection.mark_progress();
}
@@ -290,7 +328,13 @@ async fn run_multiplex(
}
DriverEvent::Liveness => {
let started = Instant::now();
send(socket, runtime, Message::Ping(Bytes::new()), &cancellation).await?;
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
@@ -322,15 +366,12 @@ async fn run_lane(
let mut read_budget = None;
let liveness_interval = connection.liveness_interval();
let mut next_ping = Instant::now() + liveness_interval;
let open_deadline = Instant::now()
+ Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_open_secs,
);
let open_deadline =
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
let backpressure_timeout =
Duration::from_secs(session.timeouts().websocket_backpressure_secs);
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
let maximum_message = session.limits().carrier_batch_bytes;
let mut active = false;
loop {
let down = session.poll_down_lane(reservation.lane_id(), cursor);
@@ -345,6 +386,8 @@ async fn run_lane(
session.profile_key(),
&cancellation,
&mut read_budget,
maximum_message,
backpressure_timeout,
) => {
DriverEvent::Incoming(incoming?)
}
@@ -361,6 +404,7 @@ async fn run_lane(
sequence,
&body,
&cancellation,
backpressure_timeout,
)
.await;
record_message(
@@ -371,14 +415,17 @@ async fn run_lane(
&body,
started,
);
result?;
if acknowledge_commit && sequence == 1 && session.is_carrier_committed() {
let progressed = result?;
if acknowledge_commit && sequence == 1 {
if !session.needs_websocket_commit_ack(connection.id()) {
return Err(());
}
let started = Instant::now();
if send(
socket,
runtime,
Message::Binary(Bytes::new()),
&cancellation,
write_timeout,
)
.await
.is_err()
@@ -394,9 +441,19 @@ async fn run_lane(
&[],
started,
);
if !session.websocket_commit_ack_written(connection.id()) {
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
}
if !active {
connection.mark_active();
if !active && progressed {
if !connection.mark_active() {
return Err(());
}
active = true;
}
sequence = sequence.checked_add(1).ok_or(())?;
@@ -417,7 +474,7 @@ async fn run_lane(
}
Message::Ping(payload) => {
let started = Instant::now();
flush(socket, runtime, &cancellation).await?;
flush(socket, &cancellation, write_timeout).await?;
record_message(
runtime,
trace,
@@ -467,7 +524,13 @@ async fn run_lane(
}
if result.body.is_empty() {
let started = Instant::now();
send(socket, runtime, Message::Ping(Bytes::new()), &cancellation).await?;
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
@@ -483,12 +546,19 @@ async fn run_lane(
session.profile_key(),
result.body.len(),
&cancellation,
backpressure_timeout,
)
.await?;
let body = result.body;
let started = Instant::now();
if trace.is_some() {
send(socket, runtime, Message::Binary(body.clone()), &cancellation).await?;
send(
socket,
Message::Binary(body.clone()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
@@ -498,7 +568,13 @@ async fn run_lane(
started,
);
} else {
send(socket, runtime, Message::Binary(body), &cancellation).await?;
send(
socket,
Message::Binary(body),
&cancellation,
write_timeout,
)
.await?;
}
connection.mark_progress();
}
@@ -506,7 +582,13 @@ async fn run_lane(
}
DriverEvent::Liveness => {
let started = Instant::now();
send(socket, runtime, Message::Ping(Bytes::new()), &cancellation).await?;
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
+36 -58
View File
@@ -16,19 +16,24 @@ pub(super) async fn read_message(
owner: crate::web::manager::ProfileKey,
cancellation: &CancellationToken,
retained_budget: &mut Option<WebSocketBudgetLease>,
maximum: usize,
backpressure_timeout: Duration,
) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
tokio::select! {
_ = cancellation.cancelled() => return Err(()),
ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
}
if retained_budget.is_none() {
let maximum = runtime
.active_generation()
.config()
.web
.limits
.carrier_batch_bytes;
*retained_budget = Some(reserve_data(runtime, owner, maximum, cancellation).await?);
*retained_budget = Some(
reserve_data(
runtime,
owner,
maximum,
cancellation,
backpressure_timeout,
)
.await?,
);
}
let message = tokio::select! {
_ = cancellation.cancelled() => return Err(()),
@@ -47,17 +52,13 @@ pub(super) async fn reserve_data(
owner: crate::web::manager::ProfileKey,
bytes: usize,
cancellation: &CancellationToken,
timeout: Duration,
) -> Result<WebSocketBudgetLease, ()> {
let timeout = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_backpressure_secs,
);
tokio::time::timeout(timeout, async {
loop {
if cancellation.is_cancelled() {
return Err(());
}
let notify = runtime.budget_notify();
let notified = notify.notified();
if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
@@ -79,9 +80,10 @@ pub(super) async fn process_multiplex(
sequence: u64,
body: &[u8],
cancellation: &CancellationToken,
) -> Result<(), ()> {
retry_backpressure(runtime, cancellation, || {
session.process_up(sequence, body).map(|_| ())
timeout: Duration,
) -> Result<bool, ()> {
retry_backpressure(runtime, cancellation, timeout, || {
session.process_websocket_multiplex(sequence, body)
})
.await
}
@@ -93,21 +95,17 @@ pub(super) async fn process_lane(
sequence: u64,
body: &[u8],
cancellation: &CancellationToken,
) -> Result<(), ()> {
let timeout = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_backpressure_secs,
);
timeout: Duration,
) -> Result<bool, ()> {
tokio::time::timeout(timeout, async {
loop {
if cancellation.is_cancelled() {
return Err(());
}
let notify = runtime.budget_notify();
let notified = notify.notified();
match session.process_websocket_lane(reservation, sequence, body) {
Ok(()) => return Ok(()),
Ok(progressed) => return Ok(progressed),
Err(ManagerError::Backpressure) => {}
Err(_) => return Err(()),
}
@@ -121,28 +119,24 @@ pub(super) async fn process_lane(
.map_err(|_| ())?
}
async fn retry_backpressure<F>(
async fn retry_backpressure<F, T>(
runtime: &Arc<WebProcessRuntime>,
cancellation: &CancellationToken,
timeout: Duration,
mut operation: F,
) -> Result<(), ()>
) -> Result<T, ()>
where
F: FnMut() -> Result<(), ManagerError>,
F: FnMut() -> Result<T, ManagerError>,
{
let timeout = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_backpressure_secs,
);
tokio::time::timeout(timeout, async {
loop {
if cancellation.is_cancelled() {
return Err(());
}
let notify = runtime.budget_notify();
let notified = notify.notified();
match operation() {
Ok(()) => return Ok(()),
Ok(value) => return Ok(value),
Err(ManagerError::Backpressure) => {}
Err(_) => return Err(()),
}
@@ -158,18 +152,10 @@ where
pub(super) async fn send(
socket: &mut CarrierSocket,
runtime: &WebProcessRuntime,
message: Message,
cancellation: &CancellationToken,
timeout: Duration,
) -> Result<(), ()> {
let timeout = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_write_secs,
);
tokio::select! {
_ = cancellation.cancelled() => Err(()),
result = tokio::time::timeout(timeout, socket.send(message)) => {
@@ -180,17 +166,9 @@ pub(super) async fn send(
pub(super) async fn flush(
socket: &mut CarrierSocket,
runtime: &WebProcessRuntime,
cancellation: &CancellationToken,
timeout: Duration,
) -> Result<(), ()> {
let timeout = Duration::from_secs(
runtime
.active_generation()
.config()
.web
.timeouts
.websocket_write_secs,
);
tokio::select! {
_ = cancellation.cancelled() => Err(()),
result = tokio::time::timeout(timeout, socket.flush()) => {
+32 -1
View File
@@ -160,6 +160,7 @@ fn create_automatic_session(
None,
[9; 32],
),
false,
)
.unwrap()
.token;
@@ -194,6 +195,7 @@ fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
client_ip,
&hello,
CarrierRequest::legacy([0; 32]),
false,
)
.unwrap()
.token;
@@ -382,7 +384,7 @@ async fn malformed_websocket_lane_closes_only_that_lane() {
}
#[tokio::test]
async fn automatic_websocket_carriers_ack_the_first_committing_message() {
async fn automatic_websocket_carriers_commit_after_acknowledged_peer_progress() {
for carrier in [WebCarrier::Websocket, WebCarrier::WebsocketLanes] {
let live = live_negotiation_runtime(carrier, Arc::from([carrier]));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -409,6 +411,34 @@ async fn automatic_websocket_carriers_ack_the_first_committing_message() {
.unwrap()
.is_carrier_committed()
);
socket
.send(Message::Binary(frame::encode(
FrameType::Window,
7,
&frame::window_payload(1),
)))
.await
.unwrap();
socket
.send(Message::Ping(Bytes::from_static(b"commit")))
.await
.unwrap();
loop {
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap();
if message == Message::Pong(Bytes::from_static(b"commit")) {
break;
}
}
assert!(
live.runtime
.get_session(session_hash, "proxy.example.com")
.unwrap()
.is_carrier_committed()
);
let _ = socket.close(None).await;
live.shutdown().await;
@@ -448,6 +478,7 @@ async fn failed_automatic_multiplex_socket_remains_supersedable() {
Some(CarrierFailure::Upgrade),
[9; 32],
),
false,
)
.unwrap();
assert_eq!(replacement.carrier, WebCarrier::Https);