WEB Carriers negotiation and lane lifecycle bounds hardened

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-26 23:08:53 +03:00
parent 34eeb2d856
commit e72b1f04d1
15 changed files with 645 additions and 489 deletions
+213
View File
@@ -0,0 +1,213 @@
use std::net::IpAddr;
use std::sync::Arc;
use hyper::header::{self, HeaderName, HeaderValue};
use hyper::{Method, Request, StatusCode};
use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
use super::decoy::serve_decoy;
use super::request::{binary_content_type, carrier_ip_learning_eligible, carrier_request};
use super::response::{
carrier_empty, carrier_headers, full_response, insert_header, service_unavailable,
};
use super::{HttpResponse, request_trace};
use crate::config::WebRuntimeVhost;
use crate::web::frame::{self, FrameType};
use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
use crate::web::trace::{TraceDirection, TraceLifecycleEvent, TraceRoute};
const CREATE_BODY_LIMIT: usize = 64;
/// Handles session creation, replacement replay, and authenticated closure.
pub(super) async fn handle_session(
request: Request<RequestBody>,
runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>,
token_hash: TokenHash,
client_ip: IpAddr,
) -> HttpResponse {
if request.headers().contains_key("x-lane-id") {
return serve_decoy(request, vhost, true, &runtime).await;
}
if request.method() == Method::DELETE {
if request.headers().contains_key(header::CONTENT_TYPE) {
return serve_decoy(request, vhost, true, &runtime).await;
}
if let Some(trace) = request_trace(&request)
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
{
trace.set_route(TraceRoute::Session);
trace.bind_identity(session.trace_identity());
}
let CollectedBody {
request,
body,
_body_budget,
} = match collect_body(request, &runtime, 1, true).await {
Ok(result) => result,
Err(CollectBodyError::Limit) => return service_unavailable(),
Err(CollectBodyError::Invalid(request)) => {
return serve_decoy(request, vhost, true, &runtime).await;
}
};
if !body.is_empty() || runtime.close_token(token_hash, &vhost.host).is_err() {
return serve_decoy(request, vhost, true, &runtime).await;
}
return carrier_empty(StatusCode::NO_CONTENT);
}
if request.method() != Method::POST || !binary_content_type(&request) {
return serve_decoy(request, vhost, true, &runtime).await;
}
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
let Some((trace_session_id, profile)) =
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
else {
return serve_decoy(request, vhost, true, &runtime).await;
};
if let Some(trace) = request_trace(&request) {
trace.set_route(TraceRoute::Session);
trace.bind_profile(&profile, trace_session_id);
}
let CollectedBody {
request,
body,
_body_budget,
} = match collect_body(request, &runtime, CREATE_BODY_LIMIT, false).await {
Ok(result) => result,
Err(CollectBodyError::Limit) => return service_unavailable(),
Err(CollectBodyError::Invalid(request)) => {
return serve_decoy(request, vhost, true, &runtime).await;
}
};
if let Some(trace) = request_trace(&request) {
trace.record_frames(
TraceDirection::Request,
&body,
&runtime.active_generation().config().web.limits,
);
}
match runtime.create_session(
token_hash,
&vhost.host,
client_ip,
&body,
carrier_request,
ip_learning_eligible,
) {
Ok(result) => {
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
if let Some(trace) = request_trace(&request) {
trace.register_redaction(result.token.as_bytes());
trace.record_frames(
TraceDirection::Response,
&welcome,
&runtime.active_generation().config().web.limits,
);
}
let mut response = full_response(StatusCode::OK, welcome);
carrier_headers(&mut response);
insert_header(
&mut response,
HeaderName::from_static("x-session-token"),
&result.token,
);
response.headers_mut().insert(
HeaderName::from_static("x-carrier-mode"),
HeaderValue::from_static(result.carrier.as_str()),
);
response.headers_mut().insert(
HeaderName::from_static("x-down-cursor"),
HeaderValue::from_static("0"),
);
if let Some(attempt) = result.attempt {
insert_header(
&mut response,
HeaderName::from_static("x-carrier-attempt"),
&attempt.to_string(),
);
}
if let Some(candidate_count) = result.candidate_count {
insert_header(
&mut response,
HeaderName::from_static("x-carrier-candidate-count"),
&candidate_count.to_string(),
);
insert_header(
&mut response,
HeaderName::from_static("x-carrier-deadline"),
&result.deadline_secs.unwrap_or_default().to_string(),
);
if let Some(state) = result.carrier_state {
insert_header(
&mut response,
HeaderName::from_static("x-carrier-state"),
state,
);
}
}
response
}
Err(ManagerError::Committed) => {
let mut response = carrier_empty(StatusCode::CONFLICT);
if let Some(echo) = runtime.carrier_echo(
token_hash,
&vhost.host,
client_ip,
carrier_request,
) {
response.headers_mut().insert(
HeaderName::from_static("x-carrier-mode"),
HeaderValue::from_static(echo.carrier.as_str()),
);
insert_header(
&mut response,
HeaderName::from_static("x-carrier-attempt"),
&echo.attempt.to_string(),
);
insert_header(
&mut response,
HeaderName::from_static("x-carrier-candidate-count"),
&echo.candidate_count.to_string(),
);
insert_header(
&mut response,
HeaderName::from_static("x-carrier-deadline"),
&echo.deadline_secs.to_string(),
);
insert_header(
&mut response,
HeaderName::from_static("x-carrier-state"),
echo.state,
);
}
response
}
Err(
error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent),
) => {
runtime.trace().record_profile_lifecycle(
client_ip,
Some(trace_session_id),
&profile,
TraceLifecycleEvent::SessionRejected,
None,
Some(error.as_str()),
);
service_unavailable()
}
Err(error) => {
runtime.trace().record_profile_lifecycle(
client_ip,
Some(trace_session_id),
&profile,
TraceLifecycleEvent::SessionRejected,
None,
Some(error.as_str()),
);
serve_decoy(request, vhost, true, &runtime).await
}
}
}
+4 -255
View File
@@ -21,7 +21,10 @@ const WRITE_BUFFER_BYTES: usize = 64 * 1024;
// Cancellation-safe message I/O and budget retries remain separate from carrier loops.
mod io;
use io::{flush, process_lane, process_multiplex, read_message, record_message, reserve_data, send};
// Per-lane carrier state remains isolated from the multiplexed driver.
mod lane;
use io::{flush, process_multiplex, read_message, record_message, reserve_data, send};
use lane::run_lane;
pub(super) async fn run_upgraded(
on_upgrade: hyper::upgrade::OnUpgrade,
@@ -349,260 +352,6 @@ async fn run_multiplex(
}
}
#[allow(clippy::too_many_arguments)]
async fn run_lane(
socket: &mut CarrierSocket,
runtime: &Arc<WebProcessRuntime>,
session: &Arc<WebSession>,
connection: &WebSocketConnection,
reservation: &mut WebSocketLaneReservation,
cancellation: CancellationToken,
trace: Option<&TraceWebSocketContext>,
acknowledge_commit: bool,
) -> Result<(), ()> {
let mut sequence = 1u64;
let mut cursor = 0u64;
// Lane reads use the same cancellation-safe fragmented-message ownership.
let mut read_budget = None;
let liveness_interval = connection.liveness_interval();
let mut next_ping = Instant::now() + liveness_interval;
let open_deadline =
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
let backpressure_timeout =
Duration::from_secs(session.timeouts().websocket_backpressure_secs);
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
let maximum_message = session.limits().carrier_batch_bytes;
let mut active = false;
loop {
let down = session.poll_down_lane(reservation.lane_id(), cursor);
tokio::pin!(down);
let event = tokio::select! {
_ = cancellation.cancelled() => return Err(()),
_ = tokio::time::sleep_until(open_deadline.into()), if !active => return Err(()),
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
incoming = read_message(
socket,
runtime,
session.profile_key(),
&cancellation,
&mut read_budget,
maximum_message,
backpressure_timeout,
) => {
DriverEvent::Incoming(incoming?)
}
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
};
match event {
DriverEvent::Incoming((message, _budget)) => match message {
Message::Binary(body) => {
let started = Instant::now();
let result = process_lane(
runtime,
session,
reservation,
sequence,
&body,
&cancellation,
backpressure_timeout,
)
.await;
record_message(
runtime,
trace,
TraceDirection::Request,
"binary",
&body,
started,
);
let progressed = result?;
if acknowledge_commit && sequence == 1 {
if !session.needs_websocket_commit_ack(connection.id()) {
return Err(());
}
let started = Instant::now();
if send(
socket,
Message::Binary(Bytes::new()),
&cancellation,
write_timeout,
)
.await
.is_err()
{
session.close();
return Err(());
}
record_message(
runtime,
trace,
TraceDirection::Response,
"carrier-ack",
&[],
started,
);
if !session.websocket_commit_ack_written(connection.id()) {
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
}
if !active && progressed {
if !connection.mark_active() {
return Err(());
}
active = true;
}
sequence = sequence.checked_add(1).ok_or(())?;
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Pong(payload) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"pong",
&payload,
Instant::now(),
);
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Ping(payload) => {
let started = Instant::now();
flush(socket, &cancellation, write_timeout).await?;
record_message(
runtime,
trace,
TraceDirection::Request,
"ping",
&payload,
started,
);
record_message(
runtime,
trace,
TraceDirection::Response,
"pong",
&payload,
started,
);
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Close(_) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"close",
&[],
Instant::now(),
);
return Ok(());
}
Message::Text(text) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"text",
text.as_bytes(),
Instant::now(),
);
return Err(());
}
Message::Frame(_) => return Err(()),
},
DriverEvent::Down(result) => {
if result.lane_closed {
return Ok(());
}
if result.body.is_empty() {
let started = Instant::now();
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"ping",
&[],
started,
);
next_ping = Instant::now() + liveness_interval;
} else {
let _budget = reserve_data(
runtime,
session.profile_key(),
result.body.len(),
&cancellation,
backpressure_timeout,
)
.await?;
let body = result.body;
let started = Instant::now();
if trace.is_some() {
send(
socket,
Message::Binary(body.clone()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"binary",
&body,
started,
);
} else {
send(
socket,
Message::Binary(body),
&cancellation,
write_timeout,
)
.await?;
}
connection.mark_progress();
}
cursor = result.next_cursor;
}
DriverEvent::Liveness => {
let started = Instant::now();
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"ping",
&[],
started,
);
next_ping = Instant::now() + liveness_interval;
}
}
}
}
enum DriverEvent {
Incoming((Message, Option<WebSocketBudgetLease>)),
Down(crate::web::session::PollResult),
+274
View File
@@ -0,0 +1,274 @@
use std::sync::Arc;
use std::time::{Duration, Instant};
use bytes::Bytes;
use tokio_tungstenite::tungstenite::protocol::Message;
use tokio_util::sync::CancellationToken;
use super::CarrierSocket;
use super::io::{flush, process_lane, read_message, record_message, reserve_data, send};
use crate::web::manager::{
WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
};
use crate::web::session::{WebSession, WebSocketLaneReservation};
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
#[allow(clippy::too_many_arguments)]
pub(super) async fn run_lane(
socket: &mut CarrierSocket,
runtime: &Arc<WebProcessRuntime>,
session: &Arc<WebSession>,
connection: &WebSocketConnection,
reservation: &mut WebSocketLaneReservation,
cancellation: CancellationToken,
trace: Option<&TraceWebSocketContext>,
acknowledge_commit: bool,
) -> Result<(), ()> {
let mut sequence = 1u64;
let mut cursor = 0u64;
// Lane reads use the same cancellation-safe fragmented-message ownership.
let mut read_budget = None;
let liveness_interval = connection.liveness_interval();
let mut next_ping = Instant::now() + liveness_interval;
let open_deadline =
Instant::now() + Duration::from_secs(session.timeouts().websocket_open_secs);
let backpressure_timeout =
Duration::from_secs(session.timeouts().websocket_backpressure_secs);
let write_timeout = Duration::from_secs(session.timeouts().websocket_write_secs);
let maximum_message = session.limits().carrier_batch_bytes;
let mut active = false;
loop {
let down = session.poll_down_lane(reservation.lane_id(), cursor);
tokio::pin!(down);
let event = tokio::select! {
_ = cancellation.cancelled() => return Err(()),
_ = tokio::time::sleep_until(open_deadline.into()), if !active => return Err(()),
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
incoming = read_message(
socket,
runtime,
session.profile_key(),
&cancellation,
&mut read_budget,
maximum_message,
backpressure_timeout,
) => {
DriverEvent::Incoming(incoming?)
}
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
};
match event {
DriverEvent::Incoming((message, _budget)) => match message {
Message::Binary(body) => {
let started = Instant::now();
let result = process_lane(
runtime,
session,
reservation,
sequence,
&body,
&cancellation,
backpressure_timeout,
)
.await;
record_message(
runtime,
trace,
TraceDirection::Request,
"binary",
&body,
started,
);
let progressed = result?;
if acknowledge_commit && sequence == 1 {
if !session.needs_websocket_commit_ack(connection.id()) {
return Err(());
}
let started = Instant::now();
if send(
socket,
Message::Binary(Bytes::new()),
&cancellation,
write_timeout,
)
.await
.is_err()
{
session.close();
return Err(());
}
record_message(
runtime,
trace,
TraceDirection::Response,
"carrier-ack",
&[],
started,
);
if !session.websocket_commit_ack_written(connection.id()) {
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
}
if !active && progressed {
if !connection.mark_active() {
return Err(());
}
active = true;
}
sequence = sequence.checked_add(1).ok_or(())?;
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Pong(payload) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"pong",
&payload,
Instant::now(),
);
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Ping(payload) => {
let started = Instant::now();
flush(socket, &cancellation, write_timeout).await?;
record_message(
runtime,
trace,
TraceDirection::Request,
"ping",
&payload,
started,
);
record_message(
runtime,
trace,
TraceDirection::Response,
"pong",
&payload,
started,
);
connection.mark_peer_activity();
next_ping = Instant::now() + liveness_interval;
}
Message::Close(_) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"close",
&[],
Instant::now(),
);
return Ok(());
}
Message::Text(text) => {
record_message(
runtime,
trace,
TraceDirection::Request,
"text",
text.as_bytes(),
Instant::now(),
);
return Err(());
}
Message::Frame(_) => return Err(()),
},
DriverEvent::Down(result) => {
if result.lane_closed {
return Ok(());
}
if result.body.is_empty() {
let started = Instant::now();
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"ping",
&[],
started,
);
next_ping = Instant::now() + liveness_interval;
} else {
let _budget = reserve_data(
runtime,
session.profile_key(),
result.body.len(),
&cancellation,
backpressure_timeout,
)
.await?;
let body = result.body;
let started = Instant::now();
if trace.is_some() {
send(
socket,
Message::Binary(body.clone()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"binary",
&body,
started,
);
} else {
send(
socket,
Message::Binary(body),
&cancellation,
write_timeout,
)
.await?;
}
connection.mark_progress();
}
cursor = result.next_cursor;
}
DriverEvent::Liveness => {
let started = Instant::now();
send(
socket,
Message::Ping(Bytes::new()),
&cancellation,
write_timeout,
)
.await?;
record_message(
runtime,
trace,
TraceDirection::Response,
"ping",
&[],
started,
);
next_ping = Instant::now() + liveness_interval;
}
}
}
}
enum DriverEvent {
Incoming((Message, Option<WebSocketBudgetLease>)),
Down(crate::web::session::PollResult),
Liveness,
}