WEB: Lifecycle + Lane ownership + Diag fixes

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-27 09:02:19 +03:00
parent c75cf5cc9d
commit f73f52a033
25 changed files with 2050 additions and 245 deletions
+326 -8
View File
@@ -1,31 +1,291 @@
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;
use std::time::Duration;
use bytes::Bytes;
use hyper::body::{Body, Frame, SizeHint};
use parking_lot::Mutex;
use tokio::time::Instant;
use super::{BoxError, HttpBody};
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
#[derive(Clone, Copy)]
struct DeadlineSlot {
id: u64,
deadline: Instant,
}
struct RequestSlot {
id: u64,
deadline: Option<DeadlineSlot>,
}
struct UpgradeSlot {
request_id: u64,
deadline: DeadlineSlot,
}
struct ActivityState {
last_progress: Instant,
next_request_id: u64,
next_deadline_id: u64,
request: Option<RequestSlot>,
upgrade: Option<UpgradeSlot>,
failed: bool,
}
/// Shared liveness state for one accepted HTTP connection.
#[derive(Clone)]
pub(super) struct ConnectionActivity {
state: Arc<Mutex<ActivityState>>,
}
impl ConnectionActivity {
/// Creates activity state at the connection acceptance boundary.
pub(super) fn new() -> Self {
Self {
state: Arc::new(Mutex::new(ActivityState {
last_progress: Instant::now(),
next_request_id: 1,
next_deadline_id: 1,
request: None,
upgrade: None,
failed: false,
})),
}
}
/// Returns whether the connection has no protected operation or recent progress.
pub(super) fn should_close(&self, now: Instant, idle: Duration) -> bool {
let state = self.state.lock();
if state.failed {
return true;
}
let request_protected = state
.request
.as_ref()
.and_then(|request| request.deadline)
.is_some_and(|deadline| now <= deadline.deadline);
let upgrade_protected = state
.upgrade
.as_ref()
.is_some_and(|upgrade| now <= upgrade.deadline.deadline);
!request_protected
&& !upgrade_protected
&& now.saturating_duration_since(state.last_progress) >= idle
}
fn fail(&self) {
self.state.lock().failed = true;
}
}
/// Cloneable authority for protecting one request's explicitly bounded awaits.
#[derive(Clone)]
pub(super) struct RequestDeadlineHandle {
activity: ConnectionActivity,
request_id: u64,
}
impl RequestDeadlineHandle {
/// Protects the current bounded request operation until its absolute deadline.
pub(super) fn lease_until(&self, deadline: Instant) -> Option<RequestDeadlineLease> {
let now = Instant::now();
let mut state = self.activity.state.lock();
if state.failed {
return None;
}
let Some(current) = state.request.as_ref() else {
state.failed = true;
return None;
};
if current.id != self.request_id
|| current
.deadline
.is_some_and(|active| now <= active.deadline)
{
state.failed = true;
return None;
}
let id = state.next_deadline_id;
let Some(next) = id.checked_add(1) else {
state.failed = true;
return None;
};
state.next_deadline_id = next;
let Some(current) = state.request.as_mut() else {
state.failed = true;
return None;
};
current.deadline = Some(DeadlineSlot { id, deadline });
Some(RequestDeadlineLease {
handle: self.clone(),
deadline_id: id,
})
}
/// Protects the current bounded request operation for one checked duration.
pub(super) fn lease_for(&self, duration: Duration) -> Option<RequestDeadlineLease> {
let Some(deadline) = Instant::now().checked_add(duration) else {
self.activity.fail();
return None;
};
self.lease_until(deadline)
}
/// Transfers idle protection to a pending Hyper upgrade operation.
pub(super) fn upgrade_until(&self, deadline: Instant) -> Option<UpgradeDeadlineLease> {
let now = Instant::now();
let mut state = self.activity.state.lock();
if state.failed
|| state
.request
.as_ref()
.is_none_or(|request| request.id != self.request_id)
|| state
.upgrade
.as_ref()
.is_some_and(|upgrade| now <= upgrade.deadline.deadline)
{
state.failed = true;
return None;
}
let id = state.next_deadline_id;
let Some(next) = id.checked_add(1) else {
state.failed = true;
return None;
};
state.next_deadline_id = next;
state.upgrade = Some(UpgradeSlot {
request_id: self.request_id,
deadline: DeadlineSlot { id, deadline },
});
Some(UpgradeDeadlineLease {
activity: self.activity.clone(),
request_id: self.request_id,
deadline_id: id,
deadline,
})
}
}
/// Exact request-operation lease that cannot clear a newer deadline.
pub(super) struct RequestDeadlineLease {
handle: RequestDeadlineHandle,
deadline_id: u64,
}
impl Drop for RequestDeadlineLease {
fn drop(&mut self) {
let mut state = self.handle.activity.state.lock();
let matches = state.request.as_ref().is_some_and(|request| {
request.id == self.handle.request_id
&& request
.deadline
.is_some_and(|deadline| deadline.id == self.deadline_id)
});
if matches {
if let Some(request) = state.request.as_mut() {
request.deadline = None;
}
state.last_progress = Instant::now();
}
}
}
/// Exact pending-upgrade lease retained by the spawned upgrade future.
pub(super) struct UpgradeDeadlineLease {
activity: ConnectionActivity,
request_id: u64,
deadline_id: u64,
deadline: Instant,
}
impl UpgradeDeadlineLease {
/// Returns the absolute deadline shared with the upgrade timeout.
pub(super) fn deadline(&self) -> Instant {
self.deadline
}
}
impl Drop for UpgradeDeadlineLease {
fn drop(&mut self) {
let mut state = self.activity.state.lock();
let matches = state.upgrade.as_ref().is_some_and(|upgrade| {
upgrade.request_id == self.request_id && upgrade.deadline.id == self.deadline_id
});
if matches {
state.upgrade = None;
state.last_progress = Instant::now();
}
}
}
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
pub(super) struct RequestActivity {
last_activity: Arc<Mutex<Instant>>,
handle: RequestDeadlineHandle,
}
impl RequestActivity {
/// Starts activity accounting for one HTTP request.
pub(super) fn begin(last_activity: Arc<Mutex<Instant>>) -> Self {
*last_activity.lock() = Instant::now();
Self { last_activity }
pub(super) fn begin(activity: ConnectionActivity) -> Option<Self> {
let mut state = activity.state.lock();
if state.failed || state.request.is_some() {
state.failed = true;
return None;
}
let id = state.next_request_id;
let Some(next) = id.checked_add(1) else {
state.failed = true;
return None;
};
state.next_request_id = next;
state.last_progress = Instant::now();
state.request = Some(RequestSlot { id, deadline: None });
drop(state);
Some(Self {
handle: RequestDeadlineHandle {
activity,
request_id: id,
},
})
}
/// Returns the authority copied into request extensions for bounded awaits.
pub(super) fn deadline_handle(&self) -> RequestDeadlineHandle {
self.handle.clone()
}
fn progress(&self) {
self.handle.activity.state.lock().last_progress = Instant::now();
}
fn enter_response(&mut self) {
let mut state = self.handle.activity.state.lock();
if let Some(request) = state
.request
.as_mut()
.filter(|request| request.id == self.handle.request_id)
{
request.deadline = None;
state.last_progress = Instant::now();
}
}
}
impl Drop for RequestActivity {
fn drop(&mut self) {
*self.last_activity.lock() = Instant::now();
let mut state = self.handle.activity.state.lock();
if state
.request
.as_ref()
.is_some_and(|request| request.id == self.handle.request_id)
{
state.request = None;
state.last_progress = Instant::now();
}
}
}
@@ -41,9 +301,10 @@ impl ActivityBody {
/// Binds one response body to its request activity guard.
pub(super) fn new(
inner: HttpBody,
activity: RequestActivity,
mut activity: RequestActivity,
trace: Option<Arc<HttpTraceExchange>>,
) -> Self {
activity.enter_response();
Self {
inner,
activity,
@@ -88,7 +349,7 @@ impl Body for ActivityBody {
Poll::Pending => {}
}
if result.is_ready() {
*self.activity.last_activity.lock() = Instant::now();
self.activity.progress();
}
result
}
@@ -107,3 +368,60 @@ impl Drop for ActivityBody {
self.finish(TraceBodyState::Aborted);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bounded_request_deadline_suspends_only_idle_expiry() {
let activity = ConnectionActivity::new();
let request = RequestActivity::begin(activity.clone()).unwrap();
let now = Instant::now();
let lease = request
.deadline_handle()
.lease_until(now + Duration::from_secs(5))
.unwrap();
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
assert!(activity.should_close(now + Duration::from_secs(6), Duration::from_secs(1)));
drop(lease);
assert!(!activity.should_close(Instant::now(), Duration::from_secs(1)));
}
#[test]
fn stale_request_lease_cannot_clear_a_new_request_deadline() {
let activity = ConnectionActivity::new();
let request_a = RequestActivity::begin(activity.clone()).unwrap();
let now = Instant::now();
let lease_a = request_a
.deadline_handle()
.lease_until(now - Duration::from_secs(1))
.unwrap();
drop(request_a);
let request_b = RequestActivity::begin(activity.clone()).unwrap();
let _lease_b = request_b
.deadline_handle()
.lease_until(now + Duration::from_secs(5))
.unwrap();
drop(lease_a);
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
}
#[test]
fn stale_upgrade_lease_cannot_clear_its_replacement() {
let activity = ConnectionActivity::new();
let request = RequestActivity::begin(activity.clone()).unwrap();
let handle = request.deadline_handle();
let now = Instant::now();
let lease_a = handle.upgrade_until(now - Duration::from_secs(1)).unwrap();
let _lease_b = handle.upgrade_until(now + Duration::from_secs(5)).unwrap();
drop(lease_a);
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
}
}
+9
View File
@@ -118,6 +118,7 @@ pub(super) async fn collect_body(
limit: usize,
allow_empty: bool,
) -> Result<CollectedBody, CollectBodyError> {
let request_deadline = super::request_deadline(&request);
let exceeds_limit = request.body().size_hint().lower() > limit as u64
|| request
.body()
@@ -134,6 +135,14 @@ pub(super) async fn collect_body(
let Some((reader_budget, body_budget)) = runtime.try_body_budget(limit) else {
return Err(CollectBodyError::Limit);
};
let _deadline_lease = match request_deadline {
Some(deadline) => Some(
deadline
.lease_for(body_timeout)
.ok_or(CollectBodyError::Limit)?,
),
None => None,
};
let body = match tokio::time::timeout(body_timeout, Limited::new(body, limit).collect()).await {
Ok(Ok(body)) => body.to_bytes(),
_ => {
+26
View File
@@ -184,6 +184,7 @@ async fn proxy_to_upstream(
header_timeout: Duration,
runtime: &WebProcessRuntime,
) -> HttpResponse {
let request_deadline = super::request_deadline(&request);
remove_hop_by_hop(request.headers_mut());
if let Ok(host) = HeaderValue::from_str(authority) {
request.headers_mut().insert(header::HOST, host);
@@ -197,10 +198,15 @@ async fn proxy_to_upstream(
return bad_gateway();
};
*request.uri_mut() = uri;
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
};
let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => stream,
_ => return bad_gateway(),
};
drop(_deadline_lease);
let max_header_bytes = runtime
.active_generation()
.config()
@@ -209,19 +215,29 @@ async fn proxy_to_upstream(
.max_header_bytes;
let mut builder = hyper::client::conn::http1::Builder::new();
builder.max_buf_size(max_header_bytes);
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
};
let (mut sender, connection) =
match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
Ok(Ok(parts)) => parts,
_ => return bad_gateway(),
};
drop(_deadline_lease);
runtime.spawn_auxiliary(async move {
let _ = connection.await;
});
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
};
let mut response =
match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
Ok(Ok(response)) => response,
_ => return bad_gateway(),
};
drop(_deadline_lease);
remove_hop_by_hop(response.headers_mut());
response.map(|body| {
body.map_err(|error| -> BoxError { Box::new(error) })
@@ -229,6 +245,16 @@ async fn proxy_to_upstream(
})
}
fn lease_deadline(
deadline: Option<&super::activity::RequestDeadlineHandle>,
timeout: Duration,
) -> Result<Option<super::activity::RequestDeadlineLease>, ()> {
match deadline {
Some(deadline) => deadline.lease_for(timeout).map(Some).ok_or(()),
None => Ok(None),
}
}
fn sanitize_transport_request<B>(request: &mut Request<B>) {
for name in [
header::AUTHORIZATION,
+16
View File
@@ -76,10 +76,26 @@ pub(super) async fn handle_down(
} else {
None
};
let poll_timeout = match lane_id {
Some(_) => Duration::from_secs(session.timeouts().lane_open_wait_secs)
.checked_add(Duration::from_secs(session.timeouts().long_poll_secs)),
None => Some(Duration::from_secs(session.timeouts().long_poll_secs)),
};
let Some(poll_timeout) = poll_timeout else {
return service_unavailable();
};
let _deadline_lease = match super::request_deadline(&request) {
Some(deadline) => match deadline.lease_for(poll_timeout) {
Some(lease) => Some(lease),
None => return service_unavailable(),
},
None => None,
};
let result = match lane_id {
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
None => session.poll_down(cursor).await,
};
drop(_deadline_lease);
match result {
Ok(result) if result.body.is_empty() => {
let mut response = carrier_empty(StatusCode::NO_CONTENT);
+138
View File
@@ -1,5 +1,50 @@
use super::*;
async fn open_keepalive(
listener: &TcpListener,
runtime: &Arc<WebProcessRuntime>,
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
let addr = listener.local_addr().unwrap();
let (accepted, client) = tokio::join!(listener.accept(), TcpStream::connect(addr));
let (server, peer) = accepted.unwrap();
let cancellation = CancellationToken::new();
let permit = runtime.try_http_connection().unwrap();
let task = tokio::spawn(serve_connection(
server,
peer,
WebClientIpSource::XForwardedFor,
Arc::from(["127.0.0.1/32".parse().unwrap()]),
Arc::clone(runtime),
cancellation.clone(),
permit,
));
(client.unwrap(), cancellation, task)
}
async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
let mut response = Vec::new();
while !response.ends_with(b"\r\n\r\n") {
assert!(response.len() < 16 * 1024);
response.push(client.read_u8().await.unwrap());
}
let content_length = std::str::from_utf8(&response)
.unwrap()
.lines()
.filter_map(|line| line.split_once(':'))
.find_map(|(name, value)| {
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().unwrap())
})
.unwrap_or(0);
let body_start = response.len();
response.resize(body_start + content_length, 0);
client
.read_exact(&mut response[body_start..])
.await
.unwrap();
response
}
async fn request_with_body_delay(
listener: &TcpListener,
runtime: &Arc<WebProcessRuntime>,
@@ -35,6 +80,9 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
let capability = [21u8; 32];
let mut initial_config = runtime_config(capability, WebCarrier::Https);
initial_config.web.timeouts.body_secs = 3;
initial_config.web.timeouts.header_secs = 1;
initial_config.web.timeouts.http_idle_secs = 4;
initial_config.web.timeouts.long_poll_secs = 3;
initial_config.web.timeouts.bootstrap_lifetime_secs = 5;
let generation = test_runtime_generation(1, initial_config);
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
@@ -68,6 +116,9 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
replacement_config.web.timeouts.body_secs = 1;
replacement_config.web.timeouts.header_secs = 1;
replacement_config.web.timeouts.http_idle_secs = 2;
replacement_config.web.timeouts.long_poll_secs = 1;
replacement_config.web.timeouts.bootstrap_lifetime_secs = 1;
let replacement = test_runtime_generation(2, replacement_config);
active_runtime.store(Arc::clone(&replacement));
@@ -84,6 +135,13 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
assert!(retry_headers.starts_with(b"HTTP/1.1 200"));
assert_eq!(response_header(retry_headers, "x-session-token"), session);
let down = format!(
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nX-Down-Cursor: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
.into_bytes();
let down_response = request(&listener, &runtime, down).await;
assert!(down_response.starts_with(b"HTTP/1.1 204"));
let close = format!(
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
@@ -100,3 +158,83 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
replacement.stop_sessions().await;
replacement.stop_background_tasks().await;
}
#[tokio::test]
async fn active_body_deadline_survives_reload_on_old_keepalive_connection() {
let capability = [22u8; 32];
let mut initial_config = runtime_config(capability, WebCarrier::Https);
initial_config.web.timeouts.header_secs = 1;
initial_config.web.timeouts.body_secs = 1;
initial_config.web.timeouts.long_poll_secs = 1;
initial_config.web.timeouts.http_idle_secs = 2;
let generation = test_runtime_generation(1, initial_config);
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let (mut client, cancellation, task) = open_keepalive(&listener, &runtime).await;
client
.write_all(
b"GET / HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n",
)
.await
.unwrap();
assert!(
read_http_response(&mut client)
.await
.starts_with(b"HTTP/1.1 200")
);
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
replacement_config.web.timeouts.header_secs = 1;
replacement_config.web.timeouts.body_secs = 3;
replacement_config.web.timeouts.long_poll_secs = 1;
replacement_config.web.timeouts.http_idle_secs = 4;
let replacement = test_runtime_generation(2, replacement_config);
active_runtime.store(Arc::clone(&replacement));
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
client
.write_all(
format!(
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n"
)
.as_bytes(),
)
.await
.unwrap();
let bridge = read_http_response(&mut client).await;
let (_, bridge_body) = split_response(&bridge);
let bootstrap = std::str::from_utf8(bridge_body)
.unwrap()
.split_once("bootstrap=\"")
.and_then(|(_, suffix)| suffix.split_once('"'))
.map(|(token, _)| token.to_string())
.unwrap();
let hello = frame::encode(FrameType::Hello, 0, &[1]);
client
.write_all(
format!(
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
hello.len()
)
.as_bytes(),
)
.await
.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(2200)).await;
client.write_all(&hello).await.unwrap();
assert!(
read_http_response(&mut client)
.await
.starts_with(b"HTTP/1.1 200")
);
cancellation.cancel();
drop(client);
task.await.unwrap();
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
replacement.stop_sessions().await;
replacement.stop_background_tasks().await;
}
+34 -3
View File
@@ -254,6 +254,7 @@ pub(super) async fn handle(
runtime: Arc<WebProcessRuntime>,
vhost: Arc<WebRuntimeVhost>,
) -> HttpResponse {
let request_deadline = super::request_deadline(&request);
let Some(parsed) = parse_upgrade(&request) else {
return serve_decoy(request, vhost, true, &runtime).await;
};
@@ -284,7 +285,16 @@ pub(super) async fn handle(
},
};
let timeouts = session.timeouts().clone();
let connection = match runtime
let admission_lease = match request_deadline.as_ref() {
Some(deadline) => {
match deadline.lease_for(Duration::from_secs(timeouts.websocket_eviction_secs)) {
Some(lease) => Some(lease),
None => return serve_decoy(request, vhost, true, &runtime).await,
}
}
None => None,
};
let admitted = runtime
.admit_websocket(
session.profile_key(),
session.trace_session_id(),
@@ -296,11 +306,17 @@ pub(super) async fn handle(
Duration::from_secs(timeouts.websocket_eviction_secs),
session.carrier_cancellation(),
)
.await
{
.await;
drop(admission_lease);
let connection = match admitted {
Ok(connection) => connection,
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
};
if let Some(reservation) = lane_reservation.as_mut()
&& reservation.bind(connection.id()).is_err()
{
return serve_decoy(request, vhost, true, &runtime).await;
}
if let Some(reservation) = probe_reservation.as_mut()
&& reservation.bind(connection.id()).is_err()
{
@@ -323,6 +339,20 @@ pub(super) async fn handle(
trace.bind_identity(session.trace_identity());
trace.register_redaction(parsed.protocol.as_bytes());
}
let upgrade_deadline = match request_deadline.as_ref() {
Some(deadline) => {
let Some(until) = tokio::time::Instant::now()
.checked_add(Duration::from_secs(timeouts.websocket_upgrade_secs))
else {
return serve_decoy(request, vhost, true, &runtime).await;
};
match deadline.upgrade_until(until) {
Some(lease) => Some(lease),
None => return serve_decoy(request, vhost, true, &runtime).await,
}
}
None => None,
};
let on_upgrade = hyper::upgrade::on(&mut request);
let protocol = parsed.protocol;
let accept = parsed.accept;
@@ -331,6 +361,7 @@ pub(super) async fn handle(
runtime.spawn_auxiliary(async move {
driver::run_upgraded(
on_upgrade,
upgrade_deadline,
driver_runtime,
driver_session,
connection,
+17 -10
View File
@@ -8,6 +8,7 @@ use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
use tokio_util::sync::CancellationToken;
use super::ConnectionIo;
use crate::web::http::activity::UpgradeDeadlineLease;
use crate::web::manager::{WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection};
use crate::web::session::{WebSession, WebSocketLaneReservation, WebSocketProbeReservation};
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
@@ -22,8 +23,11 @@ mod lane;
use io::{flush, process_multiplex, read_message, record_message, reserve_data, send};
use lane::run_lane;
// Upgrade ownership remains explicit across cancellation and reservation boundaries.
#[allow(clippy::too_many_arguments)]
pub(super) async fn run_upgraded(
on_upgrade: hyper::upgrade::OnUpgrade,
upgrade_deadline: Option<UpgradeDeadlineLease>,
runtime: Arc<WebProcessRuntime>,
session: Arc<WebSession>,
connection: WebSocketConnection,
@@ -34,13 +38,15 @@ pub(super) async fn run_upgraded(
) {
let cancellation = connection.cancellation();
let timeouts = session.timeouts().clone();
let deadline = upgrade_deadline.as_ref().map_or_else(
|| tokio::time::Instant::now() + Duration::from_secs(timeouts.websocket_upgrade_secs),
UpgradeDeadlineLease::deadline,
);
let upgraded = tokio::select! {
_ = cancellation.cancelled() => return,
result = tokio::time::timeout(
Duration::from_secs(timeouts.websocket_upgrade_secs),
on_upgrade,
) => result,
result = tokio::time::timeout_at(deadline, on_upgrade) => result,
};
drop(upgrade_deadline);
let Ok(Ok(upgraded)) = upgraded else {
return;
};
@@ -95,8 +101,7 @@ pub(super) async fn run_upgraded(
_ = tokio::time::timeout(eviction, socket.close(None)) => {}
}
if let Some(reservation) = lane_reservation {
session.close_websocket_lane(reservation.lane_id());
drop(reservation);
session.close_websocket_lane(reservation);
} else if !acknowledge_commit || session.is_carrier_committed() {
session.close();
}
@@ -192,10 +197,12 @@ async fn run_multiplex(
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
} else if acknowledge_commit
&& sequence > 1
&& progressed
&& !session.websocket_peer_after_commit_ack(connection.id())
{
return Err(());
}
if !active && progressed {
if !connection.mark_active() {
+8 -5
View File
@@ -12,6 +12,7 @@ use crate::web::session::{WebSession, WebSocketLaneReservation};
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
#[allow(clippy::too_many_arguments)]
/// Drives one exact WebSocket lane until its isolated failure boundary closes.
pub(super) async fn run_lane(
socket: &mut CarrierSocket,
runtime: &Arc<WebProcessRuntime>,
@@ -35,7 +36,7 @@ pub(super) async fn run_lane(
let maximum_message = session.limits().carrier_batch_bytes;
let mut active = false;
loop {
let down = session.poll_down_lane(reservation.lane_id(), cursor);
let down = session.poll_down_websocket_lane(reservation.lane_identity(), cursor);
tokio::pin!(down);
let event = tokio::select! {
_ = cancellation.cancelled() => return Err(()),
@@ -106,10 +107,12 @@ pub(super) async fn run_lane(
session.close();
return Err(());
}
} else if acknowledge_commit && sequence > 1 && progressed {
if !session.websocket_peer_after_commit_ack(connection.id()) {
return Err(());
}
} else if acknowledge_commit
&& sequence > 1
&& progressed
&& !session.websocket_peer_after_commit_ack(connection.id())
{
return Err(());
}
if !active && progressed {
if !connection.mark_active() {