mirror of
https://github.com/telemt/telemt.git
synced 2026-09-15 06:54:10 +03:00
WEB: Lifecycle + Lane ownership + Diag fixes
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
+326
-8
@@ -1,31 +1,291 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use hyper::body::{Body, Frame, SizeHint};
|
||||
use parking_lot::Mutex;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::{BoxError, HttpBody};
|
||||
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct DeadlineSlot {
|
||||
id: u64,
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
struct RequestSlot {
|
||||
id: u64,
|
||||
deadline: Option<DeadlineSlot>,
|
||||
}
|
||||
|
||||
struct UpgradeSlot {
|
||||
request_id: u64,
|
||||
deadline: DeadlineSlot,
|
||||
}
|
||||
|
||||
struct ActivityState {
|
||||
last_progress: Instant,
|
||||
next_request_id: u64,
|
||||
next_deadline_id: u64,
|
||||
request: Option<RequestSlot>,
|
||||
upgrade: Option<UpgradeSlot>,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
/// Shared liveness state for one accepted HTTP connection.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct ConnectionActivity {
|
||||
state: Arc<Mutex<ActivityState>>,
|
||||
}
|
||||
|
||||
impl ConnectionActivity {
|
||||
/// Creates activity state at the connection acceptance boundary.
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(ActivityState {
|
||||
last_progress: Instant::now(),
|
||||
next_request_id: 1,
|
||||
next_deadline_id: 1,
|
||||
request: None,
|
||||
upgrade: None,
|
||||
failed: false,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the connection has no protected operation or recent progress.
|
||||
pub(super) fn should_close(&self, now: Instant, idle: Duration) -> bool {
|
||||
let state = self.state.lock();
|
||||
if state.failed {
|
||||
return true;
|
||||
}
|
||||
let request_protected = state
|
||||
.request
|
||||
.as_ref()
|
||||
.and_then(|request| request.deadline)
|
||||
.is_some_and(|deadline| now <= deadline.deadline);
|
||||
let upgrade_protected = state
|
||||
.upgrade
|
||||
.as_ref()
|
||||
.is_some_and(|upgrade| now <= upgrade.deadline.deadline);
|
||||
!request_protected
|
||||
&& !upgrade_protected
|
||||
&& now.saturating_duration_since(state.last_progress) >= idle
|
||||
}
|
||||
|
||||
fn fail(&self) {
|
||||
self.state.lock().failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable authority for protecting one request's explicitly bounded awaits.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RequestDeadlineHandle {
|
||||
activity: ConnectionActivity,
|
||||
request_id: u64,
|
||||
}
|
||||
|
||||
impl RequestDeadlineHandle {
|
||||
/// Protects the current bounded request operation until its absolute deadline.
|
||||
pub(super) fn lease_until(&self, deadline: Instant) -> Option<RequestDeadlineLease> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.activity.state.lock();
|
||||
if state.failed {
|
||||
return None;
|
||||
}
|
||||
let Some(current) = state.request.as_ref() else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
if current.id != self.request_id
|
||||
|| current
|
||||
.deadline
|
||||
.is_some_and(|active| now <= active.deadline)
|
||||
{
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_deadline_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_deadline_id = next;
|
||||
let Some(current) = state.request.as_mut() else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
current.deadline = Some(DeadlineSlot { id, deadline });
|
||||
Some(RequestDeadlineLease {
|
||||
handle: self.clone(),
|
||||
deadline_id: id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Protects the current bounded request operation for one checked duration.
|
||||
pub(super) fn lease_for(&self, duration: Duration) -> Option<RequestDeadlineLease> {
|
||||
let Some(deadline) = Instant::now().checked_add(duration) else {
|
||||
self.activity.fail();
|
||||
return None;
|
||||
};
|
||||
self.lease_until(deadline)
|
||||
}
|
||||
|
||||
/// Transfers idle protection to a pending Hyper upgrade operation.
|
||||
pub(super) fn upgrade_until(&self, deadline: Instant) -> Option<UpgradeDeadlineLease> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.activity.state.lock();
|
||||
if state.failed
|
||||
|| state
|
||||
.request
|
||||
.as_ref()
|
||||
.is_none_or(|request| request.id != self.request_id)
|
||||
|| state
|
||||
.upgrade
|
||||
.as_ref()
|
||||
.is_some_and(|upgrade| now <= upgrade.deadline.deadline)
|
||||
{
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_deadline_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_deadline_id = next;
|
||||
state.upgrade = Some(UpgradeSlot {
|
||||
request_id: self.request_id,
|
||||
deadline: DeadlineSlot { id, deadline },
|
||||
});
|
||||
Some(UpgradeDeadlineLease {
|
||||
activity: self.activity.clone(),
|
||||
request_id: self.request_id,
|
||||
deadline_id: id,
|
||||
deadline,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact request-operation lease that cannot clear a newer deadline.
|
||||
pub(super) struct RequestDeadlineLease {
|
||||
handle: RequestDeadlineHandle,
|
||||
deadline_id: u64,
|
||||
}
|
||||
|
||||
impl Drop for RequestDeadlineLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
let matches = state.request.as_ref().is_some_and(|request| {
|
||||
request.id == self.handle.request_id
|
||||
&& request
|
||||
.deadline
|
||||
.is_some_and(|deadline| deadline.id == self.deadline_id)
|
||||
});
|
||||
if matches {
|
||||
if let Some(request) = state.request.as_mut() {
|
||||
request.deadline = None;
|
||||
}
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact pending-upgrade lease retained by the spawned upgrade future.
|
||||
pub(super) struct UpgradeDeadlineLease {
|
||||
activity: ConnectionActivity,
|
||||
request_id: u64,
|
||||
deadline_id: u64,
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
impl UpgradeDeadlineLease {
|
||||
/// Returns the absolute deadline shared with the upgrade timeout.
|
||||
pub(super) fn deadline(&self) -> Instant {
|
||||
self.deadline
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UpgradeDeadlineLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.activity.state.lock();
|
||||
let matches = state.upgrade.as_ref().is_some_and(|upgrade| {
|
||||
upgrade.request_id == self.request_id && upgrade.deadline.id == self.deadline_id
|
||||
});
|
||||
if matches {
|
||||
state.upgrade = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
|
||||
pub(super) struct RequestActivity {
|
||||
last_activity: Arc<Mutex<Instant>>,
|
||||
handle: RequestDeadlineHandle,
|
||||
}
|
||||
|
||||
impl RequestActivity {
|
||||
/// Starts activity accounting for one HTTP request.
|
||||
pub(super) fn begin(last_activity: Arc<Mutex<Instant>>) -> Self {
|
||||
*last_activity.lock() = Instant::now();
|
||||
Self { last_activity }
|
||||
pub(super) fn begin(activity: ConnectionActivity) -> Option<Self> {
|
||||
let mut state = activity.state.lock();
|
||||
if state.failed || state.request.is_some() {
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_request_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_request_id = next;
|
||||
state.last_progress = Instant::now();
|
||||
state.request = Some(RequestSlot { id, deadline: None });
|
||||
drop(state);
|
||||
Some(Self {
|
||||
handle: RequestDeadlineHandle {
|
||||
activity,
|
||||
request_id: id,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the authority copied into request extensions for bounded awaits.
|
||||
pub(super) fn deadline_handle(&self) -> RequestDeadlineHandle {
|
||||
self.handle.clone()
|
||||
}
|
||||
|
||||
fn progress(&self) {
|
||||
self.handle.activity.state.lock().last_progress = Instant::now();
|
||||
}
|
||||
|
||||
fn enter_response(&mut self) {
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
if let Some(request) = state
|
||||
.request
|
||||
.as_mut()
|
||||
.filter(|request| request.id == self.handle.request_id)
|
||||
{
|
||||
request.deadline = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestActivity {
|
||||
fn drop(&mut self) {
|
||||
*self.last_activity.lock() = Instant::now();
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
if state
|
||||
.request
|
||||
.as_ref()
|
||||
.is_some_and(|request| request.id == self.handle.request_id)
|
||||
{
|
||||
state.request = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +301,10 @@ impl ActivityBody {
|
||||
/// Binds one response body to its request activity guard.
|
||||
pub(super) fn new(
|
||||
inner: HttpBody,
|
||||
activity: RequestActivity,
|
||||
mut activity: RequestActivity,
|
||||
trace: Option<Arc<HttpTraceExchange>>,
|
||||
) -> Self {
|
||||
activity.enter_response();
|
||||
Self {
|
||||
inner,
|
||||
activity,
|
||||
@@ -88,7 +349,7 @@ impl Body for ActivityBody {
|
||||
Poll::Pending => {}
|
||||
}
|
||||
if result.is_ready() {
|
||||
*self.activity.last_activity.lock() = Instant::now();
|
||||
self.activity.progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -107,3 +368,60 @@ impl Drop for ActivityBody {
|
||||
self.finish(TraceBodyState::Aborted);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bounded_request_deadline_suspends_only_idle_expiry() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let now = Instant::now();
|
||||
let lease = request
|
||||
.deadline_handle()
|
||||
.lease_until(now + Duration::from_secs(5))
|
||||
.unwrap();
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
assert!(activity.should_close(now + Duration::from_secs(6), Duration::from_secs(1)));
|
||||
|
||||
drop(lease);
|
||||
assert!(!activity.should_close(Instant::now(), Duration::from_secs(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_request_lease_cannot_clear_a_new_request_deadline() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request_a = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let now = Instant::now();
|
||||
let lease_a = request_a
|
||||
.deadline_handle()
|
||||
.lease_until(now - Duration::from_secs(1))
|
||||
.unwrap();
|
||||
drop(request_a);
|
||||
let request_b = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let _lease_b = request_b
|
||||
.deadline_handle()
|
||||
.lease_until(now + Duration::from_secs(5))
|
||||
.unwrap();
|
||||
|
||||
drop(lease_a);
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_upgrade_lease_cannot_clear_its_replacement() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let handle = request.deadline_handle();
|
||||
let now = Instant::now();
|
||||
let lease_a = handle.upgrade_until(now - Duration::from_secs(1)).unwrap();
|
||||
let _lease_b = handle.upgrade_until(now + Duration::from_secs(5)).unwrap();
|
||||
|
||||
drop(lease_a);
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ pub(super) async fn collect_body(
|
||||
limit: usize,
|
||||
allow_empty: bool,
|
||||
) -> Result<CollectedBody, CollectBodyError> {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
let exceeds_limit = request.body().size_hint().lower() > limit as u64
|
||||
|| request
|
||||
.body()
|
||||
@@ -134,6 +135,14 @@ pub(super) async fn collect_body(
|
||||
let Some((reader_budget, body_budget)) = runtime.try_body_budget(limit) else {
|
||||
return Err(CollectBodyError::Limit);
|
||||
};
|
||||
let _deadline_lease = match request_deadline {
|
||||
Some(deadline) => Some(
|
||||
deadline
|
||||
.lease_for(body_timeout)
|
||||
.ok_or(CollectBodyError::Limit)?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
let body = match tokio::time::timeout(body_timeout, Limited::new(body, limit).collect()).await {
|
||||
Ok(Ok(body)) => body.to_bytes(),
|
||||
_ => {
|
||||
|
||||
@@ -184,6 +184,7 @@ async fn proxy_to_upstream(
|
||||
header_timeout: Duration,
|
||||
runtime: &WebProcessRuntime,
|
||||
) -> HttpResponse {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
remove_hop_by_hop(request.headers_mut());
|
||||
if let Ok(host) = HeaderValue::from_str(authority) {
|
||||
request.headers_mut().insert(header::HOST, host);
|
||||
@@ -197,10 +198,15 @@ async fn proxy_to_upstream(
|
||||
return bad_gateway();
|
||||
};
|
||||
*request.uri_mut() = uri;
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
|
||||
Ok(Ok(stream)) => stream,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
let max_header_bytes = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
@@ -209,19 +215,29 @@ async fn proxy_to_upstream(
|
||||
.max_header_bytes;
|
||||
let mut builder = hyper::client::conn::http1::Builder::new();
|
||||
builder.max_buf_size(max_header_bytes);
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let (mut sender, connection) =
|
||||
match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
|
||||
Ok(Ok(parts)) => parts,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
runtime.spawn_auxiliary(async move {
|
||||
let _ = connection.await;
|
||||
});
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let mut response =
|
||||
match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
|
||||
Ok(Ok(response)) => response,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
remove_hop_by_hop(response.headers_mut());
|
||||
response.map(|body| {
|
||||
body.map_err(|error| -> BoxError { Box::new(error) })
|
||||
@@ -229,6 +245,16 @@ async fn proxy_to_upstream(
|
||||
})
|
||||
}
|
||||
|
||||
fn lease_deadline(
|
||||
deadline: Option<&super::activity::RequestDeadlineHandle>,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<super::activity::RequestDeadlineLease>, ()> {
|
||||
match deadline {
|
||||
Some(deadline) => deadline.lease_for(timeout).map(Some).ok_or(()),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_transport_request<B>(request: &mut Request<B>) {
|
||||
for name in [
|
||||
header::AUTHORIZATION,
|
||||
|
||||
@@ -76,10 +76,26 @@ pub(super) async fn handle_down(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let poll_timeout = match lane_id {
|
||||
Some(_) => Duration::from_secs(session.timeouts().lane_open_wait_secs)
|
||||
.checked_add(Duration::from_secs(session.timeouts().long_poll_secs)),
|
||||
None => Some(Duration::from_secs(session.timeouts().long_poll_secs)),
|
||||
};
|
||||
let Some(poll_timeout) = poll_timeout else {
|
||||
return service_unavailable();
|
||||
};
|
||||
let _deadline_lease = match super::request_deadline(&request) {
|
||||
Some(deadline) => match deadline.lease_for(poll_timeout) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return service_unavailable(),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
||||
None => session.poll_down(cursor).await,
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
match result {
|
||||
Ok(result) if result.body.is_empty() => {
|
||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||
|
||||
@@ -1,5 +1,50 @@
|
||||
use super::*;
|
||||
|
||||
async fn open_keepalive(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (accepted, client) = tokio::join!(listener.accept(), TcpStream::connect(addr));
|
||||
let (server, peer) = accepted.unwrap();
|
||||
let cancellation = CancellationToken::new();
|
||||
let permit = runtime.try_http_connection().unwrap();
|
||||
let task = tokio::spawn(serve_connection(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
Arc::from(["127.0.0.1/32".parse().unwrap()]),
|
||||
Arc::clone(runtime),
|
||||
cancellation.clone(),
|
||||
permit,
|
||||
));
|
||||
(client.unwrap(), cancellation, task)
|
||||
}
|
||||
|
||||
async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
||||
let mut response = Vec::new();
|
||||
while !response.ends_with(b"\r\n\r\n") {
|
||||
assert!(response.len() < 16 * 1024);
|
||||
response.push(client.read_u8().await.unwrap());
|
||||
}
|
||||
let content_length = std::str::from_utf8(&response)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find_map(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let body_start = response.len();
|
||||
response.resize(body_start + content_length, 0);
|
||||
client
|
||||
.read_exact(&mut response[body_start..])
|
||||
.await
|
||||
.unwrap();
|
||||
response
|
||||
}
|
||||
|
||||
async fn request_with_body_delay(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
@@ -35,6 +80,9 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||
let capability = [21u8; 32];
|
||||
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||
initial_config.web.timeouts.body_secs = 3;
|
||||
initial_config.web.timeouts.header_secs = 1;
|
||||
initial_config.web.timeouts.http_idle_secs = 4;
|
||||
initial_config.web.timeouts.long_poll_secs = 3;
|
||||
initial_config.web.timeouts.bootstrap_lifetime_secs = 5;
|
||||
let generation = test_runtime_generation(1, initial_config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
@@ -68,6 +116,9 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||
|
||||
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||
replacement_config.web.timeouts.body_secs = 1;
|
||||
replacement_config.web.timeouts.header_secs = 1;
|
||||
replacement_config.web.timeouts.http_idle_secs = 2;
|
||||
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||
replacement_config.web.timeouts.bootstrap_lifetime_secs = 1;
|
||||
let replacement = test_runtime_generation(2, replacement_config);
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
@@ -84,6 +135,13 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||
assert!(retry_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(retry_headers, "x-session-token"), session);
|
||||
|
||||
let down = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nX-Down-Cursor: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let down_response = request(&listener, &runtime, down).await;
|
||||
assert!(down_response.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
let close = format!(
|
||||
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
@@ -100,3 +158,83 @@ async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_body_deadline_survives_reload_on_old_keepalive_connection() {
|
||||
let capability = [22u8; 32];
|
||||
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||
initial_config.web.timeouts.header_secs = 1;
|
||||
initial_config.web.timeouts.body_secs = 1;
|
||||
initial_config.web.timeouts.long_poll_secs = 1;
|
||||
initial_config.web.timeouts.http_idle_secs = 2;
|
||||
let generation = test_runtime_generation(1, initial_config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (mut client, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||
client
|
||||
.write_all(
|
||||
b"GET / HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
read_http_response(&mut client)
|
||||
.await
|
||||
.starts_with(b"HTTP/1.1 200")
|
||||
);
|
||||
|
||||
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||
replacement_config.web.timeouts.header_secs = 1;
|
||||
replacement_config.web.timeouts.body_secs = 3;
|
||||
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||
replacement_config.web.timeouts.http_idle_secs = 4;
|
||||
let replacement = test_runtime_generation(2, replacement_config);
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
client
|
||||
.write_all(
|
||||
format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let bridge = read_http_response(&mut client).await;
|
||||
let (_, bridge_body) = split_response(&bridge);
|
||||
let bootstrap = std::str::from_utf8(bridge_body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token.to_string())
|
||||
.unwrap();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
client
|
||||
.write_all(
|
||||
format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2200)).await;
|
||||
client.write_all(&hello).await.unwrap();
|
||||
|
||||
assert!(
|
||||
read_http_response(&mut client)
|
||||
.await
|
||||
.starts_with(b"HTTP/1.1 200")
|
||||
);
|
||||
|
||||
cancellation.cancel();
|
||||
drop(client);
|
||||
task.await.unwrap();
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
@@ -254,6 +254,7 @@ pub(super) async fn handle(
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
) -> HttpResponse {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
let Some(parsed) = parse_upgrade(&request) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
@@ -284,7 +285,16 @@ pub(super) async fn handle(
|
||||
},
|
||||
};
|
||||
let timeouts = session.timeouts().clone();
|
||||
let connection = match runtime
|
||||
let admission_lease = match request_deadline.as_ref() {
|
||||
Some(deadline) => {
|
||||
match deadline.lease_for(Duration::from_secs(timeouts.websocket_eviction_secs)) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let admitted = runtime
|
||||
.admit_websocket(
|
||||
session.profile_key(),
|
||||
session.trace_session_id(),
|
||||
@@ -296,11 +306,17 @@ pub(super) async fn handle(
|
||||
Duration::from_secs(timeouts.websocket_eviction_secs),
|
||||
session.carrier_cancellation(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
drop(admission_lease);
|
||||
let connection = match admitted {
|
||||
Ok(connection) => connection,
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
if let Some(reservation) = lane_reservation.as_mut()
|
||||
&& reservation.bind(connection.id()).is_err()
|
||||
{
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(reservation) = probe_reservation.as_mut()
|
||||
&& reservation.bind(connection.id()).is_err()
|
||||
{
|
||||
@@ -323,6 +339,20 @@ pub(super) async fn handle(
|
||||
trace.bind_identity(session.trace_identity());
|
||||
trace.register_redaction(parsed.protocol.as_bytes());
|
||||
}
|
||||
let upgrade_deadline = match request_deadline.as_ref() {
|
||||
Some(deadline) => {
|
||||
let Some(until) = tokio::time::Instant::now()
|
||||
.checked_add(Duration::from_secs(timeouts.websocket_upgrade_secs))
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
match deadline.upgrade_until(until) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let on_upgrade = hyper::upgrade::on(&mut request);
|
||||
let protocol = parsed.protocol;
|
||||
let accept = parsed.accept;
|
||||
@@ -331,6 +361,7 @@ pub(super) async fn handle(
|
||||
runtime.spawn_auxiliary(async move {
|
||||
driver::run_upgraded(
|
||||
on_upgrade,
|
||||
upgrade_deadline,
|
||||
driver_runtime,
|
||||
driver_session,
|
||||
connection,
|
||||
|
||||
@@ -8,6 +8,7 @@ use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::ConnectionIo;
|
||||
use crate::web::http::activity::UpgradeDeadlineLease;
|
||||
use crate::web::manager::{WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation, WebSocketProbeReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
@@ -22,8 +23,11 @@ mod lane;
|
||||
use io::{flush, process_multiplex, read_message, record_message, reserve_data, send};
|
||||
use lane::run_lane;
|
||||
|
||||
// Upgrade ownership remains explicit across cancellation and reservation boundaries.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_upgraded(
|
||||
on_upgrade: hyper::upgrade::OnUpgrade,
|
||||
upgrade_deadline: Option<UpgradeDeadlineLease>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
session: Arc<WebSession>,
|
||||
connection: WebSocketConnection,
|
||||
@@ -34,13 +38,15 @@ pub(super) async fn run_upgraded(
|
||||
) {
|
||||
let cancellation = connection.cancellation();
|
||||
let timeouts = session.timeouts().clone();
|
||||
let deadline = upgrade_deadline.as_ref().map_or_else(
|
||||
|| tokio::time::Instant::now() + Duration::from_secs(timeouts.websocket_upgrade_secs),
|
||||
UpgradeDeadlineLease::deadline,
|
||||
);
|
||||
let upgraded = tokio::select! {
|
||||
_ = cancellation.cancelled() => return,
|
||||
result = tokio::time::timeout(
|
||||
Duration::from_secs(timeouts.websocket_upgrade_secs),
|
||||
on_upgrade,
|
||||
) => result,
|
||||
result = tokio::time::timeout_at(deadline, on_upgrade) => result,
|
||||
};
|
||||
drop(upgrade_deadline);
|
||||
let Ok(Ok(upgraded)) = upgraded else {
|
||||
return;
|
||||
};
|
||||
@@ -95,8 +101,7 @@ pub(super) async fn run_upgraded(
|
||||
_ = tokio::time::timeout(eviction, socket.close(None)) => {}
|
||||
}
|
||||
if let Some(reservation) = lane_reservation {
|
||||
session.close_websocket_lane(reservation.lane_id());
|
||||
drop(reservation);
|
||||
session.close_websocket_lane(reservation);
|
||||
} else if !acknowledge_commit || session.is_carrier_committed() {
|
||||
session.close();
|
||||
}
|
||||
@@ -192,10 +197,12 @@ async fn run_multiplex(
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit
|
||||
&& sequence > 1
|
||||
&& progressed
|
||||
&& !session.websocket_peer_after_commit_ack(connection.id())
|
||||
{
|
||||
return Err(());
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Drives one exact WebSocket lane until its isolated failure boundary closes.
|
||||
pub(super) async fn run_lane(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
@@ -35,7 +36,7 @@ pub(super) async fn run_lane(
|
||||
let maximum_message = session.limits().carrier_batch_bytes;
|
||||
let mut active = false;
|
||||
loop {
|
||||
let down = session.poll_down_lane(reservation.lane_id(), cursor);
|
||||
let down = session.poll_down_websocket_lane(reservation.lane_identity(), cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
@@ -106,10 +107,12 @@ pub(super) async fn run_lane(
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit
|
||||
&& sequence > 1
|
||||
&& progressed
|
||||
&& !session.websocket_peer_after_commit_ack(connection.id())
|
||||
{
|
||||
return Err(());
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
|
||||
Reference in New Issue
Block a user