mirror of
https://github.com/telemt/telemt.git
synced 2026-09-14 14:34:11 +03:00
WEB Session Lifecycle Observability + Bridge Recovery Drafts
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Session-local activity clocks with distinct lease and diagnostic authority.
|
||||
pub(super) struct SessionActivity {
|
||||
last_peer: Instant,
|
||||
last_progress: Instant,
|
||||
}
|
||||
|
||||
impl SessionActivity {
|
||||
/// Starts both activity clocks at the same session creation instant.
|
||||
pub(super) fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
last_peer: now,
|
||||
last_progress: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// Records one validated peer operation and returns the preceding peer gap.
|
||||
pub(super) fn touch_peer(&mut self, now: Instant) -> Duration {
|
||||
let gap = now.saturating_duration_since(self.last_peer);
|
||||
self.last_peer = now;
|
||||
self.last_progress = now;
|
||||
gap
|
||||
}
|
||||
|
||||
/// Records server-side carrier progress without extending the peer lease.
|
||||
pub(super) fn touch_progress(&mut self, now: Instant) {
|
||||
self.last_progress = now;
|
||||
}
|
||||
|
||||
/// Returns elapsed time since the latest validated peer operation.
|
||||
pub(super) fn peer_idle(&self, now: Instant) -> Duration {
|
||||
now.saturating_duration_since(self.last_peer)
|
||||
}
|
||||
|
||||
/// Returns elapsed time since the latest carrier-side progress.
|
||||
pub(super) fn progress_idle(&self, now: Instant) -> Duration {
|
||||
now.saturating_duration_since(self.last_progress)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ use crate::proxy::shared_state::ConntrackClosePolicy;
|
||||
use crate::web::frame::FrameType;
|
||||
use crate::web::stream::WebLogicalStream;
|
||||
|
||||
use super::{StreamIdentity, WebSession, inbound_queue_cost};
|
||||
use super::{SessionCloseReason, StreamIdentity, WebSession, inbound_queue_cost};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
@@ -119,7 +119,7 @@ impl WebSession {
|
||||
})
|
||||
};
|
||||
if queued.is_some_and(|queued| !queued) {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Backpressure);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ impl WebSession {
|
||||
}
|
||||
if let Some(queued) = queued {
|
||||
if !queued {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Backpressure);
|
||||
}
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
|
||||
@@ -47,7 +47,7 @@ impl TestRuntime {
|
||||
}
|
||||
|
||||
async fn shutdown(self) {
|
||||
self.session.close();
|
||||
self.session.close(super::SessionCloseReason::ApiClose);
|
||||
self.session.wait().await;
|
||||
self.manager.shutdown().await;
|
||||
self.generation.stop_sessions().await;
|
||||
@@ -402,7 +402,7 @@ async fn cancellation_while_waiting_for_data_releases_stream_ownership() {
|
||||
settle_tasks().await;
|
||||
assert_eq!(runtime.session.tasks_live.load(Ordering::Acquire), 1);
|
||||
|
||||
runtime.session.close();
|
||||
runtime.session.close(super::SessionCloseReason::ApiClose);
|
||||
runtime.session.wait().await;
|
||||
|
||||
assert_eq!(runtime.session.tasks_live.load(Ordering::Acquire), 0);
|
||||
|
||||
@@ -5,7 +5,8 @@ use bytes::{BufMut, Bytes, BytesMut};
|
||||
|
||||
use super::resident::{OwnedBatchBody, PendingCounts, PendingResponseLease};
|
||||
use super::{
|
||||
DownBatch, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState, WebSession,
|
||||
DownBatch, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionCloseReason,
|
||||
SessionState, WebSession,
|
||||
};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::ManagerError;
|
||||
@@ -21,7 +22,7 @@ impl WebSession {
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_peer(Instant::now());
|
||||
if let Some(unacked) = &state.unacked {
|
||||
if cursor == unacked.base_cursor {
|
||||
return Ok(PollResult {
|
||||
@@ -32,7 +33,7 @@ impl WebSession {
|
||||
}
|
||||
if cursor != unacked.next_cursor {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let carrier_health_eligible = unacked.carrier_health_eligible;
|
||||
@@ -43,12 +44,12 @@ impl WebSession {
|
||||
}
|
||||
} else if cursor != state.down_cursor {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let Some(epoch) = state.down_epoch.checked_add(1) else {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
state.down_epoch = epoch;
|
||||
@@ -83,7 +84,7 @@ impl WebSession {
|
||||
}
|
||||
Err(error) => {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -110,7 +111,7 @@ impl WebSession {
|
||||
Err(_) => {
|
||||
let mut state = self.state.lock();
|
||||
if state.down_epoch == epoch {
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_progress(Instant::now());
|
||||
}
|
||||
Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
|
||||
@@ -69,7 +69,7 @@ async fn downlink_replays_unacknowledged_batch_byte_for_byte() {
|
||||
assert_eq!(first.body, replay.body);
|
||||
drop(first);
|
||||
drop(replay);
|
||||
session.close();
|
||||
session.close(super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ async fn acknowledged_response_stays_resident_until_the_last_body_clone_drops()
|
||||
assert!(session.resident.snapshot().bytes() > 0);
|
||||
drop(retained);
|
||||
assert_eq!(session.resident.snapshot().bytes(), 0);
|
||||
session.close();
|
||||
session.close(super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -139,6 +139,6 @@ async fn newer_poll_supersedes_older_poll_without_closing_session() {
|
||||
assert_eq!(superseded.next_cursor, 0);
|
||||
assert!(!session.state.lock().closed);
|
||||
second.abort();
|
||||
session.close();
|
||||
session.close(super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use super::uplink::{AppliedProgress, inbound_reservation, validate_batch};
|
||||
use super::{PendingClass, WebSession, insert_carrier_lane};
|
||||
use super::{PendingClass, SessionCloseReason, WebSession, insert_carrier_lane};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
@@ -24,7 +24,7 @@ impl WebSession {
|
||||
let frames = match frame::parse_all(body, &self.limits) {
|
||||
Ok(frames) => frames,
|
||||
Err(_) => {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
};
|
||||
@@ -33,7 +33,7 @@ impl WebSession {
|
||||
.copied()
|
||||
.any(|value| value.stream_id != lane_id || frame::validate_client_shape(value).is_err())
|
||||
{
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
@@ -46,7 +46,7 @@ impl WebSession {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_peer(Instant::now());
|
||||
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
||||
if new_lane {
|
||||
if lane_id != 0
|
||||
@@ -69,7 +69,7 @@ impl WebSession {
|
||||
.is_none_or(|value| value.frame_type != FrameType::Open)
|
||||
{
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let lane_limit = self
|
||||
@@ -92,13 +92,13 @@ impl WebSession {
|
||||
Ok(sequence)
|
||||
} else {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
Err(ManagerError::Protocol)
|
||||
};
|
||||
}
|
||||
if sequence == 0 || sequence != last_sequence.saturating_add(1) {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if up_active {
|
||||
@@ -106,7 +106,7 @@ impl WebSession {
|
||||
}
|
||||
if !validate_batch(&state, &frames) {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||
@@ -124,7 +124,7 @@ impl WebSession {
|
||||
if new_lane && insert_carrier_lane(&mut state, lane_id).is_none() {
|
||||
self.release_locked(&mut state, reserve_bytes, reserve_items, false);
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
@@ -161,7 +161,7 @@ impl WebSession {
|
||||
return result;
|
||||
}
|
||||
if result.is_err() {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
drop(opened);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -6,8 +6,8 @@ use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
use super::lane_downlink::take_lane_down_batch;
|
||||
use super::{
|
||||
CarrierLaneIdentity, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState,
|
||||
WebSession, remember_closed,
|
||||
CarrierLaneIdentity, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionCloseReason,
|
||||
SessionState, WebSession, remember_closed,
|
||||
};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::ManagerError;
|
||||
@@ -65,7 +65,7 @@ impl WebSession {
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_peer(Instant::now());
|
||||
let acknowledged = {
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
return Ok(PollResult {
|
||||
@@ -91,14 +91,14 @@ impl WebSession {
|
||||
}
|
||||
if cursor != unacked.next_cursor {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
lane.unacked.take()
|
||||
} else {
|
||||
if cursor != lane.down_cursor {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
None
|
||||
@@ -133,7 +133,7 @@ impl WebSession {
|
||||
.ok_or(ManagerError::Protocol)?;
|
||||
let Some(epoch) = lane.down_epoch.checked_add(1) else {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
lane.down_epoch = epoch;
|
||||
@@ -195,7 +195,7 @@ impl WebSession {
|
||||
}
|
||||
Err(error) => {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -258,7 +258,7 @@ impl WebSession {
|
||||
});
|
||||
}
|
||||
if lane.down_epoch == epoch {
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_progress(Instant::now());
|
||||
}
|
||||
}
|
||||
Ok(PollResult {
|
||||
@@ -281,7 +281,7 @@ impl WebSession {
|
||||
}
|
||||
if cursor != 0 || lane_id == 0 {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if state.closed_streams.contains(&lane_id)
|
||||
|
||||
@@ -108,7 +108,7 @@ async fn early_down_waits_without_creating_a_provisional_lane() {
|
||||
assert!(!result.body.is_empty());
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
drop(result);
|
||||
session.close();
|
||||
session.close(super::super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ async fn early_down_timeout_is_empty_and_releases_its_session_slot() {
|
||||
assert_eq!(result.next_cursor, 0);
|
||||
assert!(!result.lane_closed);
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
session.close();
|
||||
session.close(super::super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ async fn early_down_admission_is_bounded_and_cancellation_safe() {
|
||||
let _ = wait.await;
|
||||
}
|
||||
assert_eq!(session.state.lock().lane_open_waits, 0);
|
||||
session.close();
|
||||
session.close(super::super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -168,7 +168,7 @@ async fn session_close_wakes_early_down_with_closed_state() {
|
||||
while session.state.lock().lane_open_waits == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
session.close();
|
||||
session.close(super::super::SessionCloseReason::ApiClose);
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(Duration::from_secs(1), poll)
|
||||
.await
|
||||
@@ -229,7 +229,7 @@ async fn drained_closed_lane_replays_then_signals_completion() {
|
||||
assert!(finished.lane_closed);
|
||||
drop(first);
|
||||
drop(replay);
|
||||
session.close();
|
||||
session.close(super::super::SessionCloseReason::ApiClose);
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
|
||||
+182
-55
@@ -1,7 +1,95 @@
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::{SessionNegotiationPhase, WebSession};
|
||||
use super::WebSession;
|
||||
|
||||
/// Stable terminal cause assigned by the first session-close winner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[repr(usize)]
|
||||
pub(crate) enum SessionCloseReason {
|
||||
/// An authenticated client explicitly deleted its current session.
|
||||
ClientDelete,
|
||||
/// A surviving bridge replaced an unreachable carrier incarnation.
|
||||
BridgeRecovery,
|
||||
/// No validated peer operation arrived within the frozen reconnect grace.
|
||||
PeerIdle,
|
||||
/// Automatic carrier negotiation exhausted its absolute deadline.
|
||||
NegotiationTimeout,
|
||||
/// A successful negotiation replacement retired this incarnation.
|
||||
CarrierSuperseded,
|
||||
/// Authenticated carrier framing or sequencing violated the protocol.
|
||||
Protocol,
|
||||
/// Mandatory bounded control state could not be retained.
|
||||
Backpressure,
|
||||
/// A committed WebSocket carrier ended.
|
||||
WebSocketEnded,
|
||||
/// An authenticated control-plane request selected this session.
|
||||
ApiClose,
|
||||
/// A graceful operator drain reached its force-close deadline.
|
||||
OperatorForce,
|
||||
/// Terminal process shutdown closed all remaining sessions.
|
||||
RuntimeShutdown,
|
||||
}
|
||||
|
||||
impl SessionCloseReason {
|
||||
/// Complete fixed reason set in stable API and metric order.
|
||||
pub(crate) const ALL: [Self; 11] = [
|
||||
Self::ClientDelete,
|
||||
Self::BridgeRecovery,
|
||||
Self::PeerIdle,
|
||||
Self::NegotiationTimeout,
|
||||
Self::CarrierSuperseded,
|
||||
Self::Protocol,
|
||||
Self::Backpressure,
|
||||
Self::WebSocketEnded,
|
||||
Self::ApiClose,
|
||||
Self::OperatorForce,
|
||||
Self::RuntimeShutdown,
|
||||
];
|
||||
|
||||
/// Returns the stable API, trace, and Prometheus token.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ClientDelete => "client_delete",
|
||||
Self::BridgeRecovery => "bridge_recovery",
|
||||
Self::PeerIdle => "peer_idle",
|
||||
Self::NegotiationTimeout => "negotiation_timeout",
|
||||
Self::CarrierSuperseded => "carrier_superseded",
|
||||
Self::Protocol => "protocol",
|
||||
Self::Backpressure => "backpressure",
|
||||
Self::WebSocketEnded => "websocket_ended",
|
||||
Self::ApiClose => "api_close",
|
||||
Self::OperatorForce => "operator_force",
|
||||
Self::RuntimeShutdown => "runtime_shutdown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one first-writer-wins close request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SessionCloseOutcome {
|
||||
/// This caller closed the session synchronously.
|
||||
Closed,
|
||||
/// This caller owns a close deferred behind carrier replacement.
|
||||
Deferred,
|
||||
/// An earlier caller already owns or completed session closure.
|
||||
AlreadyClosing,
|
||||
}
|
||||
|
||||
impl SessionCloseOutcome {
|
||||
/// Returns whether this caller won the terminal cause.
|
||||
pub(crate) const fn accepted(self) -> bool {
|
||||
!matches!(self, Self::AlreadyClosing)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum SessionNegotiationPhase {
|
||||
Uncommitted,
|
||||
Replacing,
|
||||
Committed,
|
||||
Superseded,
|
||||
}
|
||||
|
||||
struct ReleasedQueues {
|
||||
data_bytes: usize,
|
||||
@@ -9,6 +97,7 @@ struct ReleasedQueues {
|
||||
control_bytes: usize,
|
||||
control_items: usize,
|
||||
closed_before_health: bool,
|
||||
reason: SessionCloseReason,
|
||||
}
|
||||
|
||||
/// Deferred queue release after manager publication linearizes a supersede.
|
||||
@@ -21,24 +110,31 @@ pub(crate) struct CarrierSupersedeCompletion<'a> {
|
||||
impl CarrierSupersedeCompletion<'_> {
|
||||
/// Releases process budgets and signals cancellation after manager locks are dropped.
|
||||
pub(crate) fn finish(self) {
|
||||
self.session.finish_close(self.released, true);
|
||||
self.session.finish_close(self.released);
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||
pub(crate) fn close(&self) -> bool {
|
||||
let Some(released) = self.begin_close(false, None) else {
|
||||
return false;
|
||||
};
|
||||
self.finish_close(released, false);
|
||||
true
|
||||
pub(crate) fn close(&self, reason: SessionCloseReason) -> SessionCloseOutcome {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || state.close_requested.is_some() {
|
||||
return SessionCloseOutcome::AlreadyClosing;
|
||||
}
|
||||
if state.negotiation_phase == SessionNegotiationPhase::Replacing {
|
||||
state.close_requested = Some(reason);
|
||||
return SessionCloseOutcome::Deferred;
|
||||
}
|
||||
let released = self.release_on_close_locked(&mut state, reason);
|
||||
drop(state);
|
||||
self.finish_close(released);
|
||||
SessionCloseOutcome::Closed
|
||||
}
|
||||
|
||||
/// Atomically prevents first-frame commit while one successor is prepared.
|
||||
pub(crate) fn begin_carrier_supersede(&self) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || state.close_requested {
|
||||
if state.closed || state.close_requested.is_some() {
|
||||
return false;
|
||||
}
|
||||
match state.negotiation_phase {
|
||||
@@ -54,21 +150,32 @@ impl WebSession {
|
||||
|
||||
/// Restores an uncommitted attempt after successor admission failed.
|
||||
pub(crate) fn cancel_carrier_supersede(&self) {
|
||||
let close_requested = {
|
||||
let released = {
|
||||
let mut state = self.state.lock();
|
||||
if !state.closed && state.negotiation_phase == SessionNegotiationPhase::Replacing {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Uncommitted;
|
||||
}
|
||||
state.close_requested
|
||||
state
|
||||
.close_requested
|
||||
.filter(|_| !state.closed)
|
||||
.map(|reason| self.release_on_close_locked(&mut state, reason))
|
||||
};
|
||||
if close_requested {
|
||||
self.close();
|
||||
if let Some(released) = released {
|
||||
self.finish_close(released);
|
||||
}
|
||||
}
|
||||
|
||||
/// Linearizes manager publication against close requests on the old token.
|
||||
pub(crate) fn prepare_carrier_supersede(&self) -> Option<CarrierSupersedeCompletion<'_>> {
|
||||
let released = self.begin_close(true, None)?;
|
||||
let mut state = self.state.lock();
|
||||
if state.closed
|
||||
|| state.negotiation_phase != SessionNegotiationPhase::Replacing
|
||||
|| state.close_requested.is_some()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let released =
|
||||
self.release_on_close_locked(&mut state, SessionCloseReason::CarrierSuperseded);
|
||||
Some(CarrierSupersedeCompletion {
|
||||
session: self,
|
||||
released,
|
||||
@@ -88,6 +195,19 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Waits until registry removal and close telemetry have completed.
|
||||
pub(crate) async fn wait_close_complete(&self) {
|
||||
loop {
|
||||
let notified = self.close_notify.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.close_complete.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the current number of registered logical-stream tasks.
|
||||
pub(crate) fn tasks_live(&self) -> usize {
|
||||
self.tasks_live.load(Ordering::Acquire)
|
||||
@@ -102,38 +222,38 @@ impl WebSession {
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
let Some(released) = self.begin_close(false, Some(now)) else {
|
||||
let Some(released) = self.begin_idle_close(now) else {
|
||||
return false;
|
||||
};
|
||||
self.finish_close(released, false);
|
||||
self.finish_close(released);
|
||||
true
|
||||
}
|
||||
|
||||
fn begin_close(&self, superseded: bool, idle_now: Option<Instant>) -> Option<ReleasedQueues> {
|
||||
fn begin_idle_close(&self, now: Instant) -> Option<ReleasedQueues> {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed
|
||||
|| (superseded
|
||||
&& (state.negotiation_phase != SessionNegotiationPhase::Replacing
|
||||
|| state.close_requested))
|
||||
if state.closed || state.close_requested.is_some() {
|
||||
return None;
|
||||
}
|
||||
if state.negotiation_phase == SessionNegotiationPhase::Replacing
|
||||
|| state.activity.peer_idle(now)
|
||||
< Duration::from_secs(self.timeouts.reconnect_grace_secs)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(now) = idle_now
|
||||
&& (state.negotiation_phase == SessionNegotiationPhase::Replacing
|
||||
|| now.saturating_duration_since(state.last_activity)
|
||||
< Duration::from_secs(self.timeouts.reconnect_grace_secs))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if !superseded && state.negotiation_phase == SessionNegotiationPhase::Replacing {
|
||||
state.close_requested = true;
|
||||
return None;
|
||||
}
|
||||
Some(self.release_on_close_locked(&mut state, SessionCloseReason::PeerIdle))
|
||||
}
|
||||
|
||||
fn release_on_close_locked(
|
||||
&self,
|
||||
state: &mut super::SessionState,
|
||||
reason: SessionCloseReason,
|
||||
) -> ReleasedQueues {
|
||||
let closed_before_health = self.automatic_carrier
|
||||
&& state.negotiation_phase == SessionNegotiationPhase::Committed
|
||||
&& self.reject_carrier_health_on_close();
|
||||
state.close_requested = Some(reason);
|
||||
state.closed = true;
|
||||
if superseded {
|
||||
if reason == SessionCloseReason::CarrierSuperseded {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Superseded;
|
||||
}
|
||||
for stream in state.streams.values_mut() {
|
||||
@@ -149,8 +269,8 @@ impl WebSession {
|
||||
state.pending_windows.clear();
|
||||
if let Some(batch) = state.unacked.take() {
|
||||
batch.lease.detach();
|
||||
self.release_local_locked(&mut state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(&mut state, batch.control_bytes, batch.control_items, true);
|
||||
self.release_local_locked(state, batch.data_bytes, batch.data_items, false);
|
||||
self.release_local_locked(state, batch.control_bytes, batch.control_items, true);
|
||||
}
|
||||
let mut lane_data_bytes = 0usize;
|
||||
let mut lane_data_items = 0usize;
|
||||
@@ -166,8 +286,8 @@ impl WebSession {
|
||||
lane_control_items = lane_control_items.saturating_add(batch.control_items);
|
||||
}
|
||||
}
|
||||
self.release_local_locked(&mut state, lane_data_bytes, lane_data_items, false);
|
||||
self.release_local_locked(&mut state, lane_control_bytes, lane_control_items, true);
|
||||
self.release_local_locked(state, lane_data_bytes, lane_data_items, false);
|
||||
self.release_local_locked(state, lane_control_bytes, lane_control_items, true);
|
||||
state.carrier_lanes.clear();
|
||||
let control_bytes = state.pending_control_bytes;
|
||||
let control_items = state.pending_control_items;
|
||||
@@ -177,16 +297,17 @@ impl WebSession {
|
||||
state.pending_items = 0;
|
||||
state.pending_control_bytes = 0;
|
||||
state.pending_control_items = 0;
|
||||
Some(ReleasedQueues {
|
||||
ReleasedQueues {
|
||||
data_bytes,
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
closed_before_health,
|
||||
})
|
||||
reason,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_close(&self, released: ReleasedQueues, superseded: bool) {
|
||||
fn finish_close(&self, released: ReleasedQueues) {
|
||||
self.cancel.cancel();
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
@@ -194,7 +315,8 @@ impl WebSession {
|
||||
if self.carrier().uses_lanes() {
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
let manager = self.manager.upgrade();
|
||||
if let Some(manager) = &manager {
|
||||
if released.closed_before_health {
|
||||
manager.telemetry().record_carrier_learning(
|
||||
self.selected_carrier,
|
||||
@@ -213,22 +335,27 @@ impl WebSession {
|
||||
released.control_items,
|
||||
true,
|
||||
);
|
||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||
None,
|
||||
Some(if superseded { "superseded" } else { "closed" }),
|
||||
}
|
||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||
None,
|
||||
Some(released.reason.as_str()),
|
||||
);
|
||||
if released.reason != SessionCloseReason::CarrierSuperseded
|
||||
&& let Some(manager) = manager
|
||||
{
|
||||
manager.session_finished(
|
||||
self.token_hash,
|
||||
self.client_ip,
|
||||
self.profile_key,
|
||||
&self.profile.host,
|
||||
Duration::from_secs(self.timeouts.bootstrap_lifetime_secs),
|
||||
released.reason,
|
||||
);
|
||||
if !superseded {
|
||||
manager.session_finished(
|
||||
self.token_hash,
|
||||
self.client_ip,
|
||||
self.profile_key,
|
||||
&self.profile.host,
|
||||
Duration::from_secs(self.timeouts.bootstrap_lifetime_secs),
|
||||
);
|
||||
}
|
||||
}
|
||||
self.close_complete.store(true, Ordering::Release);
|
||||
self.close_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,7 +460,7 @@ mod tests {
|
||||
let close = std::thread::spawn(move || {
|
||||
close_barrier.wait();
|
||||
std::thread::yield_now();
|
||||
close_session.close();
|
||||
close_session.close(super::SessionCloseReason::ApiClose);
|
||||
});
|
||||
barrier.wait();
|
||||
health.join().unwrap();
|
||||
@@ -472,7 +472,10 @@ mod tests {
|
||||
| CarrierHealthPublicationState::Rejected
|
||||
));
|
||||
assert!(!session.publish_carrier_health());
|
||||
assert!(!session.close());
|
||||
assert_eq!(
|
||||
session.close(super::SessionCloseReason::ApiClose),
|
||||
super::SessionCloseOutcome::AlreadyClosing
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,12 @@ pub(crate) struct WebSessionStatus {
|
||||
pub(crate) age_ms: u64,
|
||||
/// Monotonic age since the latest carrier activity.
|
||||
pub(crate) idle_ms: u64,
|
||||
/// Monotonic age since the latest validated peer operation.
|
||||
pub(crate) peer_idle_ms: u64,
|
||||
/// Session-frozen authenticated peer inactivity allowance.
|
||||
pub(crate) reconnect_grace_ms: u64,
|
||||
/// Remaining time before peer inactivity makes the session eligible for cleanup.
|
||||
pub(crate) peer_deadline_remaining_ms: u64,
|
||||
/// Remaining automatic negotiation deadline.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) negotiation_remaining_ms: Option<u64>,
|
||||
@@ -66,7 +72,7 @@ impl WebSession {
|
||||
let resident = self.resident.snapshot();
|
||||
let state_name = if state.closed {
|
||||
"closed"
|
||||
} else if state.close_requested {
|
||||
} else if state.close_requested.is_some() {
|
||||
"closing"
|
||||
} else if self.carrier_health_publication_state()
|
||||
== CarrierHealthPublicationState::Published
|
||||
@@ -112,7 +118,14 @@ impl WebSession {
|
||||
.pending_control_items
|
||||
.saturating_add(resident.control_items),
|
||||
age_ms: millis(now.saturating_duration_since(self.created_at)),
|
||||
idle_ms: millis(now.saturating_duration_since(state.last_activity)),
|
||||
idle_ms: millis(state.activity.progress_idle(now)),
|
||||
peer_idle_ms: millis(state.activity.peer_idle(now)),
|
||||
reconnect_grace_ms: self.timeouts.reconnect_grace_secs.saturating_mul(1_000),
|
||||
peer_deadline_remaining_ms: self
|
||||
.timeouts
|
||||
.reconnect_grace_secs
|
||||
.saturating_mul(1_000)
|
||||
.saturating_sub(millis(state.activity.peer_idle(now))),
|
||||
negotiation_remaining_ms: self
|
||||
.carrier_deadline_at
|
||||
.map(|deadline| millis(deadline.saturating_duration_since(now))),
|
||||
|
||||
@@ -9,8 +9,8 @@ use subtle::ConstantTimeEq;
|
||||
|
||||
use super::backend::StreamCompletion;
|
||||
use super::{
|
||||
InboundChunk, PendingClass, QUEUE_ITEM_COST, SessionState, StreamIdentity, StreamState,
|
||||
WebSession, inbound_queue_cost,
|
||||
InboundChunk, PendingClass, QUEUE_ITEM_COST, SessionCloseReason, SessionState, StreamIdentity,
|
||||
StreamState, WebSession, inbound_queue_cost,
|
||||
};
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
@@ -70,7 +70,7 @@ impl WebSession {
|
||||
let frames = match frame::parse_all(body, &self.limits) {
|
||||
Ok(frames) => frames,
|
||||
Err(_) => {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
};
|
||||
@@ -79,7 +79,7 @@ impl WebSession {
|
||||
.copied()
|
||||
.any(|value| frame::validate_client_shape(value).is_err())
|
||||
{
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
@@ -92,24 +92,24 @@ impl WebSession {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_peer(Instant::now());
|
||||
if sequence == state.last_up_sequence && sequence != 0 {
|
||||
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
||||
Ok((sequence, false))
|
||||
} else {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
Err(ManagerError::Protocol)
|
||||
};
|
||||
}
|
||||
if sequence == 0 || sequence != state.last_up_sequence.saturating_add(1) {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if !validate_batch(&state, &frames) {
|
||||
drop(state);
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||
@@ -147,7 +147,7 @@ impl WebSession {
|
||||
return result;
|
||||
}
|
||||
if result.is_err() {
|
||||
self.close();
|
||||
self.close(SessionCloseReason::Protocol);
|
||||
drop(opened);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -391,7 +391,7 @@ impl WebSession {
|
||||
lane.last_up_digest = digest;
|
||||
}
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
state.activity.touch_peer(Instant::now());
|
||||
if applied {
|
||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ struct TestRuntime {
|
||||
|
||||
impl TestRuntime {
|
||||
async fn shutdown(self) {
|
||||
self.session.close();
|
||||
self.session.close(super::super::SessionCloseReason::ApiClose);
|
||||
self.session.wait().await;
|
||||
self.manager.shutdown().await;
|
||||
self.generation.stop_sessions().await;
|
||||
@@ -181,7 +181,9 @@ async fn closed_session_releases_bound_lane_quota_on_reservation_drop() {
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
|
||||
runtime.session.close();
|
||||
runtime
|
||||
.session
|
||||
.close(super::super::SessionCloseReason::ApiClose);
|
||||
drop(reservation);
|
||||
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
@@ -220,7 +222,9 @@ async fn closed_session_releases_transferred_rejected_lane_quota() {
|
||||
WebSocketLaneReservationPhase::Transferred
|
||||
);
|
||||
|
||||
runtime.session.close();
|
||||
runtime
|
||||
.session
|
||||
.close(super::super::SessionCloseReason::ApiClose);
|
||||
drop(reservation);
|
||||
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
@@ -259,7 +263,9 @@ async fn closed_session_keeps_stream_owned_quota_until_task_completion() {
|
||||
WebSocketLaneReservationPhase::StreamOwned
|
||||
);
|
||||
|
||||
runtime.session.close();
|
||||
runtime
|
||||
.session
|
||||
.close(super::super::SessionCloseReason::ApiClose);
|
||||
drop(reservation);
|
||||
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user