WEB Carrier Counters + Status

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-31 21:39:09 +03:00
parent 66f2b8889f
commit 718ce0847e
51 changed files with 2252 additions and 453 deletions
+2 -2
View File
@@ -55,8 +55,8 @@ impl WebSession {
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
(state.down_epoch, healthy)
};
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
self.down_notify.notify_waiters();
+3 -3
View File
@@ -39,7 +39,7 @@ impl WebSession {
let digest: TokenHash = Sha256::digest(body).into();
let mut opened = Vec::new();
let mut committed = false;
let mut healthy = false;
let mut healthy = None;
let result = {
let mut state = self.state.lock();
if state.closed {
@@ -172,8 +172,8 @@ impl WebSession {
if committed {
self.finish_carrier_commit();
}
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
self.lane_open_notify.notify_waiters();
for completion in opened {
+2 -2
View File
@@ -142,8 +142,8 @@ impl WebSession {
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
(instance, epoch, notify, healthy)
};
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
notify.notify_waiters();
+16 -4
View File
@@ -8,6 +8,7 @@ struct ReleasedQueues {
data_items: usize,
control_bytes: usize,
control_items: usize,
closed_before_health: bool,
}
/// Deferred queue release after manager publication linearizes a supersede.
@@ -26,11 +27,12 @@ impl CarrierSupersedeCompletion<'_> {
impl WebSession {
/// Closes carrier state while relay tasks retain their admission until exit.
pub(crate) fn close(&self) {
pub(crate) fn close(&self) -> bool {
let Some(released) = self.begin_close(false, None) else {
return;
return false;
};
self.finish_close(released, false);
true
}
/// Atomically prevents first-frame commit while one successor is prepared.
@@ -97,8 +99,8 @@ impl WebSession {
let mut state = self.state.lock();
self.carrier_health_ready_locked(&mut state, now)
};
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
let Some(released) = self.begin_close(false, Some(now)) else {
return false;
@@ -127,6 +129,9 @@ impl WebSession {
state.close_requested = true;
return None;
}
let closed_before_health = self.automatic_carrier
&& state.negotiation_phase == SessionNegotiationPhase::Committed
&& self.reject_carrier_health_on_close();
state.closed = true;
if superseded {
state.negotiation_phase = SessionNegotiationPhase::Superseded;
@@ -177,6 +182,7 @@ impl WebSession {
data_items,
control_bytes,
control_items,
closed_before_health,
})
}
@@ -189,6 +195,12 @@ impl WebSession {
self.lane_open_notify.notify_waiters();
}
if let Some(manager) = self.manager.upgrade() {
if released.closed_before_health {
manager.telemetry().record_carrier_learning(
self.selected_carrier,
crate::web::telemetry::WebCarrierLearningOutcome::ClosedBeforeHealth,
);
}
manager.release_pending(
self.profile_key,
released.data_bytes,
+204 -22
View File
@@ -3,6 +3,26 @@ use std::time::{Duration, Instant};
use super::uplink::AppliedProgress;
use super::{SessionNegotiationPhase, SessionState, WebSession};
/// Fixed ownership state for one carrier-health publication attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub(super) enum CarrierHealthPublicationState {
/// No eligible callback has claimed publication.
Awaiting,
/// One callback is validating manager and transport ownership.
Publishing,
/// Manager state accepted the health transition.
Published,
/// Close or ownership validation permanently rejected publication.
Rejected,
}
/// Transport owner captured by the single publication claimant.
#[derive(Clone, Copy)]
pub(super) struct CarrierHealthClaim {
websocket_owner: Option<u64>,
}
impl WebSession {
/// Returns whether accepted carrier progress made this attempt immutable.
pub(crate) fn is_carrier_committed(&self) -> bool {
@@ -48,21 +68,21 @@ impl WebSession {
let healthy = {
let mut state = self.state.lock();
if state.closed || state.negotiation_phase != SessionNegotiationPhase::Committed {
false
None
} else {
state.carrier_commit_published = true;
self.carrier_health_ready_locked(&mut state, Instant::now())
}
};
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
}
/// Publishes complete transport-specific health evidence to process state.
pub(super) fn finish_carrier_health(&self) {
pub(super) fn finish_carrier_health(&self, claim: CarrierHealthClaim) {
if let Some(manager) = self.manager.upgrade() {
manager.carrier_became_healthy(
let outcome = manager.carrier_became_healthy(
self.bootstrap_hash,
self.token_hash,
self.carrier_attempt,
@@ -71,7 +91,13 @@ impl WebSession {
self.learning_context,
self.client_ip,
self.trace_identity(),
claim.websocket_owner,
);
if !outcome.published() {
self.reject_carrier_health_publication();
}
} else {
self.reject_carrier_health_publication();
}
}
@@ -80,9 +106,9 @@ impl WebSession {
&self,
state: &mut SessionState,
progress: AppliedProgress,
) -> (bool, bool) {
) -> (bool, Option<CarrierHealthClaim>) {
if !self.automatic_carrier || !progress.any() {
return (false, false);
return (false, None);
}
if self.selected_carrier.uses_websocket() {
state.websocket_carrier_active = true;
@@ -109,15 +135,14 @@ impl WebSession {
&self,
state: &mut SessionState,
now: Instant,
) -> bool {
) -> Option<CarrierHealthClaim> {
if !self.automatic_carrier
|| state.closed
|| state.negotiation_phase != SessionNegotiationPhase::Committed
|| !state.carrier_commit_published
|| state.carrier_health_reported
|| state.carrier_health_due_at.is_none_or(|due| now < due)
{
return false;
return None;
}
let evidence = if state.websocket_carrier_active {
state.websocket_probe_claimed
@@ -132,10 +157,23 @@ impl WebSession {
.zip(state.carrier_health_due_at)
.is_some_and(|(activity, due)| activity >= due)
};
if evidence {
state.carrier_health_reported = true;
if !evidence {
return None;
}
evidence
self.carrier_health_publication
.compare_exchange(
CarrierHealthPublicationState::Awaiting as u8,
CarrierHealthPublicationState::Publishing as u8,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.ok()
.map(|_| CarrierHealthClaim {
websocket_owner: state
.websocket_carrier_active
.then_some(state.websocket_commit_ack_owner)
.flatten(),
})
}
/// Returns whether the exact automatic WebSocket owner must receive a commit acknowledgement.
@@ -175,17 +213,85 @@ impl WebSession {
state.carrier_health_activity_at = Some(now);
self.carrier_health_ready_locked(&mut state, now)
};
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
true
}
/// Confirms manager ownership as the health publication linearization point.
pub(crate) fn publish_carrier_health(&self) -> bool {
self.carrier_health_publication
.compare_exchange(
CarrierHealthPublicationState::Publishing as u8,
CarrierHealthPublicationState::Published as u8,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
}
/// Rejects an in-flight publication after manager validation fails.
pub(super) fn reject_carrier_health_publication(&self) {
let _ = self.carrier_health_publication.compare_exchange(
CarrierHealthPublicationState::Publishing as u8,
CarrierHealthPublicationState::Rejected as u8,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
);
}
/// Rejects pending health on close and reports whether no callback was in flight.
pub(super) fn reject_carrier_health_on_close(&self) -> bool {
loop {
let current = self
.carrier_health_publication
.load(std::sync::atomic::Ordering::Acquire);
let count_locally = current == CarrierHealthPublicationState::Awaiting as u8;
if current != CarrierHealthPublicationState::Awaiting as u8
&& current != CarrierHealthPublicationState::Publishing as u8
{
return false;
}
if self
.carrier_health_publication
.compare_exchange(
current,
CarrierHealthPublicationState::Rejected as u8,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
{
return count_locally;
}
}
}
/// Returns the current fixed health-publication state.
pub(super) fn carrier_health_publication_state(&self) -> CarrierHealthPublicationState {
match self
.carrier_health_publication
.load(std::sync::atomic::Ordering::Acquire)
{
value if value == CarrierHealthPublicationState::Awaiting as u8 => {
CarrierHealthPublicationState::Awaiting
}
value if value == CarrierHealthPublicationState::Publishing as u8 => {
CarrierHealthPublicationState::Publishing
}
value if value == CarrierHealthPublicationState::Published as u8 => {
CarrierHealthPublicationState::Published
}
_ => CarrierHealthPublicationState::Rejected,
}
}
}
#[cfg(test)]
mod tests {
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::{Arc, Barrier};
use super::*;
use crate::config::{
@@ -229,6 +335,16 @@ mod tests {
)
}
fn arm_http_health(session: &WebSession, now: Instant) {
let mut state = session.state.lock();
state.negotiation_phase = SessionNegotiationPhase::Committed;
state.carrier_commit_published = true;
state.carrier_health_due_at = Some(now - Duration::from_secs(1));
state.carrier_health_uplink = true;
state.carrier_health_downlink = true;
state.carrier_health_activity_at = Some(now);
}
#[test]
fn final_deadline_refuses_uncommitted_progress() {
let session = session(WebCarrier::Https, Instant::now() - Duration::from_secs(1));
@@ -254,9 +370,9 @@ mod tests {
state.carrier_health_uplink = true;
state.carrier_health_downlink = true;
state.carrier_health_activity_at = Some(now - Duration::from_secs(2));
assert!(!session.carrier_health_ready_locked(&mut state, now));
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
state.carrier_health_activity_at = Some(now);
assert!(session.carrier_health_ready_locked(&mut state, now));
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
}
#[test]
@@ -274,9 +390,9 @@ mod tests {
state.websocket_commit_ack_owner = Some(7);
state.websocket_commit_ack_written = true;
state.carrier_health_uplink = true;
assert!(!session.carrier_health_ready_locked(&mut state, now));
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
state.websocket_probe_claimed = true;
assert!(session.carrier_health_ready_locked(&mut state, now));
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
}
#[test]
@@ -290,8 +406,74 @@ mod tests {
state.carrier_health_downlink = true;
state.carrier_health_activity_at = Some(now);
assert!(!session.carrier_health_ready_locked(&mut state, now));
assert!(!state.carrier_health_reported);
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
assert_eq!(
session.carrier_health_publication_state(),
CarrierHealthPublicationState::Awaiting
);
}
#[test]
fn health_publication_claim_is_single_shot() {
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
let now = Instant::now();
arm_http_health(&session, now);
let mut state = session.state.lock();
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
drop(state);
assert_eq!(
session.carrier_health_publication_state(),
CarrierHealthPublicationState::Publishing
);
assert!(session.publish_carrier_health());
assert!(!session.publish_carrier_health());
assert_eq!(
session.carrier_health_publication_state(),
CarrierHealthPublicationState::Published
);
}
#[test]
fn concurrent_health_and_close_always_reach_one_terminal_state() {
for _ in 0..512 {
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
let now = Instant::now();
arm_http_health(&session, now);
let barrier = Arc::new(Barrier::new(3));
let health_session = Arc::clone(&session);
let health_barrier = Arc::clone(&barrier);
let health = std::thread::spawn(move || {
health_barrier.wait();
std::thread::yield_now();
let claim = {
let mut state = health_session.state.lock();
health_session.carrier_health_ready_locked(&mut state, now)
};
if claim.is_some() {
health_session.publish_carrier_health();
}
});
let close_session = Arc::clone(&session);
let close_barrier = Arc::clone(&barrier);
let close = std::thread::spawn(move || {
close_barrier.wait();
std::thread::yield_now();
close_session.close();
});
barrier.wait();
health.join().unwrap();
close.join().unwrap();
assert!(matches!(
session.carrier_health_publication_state(),
CarrierHealthPublicationState::Published
| CarrierHealthPublicationState::Rejected
));
assert!(!session.publish_carrier_health());
assert!(!session.close());
}
}
#[test]
+12 -2
View File
@@ -2,7 +2,7 @@ use std::time::Instant;
use serde::Serialize;
use super::{SessionNegotiationPhase, WebSession};
use super::{CarrierHealthPublicationState, SessionNegotiationPhase, WebSession};
use crate::config::WebCarrier;
/// One bounded point-in-time session snapshot without bearer identity.
@@ -28,6 +28,8 @@ pub(crate) struct WebSessionStatus {
pub(crate) automatic: bool,
/// Current session lifecycle token.
pub(crate) state: &'static str,
/// Current manager-confirmed health publication phase.
pub(crate) health_publication: &'static str,
/// Live logical streams.
pub(crate) streams: usize,
/// Stream relay tasks that have not exited.
@@ -66,7 +68,9 @@ impl WebSession {
"closed"
} else if state.close_requested {
"closing"
} else if state.carrier_health_reported {
} else if self.carrier_health_publication_state()
== CarrierHealthPublicationState::Published
{
"healthy"
} else {
match state.negotiation_phase {
@@ -87,6 +91,12 @@ impl WebSession {
client_class: self.carrier_class.as_str(),
automatic: self.automatic_carrier,
state: state_name,
health_publication: match self.carrier_health_publication_state() {
CarrierHealthPublicationState::Awaiting => "awaiting",
CarrierHealthPublicationState::Publishing => "publishing",
CarrierHealthPublicationState::Published => "published",
CarrierHealthPublicationState::Rejected => "rejected",
},
streams: state.streams.len(),
tasks: self.tasks_live(),
lanes: state.carrier_lanes.len(),
+3 -3
View File
@@ -85,7 +85,7 @@ impl WebSession {
let digest: TokenHash = Sha256::digest(body).into();
let mut opened = Vec::new();
let mut committed = false;
let mut healthy = false;
let mut healthy = None;
let result = {
let mut state = self.state.lock();
if state.closed {
@@ -154,8 +154,8 @@ impl WebSession {
if committed {
self.finish_carrier_commit();
}
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
for completion in opened {
self.spawn_stream(completion, false);
+6 -4
View File
@@ -58,7 +58,9 @@ impl Drop for WebSocketProbeReservation {
state.websocket_probe_claimed = false;
if state.websocket_commit_ack_owner == self.owner {
state.websocket_commit_ack_owner = None;
if !state.carrier_health_reported {
if self.session.carrier_health_publication_state()
!= super::CarrierHealthPublicationState::Published
{
state.websocket_commit_ack_written = false;
state.carrier_health_uplink = false;
state.carrier_health_activity_at = None;
@@ -316,7 +318,7 @@ impl WebSession {
let digest = Sha256::digest(body).into();
let mut opened = Vec::new();
let mut committed = false;
let mut healthy = false;
let mut healthy = None;
let result = {
let mut state = self.state.lock();
if state.closed {
@@ -401,8 +403,8 @@ impl WebSession {
if committed {
self.finish_carrier_commit();
}
if healthy {
self.finish_carrier_health();
if let Some(claim) = healthy {
self.finish_carrier_health(claim);
}
for completion in opened {
let stream = completion.stream;