mirror of
https://github.com/telemt/telemt.git
synced 2026-09-19 00:38:31 +03:00
WEB Carrier Counters + Status
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -55,8 +55,8 @@ impl WebSession {
|
||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||
(state.down_epoch, healthy)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
self.down_notify.notify_waiters();
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ impl WebSession {
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let mut healthy = None;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
@@ -172,8 +172,8 @@ impl WebSession {
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
self.lane_open_notify.notify_waiters();
|
||||
for completion in opened {
|
||||
|
||||
@@ -142,8 +142,8 @@ impl WebSession {
|
||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||
(instance, epoch, notify, healthy)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
notify.notify_waiters();
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ struct ReleasedQueues {
|
||||
data_items: usize,
|
||||
control_bytes: usize,
|
||||
control_items: usize,
|
||||
closed_before_health: bool,
|
||||
}
|
||||
|
||||
/// Deferred queue release after manager publication linearizes a supersede.
|
||||
@@ -26,11 +27,12 @@ impl CarrierSupersedeCompletion<'_> {
|
||||
|
||||
impl WebSession {
|
||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||
pub(crate) fn close(&self) {
|
||||
pub(crate) fn close(&self) -> bool {
|
||||
let Some(released) = self.begin_close(false, None) else {
|
||||
return;
|
||||
return false;
|
||||
};
|
||||
self.finish_close(released, false);
|
||||
true
|
||||
}
|
||||
|
||||
/// Atomically prevents first-frame commit while one successor is prepared.
|
||||
@@ -97,8 +99,8 @@ impl WebSession {
|
||||
let mut state = self.state.lock();
|
||||
self.carrier_health_ready_locked(&mut state, now)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
let Some(released) = self.begin_close(false, Some(now)) else {
|
||||
return false;
|
||||
@@ -127,6 +129,9 @@ impl WebSession {
|
||||
state.close_requested = true;
|
||||
return None;
|
||||
}
|
||||
let closed_before_health = self.automatic_carrier
|
||||
&& state.negotiation_phase == SessionNegotiationPhase::Committed
|
||||
&& self.reject_carrier_health_on_close();
|
||||
state.closed = true;
|
||||
if superseded {
|
||||
state.negotiation_phase = SessionNegotiationPhase::Superseded;
|
||||
@@ -177,6 +182,7 @@ impl WebSession {
|
||||
data_items,
|
||||
control_bytes,
|
||||
control_items,
|
||||
closed_before_health,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -189,6 +195,12 @@ impl WebSession {
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
if released.closed_before_health {
|
||||
manager.telemetry().record_carrier_learning(
|
||||
self.selected_carrier,
|
||||
crate::web::telemetry::WebCarrierLearningOutcome::ClosedBeforeHealth,
|
||||
);
|
||||
}
|
||||
manager.release_pending(
|
||||
self.profile_key,
|
||||
released.data_bytes,
|
||||
|
||||
+204
-22
@@ -3,6 +3,26 @@ use std::time::{Duration, Instant};
|
||||
use super::uplink::AppliedProgress;
|
||||
use super::{SessionNegotiationPhase, SessionState, WebSession};
|
||||
|
||||
/// Fixed ownership state for one carrier-health publication attempt.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub(super) enum CarrierHealthPublicationState {
|
||||
/// No eligible callback has claimed publication.
|
||||
Awaiting,
|
||||
/// One callback is validating manager and transport ownership.
|
||||
Publishing,
|
||||
/// Manager state accepted the health transition.
|
||||
Published,
|
||||
/// Close or ownership validation permanently rejected publication.
|
||||
Rejected,
|
||||
}
|
||||
|
||||
/// Transport owner captured by the single publication claimant.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct CarrierHealthClaim {
|
||||
websocket_owner: Option<u64>,
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Returns whether accepted carrier progress made this attempt immutable.
|
||||
pub(crate) fn is_carrier_committed(&self) -> bool {
|
||||
@@ -48,21 +68,21 @@ impl WebSession {
|
||||
let healthy = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || state.negotiation_phase != SessionNegotiationPhase::Committed {
|
||||
false
|
||||
None
|
||||
} else {
|
||||
state.carrier_commit_published = true;
|
||||
self.carrier_health_ready_locked(&mut state, Instant::now())
|
||||
}
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes complete transport-specific health evidence to process state.
|
||||
pub(super) fn finish_carrier_health(&self) {
|
||||
pub(super) fn finish_carrier_health(&self, claim: CarrierHealthClaim) {
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.carrier_became_healthy(
|
||||
let outcome = manager.carrier_became_healthy(
|
||||
self.bootstrap_hash,
|
||||
self.token_hash,
|
||||
self.carrier_attempt,
|
||||
@@ -71,7 +91,13 @@ impl WebSession {
|
||||
self.learning_context,
|
||||
self.client_ip,
|
||||
self.trace_identity(),
|
||||
claim.websocket_owner,
|
||||
);
|
||||
if !outcome.published() {
|
||||
self.reject_carrier_health_publication();
|
||||
}
|
||||
} else {
|
||||
self.reject_carrier_health_publication();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,9 +106,9 @@ impl WebSession {
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
progress: AppliedProgress,
|
||||
) -> (bool, bool) {
|
||||
) -> (bool, Option<CarrierHealthClaim>) {
|
||||
if !self.automatic_carrier || !progress.any() {
|
||||
return (false, false);
|
||||
return (false, None);
|
||||
}
|
||||
if self.selected_carrier.uses_websocket() {
|
||||
state.websocket_carrier_active = true;
|
||||
@@ -109,15 +135,14 @@ impl WebSession {
|
||||
&self,
|
||||
state: &mut SessionState,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
) -> Option<CarrierHealthClaim> {
|
||||
if !self.automatic_carrier
|
||||
|| state.closed
|
||||
|| state.negotiation_phase != SessionNegotiationPhase::Committed
|
||||
|| !state.carrier_commit_published
|
||||
|| state.carrier_health_reported
|
||||
|| state.carrier_health_due_at.is_none_or(|due| now < due)
|
||||
{
|
||||
return false;
|
||||
return None;
|
||||
}
|
||||
let evidence = if state.websocket_carrier_active {
|
||||
state.websocket_probe_claimed
|
||||
@@ -132,10 +157,23 @@ impl WebSession {
|
||||
.zip(state.carrier_health_due_at)
|
||||
.is_some_and(|(activity, due)| activity >= due)
|
||||
};
|
||||
if evidence {
|
||||
state.carrier_health_reported = true;
|
||||
if !evidence {
|
||||
return None;
|
||||
}
|
||||
evidence
|
||||
self.carrier_health_publication
|
||||
.compare_exchange(
|
||||
CarrierHealthPublicationState::Awaiting as u8,
|
||||
CarrierHealthPublicationState::Publishing as u8,
|
||||
std::sync::atomic::Ordering::AcqRel,
|
||||
std::sync::atomic::Ordering::Acquire,
|
||||
)
|
||||
.ok()
|
||||
.map(|_| CarrierHealthClaim {
|
||||
websocket_owner: state
|
||||
.websocket_carrier_active
|
||||
.then_some(state.websocket_commit_ack_owner)
|
||||
.flatten(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the exact automatic WebSocket owner must receive a commit acknowledgement.
|
||||
@@ -175,17 +213,85 @@ impl WebSession {
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
self.carrier_health_ready_locked(&mut state, now)
|
||||
};
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Confirms manager ownership as the health publication linearization point.
|
||||
pub(crate) fn publish_carrier_health(&self) -> bool {
|
||||
self.carrier_health_publication
|
||||
.compare_exchange(
|
||||
CarrierHealthPublicationState::Publishing as u8,
|
||||
CarrierHealthPublicationState::Published as u8,
|
||||
std::sync::atomic::Ordering::AcqRel,
|
||||
std::sync::atomic::Ordering::Acquire,
|
||||
)
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Rejects an in-flight publication after manager validation fails.
|
||||
pub(super) fn reject_carrier_health_publication(&self) {
|
||||
let _ = self.carrier_health_publication.compare_exchange(
|
||||
CarrierHealthPublicationState::Publishing as u8,
|
||||
CarrierHealthPublicationState::Rejected as u8,
|
||||
std::sync::atomic::Ordering::AcqRel,
|
||||
std::sync::atomic::Ordering::Acquire,
|
||||
);
|
||||
}
|
||||
|
||||
/// Rejects pending health on close and reports whether no callback was in flight.
|
||||
pub(super) fn reject_carrier_health_on_close(&self) -> bool {
|
||||
loop {
|
||||
let current = self
|
||||
.carrier_health_publication
|
||||
.load(std::sync::atomic::Ordering::Acquire);
|
||||
let count_locally = current == CarrierHealthPublicationState::Awaiting as u8;
|
||||
if current != CarrierHealthPublicationState::Awaiting as u8
|
||||
&& current != CarrierHealthPublicationState::Publishing as u8
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.carrier_health_publication
|
||||
.compare_exchange(
|
||||
current,
|
||||
CarrierHealthPublicationState::Rejected as u8,
|
||||
std::sync::atomic::Ordering::AcqRel,
|
||||
std::sync::atomic::Ordering::Acquire,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
return count_locally;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the current fixed health-publication state.
|
||||
pub(super) fn carrier_health_publication_state(&self) -> CarrierHealthPublicationState {
|
||||
match self
|
||||
.carrier_health_publication
|
||||
.load(std::sync::atomic::Ordering::Acquire)
|
||||
{
|
||||
value if value == CarrierHealthPublicationState::Awaiting as u8 => {
|
||||
CarrierHealthPublicationState::Awaiting
|
||||
}
|
||||
value if value == CarrierHealthPublicationState::Publishing as u8 => {
|
||||
CarrierHealthPublicationState::Publishing
|
||||
}
|
||||
value if value == CarrierHealthPublicationState::Published as u8 => {
|
||||
CarrierHealthPublicationState::Published
|
||||
}
|
||||
_ => CarrierHealthPublicationState::Rejected,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Barrier};
|
||||
|
||||
use super::*;
|
||||
use crate::config::{
|
||||
@@ -229,6 +335,16 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
fn arm_http_health(session: &WebSession, now: Instant) {
|
||||
let mut state = session.state.lock();
|
||||
state.negotiation_phase = SessionNegotiationPhase::Committed;
|
||||
state.carrier_commit_published = true;
|
||||
state.carrier_health_due_at = Some(now - Duration::from_secs(1));
|
||||
state.carrier_health_uplink = true;
|
||||
state.carrier_health_downlink = true;
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_deadline_refuses_uncommitted_progress() {
|
||||
let session = session(WebCarrier::Https, Instant::now() - Duration::from_secs(1));
|
||||
@@ -254,9 +370,9 @@ mod tests {
|
||||
state.carrier_health_uplink = true;
|
||||
state.carrier_health_downlink = true;
|
||||
state.carrier_health_activity_at = Some(now - Duration::from_secs(2));
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -274,9 +390,9 @@ mod tests {
|
||||
state.websocket_commit_ack_owner = Some(7);
|
||||
state.websocket_commit_ack_written = true;
|
||||
state.carrier_health_uplink = true;
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
|
||||
state.websocket_probe_claimed = true;
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -290,8 +406,74 @@ mod tests {
|
||||
state.carrier_health_downlink = true;
|
||||
state.carrier_health_activity_at = Some(now);
|
||||
|
||||
assert!(!session.carrier_health_ready_locked(&mut state, now));
|
||||
assert!(!state.carrier_health_reported);
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
|
||||
assert_eq!(
|
||||
session.carrier_health_publication_state(),
|
||||
CarrierHealthPublicationState::Awaiting
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_publication_claim_is_single_shot() {
|
||||
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
let now = Instant::now();
|
||||
arm_http_health(&session, now);
|
||||
let mut state = session.state.lock();
|
||||
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_some());
|
||||
assert!(session.carrier_health_ready_locked(&mut state, now).is_none());
|
||||
drop(state);
|
||||
assert_eq!(
|
||||
session.carrier_health_publication_state(),
|
||||
CarrierHealthPublicationState::Publishing
|
||||
);
|
||||
assert!(session.publish_carrier_health());
|
||||
assert!(!session.publish_carrier_health());
|
||||
assert_eq!(
|
||||
session.carrier_health_publication_state(),
|
||||
CarrierHealthPublicationState::Published
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_health_and_close_always_reach_one_terminal_state() {
|
||||
for _ in 0..512 {
|
||||
let session = session(WebCarrier::Https, Instant::now() + Duration::from_secs(60));
|
||||
let now = Instant::now();
|
||||
arm_http_health(&session, now);
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let health_session = Arc::clone(&session);
|
||||
let health_barrier = Arc::clone(&barrier);
|
||||
let health = std::thread::spawn(move || {
|
||||
health_barrier.wait();
|
||||
std::thread::yield_now();
|
||||
let claim = {
|
||||
let mut state = health_session.state.lock();
|
||||
health_session.carrier_health_ready_locked(&mut state, now)
|
||||
};
|
||||
if claim.is_some() {
|
||||
health_session.publish_carrier_health();
|
||||
}
|
||||
});
|
||||
let close_session = Arc::clone(&session);
|
||||
let close_barrier = Arc::clone(&barrier);
|
||||
let close = std::thread::spawn(move || {
|
||||
close_barrier.wait();
|
||||
std::thread::yield_now();
|
||||
close_session.close();
|
||||
});
|
||||
barrier.wait();
|
||||
health.join().unwrap();
|
||||
close.join().unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
session.carrier_health_publication_state(),
|
||||
CarrierHealthPublicationState::Published
|
||||
| CarrierHealthPublicationState::Rejected
|
||||
));
|
||||
assert!(!session.publish_carrier_health());
|
||||
assert!(!session.close());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::time::Instant;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{SessionNegotiationPhase, WebSession};
|
||||
use super::{CarrierHealthPublicationState, SessionNegotiationPhase, WebSession};
|
||||
use crate::config::WebCarrier;
|
||||
|
||||
/// One bounded point-in-time session snapshot without bearer identity.
|
||||
@@ -28,6 +28,8 @@ pub(crate) struct WebSessionStatus {
|
||||
pub(crate) automatic: bool,
|
||||
/// Current session lifecycle token.
|
||||
pub(crate) state: &'static str,
|
||||
/// Current manager-confirmed health publication phase.
|
||||
pub(crate) health_publication: &'static str,
|
||||
/// Live logical streams.
|
||||
pub(crate) streams: usize,
|
||||
/// Stream relay tasks that have not exited.
|
||||
@@ -66,7 +68,9 @@ impl WebSession {
|
||||
"closed"
|
||||
} else if state.close_requested {
|
||||
"closing"
|
||||
} else if state.carrier_health_reported {
|
||||
} else if self.carrier_health_publication_state()
|
||||
== CarrierHealthPublicationState::Published
|
||||
{
|
||||
"healthy"
|
||||
} else {
|
||||
match state.negotiation_phase {
|
||||
@@ -87,6 +91,12 @@ impl WebSession {
|
||||
client_class: self.carrier_class.as_str(),
|
||||
automatic: self.automatic_carrier,
|
||||
state: state_name,
|
||||
health_publication: match self.carrier_health_publication_state() {
|
||||
CarrierHealthPublicationState::Awaiting => "awaiting",
|
||||
CarrierHealthPublicationState::Publishing => "publishing",
|
||||
CarrierHealthPublicationState::Published => "published",
|
||||
CarrierHealthPublicationState::Rejected => "rejected",
|
||||
},
|
||||
streams: state.streams.len(),
|
||||
tasks: self.tasks_live(),
|
||||
lanes: state.carrier_lanes.len(),
|
||||
|
||||
@@ -85,7 +85,7 @@ impl WebSession {
|
||||
let digest: TokenHash = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let mut healthy = None;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
@@ -154,8 +154,8 @@ impl WebSession {
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
for completion in opened {
|
||||
self.spawn_stream(completion, false);
|
||||
|
||||
@@ -58,7 +58,9 @@ impl Drop for WebSocketProbeReservation {
|
||||
state.websocket_probe_claimed = false;
|
||||
if state.websocket_commit_ack_owner == self.owner {
|
||||
state.websocket_commit_ack_owner = None;
|
||||
if !state.carrier_health_reported {
|
||||
if self.session.carrier_health_publication_state()
|
||||
!= super::CarrierHealthPublicationState::Published
|
||||
{
|
||||
state.websocket_commit_ack_written = false;
|
||||
state.carrier_health_uplink = false;
|
||||
state.carrier_health_activity_at = None;
|
||||
@@ -316,7 +318,7 @@ impl WebSession {
|
||||
let digest = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let mut committed = false;
|
||||
let mut healthy = false;
|
||||
let mut healthy = None;
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
@@ -401,8 +403,8 @@ impl WebSession {
|
||||
if committed {
|
||||
self.finish_carrier_commit();
|
||||
}
|
||||
if healthy {
|
||||
self.finish_carrier_health();
|
||||
if let Some(claim) = healthy {
|
||||
self.finish_carrier_health(claim);
|
||||
}
|
||||
for completion in opened {
|
||||
let stream = completion.stream;
|
||||
|
||||
Reference in New Issue
Block a user