astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-10-31 12:52:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

WEB Carrier Counters + Status

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-31 21:39:09 +03:00
parent 66f2b8889f
commit 718ce0847e
51 changed files with 2252 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/manager/carrier_learning.rs
+94 -123
View File
@@ -8,6 +8,10 @@ use super::ProfileKey;
use super::negotiation::{CarrierClientClass, CarrierLearningContext};
use crate::config::{WebCarrier, WebCarrierNegotiationAggressiveness};
#[path = "carrier_learning/status.rs"]
mod status;
pub(super) use status::{CarrierLearningEpoch, CarrierLearningRecordOutcome};
const PROFILE_WEIGHT: i16 = 32;
const USER_AGENT_WEIGHT: i16 = 32;
const IP_WEIGHT: i16 = 1;
@@ -117,6 +121,23 @@ struct LearningPolicy {
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
lifetime: Duration,
health_window: Duration,
}
/// Evidence detached under the policy lock and released by its caller.
pub(super) struct DetachedCarrierEvidence {
entries: HashMap<EvidenceKey, Evidence>,
insertion_order: VecDeque<(EvidenceKey, u64)>,
}
/// Result of one generation-fenced policy reconciliation.
pub(super) struct CarrierLearningPolicyOutcome {
/// Current epoch after the reconciliation.
pub(super) epoch: Option<u64>,
/// Whether this generation was current enough to apply.
pub(super) applied: bool,
/// Retired evidence whose allocation is released outside the policy lock.
pub(super) detached: Option<DetachedCarrierEvidence>,
}
#[derive(Clone, Copy)]
@@ -160,37 +181,10 @@ pub(super) struct CarrierLearning {
insertion_sequence: u64,
epoch: Option<u64>,
policy: Option<LearningPolicy>,
applied_generation: Option<u64>,
policy_started_at: Instant,
}
/// Bounded control-plane summary of carrier-learning state.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningStatus {
/// Whether outcome learning is active in the effective policy.
pub(crate) enabled: bool,
/// Effective evidence thresholds.
pub(crate) aggressiveness: WebCarrierNegotiationAggressiveness,
/// Current evidence epoch, or none after counter exhaustion.
pub(crate) epoch: Option<u64>,
/// Retained evidence entries.
pub(crate) entries: usize,
/// Restart-owned evidence ceiling.
pub(crate) capacity: usize,
/// Effective evidence lifetime.
pub(crate) lifetime_secs: u64,
/// Monotonic age of the current policy epoch.
pub(crate) age_ms: u64,
}
/// Result of one epoch-fenced learning reset.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningResetOutcome {
/// Evidence entries detached by the reset.
pub(crate) entries_cleared: usize,
/// New epoch fencing pre-reset outcomes.
pub(crate) epoch: u64,
}
impl CarrierLearning {
/// Creates an empty store under the restart-owned capacity ceiling.
pub(super) fn new(capacity: usize) -> Self {
@@ -201,70 +195,80 @@ impl CarrierLearning {
insertion_sequence: 1,
epoch: Some(0),
policy: None,
applied_generation: None,
policy_started_at: Instant::now(),
}
}
fn status(&self, now: Instant) -> CarrierLearningStatus {
let policy = self.policy.unwrap_or(LearningPolicy {
enabled: false,
aggressiveness: WebCarrierNegotiationAggressiveness::Conservative,
lifetime: Duration::ZERO,
});
CarrierLearningStatus {
enabled: policy.enabled,
aggressiveness: policy.aggressiveness,
epoch: self.epoch,
entries: self.entries.len(),
capacity: self.capacity,
lifetime_secs: policy.lifetime.as_secs(),
age_ms: millis(now.saturating_duration_since(self.policy_started_at)),
}
}
/// Applies hot-reloaded learning policy and returns its outcome epoch.
/// Applies one semantic policy unless a newer generation already owns the store.
pub(super) fn apply_policy(
&mut self,
now: Instant,
generation: u64,
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
lifetime: Duration,
) -> Option<u64> {
health_window: Duration,
) -> CarrierLearningPolicyOutcome {
if self
.applied_generation
.is_some_and(|applied| generation < applied)
{
return CarrierLearningPolicyOutcome {
epoch: self.epoch,
applied: false,
detached: None,
};
}
let policy = LearningPolicy {
enabled,
aggressiveness,
lifetime,
health_window,
};
self.applied_generation = Some(generation);
let mut detached = None;
if self.policy != Some(policy) {
self.entries.clear();
self.insertion_order.clear();
if !enabled {
self.entries.shrink_to_fit();
self.insertion_order.shrink_to_fit();
}
detached = Some(DetachedCarrierEvidence {
entries: std::mem::take(&mut self.entries),
insertion_order: std::mem::take(&mut self.insertion_order),
});
self.insertion_sequence = 1;
self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1));
self.policy = Some(policy);
self.policy_started_at = now;
}
self.epoch
CarrierLearningPolicyOutcome {
epoch: self.epoch,
applied: true,
detached,
}
}
/// Returns the current epoch only when the request snapshot matches owner policy.
/// Matches a request to the exact applied generation and semantic policy.
pub(super) fn epoch_for_policy(
&self,
generation: u64,
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
lifetime: Duration,
) -> Option<u64> {
(self.policy
== Some(LearningPolicy {
enabled,
aggressiveness,
lifetime,
}))
.then_some(self.epoch)
.flatten()
health_window: Duration,
) -> CarrierLearningEpoch {
if self.applied_generation != Some(generation)
|| self.policy
!= Some(LearningPolicy {
enabled,
aggressiveness,
lifetime,
health_window,
})
{
return CarrierLearningEpoch::Pending;
}
self.epoch.map_or(
CarrierLearningEpoch::Exhausted,
CarrierLearningEpoch::Ready,
)
}
/// Ranks supported configured candidates without scanning the evidence store.
@@ -348,12 +352,12 @@ impl CarrierLearning {
context: CarrierLearningContext,
failures: &[WebCarrier],
winner: WebCarrier,
) {
) -> CarrierLearningRecordOutcome {
let Some(policy) = self.policy.filter(|policy| policy.enabled) else {
return;
return CarrierLearningRecordOutcome::PolicyDisabled;
};
if Some(epoch) != self.epoch {
return;
return CarrierLearningRecordOutcome::StaleEpoch;
}
let mut deltas = [0i8; 4];
let _ = failures;
@@ -369,21 +373,39 @@ impl CarrierLearning {
(context.ip_learning_eligible && thresholds.ip.is_some())
.then_some(EvidenceKey::Ip(context.profile_key, context.client_ip)),
];
self.make_room(&keys);
let missing = keys
.iter()
.flatten()
.filter(|key| !self.entries.contains_key(key))
.count();
if self.entries.len().saturating_add(missing) > self.capacity {
return;
if self
.insertion_sequence
.checked_add(missing as u64)
.is_none()
{
return CarrierLearningRecordOutcome::SequenceExhausted;
}
let evictions_needed = self
.entries
.len()
.saturating_add(missing)
.saturating_sub(self.capacity);
let evictable = self
.entries
.keys()
.filter(|key| !keys.contains(&Some(**key)))
.count();
if evictions_needed > evictable {
return CarrierLearningRecordOutcome::CapacityRejected;
}
self.make_room(&keys);
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
let cohort = cohort_hash(context);
for (index, key) in keys.into_iter().enumerate() {
let Some(key) = key else { continue };
self.update_key(key, slot, deltas, (index == 0).then_some(cohort));
}
CarrierLearningRecordOutcome::Recorded
}
/// Reclaims a fixed number of entries outside both half-window buckets.
@@ -452,9 +474,8 @@ impl CarrierLearning {
entry.update(slot, deltas, cohort);
return;
}
let Some(insertion_sequence) = self.next_insertion_sequence() else {
return;
};
let insertion_sequence = self.insertion_sequence;
self.insertion_sequence += 1;
self.entries.insert(key, Evidence::new(insertion_sequence));
self.insertion_order.push_back((key, insertion_sequence));
if let Some(entry) = self.entries.get_mut(&key) {
@@ -462,56 +483,6 @@ impl CarrierLearning {
}
}
fn next_insertion_sequence(&mut self) -> Option<u64> {
let sequence = self.insertion_sequence;
self.insertion_sequence = sequence.checked_add(1)?;
Some(sequence)
}
}
impl super::WebProcessRuntime {
/// Captures learning state without waiting for a contended evidence lock.
pub(crate) fn try_carrier_learning_status(&self) -> Option<CarrierLearningStatus> {
self.learning
.try_lock()
.map(|learning| learning.status(Instant::now()))
}
/// Clears all evidence under a new epoch without changing the active policy.
pub(crate) fn reset_carrier_learning(
&self,
) -> Result<CarrierLearningResetOutcome, super::ManagerError> {
let control = self
.control_mutation_guard()
.map_err(|_| super::ManagerError::Closed)?;
let (outcome, retired_entries, retired_order) = {
let mut learning = self.learning.lock();
let epoch = learning
.epoch
.and_then(|epoch| epoch.checked_add(1))
.ok_or(super::ManagerError::Closed)?;
learning.epoch = Some(epoch);
learning.insertion_sequence = 1;
learning.policy_started_at = Instant::now();
let retired_entries = std::mem::take(&mut learning.entries);
let retired_order = std::mem::take(&mut learning.insertion_order);
(
CarrierLearningResetOutcome {
entries_cleared: retired_entries.len(),
epoch,
},
retired_entries,
retired_order,
)
};
drop(control);
drop((retired_entries, retired_order));
Ok(outcome)
}
}
fn millis(duration: Duration) -> u64 {
duration.as_millis().min(u128::from(u64::MAX)) as u64
}
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
src/web/manager/carrier_learning/status.rs
+140
View File
@@ -0,0 +1,140 @@
use std::time::{Duration, Instant};
use crate::config::WebCarrierNegotiationAggressiveness;
use super::{CarrierLearning, DetachedCarrierEvidence, LearningPolicy};
/// Exact result of matching one request snapshot to the applied learning policy.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(in crate::web::manager) enum CarrierLearningEpoch {
/// The request may use the returned evidence epoch.
Ready(u64),
/// The process store has not applied this exact generation and policy yet.
Pending,
/// The exact policy is active but its monotonic epoch space is exhausted.
Exhausted,
}
/// Terminal result of applying one complete attempt chain.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(in crate::web::manager) enum CarrierLearningRecordOutcome {
/// Every eligible evidence key received the complete sample.
Recorded,
/// The effective learning policy does not accept evidence.
PolicyDisabled,
/// The result belongs to a superseded or exhausted epoch.
StaleEpoch,
/// The bounded store cannot fit all evidence keys atomically.
CapacityRejected,
/// A unique insertion sequence cannot be assigned to every new key.
SequenceExhausted,
}
/// Bounded control-plane summary of carrier-learning state.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningStatus {
/// Whether outcome learning is active in the effective policy.
pub(crate) enabled: bool,
/// Effective evidence thresholds.
pub(crate) aggressiveness: WebCarrierNegotiationAggressiveness,
/// Generation whose policy is applied to the process-owned store.
pub(crate) policy_generation: Option<u64>,
/// Current evidence epoch, or none after counter exhaustion.
pub(crate) epoch: Option<u64>,
/// Retained evidence entries.
pub(crate) entries: usize,
/// Restart-owned evidence ceiling.
pub(crate) capacity: usize,
/// Effective evidence lifetime.
pub(crate) lifetime_secs: u64,
/// Effective post-commit health observation window.
pub(crate) health_secs: u64,
/// Monotonic age of the current semantic policy epoch.
pub(crate) age_ms: u64,
}
/// Result of one epoch-fenced learning reset.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningResetOutcome {
/// Evidence entries detached by the reset.
pub(crate) entries_cleared: usize,
/// New epoch fencing pre-reset outcomes.
pub(crate) epoch: u64,
}
impl CarrierLearning {
pub(super) fn status(&self, now: Instant) -> CarrierLearningStatus {
let policy = self.policy.unwrap_or(LearningPolicy {
enabled: false,
aggressiveness: WebCarrierNegotiationAggressiveness::Conservative,
lifetime: Duration::ZERO,
health_window: Duration::ZERO,
});
CarrierLearningStatus {
enabled: policy.enabled,
aggressiveness: policy.aggressiveness,
policy_generation: self.applied_generation,
epoch: self.epoch,
entries: self.entries.len(),
capacity: self.capacity,
lifetime_secs: policy.lifetime.as_secs(),
health_secs: policy.health_window.as_secs(),
age_ms: millis(now.saturating_duration_since(self.policy_started_at)),
}
}
}
impl super::super::WebProcessRuntime {
/// Captures learning state without waiting for a contended evidence lock.
pub(crate) fn try_carrier_learning_status(&self) -> Option<CarrierLearningStatus> {
self.learning
.try_lock()
.map(|learning| learning.status(Instant::now()))
}
/// Clears all evidence under a new epoch without changing the active policy.
pub(crate) fn reset_carrier_learning(
&self,
) -> Result<CarrierLearningResetOutcome, super::super::ManagerError> {
let control = self
.control_mutation_guard()
.map_err(|_| super::super::ManagerError::Closed)?;
let (outcome, detached) = {
let mut learning = self.learning.lock();
let epoch = learning
.epoch
.and_then(|epoch| epoch.checked_add(1))
.ok_or(super::super::ManagerError::Closed)?;
learning.epoch = Some(epoch);
learning.insertion_sequence = 1;
learning.policy_started_at = Instant::now();
let entries = std::mem::take(&mut learning.entries);
let entries_cleared = entries.len();
let detached = DetachedCarrierEvidence {
entries,
insertion_order: std::mem::take(&mut learning.insertion_order),
};
(
CarrierLearningResetOutcome {
entries_cleared,
epoch,
},
detached,
)
};
drop(control);
drop(detached);
Ok(outcome)
}
}
impl Drop for DetachedCarrierEvidence {
fn drop(&mut self) {
self.entries.clear();
self.insertion_order.clear();
}
}
fn millis(duration: Duration) -> u64 {
duration.as_millis().min(u128::from(u64::MAX)) as u64
}
src/web/manager/carrier_learning/tests.rs
+322 -93
View File
@@ -22,28 +22,45 @@ fn context(hash: u8) -> CarrierLearningContext {
}
}
fn apply_epoch(
learning: &mut CarrierLearning,
now: Instant,
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
lifetime: Duration,
) -> u64 {
let outcome = learning.apply_policy(
now,
1,
enabled,
aggressiveness,
lifetime,
Duration::from_secs(3),
);
drop(outcome.detached);
outcome.epoch.unwrap()
}
#[test]
fn policy_epoch_rejects_late_outcomes_and_clears_state() {
let now = Instant::now();
let mut learning = CarrierLearning::new(6);
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let epoch = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Websocket);
assert_eq!(learning.entries.len(), 3);
let next = learning
.apply_policy(
now,
false,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let next = apply_epoch(
&mut learning,
now,
false,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
assert_ne!(epoch, next);
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Https);
assert!(learning.entries.is_empty());
@@ -53,14 +70,13 @@ fn policy_epoch_rejects_late_outcomes_and_clears_state() {
fn aggressive_policy_ranks_one_atomic_chain_sample() {
let now = Instant::now();
let mut learning = CarrierLearning::new(6);
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let epoch = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
learning.record_chain(now, epoch, context(2), &[], WebCarrier::Websocket);
let (ranked, scores) = learning.rank(
now,
@@ -89,14 +105,13 @@ fn aggressive_policy_ranks_one_atomic_chain_sample() {
fn two_half_windows_expire_without_sliding_updates() {
let start = Instant::now();
let mut learning = CarrierLearning::new(6);
let epoch = learning
.apply_policy(
start,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let epoch = apply_epoch(
&mut learning,
start,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
learning.record_chain(start, epoch, context(3), &[], WebCarrier::Websocket);
learning.record_chain(
start + Duration::from_secs(6),
@@ -116,36 +131,43 @@ fn exhausted_epoch_and_insertion_identifiers_fail_closed() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
learning.epoch = Some(u64::MAX);
assert_eq!(
learning.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
),
None
let exhausted = learning.apply_policy(
now,
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
);
assert_eq!(exhausted.epoch, None);
drop(exhausted.detached);
assert_eq!(
learning.record_chain(now, u64::MAX, context(4), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::StaleEpoch
);
learning.record_chain(now, u64::MAX, context(4), &[], WebCarrier::Https);
assert!(learning.entries.is_empty());
learning.epoch = Some(1);
learning.insertion_sequence = u64::MAX;
learning.record_chain(now, 1, context(4), &[], WebCarrier::Https);
assert_eq!(
learning.record_chain(now, 1, context(4), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::SequenceExhausted
);
assert!(learning.entries.is_empty());
assert!(learning.insertion_order.is_empty());
}
#[test]
fn client_reported_failures_do_not_create_negative_evidence() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let epoch = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
learning.record_chain(
now,
epoch,
@@ -168,52 +190,208 @@ fn client_reported_failures_do_not_create_negative_evidence() {
fn old_new_old_policy_rejects_both_stale_epochs() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let old = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let middle = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Balanced,
Duration::from_secs(10),
)
.unwrap();
let current = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let old = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
let middle = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Balanced,
Duration::from_secs(10),
);
let current = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
assert_ne!(old, middle);
assert_ne!(middle, current);
assert_ne!(old, current);
learning.record_chain(now, old, context(6), &[], WebCarrier::Https);
learning.record_chain(now, middle, context(6), &[], WebCarrier::Https);
assert_eq!(
learning.record_chain(now, old, context(6), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::StaleEpoch
);
assert_eq!(
learning.record_chain(now, middle, context(6), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::StaleEpoch
);
assert!(learning.entries.is_empty());
learning.record_chain(now, current, context(6), &[], WebCarrier::Https);
assert_eq!(
learning.record_chain(now, current, context(6), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::Recorded
);
assert_eq!(learning.entries.len(), 3);
}
#[test]
fn generation_rollover_preserves_same_policy_evidence() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let first = learning.apply_policy(
now,
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
);
let epoch = first.epoch.unwrap();
assert!(first.detached.is_some());
drop(first.detached);
assert_eq!(
learning.record_chain(now, epoch, context(8), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::Recorded
);
let second = learning.apply_policy(
now,
2,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
);
assert!(second.applied);
assert_eq!(second.epoch, Some(epoch));
assert!(second.detached.is_none());
assert_eq!(learning.entries.len(), 3);
assert_eq!(
learning.epoch_for_policy(
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
),
CarrierLearningEpoch::Pending
);
assert_eq!(
learning.epoch_for_policy(
2,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
),
CarrierLearningEpoch::Ready(epoch)
);
}
#[test]
fn stale_generation_cannot_restore_an_old_policy() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let current = learning.apply_policy(
now,
2,
true,
WebCarrierNegotiationAggressiveness::Balanced,
Duration::from_secs(10),
Duration::from_secs(3),
);
let epoch = current.epoch;
drop(current.detached);
let stale = learning.apply_policy(
now,
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(20),
Duration::from_secs(4),
);
let status = learning.status(now);
assert!(!stale.applied);
assert_eq!(stale.epoch, epoch);
assert!(stale.detached.is_none());
assert_eq!(status.policy_generation, Some(2));
assert_eq!(
status.aggressiveness,
WebCarrierNegotiationAggressiveness::Balanced
);
assert_eq!(status.lifetime_secs, 10);
assert_eq!(status.health_secs, 3);
}
#[test]
fn health_window_change_starts_a_new_empty_epoch() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let first = learning.apply_policy(
now,
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
);
let epoch = first.epoch.unwrap();
drop(first.detached);
assert_eq!(
learning.record_chain(now, epoch, context(9), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::Recorded
);
let changed = learning.apply_policy(
now,
2,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(4),
);
assert_ne!(changed.epoch, Some(epoch));
assert_eq!(
changed.detached.as_ref().map(|detached| detached.entries.len()),
Some(3)
);
assert!(learning.entries.is_empty());
drop(changed.detached);
}
#[test]
fn capacity_rejection_does_not_partially_mutate_the_store() {
let now = Instant::now();
let mut learning = CarrierLearning::new(2);
let epoch = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
let sequence = learning.insertion_sequence;
assert_eq!(
learning.record_chain(now, epoch, context(10), &[], WebCarrier::Https),
CarrierLearningRecordOutcome::CapacityRejected
);
assert!(learning.entries.is_empty());
assert!(learning.insertion_order.is_empty());
assert_eq!(learning.insertion_sequence, sequence);
}
#[test]
fn fifo_metadata_stays_within_the_entry_capacity() {
let now = Instant::now();
let mut learning = CarrierLearning::new(3);
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let epoch = apply_epoch(
&mut learning,
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
);
for hash in 1..=32 {
learning.record_chain(now, epoch, context(hash), &[], WebCarrier::Https);
assert!(learning.entries.len() <= 3);
@@ -221,6 +399,55 @@ fn fifo_metadata_stays_within_the_entry_capacity() {
}
}
#[tokio::test]
async fn runtime_activation_publishes_matching_policy_and_generation() {
let mut config = crate::config::ProxyConfig::default();
config.web.carriers = crate::config::WebCarriers::Enabled(vec![WebCarrier::Websocket]);
config.web.carrier_learning = true;
config.web.carrier_negotiation_aggressiveness =
WebCarrierNegotiationAggressiveness::Aggressive;
config.web.timeouts.carrier_learning_secs = 10;
config.web.timeouts.carrier_health_secs = 3;
let first = crate::maestro::generation::test_runtime_generation(1, config.clone());
let runtime = crate::web::manager::WebProcessRuntime::start(std::sync::Arc::new(
arc_swap::ArcSwap::from(first.clone()),
));
let epoch = {
let mut learning = runtime.learning.lock();
let epoch = learning.status(Instant::now()).epoch.unwrap();
assert_eq!(
learning.record_chain(
Instant::now(),
epoch,
context(11),
&[],
WebCarrier::Websocket,
),
CarrierLearningRecordOutcome::Recorded
);
epoch
};
let second = crate::maestro::generation::test_runtime_generation(2, config);
let replaced = runtime.activate_generation(second.clone());
assert!(std::sync::Arc::ptr_eq(&replaced, &first));
assert_eq!(runtime.active_generation().id, 2);
{
let learning = runtime.learning.lock();
let status = learning.status(Instant::now());
assert_eq!(status.policy_generation, Some(2));
assert_eq!(status.epoch, Some(epoch));
assert_eq!(status.entries, 3);
}
runtime.shutdown().await;
first.stop_sessions().await;
first.stop_background_tasks().await;
second.stop_sessions().await;
second.stop_background_tasks().await;
}
#[tokio::test]
async fn explicit_reset_preserves_policy_and_rejects_old_epoch_outcomes() {
let generation = crate::maestro::generation::test_runtime_generation(
@@ -233,14 +460,16 @@ async fn explicit_reset_preserves_policy_and_rejects_old_epoch_outcomes() {
let now = Instant::now();
let old_epoch = {
let mut learning = runtime.learning.lock();
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
let outcome = learning.apply_policy(
now,
1,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
Duration::from_secs(3),
);
let epoch = outcome.epoch.unwrap();
drop(outcome.detached);
learning.record_chain(now, epoch, context(7), &[], WebCarrier::Websocket);
epoch
};
src/web/manager/carrier_outcome.rs
+80 -17
View File
@@ -9,8 +9,25 @@ use super::{
};
use crate::config::WebCarrier;
use crate::web::session::WebSession;
use crate::web::telemetry::WebCarrierLearningOutcome;
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
/// Typed result of publishing one exact session health transition.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum CarrierHealthPublicationOutcome {
/// Manager ownership was confirmed and the attempt became healthy.
Published(WebCarrierLearningOutcome),
/// Manager or transport ownership rejected the publication.
Rejected(WebCarrierLearningOutcome),
}
impl CarrierHealthPublicationOutcome {
/// Returns whether manager state accepted the health transition.
pub(crate) const fn published(self) -> bool {
matches!(self, Self::Published(_))
}
}
impl WebProcessRuntime {
/// Returns authenticated current chain metadata after a committed retry conflict.
pub(crate) fn carrier_echo(
@@ -123,30 +140,75 @@ impl WebProcessRuntime {
learning_context: Option<CarrierLearningContext>,
client_ip: IpAddr,
identity: TraceIdentity,
) {
let (scores, failures) = {
websocket_owner: Option<u64>,
) -> CarrierHealthPublicationOutcome {
if carrier.uses_websocket()
&& websocket_owner.is_none_or(|owner| {
!self.claim_websocket_health(owner, session_hash)
})
{
let outcome = WebCarrierLearningOutcome::OwnerNotLive;
self.telemetry.record_carrier_learning(carrier, outcome);
return CarrierHealthPublicationOutcome::Rejected(outcome);
}
let scores = {
let mut state = self.state.lock();
let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash) else {
return;
let outcome = WebCarrierLearningOutcome::MissingChain;
self.telemetry.record_carrier_learning(carrier, outcome);
return CarrierHealthPublicationOutcome::Rejected(outcome);
};
if entry.carrier_attempt != attempt
|| entry.carrier_phase != CarrierChainPhase::CommittedPendingHealth
|| entry
.session
.as_ref()
.is_none_or(|session| session.token_hash() != session_hash)
{
return;
if entry.carrier_phase != CarrierChainPhase::CommittedPendingHealth {
let outcome = WebCarrierLearningOutcome::PhaseMismatch;
self.telemetry.record_carrier_learning(carrier, outcome);
return CarrierHealthPublicationOutcome::Rejected(outcome);
}
let Some(session) = entry.session.as_ref().filter(|session| {
entry.carrier_attempt == attempt && session.token_hash() == session_hash
}) else {
let outcome = WebCarrierLearningOutcome::SessionMismatch;
self.telemetry.record_carrier_learning(carrier, outcome);
return CarrierHealthPublicationOutcome::Rejected(outcome);
};
if !session.publish_carrier_health() {
let outcome = WebCarrierLearningOutcome::ClosedBeforeHealth;
self.telemetry.record_carrier_learning(carrier, outcome);
return CarrierHealthPublicationOutcome::Rejected(outcome);
}
entry.carrier_phase = CarrierChainPhase::Healthy;
(entry.carrier_scores, entry.carrier_failures)
entry.carrier_scores
};
if let Some(context) = learning_context {
let learning_outcome = if let Some(context) = learning_context {
let now = Instant::now();
let mut learning = self.learning.lock();
let failures = failures.into_iter().flatten().collect::<Vec<_>>();
learning.record_chain(now, context.epoch, context, &failures, carrier);
}
let outcome = self.learning.lock().record_chain(
now,
context.epoch,
context,
&[],
carrier,
);
match outcome {
super::learning::CarrierLearningRecordOutcome::Recorded => {
WebCarrierLearningOutcome::Recorded
}
super::learning::CarrierLearningRecordOutcome::PolicyDisabled => {
WebCarrierLearningOutcome::PolicyDisabled
}
super::learning::CarrierLearningRecordOutcome::StaleEpoch => {
WebCarrierLearningOutcome::StaleEpoch
}
super::learning::CarrierLearningRecordOutcome::CapacityRejected => {
WebCarrierLearningOutcome::CapacityRejected
}
super::learning::CarrierLearningRecordOutcome::SequenceExhausted => {
WebCarrierLearningOutcome::SequenceExhausted
}
}
} else {
WebCarrierLearningOutcome::NotEligible
};
self.telemetry
.record_carrier_learning(carrier, learning_outcome);
self.trace.record_carrier_lifecycle(
client_ip,
identity,
@@ -157,5 +219,6 @@ impl WebProcessRuntime {
scores,
None,
);
CarrierHealthPublicationOutcome::Published(learning_outcome)
}
}
src/web/manager/credentials.rs
+20 -2
View File
@@ -147,6 +147,8 @@ impl WebProcessRuntime {
carrier_deadline_at: None,
carrier_failures: [None; 3],
carrier_learning_epoch: 0,
carrier_learning_disposition:
crate::web::telemetry::WebCarrierSelectionDisposition::ProfileDisabled,
close_requested: false,
session_client_ip: None,
session_ip_learning_eligible: false,
@@ -218,6 +220,7 @@ impl WebProcessRuntime {
&self,
hash: TokenHash,
host: &str,
failure: Option<super::CarrierFailure>,
) -> std::result::Result<(), ManagerError> {
let mut state = self.state.lock();
let session = state
@@ -225,7 +228,8 @@ impl WebProcessRuntime {
.get(&hash)
.filter(|session| session.matches_host(host))
.cloned();
if session.is_some() {
let mut failure_phase = None;
if let Some(session) = &session {
for bootstrap in state.bootstraps.values_mut() {
if bootstrap
.session
@@ -233,6 +237,15 @@ impl WebProcessRuntime {
.is_some_and(|current| current.token_hash() == hash)
{
bootstrap.close_requested = true;
failure_phase = Some(if matches!(
bootstrap.carrier_phase,
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
) || session.is_carrier_committed()
{
crate::web::telemetry::WebCarrierFailurePhase::Committed
} else {
crate::web::telemetry::WebCarrierFailurePhase::Provisional
});
break;
}
}
@@ -243,7 +256,12 @@ impl WebProcessRuntime {
.is_some_and(|closed| closed.host == host);
drop(state);
if let Some(session) = session {
session.close();
if session.close()
&& let (Some(failure), Some(phase)) = (failure, failure_phase)
{
self.telemetry
.record_carrier_failure(session.carrier(), phase, failure);
}
return Ok(());
}
closed.then_some(()).ok_or(ManagerError::Authentication)
src/web/manager/lifecycle.rs
+40 -9
View File
@@ -1,4 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::time::Instant as TokioInstant;
@@ -9,6 +10,7 @@ use super::state::{
remove_bootstrap_locked, remove_expired_locked,
};
use super::{ProfileKey, TokenHash, WebProcessRuntime};
use crate::maestro::generation::RuntimeGeneration;
/// Result of draining all process-owned WEB work under one absolute deadline.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -27,6 +29,30 @@ pub(crate) struct WebShutdownDrain {
}
impl WebProcessRuntime {
/// Applies learning policy before publishing one new runtime generation.
pub(crate) fn activate_generation(
&self,
generation: Arc<RuntimeGeneration>,
) -> Arc<RuntimeGeneration> {
let config = generation.config();
let (replaced, detached) = {
let mut learning = self.learning.lock();
let outcome = learning.apply_policy(
Instant::now(),
generation.id,
config.web.carrier_negotiation_enabled() && config.web.carrier_learning,
config.web.carrier_negotiation_aggressiveness,
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
Duration::from_secs(config.web.timeouts.carrier_health_secs),
);
debug_assert!(outcome.applied, "runtime generations must increase monotonically");
let replaced = self.active_runtime.swap(generation);
(replaced, outcome.detached)
};
drop(detached);
replaced
}
/// Removes one closed session and retains a bounded host-bound replay marker.
pub(crate) fn session_finished(
&self,
@@ -150,15 +176,20 @@ impl WebProcessRuntime {
let generation = self.active_generation();
let config = &generation.config().web;
let learning_enabled = config.carrier_negotiation_enabled() && config.carrier_learning;
let mut learning = self.learning.lock();
let _ = learning.apply_policy(
now,
learning_enabled,
config.carrier_negotiation_aggressiveness,
Duration::from_secs(config.timeouts.carrier_learning_secs),
);
learning.prune(now);
drop(learning);
let detached = {
let mut learning = self.learning.lock();
let outcome = learning.apply_policy(
now,
generation.id,
learning_enabled,
config.carrier_negotiation_aggressiveness,
Duration::from_secs(config.timeouts.carrier_learning_secs),
Duration::from_secs(config.timeouts.carrier_health_secs),
);
learning.prune(now);
outcome.detached
};
drop(detached);
let (sessions, expired_chains) = {
let mut state = self.state.lock();
state.apply_issuance_policy(generation.id, config.enabled);
src/web/manager/negotiation.rs
+20
View File
@@ -43,6 +43,26 @@ pub(crate) enum CarrierFailure {
}
impl CarrierFailure {
/// Complete fixed bridge failure set in stable metric order.
pub(crate) const ALL: [Self; 5] = [
Self::Timeout,
Self::Network,
Self::Upgrade,
Self::Http,
Self::Protocol,
];
/// Returns the stable fixed-slot index used by process telemetry.
pub(crate) const fn index(self) -> usize {
match self {
Self::Timeout => 0,
Self::Network => 1,
Self::Upgrade => 2,
Self::Http => 3,
Self::Protocol => 4,
}
}
/// Parses one canonical bridge failure token.
pub(crate) const fn parse(value: &str) -> Option<Self> {
match value.as_bytes() {
src/web/manager/session_creation.rs
+65 -29
View File
@@ -19,6 +19,7 @@ use super::{
use crate::config::{WebCarrier, WebRuntimeProfile};
use crate::web::frame;
use crate::web::session::WebSession;
use crate::web::telemetry::WebCarrierSelectionDisposition;
use crate::web::trace::TraceLifecycleEvent;
struct Replacement {
@@ -31,6 +32,7 @@ struct Replacement {
request: CarrierRequest,
scores: [i16; 4],
learning_epoch: u64,
learning_disposition: WebCarrierSelectionDisposition,
ip_learning_eligible: bool,
carrier_deadline_at: Instant,
}
@@ -185,6 +187,7 @@ impl WebProcessRuntime {
request: carrier_request,
scores: entry.carrier_scores,
learning_epoch: entry.carrier_learning_epoch,
learning_disposition: entry.carrier_learning_disposition,
ip_learning_eligible,
carrier_deadline_at: entry.carrier_deadline_at.ok_or(ManagerError::Protocol)?,
};
@@ -223,51 +226,70 @@ impl WebProcessRuntime {
config.web.carrier_negotiation_enabled() && config.web.carrier_learning,
config.web.carrier_negotiation_aggressiveness,
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
Duration::from_secs(config.web.timeouts.carrier_health_secs),
);
let (candidates, scores, learning_epoch) = if capability_selection
let (candidates, scores, learning_epoch, learning_disposition) = if capability_selection
&& profile.carrier_learning
&& learning_policy.0
{
let learning = self.learning.lock();
if let Some(epoch) =
learning.epoch_for_policy(learning_policy.0, learning_policy.1, learning_policy.2)
{
let (candidates, scores) = learning.rank(
now,
&profile.carriers,
carrier_request,
profile_key,
client_ip,
ip_learning_eligible,
);
(candidates, scores, Some(epoch))
} else {
(
profile
.carriers
.iter()
.copied()
.filter(|carrier| carrier_request.supports(*carrier))
.collect(),
match learning.epoch_for_policy(
generation.id,
learning_policy.0,
learning_policy.1,
learning_policy.2,
learning_policy.3,
) {
super::learning::CarrierLearningEpoch::Ready(epoch) => {
let (candidates, scores) = learning.rank(
now,
&profile.carriers,
carrier_request,
profile_key,
client_ip,
ip_learning_eligible,
);
let disposition = if scores.iter().any(|score| *score != 0) {
WebCarrierSelectionDisposition::Applied
} else {
WebCarrierSelectionDisposition::Cold
};
(candidates, scores, Some(epoch), disposition)
}
super::learning::CarrierLearningEpoch::Pending => (
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
)
WebCarrierSelectionDisposition::PolicyPending,
),
super::learning::CarrierLearningEpoch::Exhausted => (
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
WebCarrierSelectionDisposition::EpochExhausted,
),
}
} else if capability_selection {
(
profile
.carriers
.iter()
.copied()
.filter(|carrier| carrier_request.supports(*carrier))
.collect(),
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
if learning_policy.0 {
WebCarrierSelectionDisposition::ProfileDisabled
} else {
WebCarrierSelectionDisposition::PolicyDisabled
},
)
} else if carrier_request.uses_capabilities() && !carrier_request.supports(profile.carrier)
{
return Err(ManagerError::Protocol);
} else {
(vec![profile.carrier], [0; 4], None)
(
vec![profile.carrier],
[0; 4],
None,
WebCarrierSelectionDisposition::ProfileDisabled,
)
};
let Some(carrier) = candidates.first().copied() else {
return Err(ManagerError::Protocol);
@@ -331,6 +353,7 @@ impl WebProcessRuntime {
entry.carrier_deadline_at = carrier_deadline_at;
entry.carrier_failures = [None; 3];
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
entry.carrier_learning_disposition = learning_disposition;
entry.expires_at = now + Duration::from_secs(issued_timeouts.bootstrap_lifetime_secs);
entry.session_client_ip = Some(client_ip);
entry.session_ip_learning_eligible = ip_learning_eligible;
@@ -367,6 +390,8 @@ impl WebProcessRuntime {
},
);
drop(state);
self.telemetry
.record_carrier_selection(carrier, learning_disposition);
self.trace.record_carrier_lifecycle(
client_ip,
identity.clone(),
@@ -399,5 +424,16 @@ impl WebProcessRuntime {
}
}
fn supported_candidates(
configured: &[WebCarrier],
request: CarrierRequest,
) -> Vec<WebCarrier> {
configured
.iter()
.copied()
.filter(|carrier| request.supports(*carrier))
.collect()
}
// Atomic pre-commit carrier replacement and frozen-policy transfer.
mod replacement;
src/web/manager/session_creation/replacement.rs
+11
View File
@@ -135,6 +135,17 @@ impl WebProcessRuntime {
let old_identity = replacement.old_session.trace_identity();
drop(state);
supersede.finish();
self.telemetry.record_carrier_selection(
replacement.carrier,
replacement.learning_disposition,
);
if let Some(failure) = replacement.request.failure() {
self.telemetry.record_carrier_failure(
replacement.old_session.carrier(),
crate::web::telemetry::WebCarrierFailurePhase::Provisional,
failure,
);
}
self.trace.record_carrier_lifecycle(
client_ip,
old_identity.clone(),
src/web/manager/state.rs
+3
View File
@@ -11,6 +11,7 @@ use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile, WebTimeoutsConfig};
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
use crate::web::telemetry::WebCarrierSelectionDisposition;
const WEB_PROFILE_OWNER_CONTEXT: &[u8] = b"telemt-web-profile-owner-v1\0";
@@ -75,6 +76,8 @@ pub(super) struct Bootstrap {
pub(super) carrier_failures: [Option<WebCarrier>; 3],
/// Learning-policy epoch frozen by the first automatic attempt.
pub(super) carrier_learning_epoch: u64,
/// Frozen reason learning did or did not influence carrier ordering.
pub(super) carrier_learning_disposition: WebCarrierSelectionDisposition,
/// DELETE observed before an in-flight replacement committed its swap.
pub(super) close_requested: bool,
/// Effective address frozen by the first session-creation request.
src/web/manager/status.rs
+4
View File
@@ -72,10 +72,12 @@ struct WebSocketStatus {
struct LearningStatus {
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
policy_generation: Option<u64>,
epoch: Option<u64>,
entries: usize,
capacity: usize,
lifetime_secs: u64,
health_secs: u64,
age_ms: u64,
}
@@ -267,10 +269,12 @@ impl WebProcessRuntime {
.map(|status| LearningStatus {
enabled: status.enabled,
aggressiveness: status.aggressiveness,
policy_generation: status.policy_generation,
epoch: status.epoch,
entries: status.entries,
capacity: status.capacity,
lifetime_secs: status.lifetime_secs,
health_secs: status.health_secs,
age_ms: status.age_ms,
});
if learning.is_none() {
src/web/manager/websocket.rs
+22
View File
@@ -50,6 +50,7 @@ pub(super) struct WebSocketEntry {
last_progress_tick: AtomicU64,
phase: AtomicU8,
closing: AtomicBool,
health_claimed: AtomicBool,
cancel: CancellationToken,
released: CancellationToken,
}
@@ -345,6 +346,7 @@ fn try_admit(
last_progress_tick: AtomicU64::new(now),
phase: AtomicU8::new(WebSocketPhase::Claimed as u8),
closing: AtomicBool::new(false),
health_claimed: AtomicBool::new(false),
cancel: parent_cancellation.child_token(),
released: CancellationToken::new(),
});
@@ -371,6 +373,26 @@ impl WebProcessRuntime {
self.websocket_clock.elapsed().as_millis() as u64
}
/// Claims health from one exact active owner under the eviction registry lock.
pub(super) fn claim_websocket_health(
&self,
owner: u64,
session_hash: super::TokenHash,
) -> bool {
let registry = self.websockets.lock();
let Some(entry) = registry.entries.get(&owner) else {
return false;
};
!registry.closed
&& entry.claim.session_hash == session_hash
&& entry.phase.load(Ordering::Acquire) == WebSocketPhase::Active as u8
&& !entry.closing.load(Ordering::Acquire)
&& entry
.health_claimed
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
}
pub(super) fn cleanup_websockets(&self) {
let now = self.websocket_tick();
let mut victims = claim_stale_victims(self, now);
src/web/manager/websocket/tests.rs
+52
View File
@@ -36,6 +36,7 @@ fn entry(
last_progress_tick: AtomicU64::new(progress_tick),
phase: AtomicU8::new(phase as u8),
closing: AtomicBool::new(false),
health_claimed: AtomicBool::new(false),
cancel: CancellationToken::new(),
released: CancellationToken::new(),
}
@@ -297,3 +298,54 @@ async fn concurrent_victim_claims_stay_bounded_and_return_to_zero() {
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_health_claims_publish_one_exact_live_owner() {
let generation = test_runtime_generation(1, ProxyConfig::default());
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
let owner = Arc::new(entry(
1,
[1; 32],
1,
"192.0.2.10",
WebSocketKind::Multiplex,
WebSocketPhase::Active,
1,
1,
));
let connection = WebSocketConnection {
runtime: Arc::downgrade(&runtime),
entry: Arc::clone(&owner),
slot: None,
base_budget: None,
};
{
let mut registry = runtime.websockets.lock();
registry.claims.insert(owner.claim, owner.id);
registry.entries.insert(owner.id, Arc::clone(&owner));
}
let successes = Arc::new(AtomicUsize::new(0));
let mut tasks = Vec::new();
for _ in 0..64 {
let runtime = Arc::clone(&runtime);
let successes = Arc::clone(&successes);
tasks.push(tokio::spawn(async move {
if runtime.claim_websocket_health(1, [0; 32]) {
successes.fetch_add(1, Ordering::AcqRel);
}
}));
}
for task in tasks {
task.await.unwrap();
}
assert_eq!(successes.load(Ordering::Acquire), 1);
assert!(!runtime.claim_websocket_health(1, [0; 32]));
assert!(!runtime.claim_websocket_health(2, [0; 32]));
assert!(!runtime.claim_websocket_health(1, [1; 32]));
drop(connection);
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}