mirror of
https://github.com/telemt/telemt.git
synced 2026-09-10 04:24:08 +03:00
WEB Carriers Safe-matrix Refactored
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -15,7 +15,7 @@ impl WebProcessRuntime {
|
||||
public_addr: SocketAddr,
|
||||
) -> Option<u16> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if state.closed
|
||||
|| state.streams_live >= self.limits.max_streams_global
|
||||
|| state
|
||||
@@ -54,7 +54,7 @@ impl WebProcessRuntime {
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if !release_stream_port(&mut state, client_ip, public_addr, peer_port) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -216,6 +216,10 @@ impl WebDataBudget {
|
||||
self.pressured.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
pub(super) fn restore_pressure(&self) {
|
||||
self.pressured.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::net::IpAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::negotiation::{CarrierClientClass, CarrierLearningContext};
|
||||
use super::ProfileKey;
|
||||
use crate::config::{WebCarrier, WebCarrierNegotiationAggressiveness};
|
||||
|
||||
const PROFILE_WEIGHT: i16 = 32;
|
||||
const USER_AGENT_WEIGHT: i16 = 32;
|
||||
const IP_WEIGHT: i16 = 1;
|
||||
const SCORE_MIN: i8 = -8;
|
||||
const SCORE_MAX: i8 = 8;
|
||||
const MAX_COHORTS: usize = 4;
|
||||
const PRUNE_ENTRIES_PER_TICK: usize = 64;
|
||||
const COHORT_CONTEXT: &[u8] = b"telemt-web-carrier-cohort-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum EvidenceKey {
|
||||
Profile(ProfileKey),
|
||||
UserAgent(ProfileKey, CarrierClientClass, [u8; 32]),
|
||||
Ip(ProfileKey, IpAddr),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Bucket {
|
||||
slot: u64,
|
||||
valid: bool,
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; MAX_COHORTS],
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
fn reset(&mut self, slot: u64) {
|
||||
*self = Self {
|
||||
slot,
|
||||
valid: true,
|
||||
..Self::default()
|
||||
};
|
||||
}
|
||||
|
||||
fn update(&mut self, deltas: [i8; 4], cohort: Option<[u8; 32]>) {
|
||||
for (score, delta) in self.scores.iter_mut().zip(deltas) {
|
||||
*score = score.saturating_add(delta).clamp(SCORE_MIN, SCORE_MAX);
|
||||
}
|
||||
self.outcomes = self.outcomes.saturating_add(1);
|
||||
if let Some(cohort) = cohort
|
||||
&& !self.cohorts.contains(&Some(cohort))
|
||||
&& let Some(slot) = self.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*slot = Some(cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Evidence {
|
||||
insertion_sequence: u64,
|
||||
buckets: [Bucket; 2],
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
fn new(insertion_sequence: u64) -> Self {
|
||||
Self {
|
||||
insertion_sequence,
|
||||
buckets: [Bucket::default(), Bucket::default()],
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, slot: u64, deltas: [i8; 4], cohort: Option<[u8; 32]>) {
|
||||
let index = slot as usize % self.buckets.len();
|
||||
if !self.buckets[index].valid || self.buckets[index].slot != slot {
|
||||
self.buckets[index].reset(slot);
|
||||
}
|
||||
self.buckets[index].update(deltas, cohort);
|
||||
}
|
||||
|
||||
fn aggregate(&self, slot: u64) -> Aggregate {
|
||||
let mut aggregate = Aggregate::default();
|
||||
for bucket in &self.buckets {
|
||||
if !bucket.valid || (bucket.slot != slot && bucket.slot.saturating_add(1) != slot) {
|
||||
continue;
|
||||
}
|
||||
aggregate.outcomes = aggregate.outcomes.saturating_add(bucket.outcomes);
|
||||
for (score, value) in aggregate.scores.iter_mut().zip(bucket.scores) {
|
||||
*score = score
|
||||
.saturating_add(value)
|
||||
.clamp(SCORE_MIN, SCORE_MAX);
|
||||
}
|
||||
for cohort in bucket.cohorts.iter().flatten() {
|
||||
if !aggregate.cohorts.contains(&Some(*cohort))
|
||||
&& let Some(target) = aggregate.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*target = Some(*cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
aggregate
|
||||
}
|
||||
|
||||
fn is_live(&self, slot: u64) -> bool {
|
||||
self.buckets.iter().any(|bucket| {
|
||||
bucket.valid && (bucket.slot == slot || bucket.slot.saturating_add(1) == slot)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Aggregate {
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; MAX_COHORTS],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct LearningPolicy {
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Thresholds {
|
||||
user_agent: u8,
|
||||
ip: Option<u8>,
|
||||
profile_outcomes: u8,
|
||||
profile_cohorts: usize,
|
||||
}
|
||||
|
||||
impl Thresholds {
|
||||
fn for_aggressiveness(value: WebCarrierNegotiationAggressiveness) -> Self {
|
||||
match value {
|
||||
WebCarrierNegotiationAggressiveness::Conservative => Self {
|
||||
user_agent: 3,
|
||||
ip: None,
|
||||
profile_outcomes: 8,
|
||||
profile_cohorts: 4,
|
||||
},
|
||||
WebCarrierNegotiationAggressiveness::Balanced => Self {
|
||||
user_agent: 2,
|
||||
ip: Some(3),
|
||||
profile_outcomes: 6,
|
||||
profile_cohorts: 3,
|
||||
},
|
||||
WebCarrierNegotiationAggressiveness::Aggressive => Self {
|
||||
user_agent: 1,
|
||||
ip: Some(1),
|
||||
profile_outcomes: 4,
|
||||
profile_cohorts: 2,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-local bounded two-bucket carrier evidence store.
|
||||
pub(super) struct CarrierLearning {
|
||||
entries: HashMap<EvidenceKey, Evidence>,
|
||||
insertion_order: VecDeque<(EvidenceKey, u64)>,
|
||||
capacity: usize,
|
||||
insertion_sequence: u64,
|
||||
epoch: Option<u64>,
|
||||
policy: Option<LearningPolicy>,
|
||||
policy_started_at: Instant,
|
||||
}
|
||||
|
||||
impl CarrierLearning {
|
||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||
pub(super) fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
insertion_order: VecDeque::new(),
|
||||
capacity,
|
||||
insertion_sequence: 1,
|
||||
epoch: Some(0),
|
||||
policy: None,
|
||||
policy_started_at: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies hot-reloaded learning policy and returns its outcome epoch.
|
||||
pub(super) fn apply_policy(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
) -> Option<u64> {
|
||||
let policy = LearningPolicy {
|
||||
enabled,
|
||||
aggressiveness,
|
||||
lifetime,
|
||||
};
|
||||
if self.policy != Some(policy) {
|
||||
self.entries.clear();
|
||||
self.insertion_order.clear();
|
||||
if !enabled {
|
||||
self.entries.shrink_to_fit();
|
||||
self.insertion_order.shrink_to_fit();
|
||||
}
|
||||
self.insertion_sequence = 1;
|
||||
self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1));
|
||||
self.policy = Some(policy);
|
||||
self.policy_started_at = now;
|
||||
}
|
||||
self.epoch
|
||||
}
|
||||
|
||||
/// Returns the current epoch only when the request snapshot matches owner policy.
|
||||
pub(super) fn epoch_for_policy(
|
||||
&self,
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
lifetime: Duration,
|
||||
) -> Option<u64> {
|
||||
(self.policy
|
||||
== Some(LearningPolicy {
|
||||
enabled,
|
||||
aggressiveness,
|
||||
lifetime,
|
||||
}))
|
||||
.then_some(self.epoch)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Ranks supported configured candidates without scanning the evidence store.
|
||||
pub(super) fn rank(
|
||||
&self,
|
||||
now: Instant,
|
||||
configured: &[WebCarrier],
|
||||
request: super::CarrierRequest,
|
||||
profile_key: ProfileKey,
|
||||
client_ip: IpAddr,
|
||||
ip_learning_eligible: bool,
|
||||
) -> (Vec<WebCarrier>, [i16; 4]) {
|
||||
let Some(policy) = self.policy.filter(|policy| policy.enabled) else {
|
||||
return (supported(configured, request), [0; 4]);
|
||||
};
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let thresholds = Thresholds::for_aggressiveness(policy.aggressiveness);
|
||||
let profile = self
|
||||
.entries
|
||||
.get(&EvidenceKey::Profile(profile_key))
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let user_agent = self
|
||||
.entries
|
||||
.get(&EvidenceKey::UserAgent(
|
||||
profile_key,
|
||||
request.class(),
|
||||
request.user_agent_hash(),
|
||||
))
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let ip = (ip_learning_eligible && thresholds.ip.is_some())
|
||||
.then(|| self.entries.get(&EvidenceKey::Ip(profile_key, client_ip)))
|
||||
.flatten()
|
||||
.map(|entry| entry.aggregate(slot));
|
||||
let profile_ready = profile.as_ref().is_some_and(|entry| {
|
||||
entry.outcomes >= thresholds.profile_outcomes
|
||||
&& entry.cohorts.iter().flatten().count() >= thresholds.profile_cohorts
|
||||
});
|
||||
let user_agent_ready = user_agent
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.outcomes >= thresholds.user_agent);
|
||||
let ip_ready = thresholds.ip.is_some_and(|minimum| {
|
||||
ip.as_ref()
|
||||
.is_some_and(|entry| entry.outcomes >= minimum)
|
||||
});
|
||||
let mut scores = [0i16; 4];
|
||||
for carrier in WebCarrier::ALL {
|
||||
let index = carrier.index();
|
||||
if profile_ready {
|
||||
scores[index] += i16::from(profile.as_ref().map_or(0, |value| value.scores[index]))
|
||||
* PROFILE_WEIGHT;
|
||||
}
|
||||
if user_agent_ready {
|
||||
scores[index] += i16::from(
|
||||
user_agent
|
||||
.as_ref()
|
||||
.map_or(0, |value| value.scores[index]),
|
||||
) * USER_AGENT_WEIGHT;
|
||||
}
|
||||
if ip_ready {
|
||||
scores[index] +=
|
||||
i16::from(ip.as_ref().map_or(0, |value| value.scores[index])) * IP_WEIGHT;
|
||||
}
|
||||
}
|
||||
let mut ranked = supported(configured, request);
|
||||
let fallback = configured
|
||||
.last()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier));
|
||||
if let Some(fallback) = fallback {
|
||||
ranked.retain(|carrier| *carrier != fallback);
|
||||
}
|
||||
ranked.sort_by_key(|carrier| std::cmp::Reverse(scores[carrier.index()]));
|
||||
if let Some(fallback) = fallback {
|
||||
ranked.push(fallback);
|
||||
}
|
||||
(ranked, scores)
|
||||
}
|
||||
|
||||
/// Applies one complete attempt chain as one atomic evidence sample.
|
||||
pub(super) fn record_chain(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
epoch: u64,
|
||||
context: CarrierLearningContext,
|
||||
failures: &[WebCarrier],
|
||||
winner: WebCarrier,
|
||||
) {
|
||||
let Some(policy) = self.policy.filter(|policy| policy.enabled) else {
|
||||
return;
|
||||
};
|
||||
if Some(epoch) != self.epoch {
|
||||
return;
|
||||
}
|
||||
let mut deltas = [0i8; 4];
|
||||
let _ = failures;
|
||||
deltas[winner.index()] = deltas[winner.index()].saturating_add(1);
|
||||
let thresholds = Thresholds::for_aggressiveness(policy.aggressiveness);
|
||||
let keys = [
|
||||
Some(EvidenceKey::Profile(context.profile_key)),
|
||||
Some(EvidenceKey::UserAgent(
|
||||
context.profile_key,
|
||||
context.class,
|
||||
context.user_agent_hash,
|
||||
)),
|
||||
(context.ip_learning_eligible && thresholds.ip.is_some())
|
||||
.then_some(EvidenceKey::Ip(context.profile_key, context.client_ip)),
|
||||
];
|
||||
self.make_room(&keys);
|
||||
let missing = keys
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|key| !self.entries.contains_key(key))
|
||||
.count();
|
||||
if self.entries.len().saturating_add(missing) > self.capacity {
|
||||
return;
|
||||
}
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let cohort = cohort_hash(context);
|
||||
for (index, key) in keys.into_iter().enumerate() {
|
||||
let Some(key) = key else { continue };
|
||||
self.update_key(key, slot, deltas, (index == 0).then_some(cohort));
|
||||
}
|
||||
}
|
||||
|
||||
/// Reclaims a fixed number of entries outside both half-window buckets.
|
||||
pub(super) fn prune(&mut self, now: Instant) {
|
||||
let Some(policy) = self.policy else { return };
|
||||
let slot = bucket_slot(self.policy_started_at, now, policy.lifetime);
|
||||
let budget = self.insertion_order.len().min(PRUNE_ENTRIES_PER_TICK);
|
||||
for _ in 0..budget {
|
||||
let Some((key, sequence)) = self.insertion_order.pop_front() else {
|
||||
break;
|
||||
};
|
||||
let current = self
|
||||
.entries
|
||||
.get(&key)
|
||||
.is_some_and(|entry| entry.insertion_sequence == sequence);
|
||||
if !current {
|
||||
continue;
|
||||
}
|
||||
if self.entries.get(&key).is_some_and(|entry| entry.is_live(slot)) {
|
||||
self.insertion_order.push_back((key, sequence));
|
||||
} else {
|
||||
self.entries.remove(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_room(&mut self, keys: &[Option<EvidenceKey>; 3]) {
|
||||
let missing = keys
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|key| !self.entries.contains_key(key))
|
||||
.count();
|
||||
let mut remaining = self.insertion_order.len();
|
||||
while self.entries.len().saturating_add(missing) > self.capacity && remaining > 0 {
|
||||
remaining -= 1;
|
||||
let Some((oldest, sequence)) = self.insertion_order.pop_front() else {
|
||||
break;
|
||||
};
|
||||
if self
|
||||
.entries
|
||||
.get(&oldest)
|
||||
.is_none_or(|entry| entry.insertion_sequence != sequence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if keys.contains(&Some(oldest)) {
|
||||
self.insertion_order.push_back((oldest, sequence));
|
||||
continue;
|
||||
}
|
||||
self.entries.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_key(
|
||||
&mut self,
|
||||
key: EvidenceKey,
|
||||
slot: u64,
|
||||
deltas: [i8; 4],
|
||||
cohort: Option<[u8; 32]>,
|
||||
) {
|
||||
if let Some(entry) = self.entries.get_mut(&key) {
|
||||
entry.update(slot, deltas, cohort);
|
||||
return;
|
||||
}
|
||||
let Some(insertion_sequence) = self.next_insertion_sequence() else {
|
||||
return;
|
||||
};
|
||||
self.entries
|
||||
.insert(key, Evidence::new(insertion_sequence));
|
||||
self.insertion_order.push_back((key, insertion_sequence));
|
||||
if let Some(entry) = self.entries.get_mut(&key) {
|
||||
entry.update(slot, deltas, cohort);
|
||||
}
|
||||
}
|
||||
|
||||
fn next_insertion_sequence(&mut self) -> Option<u64> {
|
||||
let sequence = self.insertion_sequence;
|
||||
self.insertion_sequence = sequence.checked_add(1)?;
|
||||
Some(sequence)
|
||||
}
|
||||
}
|
||||
|
||||
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
|
||||
configured
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bucket_slot(start: Instant, now: Instant, lifetime: Duration) -> u64 {
|
||||
let half = (lifetime / 2).max(Duration::from_nanos(1));
|
||||
let quotient = now.saturating_duration_since(start).as_nanos() / half.as_nanos();
|
||||
quotient.min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
fn cohort_hash(context: CarrierLearningContext) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(COHORT_CONTEXT);
|
||||
digest.update(context.profile_key);
|
||||
digest.update([match context.class {
|
||||
CarrierClientClass::Legacy => 0,
|
||||
CarrierClientClass::Bridge => 1,
|
||||
CarrierClientClass::BrowserHint => 2,
|
||||
CarrierClientClass::Ios => 3,
|
||||
}]);
|
||||
digest.update(context.user_agent_hash);
|
||||
match context.client_ip {
|
||||
IpAddr::V4(address) => {
|
||||
digest.update([4]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
IpAddr::V6(address) => {
|
||||
digest.update([6]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "carrier_learning/tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,222 @@
|
||||
use super::*;
|
||||
use crate::web::manager::{CarrierCapabilities, CarrierRequest};
|
||||
|
||||
fn request(hash: u8) -> CarrierRequest {
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
1,
|
||||
None,
|
||||
[hash; 32],
|
||||
)
|
||||
}
|
||||
|
||||
fn context(hash: u8) -> CarrierLearningContext {
|
||||
CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, hash)),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [hash; 32],
|
||||
epoch: 1,
|
||||
ip_learning_eligible: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_epoch_rejects_late_outcomes_and_clears_state() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Websocket);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
let next = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
false,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
assert_ne!(epoch, next);
|
||||
learning.record_chain(now, epoch, context(1), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggressive_policy_ranks_one_atomic_chain_sample() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(now, epoch, context(2), &[], WebCarrier::Websocket);
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&[
|
||||
WebCarrier::Https,
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::HttpsLanes,
|
||||
],
|
||||
request(2),
|
||||
[1; 32],
|
||||
context(2).client_ip,
|
||||
true,
|
||||
);
|
||||
assert_eq!(
|
||||
ranked,
|
||||
[
|
||||
WebCarrier::Websocket,
|
||||
WebCarrier::Https,
|
||||
WebCarrier::HttpsLanes,
|
||||
]
|
||||
);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 33);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_half_windows_expire_without_sliding_updates() {
|
||||
let start = Instant::now();
|
||||
let mut learning = CarrierLearning::new(6);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
start,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(start, epoch, context(3), &[], WebCarrier::Websocket);
|
||||
learning.record_chain(
|
||||
start + Duration::from_secs(6),
|
||||
epoch,
|
||||
context(3),
|
||||
&[],
|
||||
WebCarrier::Websocket,
|
||||
);
|
||||
learning.prune(start + Duration::from_secs(11));
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
learning.prune(start + Duration::from_secs(16));
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_epoch_and_insertion_identifiers_fail_closed() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
learning.epoch = Some(u64::MAX);
|
||||
assert_eq!(
|
||||
learning.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
),
|
||||
None
|
||||
);
|
||||
learning.record_chain(now, u64::MAX, context(4), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
|
||||
learning.epoch = Some(1);
|
||||
learning.insertion_sequence = u64::MAX;
|
||||
learning.record_chain(now, 1, context(4), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_reported_failures_do_not_create_negative_evidence() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(
|
||||
now,
|
||||
epoch,
|
||||
context(5),
|
||||
&[WebCarrier::Websocket],
|
||||
WebCarrier::Https,
|
||||
);
|
||||
let (_, scores) = learning.rank(
|
||||
now,
|
||||
&[WebCarrier::Websocket, WebCarrier::HttpsLanes],
|
||||
request(5),
|
||||
[1; 32],
|
||||
context(5).client_ip,
|
||||
true,
|
||||
);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_new_old_policy_rejects_both_stale_epochs() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let old = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
let middle = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Balanced,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
let current = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
assert_ne!(old, middle);
|
||||
assert_ne!(middle, current);
|
||||
assert_ne!(old, current);
|
||||
learning.record_chain(now, old, context(6), &[], WebCarrier::Https);
|
||||
learning.record_chain(now, middle, context(6), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.is_empty());
|
||||
learning.record_chain(now, current, context(6), &[], WebCarrier::Https);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_metadata_stays_within_the_entry_capacity() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
for hash in 1..=32 {
|
||||
learning.record_chain(now, epoch, context(hash), &[], WebCarrier::Https);
|
||||
assert!(learning.entries.len() <= 3);
|
||||
assert!(learning.insertion_order.len() <= 3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::state::CarrierChainPhase;
|
||||
use super::{
|
||||
CarrierClientClass, CarrierEcho, CarrierLearningContext, CarrierRequest, TokenHash,
|
||||
WebProcessRuntime,
|
||||
};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Returns authenticated current chain metadata after a committed retry conflict.
|
||||
pub(crate) fn carrier_echo(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
request: CarrierRequest,
|
||||
) -> Option<CarrierEcho> {
|
||||
let state = self.state.lock();
|
||||
let entry = state.bootstraps.get(&bootstrap_hash)?;
|
||||
let session = entry.session.as_ref()?;
|
||||
if !entry.used
|
||||
|| entry.profile.host != host
|
||||
|| entry.session_client_ip != Some(client_ip)
|
||||
|| entry
|
||||
.carrier_request
|
||||
.is_none_or(|current| !current.matches_client(request))
|
||||
|| !(matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
) || session.is_carrier_committed())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(CarrierEcho {
|
||||
carrier: session.carrier(),
|
||||
attempt: entry.carrier_attempt,
|
||||
candidate_count: u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
|
||||
deadline_secs: entry.profile.carrier_negotiation_deadlines_secs[3],
|
||||
state: if entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& session.is_carrier_committed()
|
||||
{
|
||||
CarrierChainPhase::CommittedPendingHealth.as_str()
|
||||
} else {
|
||||
entry.carrier_phase.as_str()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Restores the exact old attempt after successor admission fails.
|
||||
pub(super) fn cancel_replacement(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
old_session: &Arc<WebSession>,
|
||||
) {
|
||||
old_session.cancel_carrier_supersede();
|
||||
let mut state = self.state.lock();
|
||||
if let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash)
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, old_session))
|
||||
{
|
||||
entry.carrier_transitioning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Freezes replacement immediately after accepted carrier state mutation.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn carrier_committed(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
session_hash: TokenHash,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
class: CarrierClientClass,
|
||||
client_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
let scores = state.bootstraps.get_mut(&bootstrap_hash).and_then(|entry| {
|
||||
if entry.carrier_attempt == attempt
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.token_hash() == session_hash)
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
{
|
||||
entry.carrier_phase = CarrierChainPhase::CommittedPendingHealth;
|
||||
Some(entry.carrier_scores)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
drop(state);
|
||||
let Some(scores) = scores else { return };
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierCommitted,
|
||||
class.as_str(),
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
/// Promotes one exact committed attempt after transport-specific health evidence.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn carrier_became_healthy(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
session_hash: TokenHash,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
class: CarrierClientClass,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
client_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
) {
|
||||
let (scores, failures) = {
|
||||
let mut state = self.state.lock();
|
||||
let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash) else {
|
||||
return;
|
||||
};
|
||||
if entry.carrier_attempt != attempt
|
||||
|| entry.carrier_phase != CarrierChainPhase::CommittedPendingHealth
|
||||
|| entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_none_or(|session| session.token_hash() != session_hash)
|
||||
{
|
||||
return;
|
||||
}
|
||||
entry.carrier_phase = CarrierChainPhase::Healthy;
|
||||
(entry.carrier_scores, entry.carrier_failures)
|
||||
};
|
||||
if let Some(context) = learning_context {
|
||||
let now = Instant::now();
|
||||
let mut learning = self.learning.lock();
|
||||
let failures = failures.into_iter().flatten().collect::<Vec<_>>();
|
||||
learning.record_chain(now, context.epoch, context, &failures, carrier);
|
||||
}
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity,
|
||||
TraceLifecycleEvent::CarrierHealthy,
|
||||
class.as_str(),
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,8 @@ use std::time::{Duration, Instant};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::state::{
|
||||
Bootstrap, allow_rate, evict_oldest_unused_bootstrap, matching_profile, new_unique_token,
|
||||
remove_expired_locked,
|
||||
Bootstrap, CarrierChainPhase, allow_rate, evict_oldest_unused_bootstrap, matching_profile,
|
||||
new_unique_token, remove_expired_locked,
|
||||
};
|
||||
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
@@ -78,8 +78,14 @@ impl WebProcessRuntime {
|
||||
carrier_scores: [0; 4],
|
||||
carrier_attempt: 0,
|
||||
carrier_transitioning: false,
|
||||
carrier_committed: false,
|
||||
carrier_phase: CarrierChainPhase::Provisional,
|
||||
carrier_started_at: None,
|
||||
carrier_deadline_at: None,
|
||||
carrier_failures: [None; 3],
|
||||
carrier_learning_epoch: 0,
|
||||
close_requested: false,
|
||||
session_client_ip: None,
|
||||
session_ip_learning_eligible: false,
|
||||
used: false,
|
||||
},
|
||||
);
|
||||
@@ -140,12 +146,24 @@ impl WebProcessRuntime {
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<(), ManagerError> {
|
||||
let state = self.state.lock();
|
||||
let mut state = self.state.lock();
|
||||
let session = state
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.filter(|session| session.matches_host(host))
|
||||
.cloned();
|
||||
if session.is_some() {
|
||||
for bootstrap in state.bootstraps.values_mut() {
|
||||
if bootstrap
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|current| current.token_hash() == hash)
|
||||
{
|
||||
bootstrap.close_requested = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let closed = state
|
||||
.closed_tokens
|
||||
.get(&hash)
|
||||
|
||||
@@ -67,6 +67,7 @@ impl WebProcessRuntime {
|
||||
state.bootstraps_per_ip.clear();
|
||||
state.sessions.values().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
self.stream_admission.lock().closed = true;
|
||||
for session in &sessions {
|
||||
session.close();
|
||||
}
|
||||
@@ -85,10 +86,8 @@ impl WebProcessRuntime {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), waits).await;
|
||||
self.tasks.close();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), self.tasks.wait()).await;
|
||||
let (sessions_live, streams_live) = {
|
||||
let state = self.state.lock();
|
||||
(state.sessions.len(), state.streams_live)
|
||||
};
|
||||
let sessions_live = self.state.lock().sessions.len();
|
||||
let streams_live = self.stream_admission.lock().streams_live;
|
||||
let budget = self.data_budget.snapshot();
|
||||
info!(
|
||||
target: "telemt::web",
|
||||
@@ -113,12 +112,51 @@ impl WebProcessRuntime {
|
||||
pub(super) fn cleanup(&self) {
|
||||
self.cleanup_websockets();
|
||||
let now = Instant::now();
|
||||
self.learning.lock().prune(now);
|
||||
let sessions = {
|
||||
let generation = self.active_generation();
|
||||
let config = &generation.config().web;
|
||||
let learning_enabled = config.carrier_negotiation_enabled() && config.carrier_learning;
|
||||
let mut learning = self.learning.lock();
|
||||
let _ = learning.apply_policy(
|
||||
now,
|
||||
learning_enabled,
|
||||
config.carrier_negotiation_aggressiveness,
|
||||
Duration::from_secs(config.timeouts.carrier_learning_secs),
|
||||
);
|
||||
learning.prune(now);
|
||||
drop(learning);
|
||||
let (sessions, expired_chains) = {
|
||||
let mut state = self.state.lock();
|
||||
let expired = state
|
||||
.bootstraps
|
||||
.iter()
|
||||
.filter_map(|(hash, bootstrap)| {
|
||||
(bootstrap.carrier_phase == super::state::CarrierChainPhase::Provisional
|
||||
&& bootstrap
|
||||
.carrier_deadline_at
|
||||
.is_some_and(|deadline| now >= deadline)
|
||||
&& bootstrap
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| !session.is_carrier_committed()))
|
||||
.then_some((*hash, bootstrap.session.clone()))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let expired_chains = expired
|
||||
.iter()
|
||||
.filter_map(|(_, session)| session.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for (hash, _) in expired {
|
||||
remove_bootstrap_locked(&mut state, hash);
|
||||
}
|
||||
remove_expired_locked(&mut state, now);
|
||||
state.sessions.values().cloned().collect::<Vec<_>>()
|
||||
(
|
||||
state.sessions.values().cloned().collect::<Vec<_>>(),
|
||||
expired_chains,
|
||||
)
|
||||
};
|
||||
for session in expired_chains {
|
||||
session.close();
|
||||
}
|
||||
for session in sessions {
|
||||
session.close_if_due(now);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ pub(crate) enum CarrierClientClass {
|
||||
Bridge,
|
||||
/// Strict same-origin browser metadata survived while the marker did not.
|
||||
BrowserHint,
|
||||
/// A native iOS client that supports only the serialized HTTPS carrier.
|
||||
Ios,
|
||||
}
|
||||
|
||||
impl CarrierClientClass {
|
||||
@@ -20,6 +22,7 @@ impl CarrierClientClass {
|
||||
Self::Legacy => "legacy",
|
||||
Self::Bridge => "bridge",
|
||||
Self::BrowserHint => "browser-hint",
|
||||
Self::Ios => "ios",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,6 +77,11 @@ impl CarrierCapabilities {
|
||||
Self(0b1111)
|
||||
}
|
||||
|
||||
/// Returns the only carrier implemented by the native iOS client.
|
||||
pub(crate) const fn ios() -> Self {
|
||||
Self(0b0001)
|
||||
}
|
||||
|
||||
/// Builds a set from a validated bit representation.
|
||||
pub(crate) const fn from_bits(bits: u8) -> Option<Self> {
|
||||
if bits != 0 && bits & !0b1111 == 0 {
|
||||
@@ -97,6 +105,7 @@ pub(crate) struct CarrierRequest {
|
||||
attempt: Option<u8>,
|
||||
failure: Option<CarrierFailure>,
|
||||
user_agent_hash: [u8; 32],
|
||||
initial_only: bool,
|
||||
}
|
||||
|
||||
impl CarrierRequest {
|
||||
@@ -108,6 +117,19 @@ impl CarrierRequest {
|
||||
attempt: None,
|
||||
failure: None,
|
||||
user_agent_hash,
|
||||
initial_only: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a known fixed-capability client without retry negotiation.
|
||||
pub(crate) const fn ios(user_agent_hash: [u8; 32]) -> Self {
|
||||
Self {
|
||||
class: CarrierClientClass::Ios,
|
||||
capabilities: Some(CarrierCapabilities::ios()),
|
||||
attempt: None,
|
||||
failure: None,
|
||||
user_agent_hash,
|
||||
initial_only: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,11 +147,17 @@ impl CarrierRequest {
|
||||
attempt: Some(attempt),
|
||||
failure,
|
||||
user_agent_hash,
|
||||
initial_only: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this request participates in server-side negotiation.
|
||||
pub(crate) const fn is_automatic(self) -> bool {
|
||||
self.capabilities.is_some() && !self.initial_only
|
||||
}
|
||||
|
||||
/// Returns whether server capability filtering applies to this request.
|
||||
pub(crate) const fn uses_capabilities(self) -> bool {
|
||||
self.capabilities.is_some()
|
||||
}
|
||||
|
||||
@@ -166,6 +194,14 @@ impl CarrierRequest {
|
||||
self.class == other.class
|
||||
&& self.capabilities_bits() == other.capabilities_bits()
|
||||
&& self.user_agent_hash == other.user_agent_hash
|
||||
&& self.initial_only == other.initial_only
|
||||
}
|
||||
|
||||
/// Checks the complete idempotent identity of one exact attempt request.
|
||||
pub(crate) fn matches_attempt(self, other: Self) -> bool {
|
||||
self.matches_client(other)
|
||||
&& self.attempt == other.attempt
|
||||
&& self.failure == other.failure
|
||||
}
|
||||
|
||||
fn capabilities_bits(self) -> Option<u8> {
|
||||
@@ -184,4 +220,8 @@ pub(crate) struct CarrierLearningContext {
|
||||
pub(crate) class: CarrierClientClass,
|
||||
/// Domain-separated normalized User-Agent digest.
|
||||
pub(crate) user_agent_hash: [u8; 32],
|
||||
/// Hot-reload epoch that rejects late outcomes from an older policy.
|
||||
pub(crate) epoch: u64,
|
||||
/// Whether the authoritative client address is safe to use as learning evidence.
|
||||
pub(crate) ip_learning_eligible: bool,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::state::{ManagerState, allow_rate};
|
||||
use super::{ProfileKey, WebProcessRuntime};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
|
||||
/// Applies process, address, profile, and rate ceilings to one initial session.
|
||||
pub(super) fn admit_initial(
|
||||
runtime: &WebProcessRuntime,
|
||||
state: &mut ManagerState,
|
||||
now: Instant,
|
||||
client_ip: IpAddr,
|
||||
profile_key: ProfileKey,
|
||||
profile: &WebRuntimeProfile,
|
||||
) -> bool {
|
||||
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
|
||||
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
< runtime.limits.max_sessions_per_ip
|
||||
&& state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
< profile.max_sessions
|
||||
&& allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
runtime.limits.new_sessions_per_minute,
|
||||
runtime.limits.new_sessions_burst,
|
||||
);
|
||||
if !admitted {
|
||||
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
admitted
|
||||
}
|
||||
+180
-143
@@ -8,17 +8,17 @@ use subtle::ConstantTimeEq;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::state::{
|
||||
ManagerState, allow_rate, decrement_map, matching_profile, new_unique_token, profile_key,
|
||||
CarrierChainPhase, decrement_map, matching_profile, new_unique_token, profile_key,
|
||||
remember_closed_token_locked, remove_expired_locked,
|
||||
};
|
||||
use super::session_admission::admit_initial;
|
||||
use super::{
|
||||
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
|
||||
WebProcessRuntime,
|
||||
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash, WebProcessRuntime,
|
||||
};
|
||||
use crate::config::{WebCarrier, WebRuntimeProfile, WebTimeoutsConfig};
|
||||
use crate::config::{WebCarrier, WebRuntimeProfile};
|
||||
use crate::web::frame;
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||
use crate::web::trace::TraceLifecycleEvent;
|
||||
|
||||
struct Replacement {
|
||||
old_session: Arc<WebSession>,
|
||||
@@ -29,6 +29,9 @@ struct Replacement {
|
||||
carrier: WebCarrier,
|
||||
request: CarrierRequest,
|
||||
scores: [i16; 4],
|
||||
learning_epoch: u64,
|
||||
ip_learning_eligible: bool,
|
||||
carrier_deadline_at: Instant,
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
@@ -40,6 +43,7 @@ impl WebProcessRuntime {
|
||||
client_ip: IpAddr,
|
||||
body: &[u8],
|
||||
carrier_request: CarrierRequest,
|
||||
ip_learning_eligible: bool,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !frame::validate_hello(body, &self.limits) {
|
||||
return Err(ManagerError::Protocol);
|
||||
@@ -57,8 +61,25 @@ impl WebProcessRuntime {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.used {
|
||||
if entry.carrier_deadline_at.is_some_and(|deadline| now >= deadline)
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| !session.is_carrier_committed())
|
||||
{
|
||||
let session = entry.session.clone();
|
||||
drop(state);
|
||||
if let Some(session) = session {
|
||||
session.close();
|
||||
}
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if entry.close_requested {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||
let client_matches = entry.session_client_ip == Some(client_ip);
|
||||
let client_matches = entry.session_client_ip == Some(client_ip)
|
||||
&& entry.session_ip_learning_eligible == ip_learning_eligible;
|
||||
let request_matches = entry
|
||||
.carrier_request
|
||||
.is_some_and(|current| current.matches_client(carrier_request));
|
||||
@@ -71,11 +92,31 @@ impl WebProcessRuntime {
|
||||
if carrier_request.attempt() == Some(entry.carrier_attempt)
|
||||
|| (!carrier_request.is_automatic() && entry.carrier_attempt == 1)
|
||||
{
|
||||
if entry
|
||||
.carrier_request
|
||||
.is_none_or(|current| !current.matches_attempt(carrier_request))
|
||||
{
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||
let automatic = carrier_request.is_automatic();
|
||||
let carrier_state = if entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& session.is_carrier_committed()
|
||||
{
|
||||
CarrierChainPhase::CommittedPendingHealth.as_str()
|
||||
} else {
|
||||
entry.carrier_phase.as_str()
|
||||
};
|
||||
let result = CreateResult {
|
||||
token: entry.session_token.as_str().to_owned(),
|
||||
carrier: session.carrier(),
|
||||
attempt: carrier_request.attempt(),
|
||||
candidate_count: automatic.then(|| {
|
||||
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4)
|
||||
}),
|
||||
deadline_secs: automatic
|
||||
.then_some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: automatic.then_some(carrier_state),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
drop(state);
|
||||
@@ -92,9 +133,19 @@ impl WebProcessRuntime {
|
||||
let next_attempt = entry.carrier_attempt.saturating_add(1);
|
||||
if !carrier_request.is_automatic()
|
||||
|| carrier_request.attempt() != Some(next_attempt)
|
||||
|| entry.carrier_committed
|
||||
|| matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
)
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
return Err(if matches!(
|
||||
entry.carrier_phase,
|
||||
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
|
||||
) {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Protocol
|
||||
});
|
||||
}
|
||||
let Some(carrier) = entry
|
||||
.carrier_candidates
|
||||
@@ -103,6 +154,15 @@ impl WebProcessRuntime {
|
||||
else {
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
let deadline_index = usize::from(next_attempt.saturating_sub(2));
|
||||
if entry.carrier_started_at.is_some_and(|started| {
|
||||
now.saturating_duration_since(started)
|
||||
>= Duration::from_secs(
|
||||
entry.profile.carrier_negotiation_deadlines_secs[deadline_index],
|
||||
)
|
||||
}) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let old_session = entry.session.clone().ok_or(ManagerError::Authentication)?;
|
||||
let replacement = Replacement {
|
||||
profile: Arc::clone(&entry.profile),
|
||||
@@ -113,6 +173,9 @@ impl WebProcessRuntime {
|
||||
carrier,
|
||||
request: carrier_request,
|
||||
scores: entry.carrier_scores,
|
||||
learning_epoch: entry.carrier_learning_epoch,
|
||||
ip_learning_eligible,
|
||||
carrier_deadline_at: entry.carrier_deadline_at.ok_or(ManagerError::Protocol)?,
|
||||
};
|
||||
state
|
||||
.bootstraps
|
||||
@@ -120,12 +183,7 @@ impl WebProcessRuntime {
|
||||
.ok_or(ManagerError::Authentication)?
|
||||
.carrier_transitioning = true;
|
||||
drop(state);
|
||||
return self.replace_session(
|
||||
bootstrap_hash,
|
||||
client_ip,
|
||||
replacement,
|
||||
&config.web.timeouts,
|
||||
);
|
||||
return self.replace_session(bootstrap_hash, client_ip, replacement);
|
||||
}
|
||||
|
||||
if (carrier_request.is_automatic() && carrier_request.attempt() != Some(1))
|
||||
@@ -149,15 +207,44 @@ impl WebProcessRuntime {
|
||||
if carrier_request.is_automatic() && !profile.carrier_negotiation_enabled {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (candidates, scores) = if carrier_request.is_automatic() && profile.carrier_learning {
|
||||
self.learning.lock().rank(
|
||||
now,
|
||||
&profile.carriers,
|
||||
carrier_request,
|
||||
profile_key,
|
||||
client_ip,
|
||||
)
|
||||
} else if carrier_request.is_automatic() {
|
||||
let capability_selection = carrier_request.uses_capabilities()
|
||||
&& profile.carrier_negotiation_enabled;
|
||||
let learning_policy = (
|
||||
config.web.carrier_negotiation_enabled() && config.web.carrier_learning,
|
||||
config.web.carrier_negotiation_aggressiveness,
|
||||
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
|
||||
);
|
||||
let (candidates, scores, learning_epoch) = if capability_selection
|
||||
&& profile.carrier_learning
|
||||
{
|
||||
let learning = self.learning.lock();
|
||||
if let Some(epoch) = learning.epoch_for_policy(
|
||||
learning_policy.0,
|
||||
learning_policy.1,
|
||||
learning_policy.2,
|
||||
) {
|
||||
let (candidates, scores) = learning.rank(
|
||||
now,
|
||||
&profile.carriers,
|
||||
carrier_request,
|
||||
profile_key,
|
||||
client_ip,
|
||||
ip_learning_eligible,
|
||||
);
|
||||
(candidates, scores, Some(epoch))
|
||||
} else {
|
||||
(
|
||||
profile
|
||||
.carriers
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| carrier_request.supports(*carrier))
|
||||
.collect(),
|
||||
[0; 4],
|
||||
None,
|
||||
)
|
||||
}
|
||||
} else if capability_selection {
|
||||
(
|
||||
profile
|
||||
.carriers
|
||||
@@ -166,9 +253,14 @@ impl WebProcessRuntime {
|
||||
.filter(|carrier| carrier_request.supports(*carrier))
|
||||
.collect(),
|
||||
[0; 4],
|
||||
None,
|
||||
)
|
||||
} else if carrier_request.uses_capabilities()
|
||||
&& !carrier_request.supports(profile.carrier)
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
} else {
|
||||
(vec![profile.carrier], [0; 4])
|
||||
(vec![profile.carrier], [0; 4], None)
|
||||
};
|
||||
let Some(carrier) = candidates.first().copied() else {
|
||||
return Err(ManagerError::Protocol);
|
||||
@@ -180,12 +272,16 @@ impl WebProcessRuntime {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let learning_context = (carrier_request.is_automatic() && profile.carrier_learning)
|
||||
.then_some(CarrierLearningContext {
|
||||
let carrier_deadline_at = carrier_request.is_automatic().then_some(
|
||||
now + Duration::from_secs(profile.carrier_negotiation_deadlines_secs[3]),
|
||||
);
|
||||
let learning_context = learning_epoch.map(|epoch| CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip,
|
||||
class: carrier_request.class(),
|
||||
user_agent_hash: carrier_request.user_agent_hash(),
|
||||
epoch,
|
||||
ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
@@ -197,7 +293,10 @@ impl WebProcessRuntime {
|
||||
carrier,
|
||||
1,
|
||||
bootstrap_hash,
|
||||
carrier_deadline_at,
|
||||
carrier_request.class(),
|
||||
learning_context,
|
||||
carrier_request.is_automatic(),
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
);
|
||||
@@ -216,8 +315,16 @@ impl WebProcessRuntime {
|
||||
entry.carrier_candidates = candidates.into();
|
||||
entry.carrier_scores = scores;
|
||||
entry.carrier_attempt = 1;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
||||
entry.carrier_deadline_at = carrier_deadline_at;
|
||||
entry.carrier_failures = [None; 3];
|
||||
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
||||
entry.expires_at = now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs);
|
||||
entry.session_client_ip = Some(client_ip);
|
||||
entry.session_ip_learning_eligible = ip_learning_eligible;
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
let candidate_count = u8::try_from(entry.carrier_candidates.len()).unwrap_or(4);
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
let identity = session.trace_identity();
|
||||
@@ -225,6 +332,15 @@ impl WebProcessRuntime {
|
||||
token: session_token,
|
||||
carrier,
|
||||
attempt: carrier_request.attempt(),
|
||||
candidate_count: carrier_request
|
||||
.is_automatic()
|
||||
.then_some(candidate_count),
|
||||
deadline_secs: carrier_request
|
||||
.is_automatic()
|
||||
.then_some(profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: carrier_request
|
||||
.is_automatic()
|
||||
.then_some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
drop(state);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
@@ -263,11 +379,15 @@ impl WebProcessRuntime {
|
||||
bootstrap_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
replacement: Replacement,
|
||||
timeouts: &WebTimeoutsConfig,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !replacement.old_session.begin_carrier_supersede() {
|
||||
let committed = replacement.old_session.is_carrier_committed();
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Protocol);
|
||||
return Err(if committed {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Closed
|
||||
});
|
||||
}
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
@@ -276,7 +396,10 @@ impl WebProcessRuntime {
|
||||
remove_expired_locked(&mut state, now);
|
||||
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
||||
entry.carrier_transitioning
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& !entry.close_requested
|
||||
&& entry.carrier_attempt.saturating_add(1) == replacement.attempt
|
||||
&& now < replacement.carrier_deadline_at
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
@@ -302,14 +425,16 @@ impl WebProcessRuntime {
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let learning_context = replacement.profile.carrier_learning.then_some(
|
||||
CarrierLearningContext {
|
||||
let learning_context = (replacement.profile.carrier_learning
|
||||
&& replacement.learning_epoch != 0)
|
||||
.then_some(CarrierLearningContext {
|
||||
profile_key: replacement.profile_key,
|
||||
client_ip,
|
||||
class: replacement.request.class(),
|
||||
user_agent_hash: replacement.request.user_agent_hash(),
|
||||
},
|
||||
);
|
||||
epoch: replacement.learning_epoch,
|
||||
ip_learning_eligible: replacement.ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
@@ -320,10 +445,19 @@ impl WebProcessRuntime {
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
bootstrap_hash,
|
||||
Some(replacement.carrier_deadline_at),
|
||||
replacement.request.class(),
|
||||
learning_context,
|
||||
true,
|
||||
self.limits.clone(),
|
||||
timeouts.clone(),
|
||||
replacement.old_session.timeouts().clone(),
|
||||
);
|
||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
session.close();
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let old_hash = replacement.old_session.token_hash();
|
||||
state.sessions.remove(&old_hash);
|
||||
remember_closed_token_locked(
|
||||
@@ -343,23 +477,29 @@ impl WebProcessRuntime {
|
||||
entry.carrier_request = Some(replacement.request);
|
||||
entry.carrier_attempt = replacement.attempt;
|
||||
entry.carrier_transitioning = false;
|
||||
entry.carrier_committed = false;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
if let Some(slot) = entry
|
||||
.carrier_failures
|
||||
.get_mut(usize::from(replacement.attempt.saturating_sub(2)))
|
||||
{
|
||||
*slot = Some(replacement.old_session.carrier());
|
||||
}
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: replacement.carrier,
|
||||
attempt: Some(replacement.attempt),
|
||||
candidate_count: Some(
|
||||
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
|
||||
),
|
||||
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
let old_identity = replacement.old_session.trace_identity();
|
||||
drop(state);
|
||||
if replacement.old_session.finish_carrier_supersede() {
|
||||
session.close();
|
||||
}
|
||||
if let Some(context) = learning_context {
|
||||
self.record_carrier_outcome(context, replacement.old_session.carrier(), false);
|
||||
}
|
||||
supersede.finish();
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity.clone(),
|
||||
@@ -401,107 +541,4 @@ impl WebProcessRuntime {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn cancel_replacement(&self, bootstrap_hash: TokenHash, old_session: &Arc<WebSession>) {
|
||||
old_session.cancel_carrier_supersede();
|
||||
let mut state = self.state.lock();
|
||||
if let Some(entry) = state.bootstraps.get_mut(&bootstrap_hash)
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, old_session))
|
||||
{
|
||||
entry.carrier_transitioning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Commits learning only after one accepted OPEN or DATA batch.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn carrier_committed(
|
||||
&self,
|
||||
bootstrap_hash: TokenHash,
|
||||
session_hash: TokenHash,
|
||||
attempt: u8,
|
||||
carrier: WebCarrier,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
client_ip: IpAddr,
|
||||
identity: TraceIdentity,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
let scores = state.bootstraps.get_mut(&bootstrap_hash).and_then(|entry| {
|
||||
if entry.carrier_attempt == attempt
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.token_hash() == session_hash)
|
||||
{
|
||||
entry.carrier_committed = true;
|
||||
Some(entry.carrier_scores)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
drop(state);
|
||||
let Some(scores) = scores else { return };
|
||||
if let Some(context) = learning_context {
|
||||
self.record_carrier_outcome(context, carrier, true);
|
||||
}
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity,
|
||||
TraceLifecycleEvent::CarrierCommitted,
|
||||
learning_context
|
||||
.map_or("legacy", |context| context.class.as_str()),
|
||||
carrier,
|
||||
attempt,
|
||||
scores,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
fn record_carrier_outcome(
|
||||
&self,
|
||||
context: CarrierLearningContext,
|
||||
carrier: WebCarrier,
|
||||
success: bool,
|
||||
) {
|
||||
let generation = self.active_generation();
|
||||
if !generation.config().web.carrier_learning {
|
||||
return;
|
||||
}
|
||||
let lifetime = Duration::from_secs(
|
||||
generation.config().web.timeouts.carrier_learning_secs,
|
||||
);
|
||||
self.learning
|
||||
.lock()
|
||||
.record(Instant::now(), lifetime, context, carrier, success);
|
||||
}
|
||||
}
|
||||
|
||||
fn admit_initial(
|
||||
runtime: &WebProcessRuntime,
|
||||
state: &mut ManagerState,
|
||||
now: Instant,
|
||||
client_ip: IpAddr,
|
||||
profile_key: super::ProfileKey,
|
||||
profile: &WebRuntimeProfile,
|
||||
) -> bool {
|
||||
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
|
||||
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
< runtime.limits.max_sessions_per_ip
|
||||
&& state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
< profile.max_sessions
|
||||
&& allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
runtime.limits.new_sessions_per_minute,
|
||||
runtime.limits.new_sessions_burst,
|
||||
);
|
||||
if !admitted {
|
||||
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
admitted
|
||||
}
|
||||
|
||||
+48
-12
@@ -14,6 +14,23 @@ use crate::web::session::WebSession;
|
||||
|
||||
const WEB_PROFILE_OWNER_CONTEXT: &[u8] = b"telemt-web-profile-owner-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum CarrierChainPhase {
|
||||
Provisional,
|
||||
CommittedPendingHealth,
|
||||
Healthy,
|
||||
}
|
||||
|
||||
impl CarrierChainPhase {
|
||||
pub(super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Provisional => "provisional",
|
||||
Self::CommittedPendingHealth => "committed",
|
||||
Self::Healthy => "healthy",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One issued bootstrap and optional idempotent session-creation replay state.
|
||||
pub(super) struct Bootstrap {
|
||||
/// Credential and replay-state expiry deadline.
|
||||
@@ -42,10 +59,22 @@ pub(super) struct Bootstrap {
|
||||
pub(super) carrier_attempt: u8,
|
||||
/// Prevents concurrent retries from replacing the same attempt twice.
|
||||
pub(super) carrier_transitioning: bool,
|
||||
/// Records the first accepted OPEN or DATA transition exactly once.
|
||||
pub(super) carrier_committed: bool,
|
||||
/// Manager-owned attempt-chain phase used for replacement linearization.
|
||||
pub(super) carrier_phase: CarrierChainPhase,
|
||||
/// Monotonic start of the first automatic session attempt.
|
||||
pub(super) carrier_started_at: Option<Instant>,
|
||||
/// Absolute server-side end of the automatic attempt chain.
|
||||
pub(super) carrier_deadline_at: Option<Instant>,
|
||||
/// Failed candidates staged until one winner becomes healthy.
|
||||
pub(super) carrier_failures: [Option<WebCarrier>; 3],
|
||||
/// Learning-policy epoch frozen by the first automatic attempt.
|
||||
pub(super) carrier_learning_epoch: u64,
|
||||
/// DELETE observed before an in-flight replacement committed its swap.
|
||||
pub(super) close_requested: bool,
|
||||
/// Effective address frozen by the first session-creation request.
|
||||
pub(super) session_client_ip: Option<IpAddr>,
|
||||
/// Whether the first request carried an authoritative public forwarded address.
|
||||
pub(super) session_ip_learning_eligible: bool,
|
||||
/// Distinguishes unused issuance quota from completed creation replay state.
|
||||
pub(super) used: bool,
|
||||
}
|
||||
@@ -70,6 +99,20 @@ struct StreamPortState {
|
||||
next: u16,
|
||||
}
|
||||
|
||||
/// Stream admission and KDF tuple ownership isolated from credential transitions.
|
||||
#[derive(Default)]
|
||||
pub(super) struct StreamAdmissionState {
|
||||
/// Process shutdown admission latch.
|
||||
pub(super) closed: bool,
|
||||
/// Live stream counts by stable profile key.
|
||||
pub(super) streams_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Process-wide live relay-task count.
|
||||
pub(super) streams_live: usize,
|
||||
stream_ports: HashMap<(IpAddr, SocketAddr), StreamPortState>,
|
||||
/// Logical-stream creation rate limiter.
|
||||
pub(super) stream_rate: RateState,
|
||||
}
|
||||
|
||||
/// Process-wide WEB registries and quota accounting protected by one short lock.
|
||||
#[derive(Default)]
|
||||
pub(super) struct ManagerState {
|
||||
@@ -85,17 +128,10 @@ pub(super) struct ManagerState {
|
||||
pub(super) sessions_per_ip: HashMap<IpAddr, usize>,
|
||||
/// Live session counts by stable profile key.
|
||||
pub(super) sessions_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Live relay-task counts by stable profile key.
|
||||
pub(super) streams_per_profile: HashMap<ProfileKey, usize>,
|
||||
/// Process-wide live relay-task count.
|
||||
pub(super) streams_live: usize,
|
||||
stream_ports: HashMap<(IpAddr, SocketAddr), StreamPortState>,
|
||||
/// Bootstrap issuance rate limiter.
|
||||
pub(super) bootstrap_rate: RateState,
|
||||
/// Session creation rate limiter.
|
||||
pub(super) session_rate: RateState,
|
||||
/// Logical-stream creation rate limiter.
|
||||
pub(super) stream_rate: RateState,
|
||||
/// Process shutdown admission latch.
|
||||
pub(super) closed: bool,
|
||||
}
|
||||
@@ -264,7 +300,7 @@ where
|
||||
|
||||
/// Allocates a non-zero source port unique among live streams for one KDF route.
|
||||
pub(super) fn allocate_stream_port(
|
||||
state: &mut ManagerState,
|
||||
state: &mut StreamAdmissionState,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
) -> Option<u16> {
|
||||
@@ -287,7 +323,7 @@ pub(super) fn allocate_stream_port(
|
||||
|
||||
/// Releases one source port and reclaims empty per-route allocator state.
|
||||
pub(super) fn release_stream_port(
|
||||
state: &mut ManagerState,
|
||||
state: &mut StreamAdmissionState,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
@@ -309,7 +345,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn synthetic_ports_are_unique_per_live_route_and_state_is_reclaimed() {
|
||||
let mut state = ManagerState::default();
|
||||
let mut state = StreamAdmissionState::default();
|
||||
let client_ip = "192.0.2.10".parse().unwrap();
|
||||
let public_addr = "203.0.113.10:443".parse().unwrap();
|
||||
let first = allocate_stream_port(&mut state, client_ip, public_addr).unwrap();
|
||||
|
||||
@@ -82,19 +82,45 @@ impl WebSocketConnection {
|
||||
}
|
||||
|
||||
/// Marks successful ownership transfer from HTTP to the WebSocket codec.
|
||||
pub(crate) fn mark_opened(&self) {
|
||||
self.entry
|
||||
.phase
|
||||
.store(WebSocketPhase::Upgraded as u8, Ordering::Release);
|
||||
pub(crate) fn mark_opened(&self) -> bool {
|
||||
if self.entry.closing.load(Ordering::Acquire)
|
||||
|| self
|
||||
.entry
|
||||
.phase
|
||||
.compare_exchange(
|
||||
WebSocketPhase::Claimed as u8,
|
||||
WebSocketPhase::Upgraded as u8,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_err()
|
||||
|| self.entry.closing.load(Ordering::Acquire)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.mark_progress();
|
||||
true
|
||||
}
|
||||
|
||||
/// Marks the first validated carrier binary message as active progress.
|
||||
pub(crate) fn mark_active(&self) {
|
||||
self.entry
|
||||
.phase
|
||||
.store(WebSocketPhase::Active as u8, Ordering::Release);
|
||||
pub(crate) fn mark_active(&self) -> bool {
|
||||
if self.entry.closing.load(Ordering::Acquire)
|
||||
|| self
|
||||
.entry
|
||||
.phase
|
||||
.compare_exchange(
|
||||
WebSocketPhase::Upgraded as u8,
|
||||
WebSocketPhase::Active as u8,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
)
|
||||
.is_err()
|
||||
|| self.entry.closing.load(Ordering::Acquire)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.mark_peer_activity();
|
||||
true
|
||||
}
|
||||
|
||||
/// Refreshes the peer-liveness deadline after any received WebSocket message.
|
||||
@@ -125,12 +151,16 @@ impl Drop for WebSocketConnection {
|
||||
registry.claims.remove(&self.entry.claim);
|
||||
}
|
||||
if self.entry.closing.load(Ordering::Acquire) {
|
||||
registry.evictions_in_flight = registry.evictions_in_flight.saturating_sub(1);
|
||||
if registry.evictions_in_flight == 0 {
|
||||
registry.closed = true;
|
||||
} else {
|
||||
registry.evictions_in_flight -= 1;
|
||||
}
|
||||
}
|
||||
drop(registry);
|
||||
self.entry.released.cancel();
|
||||
drop(self.base_budget.take());
|
||||
drop(self.slot.take());
|
||||
self.entry.released.cancel();
|
||||
runtime.websocket_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
@@ -149,6 +179,9 @@ pub(super) async fn admit(
|
||||
eviction_timeout: Duration,
|
||||
parent_cancellation: CancellationToken,
|
||||
) -> Result<WebSocketConnection, ManagerError> {
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
|
||||
match try_admit(
|
||||
runtime,
|
||||
@@ -172,7 +205,13 @@ pub(super) async fn admit(
|
||||
};
|
||||
let released = victim.released.cancelled();
|
||||
victim.cancel.cancel();
|
||||
let _ = tokio::time::timeout(eviction_timeout, released).await;
|
||||
tokio::select! {
|
||||
_ = parent_cancellation.cancelled() => return Err(ManagerError::Closed),
|
||||
_ = tokio::time::timeout(eviction_timeout, released) => {}
|
||||
}
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
match try_admit(
|
||||
runtime,
|
||||
owner,
|
||||
@@ -211,6 +250,9 @@ fn try_admit(
|
||||
liveness_interval_ms: u64,
|
||||
parent_cancellation: &CancellationToken,
|
||||
) -> Result<WebSocketConnection, TryAdmitError> {
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
let claim = WebSocketClaimKey { session_hash, kind };
|
||||
{
|
||||
let registry = runtime.websockets.lock();
|
||||
@@ -227,6 +269,9 @@ fn try_admit(
|
||||
let base_budget = runtime
|
||||
.try_websocket_base_budget(owner, base_bytes)
|
||||
.ok_or(TryAdmitError::Capacity)?;
|
||||
if parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
let id = runtime
|
||||
.websocket_next_id
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
|
||||
@@ -251,7 +296,7 @@ fn try_admit(
|
||||
released: CancellationToken::new(),
|
||||
});
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if registry.closed {
|
||||
if registry.closed || parent_cancellation.is_cancelled() {
|
||||
return Err(TryAdmitError::Closed);
|
||||
}
|
||||
if registry.claims.contains_key(&claim) {
|
||||
@@ -276,11 +321,12 @@ impl WebProcessRuntime {
|
||||
pub(super) fn cleanup_websockets(&self) {
|
||||
let now = self.websocket_tick();
|
||||
let mut victims = claim_stale_victims(self, now);
|
||||
if victims.is_empty()
|
||||
&& self.data_budget.take_pressure()
|
||||
&& let Some(victim) = select_pressure_victim(self, now, true)
|
||||
{
|
||||
victims.push(victim);
|
||||
if victims.is_empty() && self.data_budget.take_pressure() {
|
||||
if let Some(victim) = select_pressure_victim(self, now, true) {
|
||||
victims.push(victim);
|
||||
} else {
|
||||
self.data_budget.restore_pressure();
|
||||
}
|
||||
}
|
||||
for victim in victims {
|
||||
victim.cancel.cancel();
|
||||
|
||||
@@ -1,18 +1,29 @@
|
||||
use super::*;
|
||||
|
||||
fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
|
||||
let phase = if opened {
|
||||
WebSocketPhase::Active
|
||||
} else {
|
||||
WebSocketPhase::Claimed
|
||||
};
|
||||
WebSocketEntry {
|
||||
id: 1,
|
||||
owner: [0; 32],
|
||||
session_id: 1,
|
||||
claim: WebSocketClaimKey {
|
||||
session_hash: [0; 32],
|
||||
kind,
|
||||
},
|
||||
client_ip: "192.0.2.10".parse().unwrap(),
|
||||
kind,
|
||||
liveness_interval_ms: 10,
|
||||
created_tick: 1,
|
||||
last_peer_tick: AtomicU64::new(peer_tick),
|
||||
last_progress_tick: AtomicU64::new(peer_tick),
|
||||
opened: AtomicBool::new(opened),
|
||||
phase: AtomicU8::new(phase as u8),
|
||||
closing: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
released: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user