WEB Knobs in API

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-27 21:42:14 +03:00
parent d41a8c3220
commit cf0cd08387
43 changed files with 3399 additions and 196 deletions
+26
View File
@@ -52,10 +52,18 @@ pub(crate) struct WebDataBudgetSnapshot {
pub(crate) queue_bytes: usize,
/// Total queue items currently retained.
pub(crate) queue_items: usize,
/// Control bytes included in the queue total.
pub(crate) queue_control_bytes: usize,
/// Control items included in the queue total.
pub(crate) queue_control_items: usize,
/// Total WebSocket bytes currently retained.
pub(crate) websocket_bytes: usize,
/// Largest combined byte usage observed since process start.
pub(crate) high_water_bytes: usize,
/// Distinct profile owners currently charged.
pub(crate) owners: usize,
/// Whether shutdown closed this allocation authority.
pub(crate) closed: bool,
}
/// Bounded owner-usage view captured before WebSocket registry selection.
@@ -259,11 +267,29 @@ impl WebDataBudget {
WebDataBudgetSnapshot {
queue_bytes: state.queue_bytes,
queue_items: state.queue_items,
queue_control_bytes: state.queue_control_bytes,
queue_control_items: state.queue_control_items,
websocket_bytes: state.websocket_bytes,
high_water_bytes: state.high_water_bytes,
owners: state.owner_bytes.len(),
closed: state.closed,
}
}
pub(super) fn try_snapshot(&self) -> Option<WebDataBudgetSnapshot> {
let state = self.state.try_lock()?;
Some(WebDataBudgetSnapshot {
queue_bytes: state.queue_bytes,
queue_items: state.queue_items,
queue_control_bytes: state.queue_control_bytes,
queue_control_items: state.queue_control_items,
websocket_bytes: state.websocket_bytes,
high_water_bytes: state.high_water_bytes,
owners: state.owner_bytes.len(),
closed: state.closed,
})
}
pub(super) fn close(&self) {
self.state.lock().closed = true;
self.notify.notify_waiters();
+90
View File
@@ -163,6 +163,34 @@ pub(super) struct CarrierLearning {
policy_started_at: Instant,
}
/// Bounded control-plane summary of carrier-learning state.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningStatus {
/// Whether outcome learning is active in the effective policy.
pub(crate) enabled: bool,
/// Effective evidence thresholds.
pub(crate) aggressiveness: WebCarrierNegotiationAggressiveness,
/// Current evidence epoch, or none after counter exhaustion.
pub(crate) epoch: Option<u64>,
/// Retained evidence entries.
pub(crate) entries: usize,
/// Restart-owned evidence ceiling.
pub(crate) capacity: usize,
/// Effective evidence lifetime.
pub(crate) lifetime_secs: u64,
/// Monotonic age of the current policy epoch.
pub(crate) age_ms: u64,
}
/// Result of one epoch-fenced learning reset.
#[derive(Clone, Copy)]
pub(crate) struct CarrierLearningResetOutcome {
/// Evidence entries detached by the reset.
pub(crate) entries_cleared: usize,
/// New epoch fencing pre-reset outcomes.
pub(crate) epoch: u64,
}
impl CarrierLearning {
/// Creates an empty store under the restart-owned capacity ceiling.
pub(super) fn new(capacity: usize) -> Self {
@@ -177,6 +205,23 @@ impl CarrierLearning {
}
}
fn status(&self, now: Instant) -> CarrierLearningStatus {
let policy = self.policy.unwrap_or(LearningPolicy {
enabled: false,
aggressiveness: WebCarrierNegotiationAggressiveness::Conservative,
lifetime: Duration::ZERO,
});
CarrierLearningStatus {
enabled: policy.enabled,
aggressiveness: policy.aggressiveness,
epoch: self.epoch,
entries: self.entries.len(),
capacity: self.capacity,
lifetime_secs: policy.lifetime.as_secs(),
age_ms: millis(now.saturating_duration_since(self.policy_started_at)),
}
}
/// Applies hot-reloaded learning policy and returns its outcome epoch.
pub(super) fn apply_policy(
&mut self,
@@ -424,6 +469,51 @@ impl CarrierLearning {
}
}
impl super::WebProcessRuntime {
/// Captures learning state without waiting for a contended evidence lock.
pub(crate) fn try_carrier_learning_status(&self) -> Option<CarrierLearningStatus> {
self.learning
.try_lock()
.map(|learning| learning.status(Instant::now()))
}
/// Clears all evidence under a new epoch without changing the active policy.
pub(crate) fn reset_carrier_learning(
&self,
) -> Result<CarrierLearningResetOutcome, super::ManagerError> {
let control = self
.control_mutation_guard()
.map_err(|_| super::ManagerError::Closed)?;
let (outcome, retired_entries, retired_order) = {
let mut learning = self.learning.lock();
let epoch = learning
.epoch
.and_then(|epoch| epoch.checked_add(1))
.ok_or(super::ManagerError::Closed)?;
learning.epoch = Some(epoch);
learning.insertion_sequence = 1;
learning.policy_started_at = Instant::now();
let retired_entries = std::mem::take(&mut learning.entries);
let retired_order = std::mem::take(&mut learning.insertion_order);
(
CarrierLearningResetOutcome {
entries_cleared: retired_entries.len(),
epoch,
},
retired_entries,
retired_order,
)
};
drop(control);
drop((retired_entries, retired_order));
Ok(outcome)
}
}
fn millis(duration: Duration) -> u64 {
duration.as_millis().min(u128::from(u64::MAX)) as u64
}
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
configured
.iter()
+49
View File
@@ -220,3 +220,52 @@ fn fifo_metadata_stays_within_the_entry_capacity() {
assert!(learning.insertion_order.len() <= 3);
}
}
#[tokio::test]
async fn explicit_reset_preserves_policy_and_rejects_old_epoch_outcomes() {
let generation = crate::maestro::generation::test_runtime_generation(
1,
crate::config::ProxyConfig::default(),
);
let runtime = crate::web::manager::WebProcessRuntime::start(std::sync::Arc::new(
arc_swap::ArcSwap::from(generation.clone()),
));
let now = Instant::now();
let old_epoch = {
let mut learning = runtime.learning.lock();
let epoch = learning
.apply_policy(
now,
true,
WebCarrierNegotiationAggressiveness::Aggressive,
Duration::from_secs(10),
)
.unwrap();
learning.record_chain(now, epoch, context(7), &[], WebCarrier::Websocket);
epoch
};
let outcome = runtime.reset_carrier_learning().unwrap();
{
let mut learning = runtime.learning.lock();
learning.record_chain(
Instant::now(),
old_epoch,
context(7),
&[],
WebCarrier::Https,
);
let status = learning.status(Instant::now());
assert!(status.enabled);
assert_eq!(
status.aggressiveness,
WebCarrierNegotiationAggressiveness::Aggressive
);
assert_eq!(status.entries, 0);
assert_eq!(status.epoch, Some(outcome.epoch));
}
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
+441
View File
@@ -0,0 +1,441 @@
use std::collections::{BTreeSet, VecDeque};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;
use serde::Serialize;
use super::status::immutable_matches;
use super::{SessionFilter, WebProcessRuntime};
const OPERATION_REF_VERSION: &str = "wo1";
const OPERATION_RETENTION: usize = 32;
const CLOSE_CHUNK: usize = 128;
/// Validated bulk-close selector.
#[derive(Clone)]
pub(crate) enum CloseOperationSelector {
/// Exact logical session references resolved by the API.
Refs(Vec<u64>),
/// Point-in-time sessions matching a bounded filter.
Filter(SessionFilter),
/// Every logical session at or below the submission high-water mark.
All,
}
/// Stable close-operation state.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ControlOperationState {
/// Accepted but not yet executing.
Queued,
/// Scanning the bounded point-in-time registry.
Running,
/// Finished the complete bounded scan.
Completed,
/// Stopped because process shutdown began.
Cancelled,
/// Stopped on one sanitized internal failure.
Failed,
}
/// Retained status for one asynchronous close operation.
#[derive(Clone, Serialize)]
pub(crate) struct ControlOperationStatus {
/// Opaque process-fenced operation reference.
pub(crate) operation_id: String,
/// Current lifecycle state.
pub(crate) state: ControlOperationState,
/// Highest logical session eligible for this point-in-time operation.
pub(crate) high_water_session_ref: Option<String>,
/// Exact submitted reference count, or zero for filter/all selectors.
pub(crate) requested: usize,
/// Registry candidates visited so far.
pub(crate) scanned: usize,
/// Candidates that matched the complete selector.
pub(crate) matched: usize,
/// Matching session incarnations sent a close signal.
pub(crate) close_signalled: usize,
/// Matching candidates replaced or closed before signalling.
pub(crate) conflicted: usize,
/// Wall-clock creation timestamp for operator correlation.
pub(crate) created_epoch_millis: u64,
/// Wall-clock timestamp of the latest status mutation.
pub(crate) updated_epoch_millis: u64,
/// Stable sanitized failure token.
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) failure: Option<&'static str>,
}
/// Runtime control validation failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ControlError {
/// The supplied process-instance fence is stale.
StaleInstance,
/// The selector is empty or exceeds its bound.
InvalidSelector,
/// Close-all was requested before issuance stopped.
IssuanceEnabled,
/// The single operation slot is occupied.
OperationInProgress,
/// The operation reference is not canonical.
InvalidOperation,
/// The canonical operation is outside retained history.
OperationNotFound,
/// Process shutdown has closed the mutation gate.
Closed,
}
/// Single-slot execution gate with bounded terminal history.
#[derive(Default)]
pub(super) struct ControlOperationRegistry {
active: Option<u64>,
retained: VecDeque<(u64, ControlOperationStatus)>,
closed: bool,
}
struct WorkCandidate {
trace_session_id: u64,
session: Arc<crate::web::session::WebSession>,
bootstrap_hash: super::TokenHash,
}
impl WebProcessRuntime {
/// Acquires the process mutation gate without permitting post-shutdown work.
pub(super) fn control_mutation_guard(
&self,
) -> Result<parking_lot::MutexGuard<'_, ControlOperationRegistry>, ControlError> {
let operations = self.control_operations.lock();
if operations.closed || self.shutdown.is_cancelled() {
return Err(ControlError::Closed);
}
Ok(operations)
}
/// Clears debug records under the process mutation gate.
pub(crate) fn clear_debug(&self) -> Result<crate::web::trace::TraceClearOutcome, ControlError> {
let _control = self.control_mutation_guard()?;
Ok(self.trace.clear())
}
/// Starts one bounded point-in-time close sweep.
pub(crate) fn start_close_operation(
self: &Arc<Self>,
runtime_instance: &str,
selector: CloseOperationSelector,
) -> Result<ControlOperationStatus, ControlError> {
if runtime_instance != self.runtime_instance() {
return Err(ControlError::StaleInstance);
}
if matches!(&selector, CloseOperationSelector::Refs(refs) if refs.is_empty() || refs.len() > 200)
|| matches!(&selector, CloseOperationSelector::Filter(filter) if filter.is_empty())
{
return Err(ControlError::InvalidSelector);
}
let generation = self.active_generation();
let web_enabled = generation.config().web.enabled;
let (high_water, issuance_enabled) = {
let mut state = self.state.lock();
state.apply_issuance_policy(generation.id, web_enabled);
(
state.session_index.last_key_value().map(|(id, _)| *id),
state.issuance_enabled,
)
};
if matches!(selector, CloseOperationSelector::All) && issuance_enabled {
return Err(ControlError::IssuanceEnabled);
}
if self.shutdown.is_cancelled() {
return Err(ControlError::Closed);
}
let sequence = self
.next_control_operation_id
.fetch_add(1, Ordering::Relaxed);
let operation_id = self.operation_ref(sequence);
let now = crate::web::trace::store_epoch_millis();
let requested = match &selector {
CloseOperationSelector::Refs(refs) => refs.len(),
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => 0,
};
let status = ControlOperationStatus {
operation_id,
state: ControlOperationState::Queued,
high_water_session_ref: high_water.map(|id| self.session_ref(id)),
requested,
scanned: 0,
matched: 0,
close_signalled: 0,
conflicted: 0,
created_epoch_millis: now,
updated_epoch_millis: now,
failure: None,
};
let tracked = {
let mut operations = self.control_operations.lock();
if operations.closed || self.shutdown.is_cancelled() {
return Err(ControlError::Closed);
}
if operations.active.is_some() {
return Err(ControlError::OperationInProgress);
}
operations.active = Some(sequence);
operations.retained.push_back((sequence, status.clone()));
trim_operations(&mut operations);
let weak = Arc::downgrade(self);
self.tasks.track_future(async move {
if let Some(runtime) = weak.upgrade() {
runtime
.run_close_operation(sequence, high_water, selector)
.await;
}
})
};
drop(tokio::spawn(tracked));
Ok(status)
}
/// Returns one retained operation status under the process-instance fence.
pub(crate) fn control_operation(
&self,
operation_id: &str,
) -> Result<ControlOperationStatus, ControlError> {
let sequence = self.parse_operation_ref(operation_id)?;
self.control_operations
.lock()
.retained
.iter()
.find_map(|(id, status)| (*id == sequence).then(|| status.clone()))
.ok_or(ControlError::OperationNotFound)
}
/// Prevents new control work from racing process-runtime shutdown.
pub(super) fn close_control_submission_gate(&self) {
self.control_operations.lock().closed = true;
}
async fn run_close_operation(
self: &Arc<Self>,
sequence: u64,
high_water: Option<u64>,
selector: CloseOperationSelector,
) {
self.update_operation(sequence, |status| {
status.state = ControlOperationState::Running
});
let refs = match &selector {
CloseOperationSelector::Refs(refs) => {
Some(refs.iter().copied().collect::<BTreeSet<_>>())
}
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => None,
};
let filter = match &selector {
CloseOperationSelector::Filter(filter) => Some(filter),
CloseOperationSelector::Refs(_) | CloseOperationSelector::All => None,
};
let mut cursor = None;
loop {
if self.shutdown.is_cancelled() {
self.finish_operation(sequence, ControlOperationState::Cancelled, None);
return;
}
let mut direct = Vec::new();
let mut state_filtered = Vec::new();
let (scanned, next_cursor, reached_end) = {
let mut state = self.state.lock();
let mut scanned = 0usize;
let mut next_cursor = cursor;
let mut reached_end = true;
let ids = state
.session_index
.range((
cursor.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Excluded),
std::ops::Bound::Unbounded,
))
.take(CLOSE_CHUNK)
.map(|(id, _)| *id)
.collect::<Vec<_>>();
for trace_session_id in ids {
if high_water.is_some_and(|high_water| trace_session_id > high_water) {
break;
}
scanned += 1;
next_cursor = Some(trace_session_id);
let Some(index) = state.session_index.get(&trace_session_id) else {
continue;
};
let selected = refs
.as_ref()
.is_none_or(|refs| refs.contains(&trace_session_id));
let Some(session) = selected
.then(|| state.sessions.get(&index.session_hash).cloned())
.flatten()
else {
continue;
};
if filter.is_some_and(|filter| !immutable_matches(&session, index, filter)) {
continue;
}
let candidate = WorkCandidate {
trace_session_id,
session,
bootstrap_hash: index.bootstrap_hash,
};
if filter.and_then(|filter| filter.state.as_ref()).is_some() {
state_filtered.push(candidate);
} else {
mark_close_locked(&mut state, &candidate);
direct.push(candidate);
}
}
if scanned == CLOSE_CHUNK {
reached_end = false;
}
(scanned, next_cursor, reached_end)
};
self.update_operation(sequence, |status| {
status.scanned = status.scanned.saturating_add(scanned);
status.matched = status.matched.saturating_add(direct.len());
});
for candidate in direct {
candidate.session.close();
self.update_operation(sequence, |status| {
status.close_signalled = status.close_signalled.saturating_add(1)
});
}
if let Some(expected_state) = filter.and_then(|filter| filter.state.as_deref()) {
for candidate in state_filtered {
let matches_state = candidate
.session
.try_status(Instant::now())
.is_some_and(|status| status.state == expected_state);
if !matches_state {
continue;
}
let session = {
let mut state = self.state.lock();
let current = state
.session_index
.get(&candidate.trace_session_id)
.and_then(|index| state.sessions.get(&index.session_hash))
.filter(|current| Arc::ptr_eq(current, &candidate.session))
.cloned();
if current.is_some() {
mark_close_locked(&mut state, &candidate);
}
current
};
if let Some(session) = session {
session.close();
self.update_operation(sequence, |status| {
status.matched = status.matched.saturating_add(1);
status.close_signalled = status.close_signalled.saturating_add(1);
});
} else {
self.update_operation(sequence, |status| {
status.matched = status.matched.saturating_add(1);
status.conflicted = status.conflicted.saturating_add(1);
});
}
}
}
cursor = next_cursor;
if reached_end || cursor.is_none() {
break;
}
tokio::task::yield_now().await;
}
self.finish_operation(sequence, ControlOperationState::Completed, None);
}
fn update_operation(&self, sequence: u64, update: impl FnOnce(&mut ControlOperationStatus)) {
let mut operations = self.control_operations.lock();
if let Some((_, status)) = operations
.retained
.iter_mut()
.find(|(id, _)| *id == sequence)
{
update(status);
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
}
}
fn finish_operation(
&self,
sequence: u64,
state: ControlOperationState,
failure: Option<&'static str>,
) {
let mut operations = self.control_operations.lock();
if let Some((_, status)) = operations
.retained
.iter_mut()
.find(|(id, _)| *id == sequence)
{
status.state = failure.map_or(state, |_| ControlOperationState::Failed);
status.failure = failure;
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
}
if operations.active == Some(sequence) {
operations.active = None;
}
}
fn operation_ref(&self, sequence: u64) -> String {
format!(
"{OPERATION_REF_VERSION}.{}.{sequence:016x}",
self.runtime_instance()
)
}
fn parse_operation_ref(&self, value: &str) -> Result<u64, ControlError> {
let mut parts = value.split('.');
if parts.next() != Some(OPERATION_REF_VERSION) {
return Err(ControlError::InvalidOperation);
}
let instance = parts.next().ok_or(ControlError::InvalidOperation)?;
if instance.len() != 32
|| !instance
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err(ControlError::InvalidOperation);
}
let sequence = parts
.next()
.filter(|value| {
value.len() == 16
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
})
.and_then(|value| u64::from_str_radix(value, 16).ok())
.filter(|value| *value != 0)
.ok_or(ControlError::InvalidOperation)?;
if parts.next().is_some() {
return Err(ControlError::InvalidOperation);
}
if instance != self.runtime_instance() {
return Err(ControlError::StaleInstance);
}
Ok(sequence)
}
}
fn mark_close_locked(state: &mut super::state::ManagerState, candidate: &WorkCandidate) {
if let Some(bootstrap) = state.bootstraps.get_mut(&candidate.bootstrap_hash) {
bootstrap.close_requested = true;
}
}
fn trim_operations(operations: &mut ControlOperationRegistry) {
while operations.retained.len() > OPERATION_RETENTION {
if operations
.retained
.front()
.is_some_and(|(id, _)| Some(*id) == operations.active)
{
break;
}
operations.retained.pop_front();
}
}
+75 -7
View File
@@ -3,6 +3,7 @@ use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
use zeroize::Zeroizing;
use super::state::{
@@ -11,16 +12,50 @@ use super::state::{
};
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
use crate::config::WebRuntimeProfile;
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
impl WebProcessRuntime {
/// Issues a one-use bootstrap credential for an active compatible profile.
#[cfg(test)]
pub(crate) fn issue_bootstrap(
&self,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
) -> std::result::Result<BootstrapResult, ManagerError> {
let generation = self.active_generation();
self.issue_bootstrap_inner(&generation, profile, client_ip, None)
}
/// Issues one bootstrap against the generation that selected the bridge profile.
#[cfg(test)]
pub(crate) fn issue_bootstrap_for_generation(
&self,
generation: &Arc<RuntimeGeneration>,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
) -> std::result::Result<BootstrapResult, ManagerError> {
self.issue_bootstrap_inner(generation, profile, client_ip, None)
}
/// Issues one bridge bootstrap with bounded non-secret request metadata.
pub(crate) fn issue_bootstrap_for_request(
&self,
generation: &Arc<RuntimeGeneration>,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
user_agent: Option<&str>,
) -> std::result::Result<BootstrapResult, ManagerError> {
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent)
}
fn issue_bootstrap_inner(
&self,
generation: &Arc<RuntimeGeneration>,
profile: Arc<WebRuntimeProfile>,
client_ip: IpAddr,
user_agent: Option<&str>,
) -> std::result::Result<BootstrapResult, ManagerError> {
let config = generation.config();
let profile = config
.web
@@ -34,7 +69,9 @@ impl WebProcessRuntime {
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
if state.closed
|| !state.issuance_enabled
|| state
.bootstraps_per_ip
.get(&client_ip)
@@ -57,11 +94,12 @@ impl WebProcessRuntime {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(&generation, &state) else {
let Some((token, hash)) = new_unique_token(generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
return Err(ManagerError::Limit);
};
let trace_session_id = self.trace.next_session_id();
let (user_agent, user_agent_id) = bounded_user_agent(user_agent);
state.bootstraps.insert(
hash,
Bootstrap {
@@ -69,7 +107,10 @@ impl WebProcessRuntime {
issued_at: now,
issuance_ip: client_ip,
profile,
timeouts: config.web.timeouts.clone(),
trace_session_id,
user_agent,
user_agent_id,
body_digest: [0; TOKEN_BYTES],
session_token: Zeroizing::new(String::new()),
session: None,
@@ -110,12 +151,12 @@ impl WebProcessRuntime {
})
}
/// Resolves bootstrap trace identity and the frozen live-session body timeout.
/// Resolves bootstrap trace identity and its issuance-frozen body timeout.
pub(crate) fn bootstrap_trace_identity(
&self,
hash: TokenHash,
host: &str,
) -> Option<(u64, Arc<WebRuntimeProfile>, Option<Duration>)> {
) -> Option<(u64, Arc<WebRuntimeProfile>, Duration)> {
let now = Instant::now();
self.state
.lock()
@@ -126,10 +167,10 @@ impl WebProcessRuntime {
(
entry.trace_session_id,
Arc::clone(&entry.profile),
entry
.session
.as_ref()
.map(|session| Duration::from_secs(session.timeouts().body_secs)),
entry.session.as_ref().map_or_else(
|| Duration::from_secs(entry.timeouts.body_secs),
|session| Duration::from_secs(session.timeouts().body_secs),
),
)
})
}
@@ -185,3 +226,30 @@ impl WebProcessRuntime {
closed.then_some(()).ok_or(ManagerError::Authentication)
}
}
fn bounded_user_agent(value: Option<&str>) -> (Option<Arc<str>>, Option<[u8; 16]>) {
const DISPLAY_BYTES: usize = 256;
const HASH_CONTEXT: &[u8] = b"telemt-web-user-agent-v1\0";
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
return (None, None);
};
let mut digest = Sha256::new();
digest.update(HASH_CONTEXT);
digest.update(value.as_bytes());
let digest = digest.finalize();
let mut id = [0; 16];
id.copy_from_slice(&digest[..16]);
let mut display = String::with_capacity(value.len().min(DISPLAY_BYTES));
for character in value.chars() {
let character = if character.is_control() {
'\u{fffd}'
} else {
character
};
if display.len().saturating_add(character.len_utf8()) > DISPLAY_BYTES {
break;
}
display.push(character);
}
(Some(Arc::from(display)), Some(id))
}
+60 -3
View File
@@ -6,7 +6,8 @@ use tokio::time::Instant as TokioInstant;
use tracing::{info, warn};
use super::state::{
decrement_map, remember_closed_token_locked, remove_bootstrap_locked, remove_expired_locked,
decrement_map, remember_closed_session_locked, remember_closed_token_locked,
remove_bootstrap_locked, remove_expired_locked,
};
use super::{ProfileKey, TokenHash, WebProcessRuntime};
@@ -37,9 +38,9 @@ impl WebProcessRuntime {
closed_token_lifetime: Duration,
) {
let mut state = self.state.lock();
if state.sessions.remove(&hash).is_none() {
let Some(session) = state.sessions.remove(&hash) else {
return;
}
};
decrement_map(&mut state.sessions_per_ip, &client_ip);
decrement_map(&mut state.sessions_per_profile, &profile_key);
remember_closed_token_locked(
@@ -49,6 +50,21 @@ impl WebProcessRuntime {
closed_token_lifetime,
self.limits.max_sessions_global.saturating_mul(16),
);
let trace_session_id = session.trace_session_id();
if state
.session_index
.get(&trace_session_id)
.is_some_and(|index| index.session_hash == hash)
{
state.session_index.remove(&trace_session_id);
remember_closed_session_locked(
&mut state,
trace_session_id,
session.carrier_attempt(),
closed_token_lifetime,
self.limits.max_sessions_global,
);
}
let bootstrap_hashes = state
.bootstraps
.iter()
@@ -70,6 +86,7 @@ impl WebProcessRuntime {
pub(crate) fn begin_shutdown(self: &std::sync::Arc<Self>) -> WebShutdownDrain {
let started = TokioInstant::now();
self.shutdown.cancel();
self.close_control_submission_gate();
self.close_websockets();
self.data_budget.close();
self.http_connections.close();
@@ -141,6 +158,7 @@ impl WebProcessRuntime {
drop(learning);
let (sessions, expired_chains) = {
let mut state = self.state.lock();
state.apply_issuance_policy(generation.id, config.enabled);
let expired = state
.bootstraps
.iter()
@@ -293,6 +311,7 @@ mod tests {
use super::*;
use crate::config::ProxyConfig;
use crate::maestro::generation::test_runtime_generation;
use crate::web::manager::{CloseOperationSelector, ControlError};
struct DropProbe {
polls: Arc<AtomicUsize>,
@@ -375,6 +394,44 @@ mod tests {
generation.stop_background_tasks().await;
}
#[tokio::test]
async fn initial_trace_policy_is_attributed_to_active_generation() {
let (runtime, generation) = runtime();
assert_eq!(runtime.trace().status().policy_generation, generation.id);
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
#[tokio::test]
async fn shutdown_closes_the_control_submission_gate() {
let (runtime, generation) = runtime();
let drain = runtime.begin_shutdown();
assert!(matches!(
runtime.start_close_operation(
runtime.runtime_instance(),
CloseOperationSelector::Refs(vec![1]),
),
Err(ControlError::Closed)
));
assert!(matches!(
runtime.reset_carrier_learning(),
Err(crate::web::manager::ManagerError::Closed)
));
assert!(matches!(runtime.clear_debug(), Err(ControlError::Closed)));
assert_eq!(
drain
.wait_until(TokioInstant::now() + Duration::from_secs(1))
.await,
WebShutdownOutcome::Drained
);
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
#[tokio::test(start_paused = true)]
async fn expired_deadline_still_closes_every_runtime_gate() {
let (runtime, generation) = runtime();
+46 -28
View File
@@ -10,8 +10,8 @@ use zeroize::Zeroizing;
use super::negotiation::carrier_attempt_deadline_index;
use super::session_admission::admit_initial;
use super::state::{
CarrierChainPhase, decrement_map, matching_profile, new_unique_token, profile_key,
remember_closed_token_locked, remove_expired_locked,
CarrierChainPhase, LiveSessionIndex, decrement_map, matching_profile, new_unique_token,
profile_key, remember_closed_token_locked, remove_expired_locked,
};
use super::{
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
@@ -56,6 +56,10 @@ impl WebProcessRuntime {
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
if state.closed || !state.issuance_enabled {
return Err(ManagerError::Closed);
}
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
return Err(ManagerError::Authentication);
};
@@ -201,9 +205,7 @@ impl WebProcessRuntime {
}
let trace_session_id = entry.trace_session_id;
let issued_profile = Arc::clone(&entry.profile);
if state.closed || !config.web.enabled {
return Err(ManagerError::Closed);
}
let issued_timeouts = entry.timeouts.clone();
let profile = config
.web
.runtime
@@ -303,33 +305,39 @@ impl WebProcessRuntime {
learning_context,
carrier_request.is_automatic(),
self.limits.clone(),
config.web.timeouts.clone(),
issued_timeouts.clone(),
);
state.sessions.insert(session_hash, Arc::clone(&session));
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
let entry = state
.bootstraps
.get_mut(&bootstrap_hash)
.ok_or(ManagerError::Authentication)?;
entry.used = true;
entry.body_digest = body_digest;
entry.session_token = Zeroizing::new(session_token.clone());
entry.session = Some(Arc::clone(&session));
entry.carrier_request = Some(carrier_request);
entry.carrier_candidates = candidates.into();
entry.carrier_scores = scores;
entry.carrier_attempt = 1;
entry.carrier_phase = CarrierChainPhase::Provisional;
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
entry.carrier_deadline_at = carrier_deadline_at;
entry.carrier_failures = [None; 3];
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
entry.expires_at = now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs);
entry.session_client_ip = Some(client_ip);
entry.session_ip_learning_eligible = ip_learning_eligible;
let issuance_ip = entry.issuance_ip;
let candidate_count = u8::try_from(entry.carrier_candidates.len()).unwrap_or(4);
let (issuance_ip, candidate_count, user_agent, user_agent_id) = {
let entry = state
.bootstraps
.get_mut(&bootstrap_hash)
.ok_or(ManagerError::Authentication)?;
entry.used = true;
entry.body_digest = body_digest;
entry.session_token = Zeroizing::new(session_token.clone());
entry.session = Some(Arc::clone(&session));
entry.carrier_request = Some(carrier_request);
entry.carrier_candidates = candidates.into();
entry.carrier_scores = scores;
entry.carrier_attempt = 1;
entry.carrier_phase = CarrierChainPhase::Provisional;
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
entry.carrier_deadline_at = carrier_deadline_at;
entry.carrier_failures = [None; 3];
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
entry.expires_at = now + Duration::from_secs(issued_timeouts.bootstrap_lifetime_secs);
entry.session_client_ip = Some(client_ip);
entry.session_ip_learning_eligible = ip_learning_eligible;
(
entry.issuance_ip,
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
entry.user_agent.clone(),
entry.user_agent_id,
)
};
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
let identity = session.trace_identity();
@@ -345,6 +353,16 @@ impl WebProcessRuntime {
.is_automatic()
.then_some(CarrierChainPhase::Provisional.as_str()),
};
state.session_index.insert(
trace_session_id,
LiveSessionIndex {
session_hash,
bootstrap_hash,
attempt: 1,
user_agent,
user_agent_id,
},
);
drop(state);
self.trace.record_carrier_lifecycle(
client_ip,
@@ -21,6 +21,7 @@ impl WebProcessRuntime {
let now = Instant::now();
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
entry.carrier_transitioning
&& entry.carrier_phase == CarrierChainPhase::Provisional
@@ -37,7 +38,7 @@ impl WebProcessRuntime {
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
if !valid
|| state.closed
|| !config.web.enabled
|| !state.issuance_enabled
|| !generation
.proxy_shared
.is_user_enabled(&replacement.profile.user)
@@ -121,6 +122,13 @@ impl WebProcessRuntime {
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
};
if let Some(index) = state.session_index.get_mut(&replacement.trace_session_id)
&& index.session_hash == old_hash
{
index.session_hash = session_hash;
index.bootstrap_hash = bootstrap_hash;
index.attempt = replacement.attempt;
}
let identity = session.trace_identity();
let old_identity = replacement.old_session.trace_identity();
drop(state);
+113 -3
View File
@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -8,7 +8,7 @@ use sha2::{Digest, Sha256};
use zeroize::Zeroizing;
use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile};
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile, WebTimeoutsConfig};
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
@@ -41,8 +41,14 @@ pub(super) struct Bootstrap {
pub(super) issuance_ip: IpAddr,
/// Immutable profile selected during capability validation.
pub(super) profile: Arc<WebRuntimeProfile>,
/// Request and session deadlines frozen with the generated bridge.
pub(super) timeouts: WebTimeoutsConfig,
/// Process-unique non-secret identifier shared by bootstrap and session traces.
pub(super) trace_session_id: u64,
/// Bounded display form of the issuing User-Agent.
pub(super) user_agent: Option<Arc<str>>,
/// Opaque non-secret identifier used for exact User-Agent filtering.
pub(super) user_agent_id: Option<[u8; 16]>,
/// Digest of the accepted HELLO body for idempotent retry matching.
pub(super) body_digest: TokenHash,
/// Zeroizing copy returned only for an exact session-creation retry.
@@ -87,6 +93,28 @@ pub(super) struct ClosedToken {
pub(super) host: String,
}
/// Current logical-session owner stored without exposing bearer credentials.
pub(super) struct LiveSessionIndex {
/// Current bearer hash used only for internal pointer revalidation.
pub(super) session_hash: TokenHash,
/// Bootstrap chain that owns carrier replacement and close intent.
pub(super) bootstrap_hash: TokenHash,
/// Current carrier incarnation in the logical trace session.
pub(super) attempt: u8,
/// Bounded display form of the issuing User-Agent.
pub(super) user_agent: Option<Arc<str>>,
/// Opaque non-secret identifier used for exact User-Agent filtering.
pub(super) user_agent_id: Option<[u8; 16]>,
}
/// Bounded logical-session tombstone used for exact detail semantics.
pub(super) struct ClosedSession {
/// Tombstone expiry deadline.
pub(super) expires_at: Instant,
/// Last carrier incarnation closed for this logical session.
pub(super) attempt: u8,
}
/// Token-bucket state for one process-wide creation class.
#[derive(Default)]
pub(super) struct RateState {
@@ -114,7 +142,6 @@ pub(super) struct StreamAdmissionState {
}
/// Process-wide WEB registries and quota accounting protected by one short lock.
#[derive(Default)]
pub(super) struct ManagerState {
/// Bootstrap credentials indexed by their SHA-256 token hash.
pub(super) bootstraps: HashMap<TokenHash, Bootstrap>,
@@ -122,6 +149,12 @@ pub(super) struct ManagerState {
pub(super) bootstraps_per_ip: HashMap<IpAddr, usize>,
/// Live sessions indexed by bearer-token hash.
pub(super) sessions: HashMap<TokenHash, Arc<WebSession>>,
/// Stable ordered logical-session lookup independent from bearer hashes.
pub(super) session_index: BTreeMap<u64, LiveSessionIndex>,
/// Recently closed logical sessions retained for exact detail responses.
pub(super) closed_sessions: HashMap<u64, ClosedSession>,
/// Insertion order for bounded logical-session tombstones.
pub(super) closed_session_order: VecDeque<u64>,
/// Recently closed token hashes retained for idempotent DELETE semantics.
pub(super) closed_tokens: HashMap<TokenHash, ClosedToken>,
/// Live session counts by forwarded client address.
@@ -132,10 +165,48 @@ pub(super) struct ManagerState {
pub(super) bootstrap_rate: RateState,
/// Session creation rate limiter.
pub(super) session_rate: RateState,
/// Generation-fenced issuance gate mirrored from the effective WEB policy.
pub(super) issuance_enabled: bool,
/// Generation that last authored `issuance_enabled`.
pub(super) issuance_generation: u64,
/// Process shutdown admission latch.
pub(super) closed: bool,
}
impl ManagerState {
pub(super) fn new(issuance_generation: u64, issuance_enabled: bool) -> Self {
Self {
bootstraps: HashMap::new(),
bootstraps_per_ip: HashMap::new(),
sessions: HashMap::new(),
session_index: BTreeMap::new(),
closed_sessions: HashMap::new(),
closed_session_order: VecDeque::new(),
closed_tokens: HashMap::new(),
sessions_per_ip: HashMap::new(),
sessions_per_profile: HashMap::new(),
bootstrap_rate: RateState::default(),
session_rate: RateState::default(),
issuance_enabled,
issuance_generation,
closed: false,
}
}
pub(super) fn apply_issuance_policy(&mut self, generation: u64, enabled: bool) {
if generation >= self.issuance_generation {
self.issuance_generation = generation;
self.issuance_enabled = enabled;
}
}
}
impl Default for ManagerState {
fn default() -> Self {
Self::new(0, false)
}
}
/// Generates one collision-checked credential and its stable hash key.
pub(super) fn new_unique_token(
generation: &RuntimeGeneration,
@@ -241,6 +312,16 @@ pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) {
state
.closed_tokens
.retain(|_, closed| now <= closed.expires_at);
while state
.closed_session_order
.front()
.and_then(|trace_session_id| state.closed_sessions.get(trace_session_id))
.is_some_and(|closed| now > closed.expires_at)
{
if let Some(trace_session_id) = state.closed_session_order.pop_front() {
state.closed_sessions.remove(&trace_session_id);
}
}
}
/// Removes one bootstrap and releases its per-address issuance quota when unused.
@@ -281,6 +362,35 @@ pub(super) fn remember_closed_token_locked(
}
}
/// Retains one bounded logical-session marker without storing its bearer identity.
pub(super) fn remember_closed_session_locked(
state: &mut ManagerState,
trace_session_id: u64,
attempt: u8,
lifetime: Duration,
capacity: usize,
) {
if state
.closed_sessions
.insert(
trace_session_id,
ClosedSession {
expires_at: Instant::now() + lifetime,
attempt,
},
)
.is_none()
{
state.closed_session_order.push_back(trace_session_id);
}
while state.closed_sessions.len() > capacity {
let Some(oldest) = state.closed_session_order.pop_front() else {
break;
};
state.closed_sessions.remove(&oldest);
}
}
/// Decrements one counted owner and removes its map entry at zero.
pub(super) fn decrement_map<K, Q>(values: &mut HashMap<K, usize>, key: &Q)
where
+560
View File
@@ -0,0 +1,560 @@
use std::net::IpAddr;
use std::ops::Bound::{Excluded, Unbounded};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;
use serde::Serialize;
use super::WebProcessRuntime;
use crate::config::{
WebCarrier, WebCarrierNegotiationAggressiveness, WebDebugConfig, WebLimitsConfig,
};
use crate::web::session::WebSessionStatus;
const SESSION_REF_VERSION: &str = "ws1";
const MAX_SESSION_SCAN: usize = 1000;
/// Current usage of one process-owned semaphore.
#[derive(Clone, Serialize)]
struct PermitStatus {
used: usize,
available: usize,
capacity: usize,
closed: bool,
}
/// Short-lock manager registry counts.
#[derive(Clone, Serialize)]
struct ManagerStatus {
issuance_enabled: bool,
issuance_generation: u64,
shutdown: bool,
bootstraps: usize,
sessions: usize,
closed_tokens: usize,
closed_sessions: usize,
client_ips: usize,
profiles: usize,
}
/// Logical-stream admission counters.
#[derive(Clone, Serialize)]
struct StreamStatus {
live: usize,
profiles: usize,
closed: bool,
}
/// Shared queue and WebSocket byte-budget counters.
#[derive(Clone, Serialize)]
struct BudgetStatus {
queue_bytes: usize,
queue_items: usize,
control_bytes: usize,
control_items: usize,
websocket_bytes: usize,
high_water_bytes: usize,
owners: usize,
closed: bool,
}
/// Process WebSocket registry counters.
#[derive(Clone, Serialize)]
struct WebSocketStatus {
entries: usize,
claims: usize,
evictions_in_flight: usize,
closed: bool,
}
/// Process-local carrier-learning summary.
#[derive(Clone, Serialize)]
struct LearningStatus {
enabled: bool,
aggressiveness: WebCarrierNegotiationAggressiveness,
epoch: Option<u64>,
entries: usize,
capacity: usize,
lifetime_secs: u64,
age_ms: u64,
}
/// Effective trace policy and bounded ring counters.
#[derive(Clone, Serialize)]
struct DebugStatus {
policy: WebDebugConfig,
policy_generation: u64,
epoch: u64,
records: usize,
records_capacity: usize,
used_bytes: usize,
bytes_capacity: usize,
contention_drops: u64,
evictions: u64,
byte_truncations: u64,
earliest_seq: Option<u64>,
latest_seq: Option<u64>,
}
/// One non-blocking multi-plane WEB runtime snapshot.
#[derive(Clone, Serialize)]
pub(crate) struct WebRuntimeStatus {
runtime_instance: String,
generation_id: u64,
limits: WebLimitsConfig,
manager: Option<ManagerStatus>,
streams: Option<StreamStatus>,
budget: Option<BudgetStatus>,
websockets: Option<WebSocketStatus>,
learning: Option<LearningStatus>,
debug: Option<DebugStatus>,
permits: Vec<(&'static str, PermitStatus)>,
auxiliary_tasks: usize,
session_incarnations_created: u64,
session_incarnations_closed: u64,
streams_opened: u64,
streams_rejected: u64,
bytes_up: u64,
bytes_down: u64,
limit_hits: u64,
partial: Vec<&'static str>,
}
/// Strict bounded filters for session enumeration and bulk close.
#[derive(Clone, Default)]
pub(crate) struct SessionFilter {
/// Exact process-local logical session identifier.
pub(crate) trace_session_id: Option<u64>,
/// Exact forwarded client address.
pub(crate) client_ip: Option<IpAddr>,
/// Exact canonical virtual host.
pub(crate) host: Option<String>,
/// Exact configured user label.
pub(crate) user: Option<String>,
/// Exact non-secret User-Agent identifier.
pub(crate) user_agent_id: Option<[u8; 16]>,
/// Exact non-secret profile-key fingerprint.
pub(crate) key_id: Option<String>,
/// Exact current carrier.
pub(crate) carrier: Option<WebCarrier>,
/// Exact point-in-time lifecycle token.
pub(crate) state: Option<String>,
}
impl SessionFilter {
/// Returns whether the selector would match every live session.
pub(crate) fn is_empty(&self) -> bool {
self.trace_session_id.is_none()
&& self.client_ip.is_none()
&& self.host.is_none()
&& self.user.is_none()
&& self.user_agent_id.is_none()
&& self.key_id.is_none()
&& self.carrier.is_none()
&& self.state.is_none()
}
}
/// Validated bounded list request.
pub(crate) struct SessionListRequest {
/// Maximum returned rows.
pub(crate) limit: usize,
/// Exclusive ordered logical-session cursor.
pub(crate) cursor: Option<u64>,
/// Exact bounded filters.
pub(crate) filter: SessionFilter,
}
/// One API-safe session row with optional User-Agent metadata.
#[derive(Clone, Serialize)]
pub(crate) struct SessionRow {
/// Opaque process-fenced logical-session reference.
pub(crate) session_ref: String,
/// Bounded sanitized User-Agent display value.
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) user_agent: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
user_agent_id: Option<String>,
#[serde(flatten)]
status: WebSessionStatus,
}
/// Bounded session page and continuation metadata.
#[derive(Clone, Serialize)]
pub(crate) struct SessionPage {
/// Ordered live-session rows captured without blocking.
pub(crate) sessions: Vec<SessionRow>,
next_cursor: Option<String>,
scanned: usize,
scan_truncated: bool,
partial_sessions: usize,
partial: Vec<&'static str>,
}
/// Exact detail lookup outcome.
pub(crate) enum SessionDetail {
/// One exact live-session snapshot.
Active(Box<SessionRow>),
/// One bounded retained closed-session tombstone.
Gone { attempt: u8 },
/// A required short lock was contended.
Busy,
/// Neither a live session nor a retained tombstone exists.
NotFound,
}
struct Candidate {
trace_session_id: u64,
session: Arc<crate::web::session::WebSession>,
user_agent: Option<Arc<str>>,
user_agent_id: Option<[u8; 16]>,
}
impl WebProcessRuntime {
/// Captures every independent plane without blocking on a contended lock.
pub(crate) fn try_status(&self) -> WebRuntimeStatus {
let generation_id = self.active_generation().id;
let mut partial = Vec::new();
let manager = self.state.try_lock().map(|state| ManagerStatus {
issuance_enabled: state.issuance_enabled,
issuance_generation: state.issuance_generation,
shutdown: state.closed,
bootstraps: state.bootstraps.len(),
sessions: state.sessions.len(),
closed_tokens: state.closed_tokens.len(),
closed_sessions: state.closed_sessions.len(),
client_ips: state.sessions_per_ip.len(),
profiles: state.sessions_per_profile.len(),
});
if manager.is_none() {
partial.push("manager");
}
let streams = self.stream_admission.try_lock().map(|state| StreamStatus {
live: state.streams_live,
profiles: state.streams_per_profile.len(),
closed: state.closed,
});
if streams.is_none() {
partial.push("streams");
}
let budget = self.data_budget.try_snapshot().map(|status| BudgetStatus {
queue_bytes: status.queue_bytes,
queue_items: status.queue_items,
control_bytes: status.queue_control_bytes,
control_items: status.queue_control_items,
websocket_bytes: status.websocket_bytes,
high_water_bytes: status.high_water_bytes,
owners: status.owners,
closed: status.closed,
});
if budget.is_none() {
partial.push("budget");
}
let websockets = self.websockets.try_lock().map(|registry| {
let status = registry.status();
WebSocketStatus {
entries: status.entries,
claims: status.claims,
evictions_in_flight: status.evictions_in_flight,
closed: status.closed,
}
});
if websockets.is_none() {
partial.push("websockets");
}
let learning = self
.try_carrier_learning_status()
.map(|status| LearningStatus {
enabled: status.enabled,
aggressiveness: status.aggressiveness,
epoch: status.epoch,
entries: status.entries,
capacity: status.capacity,
lifetime_secs: status.lifetime_secs,
age_ms: status.age_ms,
});
if learning.is_none() {
partial.push("learning");
}
let debug = self.trace.try_status().map(|status| DebugStatus {
policy: status.policy.as_ref().clone(),
policy_generation: status.policy_generation,
epoch: status.epoch,
records: status.records,
records_capacity: status.records_capacity,
used_bytes: status.used_bytes,
bytes_capacity: status.bytes_capacity,
contention_drops: status.contention_drops,
evictions: status.evictions,
byte_truncations: status.byte_truncations,
earliest_seq: status.earliest_seq,
latest_seq: status.latest_seq,
});
if debug.is_none() {
partial.push("debug");
}
let websocket_capacity = self
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
WebRuntimeStatus {
runtime_instance: self.runtime_instance().to_string(),
generation_id,
limits: self.limits.clone(),
manager,
streams,
budget,
websockets,
learning,
debug,
permits: vec![
(
"http_connections",
permits(&self.http_connections, self.limits.max_http_connections),
),
(
"http_handlers",
permits(&self.http_handlers, self.limits.max_http_handlers),
),
(
"lane_polls",
permits(&self.lane_polls, self.limits.max_http_handlers / 2),
),
(
"lane_aux_polls",
permits(
&self.lane_aux_polls,
(self.limits.max_http_handlers / 4).max(1),
),
),
(
"body_readers",
permits(&self.body_readers, self.limits.max_body_readers),
),
(
"body_bytes",
permits(&self.body_bytes, self.limits.max_body_bytes_global),
),
(
"stream_handshakes",
permits(&self.stream_handshakes, self.limits.max_stream_handshakes),
),
(
"websocket_connections",
permits(&self.websocket_connections, websocket_capacity),
),
],
auxiliary_tasks: self.tasks.len(),
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
partial,
}
}
/// Formats an opaque process-fenced session reference.
pub(crate) fn session_ref(&self, trace_session_id: u64) -> String {
format!(
"{SESSION_REF_VERSION}.{}.{trace_session_id:016x}",
self.runtime_instance()
)
}
/// Parses an exact reference and distinguishes stale process instances.
pub(crate) fn parse_session_ref(&self, value: &str) -> Result<u64, SessionRefError> {
let mut parts = value.split('.');
let version = parts.next();
let instance = parts.next();
let id = parts.next();
if version != Some(SESSION_REF_VERSION) || parts.next().is_some() {
return Err(SessionRefError::Invalid);
}
if !instance.is_some_and(|instance| {
instance.len() == 32
&& instance
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}) {
return Err(SessionRefError::Invalid);
}
let id = id
.filter(|id| {
id.len() == 16
&& id
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
})
.and_then(|id| u64::from_str_radix(id, 16).ok())
.filter(|id| *id != 0)
.ok_or(SessionRefError::Invalid)?;
if instance != Some(self.runtime_instance()) {
return Err(SessionRefError::StaleInstance);
}
Ok(id)
}
/// Lists live sessions under an ordered bounded scan.
pub(crate) fn list_sessions(&self, request: SessionListRequest) -> SessionPage {
let Some(state) = self.state.try_lock() else {
return SessionPage {
sessions: Vec::new(),
next_cursor: request.cursor.map(|id| self.session_ref(id)),
scanned: 0,
scan_truncated: false,
partial_sessions: 0,
partial: vec!["manager"],
};
};
let mut candidates = Vec::with_capacity(request.limit);
let mut scanned = 0usize;
let mut last_scanned = request.cursor;
for (&trace_session_id, index) in state
.session_index
.range((request.cursor.map_or(Unbounded, Excluded), Unbounded))
{
if scanned >= MAX_SESSION_SCAN || candidates.len() >= request.limit {
break;
}
scanned += 1;
last_scanned = Some(trace_session_id);
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
continue;
};
if !immutable_matches(&session, index, &request.filter) {
continue;
}
candidates.push(Candidate {
trace_session_id,
session,
user_agent: index.user_agent.clone(),
user_agent_id: index.user_agent_id,
});
}
drop(state);
let mut rows = Vec::with_capacity(candidates.len());
let mut partial_sessions = 0usize;
let now = Instant::now();
for candidate in candidates {
let Some(status) = candidate.session.try_status(now) else {
partial_sessions += 1;
continue;
};
if request
.filter
.state
.as_deref()
.is_some_and(|expected| expected != status.state)
{
continue;
}
rows.push(self.row(candidate, status));
}
let scan_truncated = scanned >= MAX_SESSION_SCAN;
SessionPage {
sessions: rows,
next_cursor: (scan_truncated || scanned >= request.limit)
.then(|| last_scanned.map(|id| self.session_ref(id)))
.flatten(),
scanned,
scan_truncated,
partial_sessions,
partial: Vec::new(),
}
}
/// Resolves one active or recently closed logical session.
pub(crate) fn session_detail(&self, trace_session_id: u64) -> SessionDetail {
let Some(state) = self.state.try_lock() else {
return SessionDetail::Busy;
};
if let Some(index) = state.session_index.get(&trace_session_id) {
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
return SessionDetail::Busy;
};
let candidate = Candidate {
trace_session_id,
session,
user_agent: index.user_agent.clone(),
user_agent_id: index.user_agent_id,
};
drop(state);
return candidate
.session
.try_status(Instant::now())
.map(|status| SessionDetail::Active(Box::new(self.row(candidate, status))))
.unwrap_or(SessionDetail::Busy);
}
let closed = state
.closed_sessions
.get(&trace_session_id)
.map(|closed| closed.attempt);
closed.map_or(SessionDetail::NotFound, |attempt| SessionDetail::Gone {
attempt,
})
}
fn row(&self, candidate: Candidate, status: WebSessionStatus) -> SessionRow {
SessionRow {
session_ref: self.session_ref(candidate.trace_session_id),
user_agent: candidate.user_agent.map(|value| value.to_string()),
user_agent_id: candidate.user_agent_id.map(hex::encode),
status,
}
}
}
/// Opaque session-reference validation failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SessionRefError {
/// The reference does not use the canonical versioned shape.
Invalid,
/// The reference belongs to another process runtime.
StaleInstance,
}
/// Tests immutable candidate fields before any optional state-lock read.
pub(super) fn immutable_matches(
session: &crate::web::session::WebSession,
index: &super::state::LiveSessionIndex,
filter: &SessionFilter,
) -> bool {
filter
.trace_session_id
.is_none_or(|value| value == session.trace_session_id())
&& filter
.client_ip
.is_none_or(|value| value == session.client_ip())
&& filter
.host
.as_deref()
.is_none_or(|value| value == session.profile_host())
&& filter
.user
.as_deref()
.is_none_or(|value| value == session.profile_user())
&& filter
.key_id
.as_deref()
.is_none_or(|value| session.key_id() == value)
&& filter
.carrier
.is_none_or(|value| value == session.carrier())
&& filter
.user_agent_id
.is_none_or(|value| index.user_agent_id == Some(value))
}
fn permits(semaphore: &Arc<tokio::sync::Semaphore>, capacity: usize) -> PermitStatus {
let available = semaphore.available_permits().min(capacity);
PermitStatus {
used: capacity.saturating_sub(available),
available,
capacity,
closed: semaphore.is_closed(),
}
}
+20
View File
@@ -61,6 +61,26 @@ pub(super) struct WebSocketRegistry {
closed: bool,
}
/// Point-in-time WebSocket registry counters.
#[derive(Clone, Copy)]
pub(super) struct WebSocketRegistryStatus {
pub(super) entries: usize,
pub(super) claims: usize,
pub(super) evictions_in_flight: usize,
pub(super) closed: bool,
}
impl WebSocketRegistry {
pub(super) fn status(&self) -> WebSocketRegistryStatus {
WebSocketRegistryStatus {
entries: self.entries.len(),
claims: self.claims.len(),
evictions_in_flight: self.evictions_in_flight,
closed: self.closed,
}
}
}
/// Exact process-owned admission retained through the upgraded socket lifetime.
pub(crate) struct WebSocketConnection {
runtime: std::sync::Weak<WebProcessRuntime>,