astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-10-31 04:50:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

API for WEB: bounded lifecycle and overload observability added

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-29 16:19:21 +03:00
parent 012dc07a98
commit 084834f5ec
45 changed files with 2440 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/manager/admission.rs
+24 -15
View File
@@ -1,9 +1,9 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{allocate_stream_port, allow_rate, decrement_map, release_stream_port};
use super::{ProfileKey, WebProcessRuntime};
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Reserves one process-wide and per-profile live logical-stream slot.
@@ -17,14 +17,16 @@ impl WebProcessRuntime {
let _operator_admission = match self.try_operator_admission() {
Ok(admission) => admission,
Err(error) => {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
return Err(error);
}
};
let now = Instant::now();
let mut state = self.stream_admission.lock();
if state.closed {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(super::ManagerError::Closed);
}
if state.streams_live >= self.limits.max_streams_global
@@ -34,25 +36,26 @@ impl WebProcessRuntime {
.copied()
.unwrap_or(0)
>= max_streams
|| !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
)
{
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamCapacity);
return Err(super::ManagerError::Limit);
}
if !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
) {
self.record_stream_rejected_reason(WebRejectionReason::StreamRate);
return Err(super::ManagerError::Limit);
}
let Some(peer_port) = allocate_stream_port(&mut state, client_ip, public_addr) else {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamTupleExhausted);
return Err(super::ManagerError::Limit);
};
state.streams_live += 1;
*state.streams_per_profile.entry(profile_key).or_insert(0) += 1;
self.streams_opened.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_opened();
Ok(peer_port)
}
@@ -76,9 +79,15 @@ impl WebProcessRuntime {
/// Records a logical stream rejected outside manager quota acquisition.
pub(crate) fn record_stream_rejected(&self) {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.record_limit_hit();
}
/// Records one logical stream rejection with its operational cause.
pub(crate) fn record_stream_rejected_reason(&self, reason: WebRejectionReason) {
self.record_stream_rejected();
self.telemetry.record_rejection(reason);
}
}
#[cfg(test)]
src/web/manager/credentials.rs
+41 -19
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -14,6 +13,7 @@ use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRun
use crate::config::WebRuntimeProfile;
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Issues a one-use bootstrap credential for an active compatible profile.
@@ -63,7 +63,14 @@ impl WebProcessRuntime {
.as_ref()
.and_then(|runtime| matching_profile(runtime, &profile))
.ok_or(ManagerError::Authentication)?;
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
if !config.web.enabled {
self.telemetry
.record_rejection(WebRejectionReason::ConfigDisabled);
return Err(ManagerError::Closed);
}
if !generation.proxy_shared.is_user_enabled(&profile.user) {
self.telemetry
.record_rejection(WebRejectionReason::UserDisabled);
return Err(ManagerError::Closed);
}
let _operator_admission = self.try_operator_admission()?;
@@ -71,32 +78,47 @@ impl WebProcessRuntime {
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
if state.closed
|| !state.issuance_enabled
|| state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
|| !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
)
if state.closed || !state.issuance_enabled {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Limit);
}
if state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
if !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
) {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapRate);
return Err(ManagerError::Limit);
}
if state.bootstraps.len() >= self.limits.max_bootstraps_global
&& !evict_oldest_unused_bootstrap(&mut state)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
};
let trace_session_id = self.trace.next_session_id();
src/web/manager/lifecycle.rs
+17 -16
View File
@@ -1,5 +1,4 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use tokio::time::Instant as TokioInstant;
@@ -79,7 +78,7 @@ impl WebProcessRuntime {
for bootstrap_hash in bootstrap_hashes {
remove_bootstrap_locked(&mut state, bootstrap_hash);
}
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_closed();
drop(state);
self.notify_operator_work_changed();
}
@@ -93,6 +92,7 @@ impl WebProcessRuntime {
self.close_websockets();
self.data_budget.close();
self.http_connections.close();
self.http_overload_connections.close();
self.http_handlers.close();
self.lane_polls.close();
self.lane_aux_polls.close();
@@ -234,28 +234,29 @@ impl WebShutdownDrain {
.saturating_duration_since(self.started)
.as_millis()
.min(u128::from(u64::MAX)) as u64;
let aggregates = self.runtime.telemetry.aggregates();
match outcome {
WebShutdownOutcome::Drained => info!(
target: "telemt::web",
shutdown_drained = true,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime stopped"
),
WebShutdownOutcome::DeadlineExceeded => warn!(
@@ -263,22 +264,22 @@ impl WebShutdownDrain {
shutdown_drained = false,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime shutdown deadline exceeded"
),
}
src/web/manager/observability.rs
+187
View File
@@ -0,0 +1,187 @@
use serde::Serialize;
use super::WebProcessRuntime;
/// Current usage of one fixed process-wide WEB resource.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacityResourceStatus {
/// Stable closed-set resource token.
pub(crate) resource: &'static str,
/// Stable unit used by API consumers and metric-family routing.
pub(crate) unit: &'static str,
/// Currently retained capacity.
pub(crate) used: usize,
/// Unretained portion of the immutable ceiling before class-specific reserves.
pub(crate) available: usize,
/// Immutable process-wide ceiling.
pub(crate) limit: usize,
/// Whether terminal shutdown closed this allocation authority.
pub(crate) closed: bool,
}
/// Non-blocking fixed-cardinality capacity snapshot.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacitySnapshot {
/// Exact fixed process-wide resources available without dynamic labels.
pub(crate) resources: Vec<WebCapacityResourceStatus>,
/// Resources with no immediately available capacity.
pub(crate) saturated_resources: Vec<&'static str>,
/// Snapshot planes omitted because their short lock was contended.
pub(crate) partial: Vec<&'static str>,
}
impl WebProcessRuntime {
/// Captures fixed global resource usage without blocking the data plane.
pub(crate) fn capacity_snapshot(&self) -> WebCapacitySnapshot {
let websocket_capacity = self
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let mut resources = vec![
semaphore_status(
"http_connections",
"slots",
&self.http_connections,
self.limits.max_http_connections,
),
semaphore_status(
"http_overload_connections",
"slots",
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
semaphore_status(
"http_handlers",
"slots",
&self.http_handlers,
self.limits.max_http_handlers,
),
semaphore_status(
"lane_polls",
"slots",
&self.lane_polls,
self.limits.max_http_handlers / 2,
),
semaphore_status(
"lane_aux_polls",
"slots",
&self.lane_aux_polls,
(self.limits.max_http_handlers / 4).max(1),
),
semaphore_status(
"body_readers",
"slots",
&self.body_readers,
self.limits.max_body_readers,
),
semaphore_status(
"body_bytes",
"bytes",
&self.body_bytes,
self.limits.max_body_bytes_global,
),
semaphore_status(
"stream_handshakes",
"slots",
&self.stream_handshakes,
self.limits.max_stream_handshakes,
),
semaphore_status(
"websocket_connections",
"slots",
&self.websocket_connections,
websocket_capacity,
),
];
let mut partial = Vec::new();
if let Some(budget) = self.data_budget.try_snapshot() {
resources.extend([
bounded_status(
"pending_bytes",
"bytes",
budget.queue_bytes.saturating_add(budget.websocket_bytes),
self.limits.pending_bytes_global,
budget.closed,
),
bounded_status(
"queue_items",
"items",
budget.queue_items,
self.limits.pending_items_global,
budget.closed,
),
bounded_status(
"websocket_bytes",
"bytes",
budget.websocket_bytes,
self.limits.websocket_bytes_global,
budget.closed,
),
]);
} else {
partial.push("budget");
}
let saturated_resources = resources
.iter()
.filter(|status| status.available == 0 && !status.closed)
.map(|status| status.resource)
.collect();
WebCapacitySnapshot {
resources,
saturated_resources,
partial,
}
}
}
fn semaphore_status(
resource: &'static str,
unit: &'static str,
semaphore: &tokio::sync::Semaphore,
limit: usize,
) -> WebCapacityResourceStatus {
let available = semaphore.available_permits().min(limit);
WebCapacityResourceStatus {
resource,
unit,
used: limit.saturating_sub(available),
available,
limit,
closed: semaphore.is_closed(),
}
}
fn bounded_status(
resource: &'static str,
unit: &'static str,
used: usize,
limit: usize,
closed: bool,
) -> WebCapacityResourceStatus {
let used = used.min(limit);
WebCapacityResourceStatus {
resource,
unit,
used,
available: limit.saturating_sub(used),
limit,
closed,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use tokio::sync::Semaphore;
#[test]
fn semaphore_status_reports_usage_and_available_capacity() {
let semaphore = Arc::new(Semaphore::new(3));
let _permit = semaphore.clone().try_acquire_owned().unwrap();
let status = super::semaphore_status("test", "slots", &semaphore, 3);
assert_eq!(status.used, 1);
assert_eq!(status.available, 2);
assert_eq!(status.limit, 3);
}
}
src/web/manager/operator_lifecycle.rs
+38 -46
View File
@@ -16,6 +16,9 @@ pub(crate) use status::{
OperatorDrainOutcome, OperatorDrainState, OperatorDrainStatus, OperatorLifecycleState,
OperatorLifecycleStatus,
};
// Mutable and published lifecycle state storage.
mod state;
use state::{ActiveDrain, OperatorLifecycleInner, OperatorSnapshot, WorkCounts};
const OPERATOR_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
const OPERATOR_REGISTRATION_COUNT: usize = OPERATOR_ADMISSION_CLOSED - 1;
@@ -30,19 +33,6 @@ pub(crate) enum OperatorLifecycleError {
OperationInProgress,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct WorkCounts {
sessions: usize,
streams: usize,
websockets: usize,
}
impl WorkCounts {
fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
struct OperatorAdmission {
state: AtomicUsize,
registrations_drained: Notify,
@@ -114,29 +104,6 @@ impl Drop for OperatorRegistration<'_> {
}
}
struct ActiveDrain {
sequence: u64,
cancellation: CancellationToken,
}
struct OperatorLifecycleInner {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
active: Option<ActiveDrain>,
drain: Option<OperatorDrainStatus>,
}
#[derive(Clone)]
struct OperatorSnapshot {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
drain: Option<OperatorDrainStatus>,
}
pub(super) struct OperatorLifecycle {
runtime_instance: Arc<str>,
admission: OperatorAdmission,
@@ -187,8 +154,8 @@ impl OperatorLifecycle {
pub(super) fn status(&self, config_enabled: bool) -> OperatorLifecycleStatus {
let snapshot = self.published.load();
let admission_open = !snapshot.terminal
&& snapshot.state == OperatorLifecycleState::Running;
let admission_open =
!snapshot.terminal && snapshot.state == OperatorLifecycleState::Running;
OperatorLifecycleStatus {
state: snapshot.state,
epoch: snapshot.epoch,
@@ -203,6 +170,30 @@ impl OperatorLifecycle {
self.published.load().terminal
}
fn rejection_reason(&self) -> crate::web::telemetry::WebRejectionReason {
let inner = self.inner.lock();
if inner.terminal {
return crate::web::telemetry::WebRejectionReason::RuntimeClosed;
}
match inner.state {
OperatorLifecycleState::Paused => {
crate::web::telemetry::WebRejectionReason::OperatorPaused
}
OperatorLifecycleState::Draining => {
crate::web::telemetry::WebRejectionReason::OperatorDraining
}
OperatorLifecycleState::ForceClosing => {
crate::web::telemetry::WebRejectionReason::OperatorForceClosing
}
OperatorLifecycleState::Drained => {
crate::web::telemetry::WebRejectionReason::OperatorDrained
}
OperatorLifecycleState::Running => {
crate::web::telemetry::WebRejectionReason::RuntimeClosed
}
}
}
fn publish_locked(&self, inner: &OperatorLifecycleInner) {
self.published.store(Arc::new(OperatorSnapshot {
state: inner.state,
@@ -213,11 +204,7 @@ impl OperatorLifecycle {
}));
}
fn transition_locked(
&self,
inner: &mut OperatorLifecycleInner,
state: OperatorLifecycleState,
) {
fn transition_locked(&self, inner: &mut OperatorLifecycleInner, state: OperatorLifecycleState) {
if inner.state != state {
inner.state = state;
inner.epoch = inner.epoch.saturating_add(1);
@@ -460,9 +447,14 @@ impl WebProcessRuntime {
pub(super) fn try_operator_admission(
&self,
) -> Result<OperatorRegistration<'_>, super::ManagerError> {
self.operator_lifecycle
.try_register()
.ok_or(super::ManagerError::AdmissionPaused)
match self.operator_lifecycle.try_register() {
Some(registration) => Ok(registration),
None => {
self.telemetry
.record_rejection(self.operator_lifecycle.rejection_reason());
Err(super::ManagerError::AdmissionPaused)
}
}
}
pub(super) fn notify_operator_work_changed(&self) {
src/web/manager/operator_lifecycle/state.rs
+62
View File
@@ -0,0 +1,62 @@
use std::time::Instant;
use tokio_util::sync::CancellationToken;
use super::{OperatorDrainStatus, OperatorLifecycleState};
/// Exact live-work counts sampled while an operator drain is active.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) struct WorkCounts {
/// Live WEB sessions.
pub(super) sessions: usize,
/// Live logical streams.
pub(super) streams: usize,
/// Live session-owned WebSockets.
pub(super) websockets: usize,
}
impl WorkCounts {
/// Returns whether every tracked work class reached zero.
pub(super) fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
/// Single active drain identity and cancellation authority.
pub(super) struct ActiveDrain {
/// Process-local operation sequence.
pub(super) sequence: u64,
/// Cancellation token owned by resume or terminal shutdown.
pub(super) cancellation: CancellationToken,
}
/// Mutex-protected lifecycle state used for atomic transitions.
pub(super) struct OperatorLifecycleInner {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Currently active drain, if any.
pub(super) active: Option<ActiveDrain>,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
/// Lock-free read snapshot published after lifecycle mutations.
#[derive(Clone)]
pub(super) struct OperatorSnapshot {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
src/web/manager/operator_lifecycle/tests.rs
+4 -1
View File
@@ -106,7 +106,10 @@ async fn pause_fence_leaves_no_late_admission_commits_under_scheduler_pressure()
#[tokio::test]
async fn empty_drain_completes_gracefully_and_stays_closed_until_resume() {
let (runtime, generation) = test_runtime();
let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
let accepted = runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
assert_eq!(accepted.state, OperatorLifecycleState::Draining);
let completed = tokio::time::timeout(Duration::from_secs(1), async {
src/web/manager/session_admission.rs
+25 -15
View File
@@ -1,10 +1,10 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{ManagerState, allow_rate};
use super::{ProfileKey, WebProcessRuntime};
use crate::config::WebRuntimeProfile;
use crate::web::telemetry::WebRejectionReason;
/// Applies process, address, profile, and rate ceilings to one initial session.
pub(super) fn admit_initial(
@@ -15,23 +15,33 @@ pub(super) fn admit_initial(
profile_key: ProfileKey,
profile: &WebRuntimeProfile,
) -> bool {
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
< runtime.limits.max_sessions_per_ip
&& state
if state.sessions.len() >= runtime.limits.max_sessions_global
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
>= runtime.limits.max_sessions_per_ip
|| state
.sessions_per_profile
.get(&profile_key)
.copied()
.unwrap_or(0)
< profile.max_sessions
&& allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
);
if !admitted {
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
>= profile.max_sessions
{
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionCapacity);
return false;
}
admitted
if !allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
) {
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionRate);
return false;
}
true
}
src/web/manager/session_creation.rs
+4 -3
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -278,7 +277,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Limit);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
return Err(ManagerError::Limit);
};
let carrier_deadline_at = carrier_request
@@ -341,7 +342,7 @@ impl WebProcessRuntime {
)
};
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
let identity = session.trace_identity();
let result = CreateResult {
token: session_token,
src/web/manager/session_creation/replacement.rs
+5 -3
View File
@@ -48,7 +48,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Closed);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
drop(state);
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
return Err(ManagerError::Limit);
@@ -112,8 +114,8 @@ impl WebProcessRuntime {
{
*slot = Some(replacement.old_session.carrier());
}
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
self.telemetry.record_session_closed();
let result = CreateResult {
token: session_token,
carrier: replacement.carrier,
src/web/manager/status.rs
+15 -8
View File
@@ -1,7 +1,6 @@
use std::net::IpAddr;
use std::ops::Bound::{Excluded, Unbounded};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;
use serde::Serialize;
@@ -298,6 +297,7 @@ impl WebProcessRuntime {
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let aggregates = self.telemetry.aggregates();
WebRuntimeStatus {
runtime_instance: self.runtime_instance().to_string(),
generation_id,
@@ -313,6 +313,13 @@ impl WebProcessRuntime {
"http_connections",
permits(&self.http_connections, self.limits.max_http_connections),
),
(
"http_overload_connections",
permits(
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
),
(
"http_handlers",
permits(&self.http_handlers, self.limits.max_http_handlers),
@@ -346,13 +353,13 @@ impl WebProcessRuntime {
),
],
auxiliary_tasks: self.tasks.len(),
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
session_incarnations_created: aggregates.sessions_created,
session_incarnations_closed: aggregates.sessions_closed,
streams_opened: aggregates.streams_opened,
streams_rejected: aggregates.streams_rejected,
bytes_up: aggregates.bytes_up,
bytes_down: aggregates.bytes_down,
limit_hits: aggregates.limit_hits,
partial,
}
}
src/web/manager/websocket.rs
+40 -14
View File
@@ -8,6 +8,7 @@ use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
use crate::web::telemetry::WebRejectionReason;
// Deterministic victim ordering remains isolated from registry mutation.
mod policy;
@@ -208,7 +209,7 @@ pub(super) async fn admit(
return Err(ManagerError::Closed);
}
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
match try_admit(
let capacity_reason = match try_admit(
runtime,
owner,
session_id,
@@ -220,12 +221,23 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => return Ok(connection),
Err(TryAdmitError::Conflict) => return Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => return Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {}
}
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
return Err(ManagerError::Concurrent);
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Closed);
}
Err(TryAdmitError::Capacity(reason)) => reason,
};
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None, true) else {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(capacity_reason);
return Err(ManagerError::Limit);
};
let released = victim.released.cancelled();
@@ -249,17 +261,28 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => Ok(connection),
Err(TryAdmitError::Conflict) => Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
Err(ManagerError::Concurrent)
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
Err(ManagerError::Closed)
}
Err(TryAdmitError::Capacity(reason)) => {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(reason);
Err(ManagerError::Limit)
}
}
}
enum TryAdmitError {
Capacity,
Capacity(WebRejectionReason),
Conflict,
Closed,
}
@@ -292,10 +315,13 @@ fn try_admit(
}
let slot = Arc::clone(&runtime.websocket_connections)
.try_acquire_owned()
.map_err(|_| TryAdmitError::Capacity)?;
let base_budget = runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let base_budget =
runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity(
WebRejectionReason::WebSocketBytesCapacity,
))?;
if parent_cancellation.is_cancelled() {
return Err(TryAdmitError::Closed);
}
@@ -304,7 +330,7 @@ fn try_admit(
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_add(1)
})
.map_err(|_| TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let now = runtime.websocket_tick();
let entry = Arc::new(WebSocketEntry {
id,