astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-09-14 06:24:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

API for WEB: bounded lifecycle and overload observability added

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-29 16:19:21 +03:00
parent 012dc07a98
commit 084834f5ec
45 changed files with 2440 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/control.rs
+23
View File
@@ -7,6 +7,7 @@ use std::time::Instant;
use tokio::sync::watch;
use super::manager::WebProcessRuntime;
use super::telemetry::WebTelemetry;
/// Process-owned WEB ingress lifecycle state.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -26,6 +27,16 @@ pub(crate) enum WebRuntimeLifecycle {
}
impl WebRuntimeLifecycle {
/// Complete fixed lifecycle set used by one-hot metrics.
pub(crate) const ALL: [Self; 6] = [
Self::Starting,
Self::NoWebListener,
Self::Running,
Self::Draining,
Self::Drained,
Self::DeadlineExceeded,
];
/// Returns the stable API token for this lifecycle state.
pub(crate) const fn as_str(self) -> &'static str {
match self {
@@ -52,28 +63,34 @@ pub(crate) struct WebRuntimePublication {
pub(crate) listeners: Arc<[SocketAddr]>,
/// Weak runtime access that never extends data-plane ownership.
pub(crate) runtime: Weak<WebProcessRuntime>,
/// Process-owned counters that remain readable after runtime release.
pub(crate) telemetry: Arc<WebTelemetry>,
}
/// Single-writer process lifecycle publisher for WEB ingress.
#[derive(Clone)]
pub(crate) struct WebRuntimeControl {
epoch: Arc<AtomicU64>,
telemetry: Arc<WebTelemetry>,
tx: watch::Sender<WebRuntimePublication>,
}
impl WebRuntimeControl {
/// Creates the process channel in the pre-listener `starting` state.
pub(crate) fn new() -> Self {
let telemetry = WebTelemetry::new();
let publication = WebRuntimePublication {
epoch: 1,
lifecycle: WebRuntimeLifecycle::Starting,
since: Instant::now(),
listeners: Arc::from([]),
runtime: Weak::new(),
telemetry: Arc::clone(&telemetry),
};
let (tx, _rx) = watch::channel(publication);
Self {
epoch: Arc::new(AtomicU64::new(1)),
telemetry,
tx,
}
}
@@ -83,6 +100,11 @@ impl WebRuntimeControl {
self.tx.subscribe()
}
/// Returns the shared process-owned WEB telemetry handle.
pub(crate) fn telemetry(&self) -> Arc<WebTelemetry> {
Arc::clone(&self.telemetry)
}
/// Publishes one lifecycle transition and optional weak runtime reference.
pub(crate) fn publish(
&self,
@@ -97,6 +119,7 @@ impl WebRuntimeControl {
since: Instant::now(),
listeners,
runtime,
telemetry: Arc::clone(&self.telemetry),
});
}
}
src/web/http/decoy.rs
+60 -9
View File
@@ -15,6 +15,7 @@ use super::{
};
use crate::config::{WebRuntimeDecoy, WebRuntimeVhost};
use crate::web::manager::WebProcessRuntime;
use crate::web::telemetry::WebDecoyUpstreamOutcome;
/// Serves the configured ordinary site after optionally removing carrier material.
/// Transport-sanitized static fallbacks remain uncacheable after query removal.
@@ -195,16 +196,18 @@ async fn proxy_to_upstream(
.map(|value| value.as_str())
.unwrap_or("/");
let Ok(uri) = path_and_query.parse::<Uri>() else {
return bad_gateway();
return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
};
*request.uri_mut() = uri;
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => stream,
_ => return bad_gateway(),
let stream = match connect_upstream(addr, header_timeout).await {
Ok(stream) => stream,
Err(outcome) => return decoy_failure(runtime, outcome),
};
drop(_deadline_lease);
let max_header_bytes = runtime
@@ -217,12 +220,19 @@ async fn proxy_to_upstream(
builder.max_buf_size(max_header_bytes);
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let (mut sender, connection) =
match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
Ok(Ok(parts)) => parts,
_ => return bad_gateway(),
Ok(Err(_)) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeError);
}
Err(_) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeTimeout);
}
};
drop(_deadline_lease);
runtime.spawn_auxiliary(async move {
@@ -230,14 +240,24 @@ async fn proxy_to_upstream(
});
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let mut response =
match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
Ok(Ok(response)) => response,
_ => return bad_gateway(),
Ok(Err(_)) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
}
Err(_) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::ResponseHeadTimeout);
}
};
drop(_deadline_lease);
runtime
.telemetry()
.record_decoy(WebDecoyUpstreamOutcome::Success);
remove_hop_by_hop(response.headers_mut());
response.map(|body| {
body.map_err(|error| -> BoxError { Box::new(error) })
@@ -245,6 +265,25 @@ async fn proxy_to_upstream(
})
}
async fn connect_upstream(
addr: SocketAddr,
timeout: Duration,
) -> Result<TcpStream, WebDecoyUpstreamOutcome> {
match tokio::time::timeout(timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(error)) if error.kind() == std::io::ErrorKind::ConnectionRefused => {
Err(WebDecoyUpstreamOutcome::ConnectRefused)
}
Ok(Err(_)) => Err(WebDecoyUpstreamOutcome::ConnectError),
Err(_) => Err(WebDecoyUpstreamOutcome::ConnectTimeout),
}
}
fn decoy_failure(runtime: &WebProcessRuntime, outcome: WebDecoyUpstreamOutcome) -> HttpResponse {
runtime.telemetry().record_decoy(outcome);
bad_gateway()
}
fn lease_deadline(
deadline: Option<&super::activity::RequestDeadlineHandle>,
timeout: Duration,
@@ -315,4 +354,16 @@ mod tests {
assert!(resolve_static_path("/../index.html", &site).is_none());
assert!(resolve_static_path("//index.html", &site).is_none());
}
#[tokio::test]
async fn closed_loopback_origin_is_classified_as_connect_refused() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
drop(listener);
assert_eq!(
connect_upstream(addr, Duration::from_secs(1)).await.err(),
Some(WebDecoyUpstreamOutcome::ConnectRefused)
);
}
}
src/web/http/operator_lifecycle_tests.rs
+29 -28
View File
@@ -46,10 +46,7 @@ async fn live_runtime() -> (
Arc<crate::maestro::generation::RuntimeGeneration>,
TcpListener,
) {
let generation = test_runtime_generation(
1,
runtime_config([71; 32], WebCarrier::Https),
);
let generation = test_runtime_generation(1, runtime_config([71; 32], WebCarrier::Https));
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
(runtime, generation, listener)
@@ -100,12 +97,7 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
let hello = frame::encode(FrameType::Hello, 0, &[1]);
runtime.pause_operator().await.unwrap();
let paused_create = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let paused_create = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (paused_headers, _) = split_response(&paused_create);
assert!(paused_headers.starts_with(b"HTTP/1.1 503"));
assert_eq!(response_header(paused_headers, "retry-after"), "1");
@@ -131,26 +123,20 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
let decoy = request(&listener, &runtime, decoy).await;
let (decoy_headers, decoy_body) = split_response(&decoy);
assert!(decoy_headers.starts_with(b"HTTP/1.1 200"));
assert!(!decoy_body.windows(11).any(|window| window == b"bootstrap=\""));
assert!(
!decoy_body
.windows(11)
.any(|window| window == b"bootstrap=\"")
);
runtime.resume_operator().await.unwrap();
let created = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let created = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (created_headers, _) = split_response(&created);
assert!(created_headers.starts_with(b"HTTP/1.1 200"));
let token = response_header(created_headers, "x-session-token").to_string();
runtime.pause_operator().await.unwrap();
let replay = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let replay = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (replay_headers, _) = split_response(&replay);
assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
assert_eq!(response_header(replay_headers, "x-session-token"), token);
@@ -230,7 +216,10 @@ async fn paused_replacement_preserves_old_session_and_attempt_replay() {
let replay = request(&listener, &runtime, first_request).await;
let (replay_headers, _) = split_response(&replay);
assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
assert_eq!(response_header(replay_headers, "x-session-token"), first_token);
assert_eq!(
response_header(replay_headers, "x-session-token"),
first_token
);
runtime.resume_operator().await.unwrap();
let replacement = request(&listener, &runtime, replacement_request).await;
@@ -249,7 +238,10 @@ async fn client_delete_during_drain_completes_naturally() {
let (runtime, generation, listener) = live_runtime().await;
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
let delete = format!(
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
@@ -281,8 +273,14 @@ async fn deadline_force_closes_all_sessions_and_requires_explicit_resume() {
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
assert_eq!(serde_json::to_value(&accepted).unwrap()["state"], "draining");
let accepted = runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
assert_eq!(
serde_json::to_value(&accepted).unwrap()["state"],
"draining"
);
assert!(matches!(
runtime.drain_operator(Duration::from_secs(30)).await,
Err(OperatorLifecycleError::OperationInProgress)
@@ -323,7 +321,10 @@ async fn resume_before_deadline_cancels_drain_without_closing_existing_session()
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
let still_draining = runtime.pause_operator().await.unwrap();
assert_eq!(
serde_json::to_value(&still_draining).unwrap()["state"],
src/web/http/session.rs
+3 -3
View File
@@ -190,9 +190,9 @@ pub(super) async fn handle_session(
}
Err(
error @ (ManagerError::Limit
| ManagerError::Backpressure
| ManagerError::Concurrent
| ManagerError::AdmissionPaused),
| ManagerError::Backpressure
| ManagerError::Concurrent
| ManagerError::AdmissionPaused),
) => {
runtime.trace().record_profile_lifecycle(
client_ip,
src/web/http/websocket/driver/io.rs
+10 -2
View File
@@ -46,7 +46,7 @@ pub(super) async fn reserve_data(
cancellation: &CancellationToken,
timeout: Duration,
) -> Result<WebSocketBudgetLease, ()> {
tokio::time::timeout(timeout, async {
match tokio::time::timeout(timeout, async {
loop {
if cancellation.is_cancelled() {
return Err(());
@@ -63,7 +63,15 @@ pub(super) async fn reserve_data(
}
})
.await
.map_err(|_| ())?
{
Ok(result) => result,
Err(_) => {
runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::WebSocketBytesCapacity,
);
Err(())
}
}
}
pub(super) async fn process_multiplex(
src/web/manager.rs
+61 -23
View File
@@ -1,7 +1,7 @@
use std::future::Future;
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::AtomicU64;
use std::time::Duration;
use arc_swap::ArcSwap;
@@ -12,6 +12,7 @@ use tokio_util::task::TaskTracker;
use crate::config::{WebCarrier, WebLimitsConfig};
use crate::maestro::generation::RuntimeGeneration;
use crate::web::telemetry::{WebRejectionReason, WebTelemetry};
use crate::web::trace::WebTraceStore;
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
@@ -37,7 +38,7 @@ pub(crate) use lifecycle::WebShutdownOutcome;
// Reversible operator admission stays independent from terminal process shutdown.
mod operator_lifecycle;
pub(crate) use operator_lifecycle::{
OperatorLifecycleError, OperatorLifecycleStatus,
OperatorLifecycleError, OperatorLifecycleState, OperatorLifecycleStatus,
};
// Queue and WebSocket allocations share one process-owned data-plane budget.
mod budget;
@@ -48,6 +49,9 @@ mod status;
pub(crate) use status::{
SessionDetail, SessionFilter, SessionListRequest, SessionRefError, WebRuntimeStatus,
};
// Fixed-cardinality capacity snapshots serve API and Prometheus observability.
mod observability;
pub(crate) use observability::{WebCapacityResourceStatus, WebCapacitySnapshot};
// Asynchronous bounded close operations isolate mutation lifecycle from HTTP requests.
mod control;
pub(crate) use budget::WebSocketBudgetLease;
@@ -152,6 +156,7 @@ pub(crate) struct WebProcessRuntime {
stream_admission: Mutex<StreamAdmissionState>,
learning: Mutex<learning::CarrierLearning>,
http_connections: Arc<Semaphore>,
http_overload_connections: Arc<Semaphore>,
http_handlers: Arc<Semaphore>,
lane_polls: Arc<Semaphore>,
lane_aux_polls: Arc<Semaphore>,
@@ -169,13 +174,7 @@ pub(crate) struct WebProcessRuntime {
next_control_operation_id: AtomicU64,
shutdown: CancellationToken,
tasks: TaskTracker,
sessions_created: AtomicU64,
sessions_closed: AtomicU64,
streams_opened: AtomicU64,
streams_rejected: AtomicU64,
bytes_up: AtomicU64,
bytes_down: AtomicU64,
limit_hits: AtomicU64,
telemetry: Arc<WebTelemetry>,
}
impl WebProcessRuntime {
@@ -184,13 +183,14 @@ impl WebProcessRuntime {
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
let config = active_runtime.load().config();
let trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
Self::start_with_trace(active_runtime, trace)
Self::start_with_trace(active_runtime, trace, WebTelemetry::new())
}
/// Starts one process-scoped manager with a shared API-visible trace store.
pub(crate) fn start_with_trace(
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
trace: Arc<WebTraceStore>,
telemetry: Arc<WebTelemetry>,
) -> Arc<Self> {
let initial_generation = active_runtime.load_full();
let config = initial_generation.config();
@@ -209,8 +209,7 @@ impl WebProcessRuntime {
.saturating_sub(limits.websocket_http_connection_reserve);
let lane_poll_limit = limits.max_http_handlers / 2;
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
let runtime_instance: Arc<str> =
Arc::from(format!("{:032x}", rand::random::<u128>()));
let runtime_instance: Arc<str> = Arc::from(format!("{:032x}", rand::random::<u128>()));
let runtime = Arc::new(Self {
operator_lifecycle: operator_lifecycle::OperatorLifecycle::new(Arc::clone(
&runtime_instance,
@@ -219,6 +218,9 @@ impl WebProcessRuntime {
active_runtime,
trace,
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
http_overload_connections: Arc::new(Semaphore::new(
limits.max_http_overload_connections,
)),
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
lane_polls: Arc::new(Semaphore::new(lane_poll_limit)),
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
@@ -239,13 +241,7 @@ impl WebProcessRuntime {
learning: Mutex::new(carrier_learning),
shutdown: CancellationToken::new(),
tasks: TaskTracker::new(),
sessions_created: AtomicU64::new(0),
sessions_closed: AtomicU64::new(0),
streams_opened: AtomicU64::new(0),
streams_rejected: AtomicU64::new(0),
bytes_up: AtomicU64::new(0),
bytes_down: AtomicU64::new(0),
limit_hits: AtomicU64::new(0),
telemetry,
});
let weak = Arc::downgrade(&runtime);
let shutdown = runtime.shutdown.clone();
@@ -287,6 +283,16 @@ impl WebProcessRuntime {
&self.trace
}
/// Returns the process-owned operational telemetry handle.
pub(crate) fn telemetry(&self) -> &Arc<WebTelemetry> {
&self.telemetry
}
/// Returns whether terminal process shutdown has started.
pub(crate) fn is_shutdown(&self) -> bool {
self.shutdown.is_cancelled()
}
/// Reserves one accepted HTTP connection.
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
@@ -296,11 +302,28 @@ impl WebProcessRuntime {
permit
}
/// Waits for one accepted HTTP connection slot after bounded overload admission.
pub(crate) async fn acquire_http_connection(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.http_connections)
.acquire_owned()
.await
.ok()
}
/// Reserves one accepted socket outside ordinary HTTP connection capacity.
pub(crate) fn try_http_overload_connection(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.http_overload_connections)
.try_acquire_owned()
.ok()
}
/// Reserves one concurrently executing HTTP request handler.
pub(crate) fn try_http_handler(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.http_handlers).try_acquire_owned().ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::HttpHandlerCapacity);
}
permit
}
@@ -315,6 +338,11 @@ impl WebProcessRuntime {
let permit = Arc::clone(slots).try_acquire_owned().ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry.record_rejection(if auxiliary {
WebRejectionReason::LaneAuxPollCapacity
} else {
WebRejectionReason::LanePollCapacity
});
}
permit
}
@@ -323,7 +351,7 @@ impl WebProcessRuntime {
pub(crate) fn try_stream_handshake(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.stream_handshakes).try_acquire_owned().ok();
if permit.is_none() {
self.record_stream_rejected();
self.record_stream_rejected_reason(WebRejectionReason::StreamHandshakeCapacity);
}
permit
}
@@ -359,10 +387,14 @@ impl WebProcessRuntime {
) -> Option<(OwnedSemaphorePermit, OwnedSemaphorePermit)> {
let Some(bytes) = u32::try_from(bytes).ok() else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
return None;
};
let Some(reader) = Arc::clone(&self.body_readers).try_acquire_owned().ok() else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyReaderCapacity);
return None;
};
let Some(body) = Arc::clone(&self.body_bytes)
@@ -370,6 +402,8 @@ impl WebProcessRuntime {
.ok()
else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
return None;
};
Some((reader, body))
@@ -383,6 +417,8 @@ impl WebProcessRuntime {
.ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
}
permit
}
@@ -401,6 +437,8 @@ impl WebProcessRuntime {
.try_reserve_queue(owner, bytes, items, control, downlink)
{
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::QueueGlobalCapacity);
return false;
}
true
@@ -473,15 +511,15 @@ impl WebProcessRuntime {
/// Accounts one successfully committed carrier uplink body.
pub(crate) fn record_up(&self, bytes: usize) {
self.bytes_up.fetch_add(bytes as u64, Ordering::Relaxed);
self.telemetry.record_up(bytes);
}
/// Accounts one emitted carrier downlink body.
pub(crate) fn record_down(&self, bytes: usize) {
self.bytes_down.fetch_add(bytes as u64, Ordering::Relaxed);
self.telemetry.record_down(bytes);
}
fn record_limit_hit(&self) {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_limit_hit();
}
}
src/web/manager/admission.rs
+24 -15
View File
@@ -1,9 +1,9 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{allocate_stream_port, allow_rate, decrement_map, release_stream_port};
use super::{ProfileKey, WebProcessRuntime};
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Reserves one process-wide and per-profile live logical-stream slot.
@@ -17,14 +17,16 @@ impl WebProcessRuntime {
let _operator_admission = match self.try_operator_admission() {
Ok(admission) => admission,
Err(error) => {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
return Err(error);
}
};
let now = Instant::now();
let mut state = self.stream_admission.lock();
if state.closed {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(super::ManagerError::Closed);
}
if state.streams_live >= self.limits.max_streams_global
@@ -34,25 +36,26 @@ impl WebProcessRuntime {
.copied()
.unwrap_or(0)
>= max_streams
|| !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
)
{
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamCapacity);
return Err(super::ManagerError::Limit);
}
if !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
) {
self.record_stream_rejected_reason(WebRejectionReason::StreamRate);
return Err(super::ManagerError::Limit);
}
let Some(peer_port) = allocate_stream_port(&mut state, client_ip, public_addr) else {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamTupleExhausted);
return Err(super::ManagerError::Limit);
};
state.streams_live += 1;
*state.streams_per_profile.entry(profile_key).or_insert(0) += 1;
self.streams_opened.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_opened();
Ok(peer_port)
}
@@ -76,9 +79,15 @@ impl WebProcessRuntime {
/// Records a logical stream rejected outside manager quota acquisition.
pub(crate) fn record_stream_rejected(&self) {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.record_limit_hit();
}
/// Records one logical stream rejection with its operational cause.
pub(crate) fn record_stream_rejected_reason(&self, reason: WebRejectionReason) {
self.record_stream_rejected();
self.telemetry.record_rejection(reason);
}
}
#[cfg(test)]
src/web/manager/credentials.rs
+41 -19
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -14,6 +13,7 @@ use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRun
use crate::config::WebRuntimeProfile;
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Issues a one-use bootstrap credential for an active compatible profile.
@@ -63,7 +63,14 @@ impl WebProcessRuntime {
.as_ref()
.and_then(|runtime| matching_profile(runtime, &profile))
.ok_or(ManagerError::Authentication)?;
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
if !config.web.enabled {
self.telemetry
.record_rejection(WebRejectionReason::ConfigDisabled);
return Err(ManagerError::Closed);
}
if !generation.proxy_shared.is_user_enabled(&profile.user) {
self.telemetry
.record_rejection(WebRejectionReason::UserDisabled);
return Err(ManagerError::Closed);
}
let _operator_admission = self.try_operator_admission()?;
@@ -71,32 +78,47 @@ impl WebProcessRuntime {
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
if state.closed
|| !state.issuance_enabled
|| state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
|| !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
)
if state.closed || !state.issuance_enabled {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Limit);
}
if state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
if !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
) {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapRate);
return Err(ManagerError::Limit);
}
if state.bootstraps.len() >= self.limits.max_bootstraps_global
&& !evict_oldest_unused_bootstrap(&mut state)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
};
let trace_session_id = self.trace.next_session_id();
src/web/manager/lifecycle.rs
+17 -16
View File
@@ -1,5 +1,4 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use tokio::time::Instant as TokioInstant;
@@ -79,7 +78,7 @@ impl WebProcessRuntime {
for bootstrap_hash in bootstrap_hashes {
remove_bootstrap_locked(&mut state, bootstrap_hash);
}
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_closed();
drop(state);
self.notify_operator_work_changed();
}
@@ -93,6 +92,7 @@ impl WebProcessRuntime {
self.close_websockets();
self.data_budget.close();
self.http_connections.close();
self.http_overload_connections.close();
self.http_handlers.close();
self.lane_polls.close();
self.lane_aux_polls.close();
@@ -234,28 +234,29 @@ impl WebShutdownDrain {
.saturating_duration_since(self.started)
.as_millis()
.min(u128::from(u64::MAX)) as u64;
let aggregates = self.runtime.telemetry.aggregates();
match outcome {
WebShutdownOutcome::Drained => info!(
target: "telemt::web",
shutdown_drained = true,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime stopped"
),
WebShutdownOutcome::DeadlineExceeded => warn!(
@@ -263,22 +264,22 @@ impl WebShutdownDrain {
shutdown_drained = false,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime shutdown deadline exceeded"
),
}
src/web/manager/observability.rs
+187
View File
@@ -0,0 +1,187 @@
use serde::Serialize;
use super::WebProcessRuntime;
/// Current usage of one fixed process-wide WEB resource.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacityResourceStatus {
/// Stable closed-set resource token.
pub(crate) resource: &'static str,
/// Stable unit used by API consumers and metric-family routing.
pub(crate) unit: &'static str,
/// Currently retained capacity.
pub(crate) used: usize,
/// Unretained portion of the immutable ceiling before class-specific reserves.
pub(crate) available: usize,
/// Immutable process-wide ceiling.
pub(crate) limit: usize,
/// Whether terminal shutdown closed this allocation authority.
pub(crate) closed: bool,
}
/// Non-blocking fixed-cardinality capacity snapshot.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacitySnapshot {
/// Exact fixed process-wide resources available without dynamic labels.
pub(crate) resources: Vec<WebCapacityResourceStatus>,
/// Resources with no immediately available capacity.
pub(crate) saturated_resources: Vec<&'static str>,
/// Snapshot planes omitted because their short lock was contended.
pub(crate) partial: Vec<&'static str>,
}
impl WebProcessRuntime {
/// Captures fixed global resource usage without blocking the data plane.
pub(crate) fn capacity_snapshot(&self) -> WebCapacitySnapshot {
let websocket_capacity = self
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let mut resources = vec![
semaphore_status(
"http_connections",
"slots",
&self.http_connections,
self.limits.max_http_connections,
),
semaphore_status(
"http_overload_connections",
"slots",
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
semaphore_status(
"http_handlers",
"slots",
&self.http_handlers,
self.limits.max_http_handlers,
),
semaphore_status(
"lane_polls",
"slots",
&self.lane_polls,
self.limits.max_http_handlers / 2,
),
semaphore_status(
"lane_aux_polls",
"slots",
&self.lane_aux_polls,
(self.limits.max_http_handlers / 4).max(1),
),
semaphore_status(
"body_readers",
"slots",
&self.body_readers,
self.limits.max_body_readers,
),
semaphore_status(
"body_bytes",
"bytes",
&self.body_bytes,
self.limits.max_body_bytes_global,
),
semaphore_status(
"stream_handshakes",
"slots",
&self.stream_handshakes,
self.limits.max_stream_handshakes,
),
semaphore_status(
"websocket_connections",
"slots",
&self.websocket_connections,
websocket_capacity,
),
];
let mut partial = Vec::new();
if let Some(budget) = self.data_budget.try_snapshot() {
resources.extend([
bounded_status(
"pending_bytes",
"bytes",
budget.queue_bytes.saturating_add(budget.websocket_bytes),
self.limits.pending_bytes_global,
budget.closed,
),
bounded_status(
"queue_items",
"items",
budget.queue_items,
self.limits.pending_items_global,
budget.closed,
),
bounded_status(
"websocket_bytes",
"bytes",
budget.websocket_bytes,
self.limits.websocket_bytes_global,
budget.closed,
),
]);
} else {
partial.push("budget");
}
let saturated_resources = resources
.iter()
.filter(|status| status.available == 0 && !status.closed)
.map(|status| status.resource)
.collect();
WebCapacitySnapshot {
resources,
saturated_resources,
partial,
}
}
}
fn semaphore_status(
resource: &'static str,
unit: &'static str,
semaphore: &tokio::sync::Semaphore,
limit: usize,
) -> WebCapacityResourceStatus {
let available = semaphore.available_permits().min(limit);
WebCapacityResourceStatus {
resource,
unit,
used: limit.saturating_sub(available),
available,
limit,
closed: semaphore.is_closed(),
}
}
fn bounded_status(
resource: &'static str,
unit: &'static str,
used: usize,
limit: usize,
closed: bool,
) -> WebCapacityResourceStatus {
let used = used.min(limit);
WebCapacityResourceStatus {
resource,
unit,
used,
available: limit.saturating_sub(used),
limit,
closed,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use tokio::sync::Semaphore;
#[test]
fn semaphore_status_reports_usage_and_available_capacity() {
let semaphore = Arc::new(Semaphore::new(3));
let _permit = semaphore.clone().try_acquire_owned().unwrap();
let status = super::semaphore_status("test", "slots", &semaphore, 3);
assert_eq!(status.used, 1);
assert_eq!(status.available, 2);
assert_eq!(status.limit, 3);
}
}
src/web/manager/operator_lifecycle.rs
+38 -46
View File
@@ -16,6 +16,9 @@ pub(crate) use status::{
OperatorDrainOutcome, OperatorDrainState, OperatorDrainStatus, OperatorLifecycleState,
OperatorLifecycleStatus,
};
// Mutable and published lifecycle state storage.
mod state;
use state::{ActiveDrain, OperatorLifecycleInner, OperatorSnapshot, WorkCounts};
const OPERATOR_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
const OPERATOR_REGISTRATION_COUNT: usize = OPERATOR_ADMISSION_CLOSED - 1;
@@ -30,19 +33,6 @@ pub(crate) enum OperatorLifecycleError {
OperationInProgress,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct WorkCounts {
sessions: usize,
streams: usize,
websockets: usize,
}
impl WorkCounts {
fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
struct OperatorAdmission {
state: AtomicUsize,
registrations_drained: Notify,
@@ -114,29 +104,6 @@ impl Drop for OperatorRegistration<'_> {
}
}
struct ActiveDrain {
sequence: u64,
cancellation: CancellationToken,
}
struct OperatorLifecycleInner {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
active: Option<ActiveDrain>,
drain: Option<OperatorDrainStatus>,
}
#[derive(Clone)]
struct OperatorSnapshot {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
drain: Option<OperatorDrainStatus>,
}
pub(super) struct OperatorLifecycle {
runtime_instance: Arc<str>,
admission: OperatorAdmission,
@@ -187,8 +154,8 @@ impl OperatorLifecycle {
pub(super) fn status(&self, config_enabled: bool) -> OperatorLifecycleStatus {
let snapshot = self.published.load();
let admission_open = !snapshot.terminal
&& snapshot.state == OperatorLifecycleState::Running;
let admission_open =
!snapshot.terminal && snapshot.state == OperatorLifecycleState::Running;
OperatorLifecycleStatus {
state: snapshot.state,
epoch: snapshot.epoch,
@@ -203,6 +170,30 @@ impl OperatorLifecycle {
self.published.load().terminal
}
fn rejection_reason(&self) -> crate::web::telemetry::WebRejectionReason {
let inner = self.inner.lock();
if inner.terminal {
return crate::web::telemetry::WebRejectionReason::RuntimeClosed;
}
match inner.state {
OperatorLifecycleState::Paused => {
crate::web::telemetry::WebRejectionReason::OperatorPaused
}
OperatorLifecycleState::Draining => {
crate::web::telemetry::WebRejectionReason::OperatorDraining
}
OperatorLifecycleState::ForceClosing => {
crate::web::telemetry::WebRejectionReason::OperatorForceClosing
}
OperatorLifecycleState::Drained => {
crate::web::telemetry::WebRejectionReason::OperatorDrained
}
OperatorLifecycleState::Running => {
crate::web::telemetry::WebRejectionReason::RuntimeClosed
}
}
}
fn publish_locked(&self, inner: &OperatorLifecycleInner) {
self.published.store(Arc::new(OperatorSnapshot {
state: inner.state,
@@ -213,11 +204,7 @@ impl OperatorLifecycle {
}));
}
fn transition_locked(
&self,
inner: &mut OperatorLifecycleInner,
state: OperatorLifecycleState,
) {
fn transition_locked(&self, inner: &mut OperatorLifecycleInner, state: OperatorLifecycleState) {
if inner.state != state {
inner.state = state;
inner.epoch = inner.epoch.saturating_add(1);
@@ -460,9 +447,14 @@ impl WebProcessRuntime {
pub(super) fn try_operator_admission(
&self,
) -> Result<OperatorRegistration<'_>, super::ManagerError> {
self.operator_lifecycle
.try_register()
.ok_or(super::ManagerError::AdmissionPaused)
match self.operator_lifecycle.try_register() {
Some(registration) => Ok(registration),
None => {
self.telemetry
.record_rejection(self.operator_lifecycle.rejection_reason());
Err(super::ManagerError::AdmissionPaused)
}
}
}
pub(super) fn notify_operator_work_changed(&self) {
src/web/manager/operator_lifecycle/state.rs
+62
View File
@@ -0,0 +1,62 @@
use std::time::Instant;
use tokio_util::sync::CancellationToken;
use super::{OperatorDrainStatus, OperatorLifecycleState};
/// Exact live-work counts sampled while an operator drain is active.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) struct WorkCounts {
/// Live WEB sessions.
pub(super) sessions: usize,
/// Live logical streams.
pub(super) streams: usize,
/// Live session-owned WebSockets.
pub(super) websockets: usize,
}
impl WorkCounts {
/// Returns whether every tracked work class reached zero.
pub(super) fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
/// Single active drain identity and cancellation authority.
pub(super) struct ActiveDrain {
/// Process-local operation sequence.
pub(super) sequence: u64,
/// Cancellation token owned by resume or terminal shutdown.
pub(super) cancellation: CancellationToken,
}
/// Mutex-protected lifecycle state used for atomic transitions.
pub(super) struct OperatorLifecycleInner {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Currently active drain, if any.
pub(super) active: Option<ActiveDrain>,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
/// Lock-free read snapshot published after lifecycle mutations.
#[derive(Clone)]
pub(super) struct OperatorSnapshot {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
src/web/manager/operator_lifecycle/tests.rs
+4 -1
View File
@@ -106,7 +106,10 @@ async fn pause_fence_leaves_no_late_admission_commits_under_scheduler_pressure()
#[tokio::test]
async fn empty_drain_completes_gracefully_and_stays_closed_until_resume() {
let (runtime, generation) = test_runtime();
let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
let accepted = runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
assert_eq!(accepted.state, OperatorLifecycleState::Draining);
let completed = tokio::time::timeout(Duration::from_secs(1), async {
src/web/manager/session_admission.rs
+25 -15
View File
@@ -1,10 +1,10 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{ManagerState, allow_rate};
use super::{ProfileKey, WebProcessRuntime};
use crate::config::WebRuntimeProfile;
use crate::web::telemetry::WebRejectionReason;
/// Applies process, address, profile, and rate ceilings to one initial session.
pub(super) fn admit_initial(
@@ -15,23 +15,33 @@ pub(super) fn admit_initial(
profile_key: ProfileKey,
profile: &WebRuntimeProfile,
) -> bool {
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
< runtime.limits.max_sessions_per_ip
&& state
if state.sessions.len() >= runtime.limits.max_sessions_global
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
>= runtime.limits.max_sessions_per_ip
|| state
.sessions_per_profile
.get(&profile_key)
.copied()
.unwrap_or(0)
< profile.max_sessions
&& allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
);
if !admitted {
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
>= profile.max_sessions
{
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionCapacity);
return false;
}
admitted
if !allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
) {
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionRate);
return false;
}
true
}
src/web/manager/session_creation.rs
+4 -3
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -278,7 +277,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Limit);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
return Err(ManagerError::Limit);
};
let carrier_deadline_at = carrier_request
@@ -341,7 +342,7 @@ impl WebProcessRuntime {
)
};
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
let identity = session.trace_identity();
let result = CreateResult {
token: session_token,
src/web/manager/session_creation/replacement.rs
+5 -3
View File
@@ -48,7 +48,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Closed);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
drop(state);
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
return Err(ManagerError::Limit);
@@ -112,8 +114,8 @@ impl WebProcessRuntime {
{
*slot = Some(replacement.old_session.carrier());
}
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
self.telemetry.record_session_closed();
let result = CreateResult {
token: session_token,
carrier: replacement.carrier,
src/web/manager/status.rs
+15 -8
View File
@@ -1,7 +1,6 @@
use std::net::IpAddr;
use std::ops::Bound::{Excluded, Unbounded};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;
use serde::Serialize;
@@ -298,6 +297,7 @@ impl WebProcessRuntime {
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let aggregates = self.telemetry.aggregates();
WebRuntimeStatus {
runtime_instance: self.runtime_instance().to_string(),
generation_id,
@@ -313,6 +313,13 @@ impl WebProcessRuntime {
"http_connections",
permits(&self.http_connections, self.limits.max_http_connections),
),
(
"http_overload_connections",
permits(
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
),
(
"http_handlers",
permits(&self.http_handlers, self.limits.max_http_handlers),
@@ -346,13 +353,13 @@ impl WebProcessRuntime {
),
],
auxiliary_tasks: self.tasks.len(),
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
session_incarnations_created: aggregates.sessions_created,
session_incarnations_closed: aggregates.sessions_closed,
streams_opened: aggregates.streams_opened,
streams_rejected: aggregates.streams_rejected,
bytes_up: aggregates.bytes_up,
bytes_down: aggregates.bytes_down,
limit_hits: aggregates.limit_hits,
partial,
}
}
src/web/manager/websocket.rs
+40 -14
View File
@@ -8,6 +8,7 @@ use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
use crate::web::telemetry::WebRejectionReason;
// Deterministic victim ordering remains isolated from registry mutation.
mod policy;
@@ -208,7 +209,7 @@ pub(super) async fn admit(
return Err(ManagerError::Closed);
}
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
match try_admit(
let capacity_reason = match try_admit(
runtime,
owner,
session_id,
@@ -220,12 +221,23 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => return Ok(connection),
Err(TryAdmitError::Conflict) => return Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => return Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {}
}
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
return Err(ManagerError::Concurrent);
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Closed);
}
Err(TryAdmitError::Capacity(reason)) => reason,
};
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None, true) else {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(capacity_reason);
return Err(ManagerError::Limit);
};
let released = victim.released.cancelled();
@@ -249,17 +261,28 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => Ok(connection),
Err(TryAdmitError::Conflict) => Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
Err(ManagerError::Concurrent)
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
Err(ManagerError::Closed)
}
Err(TryAdmitError::Capacity(reason)) => {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(reason);
Err(ManagerError::Limit)
}
}
}
enum TryAdmitError {
Capacity,
Capacity(WebRejectionReason),
Conflict,
Closed,
}
@@ -292,10 +315,13 @@ fn try_admit(
}
let slot = Arc::clone(&runtime.websocket_connections)
.try_acquire_owned()
.map_err(|_| TryAdmitError::Capacity)?;
let base_budget = runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let base_budget =
runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity(
WebRejectionReason::WebSocketBytesCapacity,
))?;
if parent_cancellation.is_cancelled() {
return Err(TryAdmitError::Closed);
}
@@ -304,7 +330,7 @@ fn try_admit(
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_add(1)
})
.map_err(|_| TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let now = runtime.websocket_tick();
let entry = Arc::new(WebSocketEntry {
id,
src/web/mod.rs
+2
View File
@@ -14,5 +14,7 @@ pub(crate) mod manager;
pub(crate) mod session;
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
pub(crate) mod stream;
/// Process-owned fixed-cardinality WEB operational telemetry.
pub(crate) mod telemetry;
/// Process-owned bounded WEB debugging records and capture lifecycle.
pub(crate) mod trace;
src/web/session/backend.rs
+9 -1
View File
@@ -31,6 +31,9 @@ impl WebSession {
};
let generation = manager.active_generation();
if !*generation.admission_rx.borrow() {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationAdmissionClosed,
);
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream.id),
@@ -70,6 +73,9 @@ impl WebSession {
}
};
if let Err(future) = generation.try_spawn_session(future) {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationScopeClosed,
);
retain_rejected.store(retain_reservation_on_reject, Ordering::Release);
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
@@ -257,7 +263,9 @@ async fn run_stream(
return;
};
let Ok(_connection_permit) = connection_permits.try_acquire_owned() else {
manager.record_stream_rejected();
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationConnectionCapacity,
);
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_identity.id),
src/web/session/downlink.rs
+5
View File
@@ -183,6 +183,11 @@ impl WebSession {
&& data_items <= item_limit - items
};
if !fits {
if let Some(manager) = self.manager.upgrade() {
manager.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::QueueSessionCapacity,
);
}
return false;
}
let Some(manager) = self.manager.upgrade() else {
src/web/session/uplink.rs
+4 -1
View File
@@ -284,10 +284,13 @@ impl WebSession {
}
fn reserve_stream_locked(&self, state: &mut SessionState) -> Option<u16> {
let manager = self.manager.upgrade()?;
if state.active_peer_ports.len() >= self.profile.max_streams_per_session {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::StreamSessionCapacity,
);
return None;
}
let manager = self.manager.upgrade()?;
let peer_port = manager
.try_acquire_stream(
self.profile_key,
src/web/telemetry.rs
+541
View File
@@ -0,0 +1,541 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::Instant;
use serde::Serialize;
/// Stable operational rejection reason recorded at the decision point.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebRejectionReason {
/// Ordinary accepted HTTP connection slots were exhausted.
HttpConnectionCapacity,
/// Concurrent HTTP request handler slots were exhausted.
HttpHandlerCapacity,
/// Primary long-poll slots were exhausted.
LanePollCapacity,
/// Auxiliary lane-open wait slots were exhausted.
LaneAuxPollCapacity,
/// Concurrent collected-body reader slots were exhausted.
BodyReaderCapacity,
/// Process-wide collected-body bytes were exhausted.
BodyBytesCapacity,
/// Bootstrap credential quota or identifier space was exhausted.
BootstrapCapacity,
/// Bootstrap issuance rate policy rejected the operation.
BootstrapRate,
/// Live session quota or identifier space was exhausted.
SessionCapacity,
/// Session creation rate policy rejected the operation.
SessionRate,
/// One session reached its logical-stream ceiling.
StreamSessionCapacity,
/// Global or profile logical-stream capacity was exhausted.
StreamCapacity,
/// Logical-stream creation rate policy rejected the operation.
StreamRate,
/// No synthetic source port remained for the exact relay tuple.
StreamTupleExhausted,
/// Concurrent inner-handshake slots were exhausted.
StreamHandshakeCapacity,
/// The active relay generation had no connection permit.
GenerationConnectionCapacity,
/// One session reached its queued data ceiling.
QueueSessionCapacity,
/// The process-wide queued data ceiling was exhausted.
QueueGlobalCapacity,
/// Process-wide WebSocket connection slots were exhausted.
WebSocketConnectionCapacity,
/// Shared WebSocket byte capacity was exhausted.
WebSocketBytesCapacity,
/// Bounded WebSocket replacement capacity was exhausted.
WebSocketEvictionCapacity,
/// Effective WEB configuration disabled new work.
ConfigDisabled,
/// Effective user policy disabled new work.
UserDisabled,
/// Operator admission was paused.
OperatorPaused,
/// Operator admission was gracefully draining.
OperatorDraining,
/// Operator drain had committed forced close signals.
OperatorForceClosing,
/// Operator drain had reached confirmed zero.
OperatorDrained,
/// The active relay generation closed new admission.
GenerationAdmissionClosed,
/// The active relay generation could not own a new task.
GenerationScopeClosed,
/// Terminal runtime closure rejected the operation.
RuntimeClosed,
/// A bounded operation could not acquire or complete its deadline.
Deadline,
/// Concurrent ownership rejected the operation.
Concurrent,
}
impl WebRejectionReason {
/// Complete fixed rejection set in stable metric order.
pub(crate) const ALL: [Self; 32] = [
Self::HttpConnectionCapacity,
Self::HttpHandlerCapacity,
Self::LanePollCapacity,
Self::LaneAuxPollCapacity,
Self::BodyReaderCapacity,
Self::BodyBytesCapacity,
Self::BootstrapCapacity,
Self::BootstrapRate,
Self::SessionCapacity,
Self::SessionRate,
Self::StreamSessionCapacity,
Self::StreamCapacity,
Self::StreamRate,
Self::StreamTupleExhausted,
Self::StreamHandshakeCapacity,
Self::GenerationConnectionCapacity,
Self::QueueSessionCapacity,
Self::QueueGlobalCapacity,
Self::WebSocketConnectionCapacity,
Self::WebSocketBytesCapacity,
Self::WebSocketEvictionCapacity,
Self::ConfigDisabled,
Self::UserDisabled,
Self::OperatorPaused,
Self::OperatorDraining,
Self::OperatorForceClosing,
Self::OperatorDrained,
Self::GenerationAdmissionClosed,
Self::GenerationScopeClosed,
Self::RuntimeClosed,
Self::Deadline,
Self::Concurrent,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::HttpConnectionCapacity => "http_connection_capacity",
Self::HttpHandlerCapacity => "http_handler_capacity",
Self::LanePollCapacity => "lane_poll_capacity",
Self::LaneAuxPollCapacity => "lane_aux_poll_capacity",
Self::BodyReaderCapacity => "body_reader_capacity",
Self::BodyBytesCapacity => "body_bytes_capacity",
Self::BootstrapCapacity => "bootstrap_capacity",
Self::BootstrapRate => "bootstrap_rate",
Self::SessionCapacity => "session_capacity",
Self::SessionRate => "session_rate",
Self::StreamSessionCapacity => "stream_session_capacity",
Self::StreamCapacity => "stream_capacity",
Self::StreamRate => "stream_rate",
Self::StreamTupleExhausted => "stream_tuple_exhausted",
Self::StreamHandshakeCapacity => "stream_handshake_capacity",
Self::GenerationConnectionCapacity => "generation_connection_capacity",
Self::QueueSessionCapacity => "queue_session_capacity",
Self::QueueGlobalCapacity => "queue_global_capacity",
Self::WebSocketConnectionCapacity => "websocket_connection_capacity",
Self::WebSocketBytesCapacity => "websocket_bytes_capacity",
Self::WebSocketEvictionCapacity => "websocket_eviction_capacity",
Self::ConfigDisabled => "config_disabled",
Self::UserDisabled => "user_disabled",
Self::OperatorPaused => "operator_paused",
Self::OperatorDraining => "operator_draining",
Self::OperatorForceClosing => "operator_force_closing",
Self::OperatorDrained => "operator_drained",
Self::GenerationAdmissionClosed => "generation_admission_closed",
Self::GenerationScopeClosed => "generation_scope_closed",
Self::RuntimeClosed => "runtime_closed",
Self::Deadline => "deadline",
Self::Concurrent => "concurrent",
}
}
}
/// Terminal result for one accepted socket that found normal HTTP capacity exhausted.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebHttpConnectionOverloadOutcome {
/// Legacy policy closed the accepted socket immediately.
Dropped,
/// Bounded waiting acquired ordinary connection capacity.
WaitAdmitted,
/// Bounded waiting expired and emitted the retryable response.
WaitTimeout503,
/// Immediate response policy emitted the retryable response.
Responded503,
/// The bounded overload socket pool was already full.
OverflowCapacityDrop,
/// Writing or closing the retryable response failed.
ResponseErrorDrop,
/// Listener or process shutdown cancelled overload handling.
ShutdownDrop,
}
impl WebHttpConnectionOverloadOutcome {
/// Complete fixed accepted-socket outcome set in stable metric order.
pub(crate) const ALL: [Self; 7] = [
Self::Dropped,
Self::WaitAdmitted,
Self::WaitTimeout503,
Self::Responded503,
Self::OverflowCapacityDrop,
Self::ResponseErrorDrop,
Self::ShutdownDrop,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Dropped => "dropped",
Self::WaitAdmitted => "wait_admitted",
Self::WaitTimeout503 => "wait_timeout_503",
Self::Responded503 => "responded_503",
Self::OverflowCapacityDrop => "overflow_capacity_drop",
Self::ResponseErrorDrop => "response_error_drop",
Self::ShutdownDrop => "shutdown_drop",
}
}
}
/// Passive outcome for one plain-HTTP decoy upstream request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebDecoyUpstreamOutcome {
/// The internal decoy origin produced a response head.
Success,
/// The request activity plane could not grant a deadline lease.
DeadlineExhausted,
/// The configured origin actively refused the TCP connection.
ConnectRefused,
/// The bounded TCP connect phase timed out.
ConnectTimeout,
/// The TCP connect failed for another I/O reason.
ConnectError,
/// The bounded HTTP client handshake timed out.
HttpHandshakeTimeout,
/// The HTTP client handshake failed.
HttpHandshakeError,
/// The origin did not produce a response head before the deadline.
ResponseHeadTimeout,
/// URI preparation or the HTTP request failed.
RequestError,
}
impl WebDecoyUpstreamOutcome {
/// Complete fixed decoy-origin outcome set in stable metric order.
pub(crate) const ALL: [Self; 9] = [
Self::Success,
Self::DeadlineExhausted,
Self::ConnectRefused,
Self::ConnectTimeout,
Self::ConnectError,
Self::HttpHandshakeTimeout,
Self::HttpHandshakeError,
Self::ResponseHeadTimeout,
Self::RequestError,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Success => "success",
Self::DeadlineExhausted => "deadline_exhausted",
Self::ConnectRefused => "connect_refused",
Self::ConnectTimeout => "connect_timeout",
Self::ConnectError => "connect_error",
Self::HttpHandshakeTimeout => "http_handshake_timeout",
Self::HttpHandshakeError => "http_handshake_error",
Self::ResponseHeadTimeout => "response_head_timeout",
Self::RequestError => "request_error",
}
}
}
/// API-safe typed rejection counter.
#[derive(Clone, Serialize)]
pub(crate) struct WebRejectionCounter {
/// Stable closed-set rejection token.
pub(crate) reason: &'static str,
/// Monotonic process-lifetime count.
pub(crate) total: u64,
}
/// API-safe typed outcome counter.
#[derive(Clone, Serialize)]
pub(crate) struct WebOutcomeCounter {
/// Stable closed-set outcome token.
pub(crate) outcome: &'static str,
/// Monotonic process-lifetime count.
pub(crate) total: u64,
}
/// Existing aggregate WEB totals retained with their original semantics.
#[derive(Clone, Copy)]
pub(crate) struct WebAggregateSnapshot {
/// Created session incarnations.
pub(crate) sessions_created: u64,
/// Closed session incarnations.
pub(crate) sessions_closed: u64,
/// Admitted logical streams.
pub(crate) streams_opened: u64,
/// Rejected logical streams covered by the legacy aggregate.
pub(crate) streams_rejected: u64,
/// Accepted carrier uplink payload bytes.
pub(crate) bytes_up: u64,
/// Emitted carrier downlink payload bytes.
pub(crate) bytes_down: u64,
/// Legacy aggregate capacity-hit count.
pub(crate) limit_hits: u64,
}
/// Process-owned operational counters shared by ingress, API, and metrics.
pub(crate) struct WebTelemetry {
started: Instant,
live_acceptors: AtomicUsize,
accepted: AtomicU64,
accept_errors: AtomicU64,
rejections: [AtomicU64; WebRejectionReason::ALL.len()],
overload_outcomes: [AtomicU64; WebHttpConnectionOverloadOutcome::ALL.len()],
decoy_outcomes: [AtomicU64; WebDecoyUpstreamOutcome::ALL.len()],
last_decoy_outcome: AtomicUsize,
last_decoy_elapsed_ms: AtomicU64,
sessions_created: AtomicU64,
sessions_closed: AtomicU64,
streams_opened: AtomicU64,
streams_rejected: AtomicU64,
bytes_up: AtomicU64,
bytes_down: AtomicU64,
limit_hits: AtomicU64,
}
impl WebTelemetry {
/// Creates zeroed process-lifetime WEB telemetry.
pub(crate) fn new() -> Arc<Self> {
Arc::new(Self {
started: Instant::now(),
live_acceptors: AtomicUsize::new(0),
accepted: AtomicU64::new(0),
accept_errors: AtomicU64::new(0),
rejections: std::array::from_fn(|_| AtomicU64::new(0)),
overload_outcomes: std::array::from_fn(|_| AtomicU64::new(0)),
decoy_outcomes: std::array::from_fn(|_| AtomicU64::new(0)),
last_decoy_outcome: AtomicUsize::new(usize::MAX),
last_decoy_elapsed_ms: AtomicU64::new(0),
sessions_created: AtomicU64::new(0),
sessions_closed: AtomicU64::new(0),
streams_opened: AtomicU64::new(0),
streams_rejected: AtomicU64::new(0),
bytes_up: AtomicU64::new(0),
bytes_down: AtomicU64::new(0),
limit_hits: AtomicU64::new(0),
})
}
/// Registers one live accept loop before its task can be cancelled unpolled.
pub(crate) fn acceptor_guard(self: &Arc<Self>) -> WebAcceptorGuard {
self.live_acceptors.fetch_add(1, Ordering::AcqRel);
WebAcceptorGuard {
telemetry: Arc::clone(self),
}
}
/// Returns currently registered WEB accept loops.
pub(crate) fn live_acceptors(&self) -> usize {
self.live_acceptors.load(Ordering::Acquire)
}
/// Records one successfully accepted WEB socket.
pub(crate) fn record_accept(&self) {
self.accepted.fetch_add(1, Ordering::Relaxed);
}
/// Returns successfully accepted WEB sockets.
pub(crate) fn accepted(&self) -> u64 {
self.accepted.load(Ordering::Relaxed)
}
/// Records one WEB listener `accept` error.
pub(crate) fn record_accept_error(&self) {
self.accept_errors.fetch_add(1, Ordering::Relaxed);
}
/// Returns WEB listener `accept` errors.
pub(crate) fn accept_errors(&self) -> u64 {
self.accept_errors.load(Ordering::Relaxed)
}
/// Records one operational rejection at its admission decision point.
pub(crate) fn record_rejection(&self, reason: WebRejectionReason) {
self.rejections[reason as usize].fetch_add(1, Ordering::Relaxed);
}
/// Returns one fixed rejection counter.
pub(crate) fn rejection_total(&self, reason: WebRejectionReason) -> u64 {
self.rejections[reason as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed rejection set for API serialization.
pub(crate) fn rejection_counters(&self) -> Vec<WebRejectionCounter> {
WebRejectionReason::ALL
.into_iter()
.map(|reason| WebRejectionCounter {
reason: reason.as_str(),
total: self.rejection_total(reason),
})
.collect()
}
/// Records one terminal accepted-socket overload outcome.
pub(crate) fn record_overload(&self, outcome: WebHttpConnectionOverloadOutcome) {
self.overload_outcomes[outcome as usize].fetch_add(1, Ordering::Relaxed);
}
/// Returns one fixed accepted-socket overload counter.
pub(crate) fn overload_total(&self, outcome: WebHttpConnectionOverloadOutcome) -> u64 {
self.overload_outcomes[outcome as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed overload outcome set for API serialization.
pub(crate) fn overload_counters(&self) -> Vec<WebOutcomeCounter> {
WebHttpConnectionOverloadOutcome::ALL
.into_iter()
.map(|outcome| WebOutcomeCounter {
outcome: outcome.as_str(),
total: self.overload_total(outcome),
})
.collect()
}
/// Records one internal plain-HTTP decoy origin outcome.
pub(crate) fn record_decoy(&self, outcome: WebDecoyUpstreamOutcome) {
self.decoy_outcomes[outcome as usize].fetch_add(1, Ordering::Relaxed);
let elapsed_ms = self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
self.last_decoy_elapsed_ms
.store(elapsed_ms.saturating_add(1), Ordering::Relaxed);
self.last_decoy_outcome
.store(outcome as usize, Ordering::Release);
}
/// Returns one fixed internal decoy origin counter.
pub(crate) fn decoy_total(&self, outcome: WebDecoyUpstreamOutcome) -> u64 {
self.decoy_outcomes[outcome as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed decoy outcome set for API serialization.
pub(crate) fn decoy_counters(&self) -> Vec<WebOutcomeCounter> {
WebDecoyUpstreamOutcome::ALL
.into_iter()
.map(|outcome| WebOutcomeCounter {
outcome: outcome.as_str(),
total: self.decoy_total(outcome),
})
.collect()
}
/// Returns the last decoy outcome and its monotonic age in milliseconds.
pub(crate) fn last_decoy(&self) -> Option<(&'static str, u64)> {
let raw = self.last_decoy_outcome.load(Ordering::Acquire);
let outcome = WebDecoyUpstreamOutcome::ALL.get(raw).copied()?;
let recorded = self.last_decoy_elapsed_ms.load(Ordering::Relaxed);
let now = self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
Some((
outcome.as_str(),
now.saturating_sub(recorded.saturating_sub(1)),
))
}
/// Records one created session incarnation.
pub(crate) fn record_session_created(&self) {
self.sessions_created.fetch_add(1, Ordering::Relaxed);
}
/// Records one closed session incarnation.
pub(crate) fn record_session_closed(&self) {
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
}
/// Records one admitted logical stream.
pub(crate) fn record_stream_opened(&self) {
self.streams_opened.fetch_add(1, Ordering::Relaxed);
}
/// Records one logical stream in the legacy rejection aggregate.
pub(crate) fn record_stream_rejected(&self) {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
}
/// Adds accepted carrier uplink payload bytes.
pub(crate) fn record_up(&self, bytes: usize) {
self.bytes_up.fetch_add(bytes as u64, Ordering::Relaxed);
}
/// Adds emitted carrier downlink payload bytes.
pub(crate) fn record_down(&self, bytes: usize) {
self.bytes_down.fetch_add(bytes as u64, Ordering::Relaxed);
}
/// Records one legacy aggregate capacity hit.
pub(crate) fn record_limit_hit(&self) {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
}
/// Captures legacy process-owned aggregate totals.
pub(crate) fn aggregates(&self) -> WebAggregateSnapshot {
WebAggregateSnapshot {
sessions_created: self.sessions_created.load(Ordering::Relaxed),
sessions_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
}
}
}
/// Cancellation-safe live-acceptor registration.
pub(crate) struct WebAcceptorGuard {
telemetry: Arc<WebTelemetry>,
}
impl Drop for WebAcceptorGuard {
fn drop(&mut self) {
self.telemetry.live_acceptors.fetch_sub(1, Ordering::AcqRel);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fixed_counter_sets_and_acceptor_guard_are_exact() {
let telemetry = WebTelemetry::new();
let guard = telemetry.acceptor_guard();
assert_eq!(telemetry.live_acceptors(), 1);
telemetry.record_rejection(WebRejectionReason::HttpConnectionCapacity);
telemetry.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
telemetry.record_decoy(WebDecoyUpstreamOutcome::ConnectRefused);
assert_eq!(
telemetry.rejection_counters().len(),
WebRejectionReason::ALL.len()
);
assert_eq!(
telemetry.overload_counters().len(),
WebHttpConnectionOverloadOutcome::ALL.len()
);
assert_eq!(
telemetry.decoy_counters().len(),
WebDecoyUpstreamOutcome::ALL.len()
);
assert_eq!(
telemetry.rejection_total(WebRejectionReason::HttpConnectionCapacity),
1
);
assert_eq!(
telemetry.last_decoy().map(|value| value.0),
Some("connect_refused")
);
drop(guard);
assert_eq!(telemetry.live_acceptors(), 0);
}
}