API for WEB: bounded lifecycle and overload observability added

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-29 16:19:21 +03:00
parent 012dc07a98
commit 084834f5ec
45 changed files with 2440 additions and 298 deletions
+16 -6
View File
@@ -13,12 +13,13 @@ use super::model::ApiFailure;
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
use crate::config::ProxyConfig;
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
use crate::web::manager::{
ControlError, OperatorLifecycleError, SessionDetail, WebProcessRuntime,
};
use crate::web::manager::{ControlError, OperatorLifecycleError, SessionDetail, WebProcessRuntime};
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
mod request;
// Ingress, capacity, and decoy telemetry remain separate availability planes.
mod observability;
use observability::{WebCapacityStatus, WebDecoyUpstreamStatus, WebIngressStatus};
use request::{
CloseRequest, DrainRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
valid_runtime_instance,
@@ -73,7 +74,7 @@ pub(super) async fn handle(
reject_query(query)?;
let publication = shared.web_runtime_rx.borrow().clone();
let runtime = publication.runtime.upgrade();
let data = WebStatusData::new(publication, runtime.as_deref(), config.web.enabled);
let data = WebStatusData::new(publication, runtime.as_deref(), config);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", SESSIONS_PATH) => {
@@ -265,6 +266,9 @@ struct WebStatusData {
reason: Option<&'static str>,
listeners: Vec<String>,
effective_config_enabled: bool,
ingress: WebIngressStatus,
capacity: WebCapacityStatus,
decoy_upstream: WebDecoyUpstreamStatus,
#[serde(skip_serializing_if = "Option::is_none")]
operator_lifecycle: Option<crate::web::manager::OperatorLifecycleStatus>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -275,7 +279,7 @@ impl WebStatusData {
fn new(
publication: WebRuntimePublication,
runtime: Option<&WebProcessRuntime>,
effective_config_enabled: bool,
config: &ProxyConfig,
) -> Self {
let available = runtime.is_some()
&& matches!(
@@ -295,6 +299,9 @@ impl WebStatusData {
})
};
let operator_lifecycle = runtime.map(WebProcessRuntime::operator_lifecycle_status);
let ingress = WebIngressStatus::new(&publication, runtime.is_some());
let capacity = WebCapacityStatus::new(&publication, runtime, config);
let decoy_upstream = WebDecoyUpstreamStatus::new(&publication);
Self {
lifecycle: publication.lifecycle.as_str(),
lifecycle_epoch: publication.epoch,
@@ -306,7 +313,10 @@ impl WebStatusData {
.iter()
.map(ToString::to_string)
.collect(),
effective_config_enabled,
effective_config_enabled: config.web.enabled,
ingress,
capacity,
decoy_upstream,
operator_lifecycle,
runtime: runtime.map(WebProcessRuntime::try_status),
}
+161
View File
@@ -0,0 +1,161 @@
use serde::Serialize;
use crate::config::{ProxyConfig, WebHttpConnectionCapacityAction};
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
use crate::web::manager::{WebCapacityResourceStatus, WebCapacitySnapshot, WebProcessRuntime};
use crate::web::telemetry::{WebOutcomeCounter, WebRejectionCounter};
/// Private WEB ingress state owned by this Telemt process.
#[derive(Serialize)]
pub(super) struct WebIngressStatus {
configured_listeners: usize,
live_acceptors: usize,
accepting_connections: bool,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
tcp_accept_total: u64,
tcp_accept_error_total: u64,
}
impl WebIngressStatus {
pub(super) fn new(publication: &WebRuntimePublication, runtime_available: bool) -> Self {
let configured_listeners = publication.listeners.len();
let live_acceptors = publication.telemetry.live_acceptors();
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
&& runtime_available
&& configured_listeners != 0
&& live_acceptors == configured_listeners;
let reason = if accepting_connections {
None
} else {
Some(match publication.lifecycle {
WebRuntimeLifecycle::Starting => "starting",
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
WebRuntimeLifecycle::Draining => "ingress_draining",
WebRuntimeLifecycle::Drained => "ingress_drained",
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
WebRuntimeLifecycle::Running if !runtime_available => "runtime_released",
WebRuntimeLifecycle::Running if configured_listeners == 0 => "no_web_listener",
WebRuntimeLifecycle::Running => "acceptor_unavailable",
})
};
Self {
configured_listeners,
live_acceptors,
accepting_connections,
reason,
tcp_accept_total: publication.telemetry.accepted(),
tcp_accept_error_total: publication.telemetry.accept_errors(),
}
}
}
/// Bounded process-wide WEB capacity and terminal rejection view.
#[derive(Serialize)]
pub(super) struct WebCapacityStatus {
http_connection_capacity_action: WebHttpConnectionCapacityAction,
max_http_overload_connections: usize,
http_overload_timeout_ms: u64,
resources: Vec<WebCapacityResourceStatus>,
saturated_resources: Vec<&'static str>,
partial: Vec<&'static str>,
rejections: Vec<WebRejectionCounter>,
http_connection_overload_outcomes: Vec<WebOutcomeCounter>,
}
impl WebCapacityStatus {
pub(super) fn new(
publication: &WebRuntimePublication,
runtime: Option<&WebProcessRuntime>,
config: &ProxyConfig,
) -> Self {
let snapshot = runtime
.map(WebProcessRuntime::capacity_snapshot)
.unwrap_or_else(runtime_unavailable_snapshot);
Self {
http_connection_capacity_action: config.web.http_connection_capacity_action,
max_http_overload_connections: config.web.limits.max_http_overload_connections,
http_overload_timeout_ms: config.web.timeouts.http_overload_timeout_ms,
resources: snapshot.resources,
saturated_resources: snapshot.saturated_resources,
partial: snapshot.partial,
rejections: publication.telemetry.rejection_counters(),
http_connection_overload_outcomes: publication.telemetry.overload_counters(),
}
}
}
fn runtime_unavailable_snapshot() -> WebCapacitySnapshot {
WebCapacitySnapshot {
resources: Vec::new(),
saturated_resources: Vec::new(),
partial: vec!["runtime"],
}
}
/// Passive health of Telemt's internal plain-HTTP decoy origin hop.
#[derive(Serialize)]
pub(super) struct WebDecoyUpstreamStatus {
outcomes: Vec<WebOutcomeCounter>,
#[serde(skip_serializing_if = "Option::is_none")]
last_outcome: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
last_outcome_age_ms: Option<u64>,
}
impl WebDecoyUpstreamStatus {
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
let last = publication.telemetry.last_decoy();
Self {
outcomes: publication.telemetry.decoy_counters(),
last_outcome: last.map(|value| value.0),
last_outcome_age_ms: last.map(|value| value.1),
}
}
}
#[cfg(test)]
mod tests {
use crate::config::ProxyConfig;
use crate::web::control::WebRuntimeControl;
#[test]
fn starting_ingress_does_not_claim_external_availability() {
let control = WebRuntimeControl::new();
let publication = control.subscribe().borrow().clone();
let value =
serde_json::to_value(super::WebIngressStatus::new(&publication, false)).unwrap();
assert_eq!(value["configured_listeners"], 0);
assert_eq!(value["live_acceptors"], 0);
assert_eq!(value["accepting_connections"], false);
assert_eq!(value["reason"], "starting");
}
#[test]
fn unavailable_runtime_keeps_fixed_counter_sets_visible() {
let control = WebRuntimeControl::new();
let publication = control.subscribe().borrow().clone();
let config = ProxyConfig::default();
let capacity =
serde_json::to_value(super::WebCapacityStatus::new(&publication, None, &config))
.unwrap();
let decoy = serde_json::to_value(super::WebDecoyUpstreamStatus::new(&publication)).unwrap();
assert_eq!(
capacity["rejections"].as_array().unwrap().len(),
crate::web::telemetry::WebRejectionReason::ALL.len()
);
assert_eq!(
capacity["http_connection_overload_outcomes"]
.as_array()
.unwrap()
.len(),
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
);
assert_eq!(
decoy["outcomes"].as_array().unwrap().len(),
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
);
assert_eq!(capacity["partial"][0], "runtime");
}
}
+5
View File
@@ -216,6 +216,11 @@ impl ProxyConfig {
runtime_web::rebuild(self)
}
/// Revalidates decoy separation after restart-only listener fields are resolved.
pub(crate) fn validate_web_decoy_listener_separation(&self) -> Result<()> {
validate_web::validate_decoy_listener_separation(self)
}
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
self.runtime_user_auth.as_deref()
}
+3
View File
@@ -265,6 +265,7 @@ const WEB_CONFIG_KEYS: &[&str] = &[
"carriers",
"carrier_learning",
"carrier_negotiation_aggressiveness",
"http_connection_capacity_action",
"debug",
"limits",
"timeouts",
@@ -278,6 +279,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
"carrier_batch_bytes",
"max_frames_per_body",
"max_http_connections",
"max_http_overload_connections",
"max_http_handlers",
"max_lane_open_waits_per_session",
"pending_bytes_per_lane",
@@ -354,6 +356,7 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"bootstrap_lifetime_secs",
"reconnect_grace_secs",
"http_idle_secs",
"http_overload_timeout_ms",
"shutdown_secs",
"decoy_header_secs",
];
+58
View File
@@ -83,9 +83,59 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
timeouts::validate(&config.web.timeouts)?;
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
validate_vhosts(config)?;
validate_decoy_listener_separation(config)?;
Ok(())
}
/// Rejects a direct decoy recursion into an effective WEB listener.
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
let web_listeners = config
.server
.listeners
.iter()
.filter(|listener| listener.transport == ListenerTransport::Web)
.filter(|listener| {
(listener.ip.is_ipv4() && config.network.ipv4)
|| (listener.ip.is_ipv6() && config.network.ipv6 != Some(false))
})
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
.collect::<Vec<_>>();
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
let WebDecoyConfig::HttpUpstream { upstream } = &vhost.decoy else {
continue;
};
let parsed = url::Url::parse(upstream).map_err(|error| {
ProxyError::Config(format!(
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
))
})?;
let Some(port) = parsed.port_or_known_default() else {
continue;
};
let upstream_ip = match parsed.host() {
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
_ => continue,
};
let upstream_addr = SocketAddr::new(upstream_ip, port);
if web_listeners
.iter()
.any(|listener| listener_covers(*listener, upstream_addr))
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
));
}
}
Ok(())
}
fn listener_covers(listener: SocketAddr, target: SocketAddr) -> bool {
listener.port() == target.port()
&& (listener.ip() == target.ip()
|| (listener.ip().is_unspecified() && listener.is_ipv4() == target.is_ipv4()))
}
fn validate_web_listener(
config: &ProxyConfig,
idx: usize,
@@ -169,6 +219,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
let positive = [
("max_http_connections", limits.max_http_connections),
(
"max_http_overload_connections",
limits.max_http_overload_connections,
),
("max_http_handlers", limits.max_http_handlers),
(
"max_lane_open_waits_per_session",
@@ -222,6 +276,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
}
for (field, value) in [
("max_http_connections", limits.max_http_connections),
(
"max_http_overload_connections",
limits.max_http_overload_connections,
),
("max_http_handlers", limits.max_http_handlers),
("max_body_readers", limits.max_body_readers),
("max_body_bytes_global", limits.max_body_bytes_global),
+8
View File
@@ -5,6 +5,7 @@ const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
const WEB_LANE_STATE_BYTES: usize = 512;
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
/// Validates process-wide body, header, queue, static, and debug reservations.
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
@@ -30,6 +31,12 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
})?;
let overload_connection_reservation = limits
.max_http_overload_connections
.checked_mul(WEB_OVERLOAD_CONNECTION_BYTES)
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP overload reservations overflow usize".to_string())
})?;
let debug_ring_index = limits
.debug_records_capacity
.checked_mul(std::mem::size_of::<usize>())
@@ -77,6 +84,7 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
.and_then(|value| value.checked_add(carrier_learning_reservation))
.and_then(|value| value.checked_add(lane_state_reservation))
.and_then(|value| value.checked_add(http_header_reservation))
.and_then(|value| value.checked_add(overload_connection_reservation))
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
if reserved > limits.memory_envelope_bytes
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
+3
View File
@@ -2,6 +2,9 @@ use super::*;
/// Validates WEB request, learning, and lifecycle timeouts.
pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
if !(1..=60_000).contains(&timeouts.http_overload_timeout_ms) {
return config_error("web.timeouts.http_overload_timeout_ms must be within [1, 60000]");
}
let values = [
("header_secs", timeouts.header_secs),
("body_secs", timeouts.body_secs),
@@ -56,6 +56,78 @@ fn web_config_builds_canonical_runtime_snapshot() {
);
}
#[test]
fn web_http_connection_capacity_policy_is_bounded_and_configurable() {
let configured = WEB_CONFIG
.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"wait\"",
)
.replace(
"[[web.vhosts]]",
"[web.limits]\nmax_http_overload_connections = 23\n\n[web.timeouts]\nhttp_overload_timeout_ms = 731\n\n[[web.vhosts]]",
);
let config = load_config_from_temp_toml(&configured);
assert_eq!(
config.web.http_connection_capacity_action,
WebHttpConnectionCapacityAction::Wait
);
assert_eq!(config.web.limits.max_http_overload_connections, 23);
assert_eq!(config.web.timeouts.http_overload_timeout_ms, 731);
let defaults = ProxyConfig::default();
assert_eq!(
defaults.web.http_connection_capacity_action,
WebHttpConnectionCapacityAction::Drop
);
assert_eq!(defaults.web.limits.max_http_overload_connections, 64);
assert_eq!(defaults.web.timeouts.http_overload_timeout_ms, 250);
}
#[test]
fn web_http_connection_capacity_policy_rejects_unknown_or_unbounded_values() {
let unknown = WEB_CONFIG.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"queue\"",
);
assert!(load_config_error_from_temp_toml(&unknown).contains("http_connection_capacity_action"));
for timeout in [0, 60_001] {
let invalid = WEB_CONFIG.replace(
"[[web.vhosts]]",
&format!("[web.timeouts]\nhttp_overload_timeout_ms = {timeout}\n\n[[web.vhosts]]"),
);
assert!(
load_config_error_from_temp_toml(&invalid)
.contains("web.timeouts.http_overload_timeout_ms")
);
}
let no_overload_slots = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.limits]\nmax_http_overload_connections = 0\n\n[[web.vhosts]]",
);
assert!(
load_config_error_from_temp_toml(&no_overload_slots)
.contains("web.limits.max_http_overload_connections")
);
}
#[test]
fn web_decoy_rejects_direct_and_wildcard_listener_loops() {
let direct = WEB_CONFIG.replace("http://127.0.0.1:18081", "http://127.0.0.1:18080");
assert!(
load_config_error_from_temp_toml(&direct).contains("decoy upstream overlaps WEB listener")
);
let wildcard = direct.replace("ip = \"127.0.0.1\"", "ip = \"0.0.0.0\"");
assert!(
load_config_error_from_temp_toml(&wildcard)
.contains("decoy upstream overlaps WEB listener")
);
}
#[test]
fn web_profile_user_labels_are_bounded_for_runtime_status() {
let user = "a".repeat(65);
+3 -2
View File
@@ -53,8 +53,9 @@ pub use server::{
};
#[allow(unused_imports)]
pub use web::{
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebLimitsConfig,
WebProfileConfig, WebSecretMode, WebTimeoutsConfig, WebVhostConfig,
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig,
WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig, WebSecretMode,
WebTimeoutsConfig, WebVhostConfig,
};
pub(crate) use web::{
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
+15
View File
@@ -12,6 +12,9 @@ use super::web_debug::WebDebugConfig;
// Serialized WEB defaults remain separate from the runtime data model.
mod defaults;
use defaults::*;
// Accepted-socket overload policy remains separate from the bulky WEB data model.
mod overload;
pub use overload::WebHttpConnectionCapacityAction;
/// Client-facing secret representation used to derive a WEB capability.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -95,6 +98,9 @@ pub struct WebLimitsConfig {
/// Process-wide accepted WEB HTTP connection ceiling.
#[serde(default = "default_web_max_http_connections")]
pub max_http_connections: usize,
/// Accepted overload sockets allowed to wait or emit a retryable response.
#[serde(default = "default_web_max_http_overload_connections")]
pub max_http_overload_connections: usize,
/// Process-wide concurrently executing HTTP handler ceiling.
#[serde(default = "default_web_max_http_handlers")]
pub max_http_handlers: usize,
@@ -226,6 +232,7 @@ impl Default for WebLimitsConfig {
carrier_batch_bytes: default_web_carrier_batch_bytes(),
max_frames_per_body: default_web_max_frames_per_body(),
max_http_connections: default_web_max_http_connections(),
max_http_overload_connections: default_web_max_http_overload_connections(),
max_http_handlers: default_web_max_http_handlers(),
max_lane_open_waits_per_session: default_web_max_lane_open_waits_per_session(),
pending_bytes_per_lane: default_web_pending_bytes_per_lane(),
@@ -333,6 +340,9 @@ pub struct WebTimeoutsConfig {
/// Maximum idle lifetime of a WEB HTTP keep-alive connection.
#[serde(default = "default_web_http_idle_secs")]
pub http_idle_secs: u64,
/// Per-phase wait or response deadline for accepted HTTP overload sockets.
#[serde(default = "default_web_http_overload_timeout_ms")]
pub http_overload_timeout_ms: u64,
/// Maximum graceful wait for WEB connections and process-owned tasks.
#[serde(default = "default_web_shutdown_secs")]
pub shutdown_secs: u64,
@@ -364,6 +374,7 @@ impl Default for WebTimeoutsConfig {
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
reconnect_grace_secs: default_web_reconnect_grace_secs(),
http_idle_secs: default_web_http_idle_secs(),
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
shutdown_secs: default_web_shutdown_secs(),
decoy_header_secs: default_web_decoy_header_timeout_secs(),
}
@@ -401,6 +412,9 @@ pub struct WebConfig {
/// Controls the evidence thresholds used by automatic carrier ranking.
#[serde(default)]
pub carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness,
/// Action applied when accepted HTTP connection capacity is exhausted.
#[serde(default)]
pub http_connection_capacity_action: WebHttpConnectionCapacityAction,
/// Hard process and protocol limits.
#[serde(default)]
pub limits: WebLimitsConfig,
@@ -447,6 +461,7 @@ impl Default for WebConfig {
carriers: WebCarriers::default(),
carrier_learning: default_web_carrier_learning(),
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
http_connection_capacity_action: WebHttpConnectionCapacityAction::default(),
limits: WebLimitsConfig::default(),
debug: WebDebugConfig::default(),
timeouts: WebTimeoutsConfig::default(),
+2
View File
@@ -40,6 +40,7 @@ usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
usize_default!(default_web_max_frames_per_body, 4096);
usize_default!(default_web_max_http_connections, 1024);
usize_default!(default_web_max_http_overload_connections, 64);
usize_default!(default_web_max_http_handlers, 512);
usize_default!(default_web_max_lane_open_waits_per_session, 16);
usize_default!(default_web_pending_bytes_per_lane, 8 * 1024 * 1024);
@@ -105,5 +106,6 @@ pub(super) fn default_web_carrier_learning() -> bool {
u64_default!(default_web_bootstrap_lifetime_secs, 120);
u64_default!(default_web_reconnect_grace_secs, 120);
u64_default!(default_web_http_idle_secs, 75);
u64_default!(default_web_http_overload_timeout_ms, 250);
u64_default!(default_web_shutdown_secs, 15);
u64_default!(default_web_decoy_header_timeout_secs, 30);
+14
View File
@@ -0,0 +1,14 @@
use serde::{Deserialize, Serialize};
/// Action applied after an accepted WEB socket finds HTTP connection capacity exhausted.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WebHttpConnectionCapacityAction {
/// Close the accepted socket without emitting an HTTP response.
#[default]
Drop,
/// Wait for ordinary HTTP connection capacity under the overload deadline.
Wait,
/// Emit a bounded retryable HTTP response without parsing the request.
Respond,
}
+2
View File
@@ -6,6 +6,7 @@
//! - `bind` prepares and activates sockets without partial startup binding.
//! - `accept` runs cancellation-aware TCP accept loops.
//! - `control` coordinates reversible listener transitions and shutdown.
//! - `web_overload` handles accepted WEB sockets outside ordinary capacity.
mod accept;
mod bind;
@@ -13,6 +14,7 @@ mod control;
mod plan;
#[cfg(unix)]
mod unix;
mod web_overload;
pub(crate) use bind::bind_listeners;
pub(crate) use control::{ListenerManager, PreparedListenerTransition};
+58 -1
View File
@@ -13,9 +13,11 @@ use crate::config::{ListenerTransport, RstOnCloseMode};
use crate::proxy::ClientHandler;
use crate::transport::socket::set_linger_zero;
use crate::web::manager::WebProcessRuntime;
use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};
use super::bind::BoundTcpListener;
use super::plan::ListenerBindSpec;
use super::web_overload;
use crate::maestro::generation::RuntimeGeneration;
use crate::maestro::helpers::{
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
@@ -190,6 +192,7 @@ async fn run_accept_loop(
web_runtime: Option<Arc<WebProcessRuntime>>,
connections: TaskTracker,
cancellation: CancellationToken,
_web_acceptor_guard: Option<WebAcceptorGuard>,
) {
loop {
let accepted = tokio::select! {
@@ -204,8 +207,45 @@ async fn run_accept_loop(
error!(addr = %spec.addr, "WEB listener has no process runtime");
return;
};
web_runtime.telemetry().record_accept();
let Some(connection_permit) = web_runtime.try_http_connection() else {
drop(stream);
let config = web_runtime.active_generation().config();
let action = config.web.http_connection_capacity_action;
let phase_timeout =
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
drop(config);
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
);
web_runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
drop(stream);
continue;
}
let Some(overload_permit) = web_runtime.try_http_overload_connection()
else {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
);
web_runtime.telemetry().record_overload(
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
);
drop(stream);
continue;
};
connections.spawn(web_overload::serve(
stream,
peer_addr,
spec.web_client_ip_source,
Arc::clone(&spec.web_trusted_proxy_cidrs),
Arc::clone(web_runtime),
cancellation.clone(),
overload_permit,
action,
phase_timeout,
));
continue;
};
connections.spawn(crate::web::http::serve_connection(
@@ -245,6 +285,9 @@ async fn run_accept_loop(
}
}
Err(error_value) => {
if let Some(web_runtime) = &web_runtime {
web_runtime.telemetry().record_accept_error();
}
error!(addr = %spec.addr, error = %error_value, "TCP accept error");
tokio::select! {
biased;
@@ -262,8 +305,16 @@ impl ListenerSlot {
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
web_runtime: Option<Arc<WebProcessRuntime>>,
) -> Self {
let web_runtime = if bound.spec.transport == ListenerTransport::Web {
web_runtime
} else {
None
};
let cancellation = CancellationToken::new();
let connections = TaskTracker::new();
let web_acceptor_guard = web_runtime
.as_ref()
.map(|runtime| runtime.telemetry().acceptor_guard());
let task = tokio::spawn(run_accept_loop(
bound.listener.clone(),
bound.spec.clone(),
@@ -271,6 +322,7 @@ impl ListenerSlot {
web_runtime.clone(),
connections.clone(),
cancellation.clone(),
web_acceptor_guard,
));
Self {
spec: bound.spec,
@@ -364,6 +416,10 @@ impl ListenerSlot {
self.active_runtime = active_runtime.clone();
self.cancellation = CancellationToken::new();
self.connections = TaskTracker::new();
let web_acceptor_guard = self
.web_runtime
.as_ref()
.map(|runtime| runtime.telemetry().acceptor_guard());
self.task = Some(tokio::spawn(run_accept_loop(
self.listener.clone(),
self.spec.clone(),
@@ -371,6 +427,7 @@ impl ListenerSlot {
self.web_runtime.clone(),
self.connections.clone(),
self.cancellation.clone(),
web_acceptor_guard,
)));
}
}
+33 -2
View File
@@ -58,8 +58,13 @@ impl ListenerManager {
.map(|listener| listener.spec.addr)
.collect();
let has_web = !web_listeners.is_empty();
let web_runtime =
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
let web_runtime = has_web.then(|| {
WebProcessRuntime::start_with_trace(
active_runtime.clone(),
trace,
web_control.telemetry(),
)
});
let mut slots = BTreeMap::new();
for listener in bound.listeners {
let addr = listener.spec.addr;
@@ -461,4 +466,30 @@ mod tests {
manager.shutdown().await.unwrap();
runtime.stop_sessions().await;
}
#[tokio::test]
async fn acceptor_liveness_counts_only_web_listeners() {
let runtime = test_runtime_generation(1, ProxyConfig::default());
let active_runtime = Arc::new(ArcSwap::from(runtime.clone()));
let (native_listener, _native_addr) = bound_listener().await;
let (mut web_listener, _web_addr) = bound_listener().await;
web_listener.spec.transport = ListenerTransport::Web;
let bound = BoundListeners {
listeners: vec![native_listener, web_listener],
#[cfg(unix)]
unix_listener: None,
};
let trace = WebTraceStore::new(
runtime.config().web.debug.clone(),
&runtime.config().web.limits,
);
let control = WebRuntimeControl::new();
let receiver = control.subscribe();
let mut manager = ListenerManager::start(bound, active_runtime, trace, control);
assert_eq!(receiver.borrow().telemetry.live_acceptors(), 1);
manager.shutdown().await.unwrap();
runtime.stop_sessions().await;
}
}
+277
View File
@@ -0,0 +1,277 @@
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use ipnetwork::IpNetwork;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpStream;
use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
use crate::web::manager::WebProcessRuntime;
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
/// Handles one accepted WEB socket outside ordinary connection capacity.
#[allow(clippy::too_many_arguments)]
pub(super) async fn serve(
stream: TcpStream,
peer: SocketAddr,
client_ip_source: WebClientIpSource,
trusted_proxy_cidrs: Arc<[IpNetwork]>,
runtime: Arc<WebProcessRuntime>,
cancellation: CancellationToken,
overload_permit: OwnedSemaphorePermit,
action: WebHttpConnectionCapacityAction,
phase_timeout: Duration,
) {
match action {
WebHttpConnectionCapacityAction::Drop => unreachable!("drop is handled before spawn"),
WebHttpConnectionCapacityAction::Respond => {
let outcome = respond(stream, &cancellation, phase_timeout).await;
record_final_capacity_rejection(&runtime, outcome);
runtime.telemetry().record_overload(outcome);
}
WebHttpConnectionCapacityAction::Wait => {
let connection_permit = tokio::select! {
biased;
_ = cancellation.cancelled() => {
runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
return;
}
permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
permit.ok().flatten()
}
};
let Some(connection_permit) = connection_permit else {
if runtime.is_shutdown() {
runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
return;
}
let outcome = match respond(stream, &cancellation, phase_timeout).await {
WebHttpConnectionOverloadOutcome::Responded503 => {
WebHttpConnectionOverloadOutcome::WaitTimeout503
}
other => other,
};
record_final_capacity_rejection(&runtime, outcome);
runtime.telemetry().record_overload(outcome);
return;
};
runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::WaitAdmitted);
drop(overload_permit);
crate::web::http::serve_connection(
stream,
peer,
client_ip_source,
trusted_proxy_cidrs,
runtime,
cancellation,
connection_permit,
)
.await;
}
}
}
fn record_final_capacity_rejection(
runtime: &WebProcessRuntime,
outcome: WebHttpConnectionOverloadOutcome,
) {
if matches!(
outcome,
WebHttpConnectionOverloadOutcome::Responded503
| WebHttpConnectionOverloadOutcome::WaitTimeout503
| WebHttpConnectionOverloadOutcome::ResponseErrorDrop
) {
runtime
.telemetry()
.record_rejection(WebRejectionReason::HttpConnectionCapacity);
}
}
async fn respond(
stream: TcpStream,
cancellation: &CancellationToken,
phase_timeout: Duration,
) -> WebHttpConnectionOverloadOutcome {
tokio::select! {
biased;
_ = cancellation.cancelled() => WebHttpConnectionOverloadOutcome::ShutdownDrop,
written = write_service_unavailable(stream, phase_timeout) => {
if written {
WebHttpConnectionOverloadOutcome::Responded503
} else {
WebHttpConnectionOverloadOutcome::ResponseErrorDrop
}
}
}
}
async fn write_service_unavailable(mut stream: TcpStream, deadline: Duration) -> bool {
tokio::time::timeout(deadline, async {
stream.write_all(SERVICE_UNAVAILABLE_RESPONSE).await?;
stream.shutdown().await
})
.await
.is_ok_and(|result| result.is_ok())
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use arc_swap::ArcSwap;
use tokio::io::AsyncReadExt;
use tokio::net::{TcpListener, TcpStream};
use tokio_util::sync::CancellationToken;
use crate::config::{ProxyConfig, WebClientIpSource, WebHttpConnectionCapacityAction};
use crate::maestro::generation::test_runtime_generation;
use crate::web::manager::WebProcessRuntime;
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
async fn tcp_pair() -> (TcpStream, TcpStream) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let client = TcpStream::connect(addr);
let server = listener.accept();
let (client, server) = tokio::join!(client, server);
(server.unwrap().0, client.unwrap())
}
#[tokio::test]
async fn overload_response_is_exact_retryable_http() {
let (server, mut client) = tcp_pair().await;
assert!(super::write_service_unavailable(server, Duration::from_secs(1)).await);
let mut bytes = Vec::new();
client.read_to_end(&mut bytes).await.unwrap();
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
}
#[tokio::test]
async fn wait_timeout_is_one_rejection_and_one_retryable_response() {
let (runtime, generation) = runtime();
let held = runtime.try_http_connection().unwrap();
let overload = runtime.try_http_overload_connection().unwrap();
let (server, mut client) = tcp_pair().await;
let peer = server.peer_addr().unwrap();
super::serve(
server,
peer,
WebClientIpSource::XForwardedFor,
trusted_loopback(),
Arc::clone(&runtime),
CancellationToken::new(),
overload,
WebHttpConnectionCapacityAction::Wait,
Duration::from_millis(10),
)
.await;
drop(held);
let mut bytes = Vec::new();
client.read_to_end(&mut bytes).await.unwrap();
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
assert_eq!(
runtime
.telemetry()
.overload_total(WebHttpConnectionOverloadOutcome::WaitTimeout503,),
1
);
assert_eq!(
runtime
.telemetry()
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
1
);
stop(runtime, generation).await;
}
#[tokio::test]
async fn admitted_wait_is_not_counted_as_a_rejection() {
let (runtime, generation) = runtime();
let held = runtime.try_http_connection().unwrap();
let overload = runtime.try_http_overload_connection().unwrap();
let (server, _client) = tcp_pair().await;
let peer = server.peer_addr().unwrap();
let cancellation = CancellationToken::new();
let task = tokio::spawn(super::serve(
server,
peer,
WebClientIpSource::XForwardedFor,
trusted_loopback(),
Arc::clone(&runtime),
cancellation.clone(),
overload,
WebHttpConnectionCapacityAction::Wait,
Duration::from_secs(1),
));
tokio::task::yield_now().await;
drop(held);
for _ in 0..100 {
if runtime
.telemetry()
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted)
== 1
{
break;
}
tokio::task::yield_now().await;
}
cancellation.cancel();
tokio::time::timeout(Duration::from_secs(1), task)
.await
.unwrap()
.unwrap();
assert_eq!(
runtime
.telemetry()
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted),
1
);
assert_eq!(
runtime
.telemetry()
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
0
);
stop(runtime, generation).await;
}
fn runtime() -> (
Arc<WebProcessRuntime>,
Arc<crate::maestro::generation::RuntimeGeneration>,
) {
let mut config = ProxyConfig::default();
config.web.limits.max_http_connections = 1;
let generation = test_runtime_generation(1, config);
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
(runtime, generation)
}
fn trusted_loopback() -> Arc<[ipnetwork::IpNetwork]> {
Arc::from(["127.0.0.1/32".parse().unwrap()])
}
async fn stop(
runtime: Arc<WebProcessRuntime>,
generation: Arc<crate::maestro::generation::RuntimeGeneration>,
) {
runtime.shutdown().await;
generation.stop_sessions().await;
generation.stop_background_tasks().await;
}
}
+1
View File
@@ -315,6 +315,7 @@ pub(super) async fn run_telemt_core(
&runtime.config,
&startup_tracker,
active_runtime.clone(),
web_runtime_control.subscribe(),
)
.await;
+3
View File
@@ -45,6 +45,9 @@ pub(crate) async fn prepare_runtime(
quota_store: Arc<QuotaStore>,
runtime_log_filter: RuntimeLogFilter,
) -> Result<PreparedRuntime, String> {
config
.validate_web_decoy_listener_separation()
.map_err(|error| error.to_string())?;
let started_at_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
+31
View File
@@ -230,3 +230,34 @@ fn synlimited_endpoint_move_remains_restart_only() {
assert_eq!(resolved.effective.server.listeners[0].port, Some(443));
assert!(!resolved.runtime_changed);
}
#[test]
fn deferred_listener_identity_cannot_create_an_effective_decoy_loop() {
let mut old = ProxyConfig::default();
old.server.listeners = vec![test_listener(18080)];
old.server.listeners[0].transport = crate::config::ListenerTransport::Web;
let mut desired = old.clone();
desired.server.listeners[0].port = Some(18081);
desired.server.listen_backlog = desired.server.listen_backlog.saturating_add(1);
desired.web.vhosts = vec![
serde_json::from_value(serde_json::json!({
"host": "proxy.example",
"public_addr": "203.0.113.10:443",
"decoy": {
"mode": "http_upstream",
"upstream": "http://127.0.0.1:18080"
},
"profiles": []
}))
.unwrap(),
];
assert!(desired.validate_web_decoy_listener_separation().is_ok());
let resolved = resolve_reload_config(&old, &desired);
assert!(
resolved
.effective
.validate_web_decoy_listener_separation()
.is_err()
);
}
+2 -1
View File
@@ -405,6 +405,7 @@ pub(crate) async fn spawn_metrics_if_configured(
config: &Arc<ProxyConfig>,
startup_tracker: &Arc<StartupTracker>,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
) {
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
let metrics_target: Option<(u16, Option<String>)> =
@@ -437,7 +438,7 @@ pub(crate) async fn spawn_metrics_if_configured(
let active_runtime = active_runtime.clone();
let listen_backlog = config.server.listen_backlog;
tokio::spawn(async move {
metrics::serve(port, listen, listen_backlog, active_runtime).await;
metrics::serve(port, listen, listen_backlog, active_runtime, web_runtime_rx).await;
});
startup_tracker
.complete_component(
+99 -79
View File
@@ -26,6 +26,9 @@ use crate::tls_front::cache;
use crate::tls_front::fetcher;
use crate::transport::{ListenOptions, create_listener};
// Process-owned WEB metrics stay isolated from the legacy renderer body.
mod web;
// Keeps `/metrics` response size bounded when per-user telemetry is enabled.
const USER_LABELED_METRICS_MAX_USERS: usize = 4096;
// Keeps TLS-front per-domain health series bounded for large generated configs.
@@ -38,6 +41,7 @@ pub async fn serve(
listen: Option<String>,
listen_backlog: u32,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
) {
// If `metrics_listen` is set, bind on that single address only.
if let Some(ref listen_addr) = listen {
@@ -54,7 +58,7 @@ pub async fn serve(
match bind_metrics_listener(addr, ipv6_only, listen_backlog) {
Ok(listener) => {
info!("Metrics endpoint: http://{}/metrics and /beobachten", addr);
serve_listener(listener, active_runtime).await;
serve_listener(listener, active_runtime, web_runtime_rx).await;
}
Err(e) => {
warn!(error = %e, "Failed to bind metrics on {}", addr);
@@ -100,14 +104,15 @@ pub async fn serve(
warn!("Metrics listener is unavailable on both IPv4 and IPv6");
}
(Some(listener), None) | (None, Some(listener)) => {
serve_listener(listener, active_runtime).await;
serve_listener(listener, active_runtime, web_runtime_rx).await;
}
(Some(listener4), Some(listener6)) => {
let active_runtime_v6 = active_runtime.clone();
let web_runtime_rx_v6 = web_runtime_rx.clone();
tokio::spawn(async move {
serve_listener(listener6, active_runtime_v6).await;
serve_listener(listener6, active_runtime_v6, web_runtime_rx_v6).await;
});
serve_listener(listener4, active_runtime).await;
serve_listener(listener4, active_runtime, web_runtime_rx).await;
}
}
}
@@ -127,7 +132,11 @@ fn bind_metrics_listener(
TcpListener::from_std(socket.into())
}
async fn serve_listener(listener: TcpListener, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
async fn serve_listener(
listener: TcpListener,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
) {
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
loop {
@@ -165,28 +174,13 @@ async fn serve_listener(listener: TcpListener, active_runtime: Arc<ArcSwap<Runti
};
let active_runtime = active_runtime.clone();
let web_runtime_rx = web_runtime_rx.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req| {
let runtime = active_runtime.load_full();
let stats = runtime.stats.clone();
let beobachten = runtime.beobachten.clone();
let shared_state = runtime.proxy_shared.clone();
let ip_tracker = runtime.ip_tracker.clone();
let tls_cache = runtime.tls_cache.clone();
let config = runtime.config();
async move {
handle(
req,
&stats,
&beobachten,
&shared_state,
&ip_tracker,
tls_cache.as_deref(),
&config,
)
.await
}
let web_publication = web_runtime_rx.borrow().clone();
async move { handle(req, &runtime, &web_publication).await }
});
match timeout(
METRICS_HTTP_CONNECTION_TIMEOUT,
@@ -212,15 +206,26 @@ async fn serve_listener(listener: TcpListener, active_runtime: Arc<ArcSwap<Runti
async fn handle<B>(
req: Request<B>,
stats: &Stats,
beobachten: &BeobachtenStore,
shared_state: &ProxySharedState,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
config: &ProxyConfig,
runtime: &RuntimeGeneration,
web_publication: &crate::web::control::WebRuntimePublication,
) -> Result<Response<Full<Bytes>>, Infallible> {
let stats = &runtime.stats;
let beobachten = &runtime.beobachten;
let shared_state = &runtime.proxy_shared;
let ip_tracker = &runtime.ip_tracker;
let tls_cache = runtime.tls_cache.as_deref();
let config = runtime.config();
if req.uri().path() == "/metrics" {
let body = render_metrics(stats, shared_state, config, ip_tracker, tls_cache).await;
let body = render_metrics(
stats,
shared_state,
&config,
ip_tracker,
tls_cache,
web_publication,
)
.await;
let resp = Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
@@ -230,7 +235,7 @@ async fn handle<B>(
}
if req.uri().path() == "/beobachten" {
let body = render_beobachten(stats, beobachten, config);
let body = render_beobachten(stats, beobachten, &config);
let resp = Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/plain; charset=utf-8")
@@ -432,6 +437,7 @@ async fn render_metrics(
config: &ProxyConfig,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
web_publication: &crate::web::control::WebRuntimePublication,
) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(4096);
@@ -3856,6 +3862,7 @@ async fn render_metrics(
unique_ip_suppressed
);
web::render(&mut out, web_publication, config);
out
}
@@ -3870,6 +3877,11 @@ mod tests {
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
};
fn test_web_publication() -> crate::web::control::WebRuntimePublication {
let control = crate::web::control::WebRuntimeControl::new();
control.subscribe().borrow().clone()
}
#[tokio::test]
async fn test_render_metrics_format() {
let stats = Arc::new(Stats::new());
@@ -3937,7 +3949,15 @@ mod tests {
.await
.unwrap();
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker, None).await;
let output = render_metrics(
&stats,
shared_state.as_ref(),
&config,
&tracker,
None,
&test_web_publication(),
)
.await;
assert!(output.contains(&format!(
"telemt_build_info{{version=\"{}\"}} 1",
@@ -4065,7 +4085,15 @@ mod tests {
)
.await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, Some(&cache)).await;
let output = render_metrics(
&stats,
&shared_state,
&config,
&tracker,
Some(&cache),
&test_web_publication(),
)
.await;
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
@@ -4113,7 +4141,15 @@ mod tests {
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
let output = render_metrics(
&stats,
&shared_state,
&config,
&tracker,
None,
&test_web_publication(),
)
.await;
assert!(output.contains("telemt_connections_total 0"));
assert!(output.contains("telemt_connections_bad_total 0"));
assert!(output.contains("telemt_handshake_timeouts_total 0"));
@@ -4137,7 +4173,15 @@ mod tests {
let mut config = ProxyConfig::default();
config.access.user_max_unique_ips_global_each = 2;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
let output = render_metrics(
&stats,
&shared_state,
&config,
&tracker,
None,
&test_web_publication(),
)
.await;
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
@@ -4149,7 +4193,15 @@ mod tests {
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
let output = render_metrics(
&stats,
&shared_state,
&config,
&tracker,
None,
&test_web_publication(),
)
.await;
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
assert!(output.contains("# TYPE telemt_connections_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
@@ -4214,27 +4266,17 @@ mod tests {
#[tokio::test]
async fn test_endpoint_integration() {
let stats = Arc::new(Stats::new());
let beobachten = Arc::new(BeobachtenStore::new());
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let mut config = ProxyConfig::default();
stats.increment_connects_all();
stats.increment_connects_all();
stats.increment_connects_all();
config.general.beobachten = true;
config.general.beobachten_minutes = 10;
let runtime = crate::maestro::generation::test_runtime_generation(1, config);
let web_publication = test_web_publication();
runtime.stats.increment_connects_all();
runtime.stats.increment_connects_all();
runtime.stats.increment_connects_all();
let req = Request::builder().uri("/metrics").body(()).unwrap();
let resp = handle(
req,
&stats,
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
.unwrap();
let resp = handle(req, &runtime, &web_publication).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = resp.into_body().collect().await.unwrap().to_bytes();
assert!(
@@ -4251,25 +4293,13 @@ mod tests {
))
);
config.general.beobachten = true;
config.general.beobachten_minutes = 10;
beobachten.record(
runtime.beobachten.record(
"TLS-scanner",
"203.0.113.10".parse::<IpAddr>().unwrap(),
Duration::from_secs(600),
);
let req_beob = Request::builder().uri("/beobachten").body(()).unwrap();
let resp_beob = handle(
req_beob,
&stats,
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
.unwrap();
let resp_beob = handle(req_beob, &runtime, &web_publication).await.unwrap();
assert_eq!(resp_beob.status(), StatusCode::OK);
let body_beob = resp_beob.into_body().collect().await.unwrap().to_bytes();
let beob_text = std::str::from_utf8(body_beob.as_ref()).unwrap();
@@ -4277,17 +4307,7 @@ mod tests {
assert!(beob_text.contains("203.0.113.10-1"));
let req404 = Request::builder().uri("/other").body(()).unwrap();
let resp404 = handle(
req404,
&stats,
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
.unwrap();
let resp404 = handle(req404, &runtime, &web_publication).await.unwrap();
assert_eq!(resp404.status(), StatusCode::NOT_FOUND);
}
}
+365
View File
@@ -0,0 +1,365 @@
use std::fmt::Write;
use crate::config::{ProxyConfig, WebHttpConnectionCapacityAction};
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
use crate::web::manager::OperatorLifecycleState;
use crate::web::telemetry::{
WebDecoyUpstreamOutcome, WebHttpConnectionOverloadOutcome, WebRejectionReason,
};
/// Renders fixed-cardinality process-owned WEB observability families.
pub(super) fn render(out: &mut String, publication: &WebRuntimePublication, config: &ProxyConfig) {
let runtime = publication.runtime.upgrade();
let configured_listeners = publication.listeners.len();
let live_acceptors = publication.telemetry.live_acceptors();
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
&& runtime.is_some()
&& configured_listeners != 0
&& live_acceptors == configured_listeners;
let _ = writeln!(
out,
"# HELP telemt_web_ingress_lifecycle_state Current process-owned WEB ingress lifecycle"
);
let _ = writeln!(out, "# TYPE telemt_web_ingress_lifecycle_state gauge");
for state in WebRuntimeLifecycle::ALL {
let _ = writeln!(
out,
"telemt_web_ingress_lifecycle_state{{state=\"{}\"}} {}",
state.as_str(),
flag(publication.lifecycle == state)
);
}
let operator_status = runtime
.as_deref()
.map(crate::web::manager::WebProcessRuntime::operator_lifecycle_status);
let _ = writeln!(
out,
"# HELP telemt_web_operator_lifecycle_state Current reversible WEB operator lifecycle"
);
let _ = writeln!(out, "# TYPE telemt_web_operator_lifecycle_state gauge");
for state in OPERATOR_STATES {
let active = match operator_status.as_ref() {
Some(status) => operator_state_token(status.state) == state,
None => state == "unavailable",
};
let _ = writeln!(
out,
"telemt_web_operator_lifecycle_state{{state=\"{state}\"}} {}",
flag(active)
);
}
let operator_admission_open = operator_status
.as_ref()
.is_some_and(|status| status.admission_open);
let effective_new_work_admission = operator_status
.as_ref()
.is_some_and(|status| status.effective_new_work_admission);
let _ = writeln!(
out,
"# HELP telemt_web_ingress_state Independent WEB ingress and admission flags"
);
let _ = writeln!(out, "# TYPE telemt_web_ingress_state gauge");
for (name, value) in [
("runtime_available", runtime.is_some()),
("accepting_connections", accepting_connections),
("config_enabled", config.web.enabled),
("operator_admission_open", operator_admission_open),
("effective_new_work_admission", effective_new_work_admission),
] {
let _ = writeln!(
out,
"telemt_web_ingress_state{{flag=\"{name}\"}} {}",
flag(value)
);
}
let _ = writeln!(
out,
"# HELP telemt_web_listeners Process-owned WEB listener and acceptor counts"
);
let _ = writeln!(out, "# TYPE telemt_web_listeners gauge");
let _ = writeln!(
out,
"telemt_web_listeners{{status=\"configured\"}} {configured_listeners}"
);
let _ = writeln!(
out,
"telemt_web_listeners{{status=\"acceptors_live\"}} {live_acceptors}"
);
let _ = writeln!(
out,
"# HELP telemt_web_tcp_accept_total Accepted sockets and accept syscall errors"
);
let _ = writeln!(out, "# TYPE telemt_web_tcp_accept_total counter");
let _ = writeln!(
out,
"telemt_web_tcp_accept_total{{result=\"accepted\"}} {}",
publication.telemetry.accepted()
);
let _ = writeln!(
out,
"telemt_web_tcp_accept_total{{result=\"error\"}} {}",
publication.telemetry.accept_errors()
);
let _ = writeln!(
out,
"# HELP telemt_web_rejections_total WEB operational rejection decisions by fixed reason"
);
let _ = writeln!(out, "# TYPE telemt_web_rejections_total counter");
for reason in WebRejectionReason::ALL {
let _ = writeln!(
out,
"telemt_web_rejections_total{{reason=\"{}\"}} {}",
reason.as_str(),
publication.telemetry.rejection_total(reason)
);
}
let _ = writeln!(
out,
"# HELP telemt_web_http_connection_overload_total Accepted saturated sockets by terminal outcome"
);
let _ = writeln!(
out,
"# TYPE telemt_web_http_connection_overload_total counter"
);
for outcome in WebHttpConnectionOverloadOutcome::ALL {
let _ = writeln!(
out,
"telemt_web_http_connection_overload_total{{outcome=\"{}\"}} {}",
outcome.as_str(),
publication.telemetry.overload_total(outcome)
);
}
let action = config.web.http_connection_capacity_action;
let _ = writeln!(
out,
"# HELP telemt_web_http_connection_capacity_action Effective overload action"
);
let _ = writeln!(
out,
"# TYPE telemt_web_http_connection_capacity_action gauge"
);
for (token, variant) in [
("drop", WebHttpConnectionCapacityAction::Drop),
("wait", WebHttpConnectionCapacityAction::Wait),
("respond", WebHttpConnectionCapacityAction::Respond),
] {
let _ = writeln!(
out,
"telemt_web_http_connection_capacity_action{{action=\"{token}\"}} {}",
flag(action == variant)
);
}
let _ = writeln!(
out,
"# HELP telemt_web_http_overload_timeout_milliseconds Effective overload phase timeout"
);
let _ = writeln!(
out,
"# TYPE telemt_web_http_overload_timeout_milliseconds gauge"
);
let _ = writeln!(
out,
"telemt_web_http_overload_timeout_milliseconds {}",
config.web.timeouts.http_overload_timeout_ms
);
if let Some(runtime) = runtime.as_deref() {
render_capacity(out, &runtime.capacity_snapshot());
}
let _ = writeln!(
out,
"# HELP telemt_web_decoy_upstream_requests_total Internal plain-HTTP decoy origin outcomes"
);
let _ = writeln!(
out,
"# TYPE telemt_web_decoy_upstream_requests_total counter"
);
for outcome in WebDecoyUpstreamOutcome::ALL {
let _ = writeln!(
out,
"telemt_web_decoy_upstream_requests_total{{outcome=\"{}\"}} {}",
outcome.as_str(),
publication.telemetry.decoy_total(outcome)
);
}
render_aggregate_totals(out, publication);
}
fn render_capacity(out: &mut String, snapshot: &crate::web::manager::WebCapacitySnapshot) {
let _ = writeln!(
out,
"# HELP telemt_web_capacity_snapshot_partial Whether a non-blocking capacity plane was omitted"
);
let _ = writeln!(out, "# TYPE telemt_web_capacity_snapshot_partial gauge");
let _ = writeln!(
out,
"telemt_web_capacity_snapshot_partial{{plane=\"budget\"}} {}",
flag(snapshot.partial.contains(&"budget"))
);
for (unit, family) in [
("slots", "telemt_web_capacity_slots"),
("bytes", "telemt_web_capacity_bytes"),
("items", "telemt_web_capacity_items"),
] {
let _ = writeln!(
out,
"# HELP {family} Current process-wide WEB capacity in {unit}"
);
let _ = writeln!(out, "# TYPE {family} gauge");
for status in snapshot
.resources
.iter()
.filter(|status| status.unit == unit)
{
for (kind, value) in [
("used", status.used),
("available", status.available),
("limit", status.limit),
] {
let _ = writeln!(
out,
"{family}{{resource=\"{}\",kind=\"{kind}\"}} {value}",
status.resource
);
}
}
}
let _ = writeln!(
out,
"# HELP telemt_web_capacity_closed Whether terminal shutdown closed a WEB capacity authority"
);
let _ = writeln!(out, "# TYPE telemt_web_capacity_closed gauge");
let _ = writeln!(
out,
"# HELP telemt_web_capacity_saturated Whether a WEB resource has no immediately available capacity"
);
let _ = writeln!(out, "# TYPE telemt_web_capacity_saturated gauge");
for status in &snapshot.resources {
let _ = writeln!(
out,
"telemt_web_capacity_closed{{resource=\"{}\"}} {}",
status.resource,
flag(status.closed)
);
let _ = writeln!(
out,
"telemt_web_capacity_saturated{{resource=\"{}\"}} {}",
status.resource,
flag(status.available == 0 && !status.closed)
);
}
}
fn render_aggregate_totals(out: &mut String, publication: &WebRuntimePublication) {
let totals = publication.telemetry.aggregates();
let _ = writeln!(
out,
"# HELP telemt_web_session_incarnations_total Process-owned WEB session lifecycle totals"
);
let _ = writeln!(out, "# TYPE telemt_web_session_incarnations_total counter");
let _ = writeln!(
out,
"telemt_web_session_incarnations_total{{event=\"created\"}} {}",
totals.sessions_created
);
let _ = writeln!(
out,
"telemt_web_session_incarnations_total{{event=\"closed\"}} {}",
totals.sessions_closed
);
let _ = writeln!(
out,
"# HELP telemt_web_streams_total Process-owned WEB logical stream totals"
);
let _ = writeln!(out, "# TYPE telemt_web_streams_total counter");
let _ = writeln!(
out,
"telemt_web_streams_total{{event=\"opened\"}} {}",
totals.streams_opened
);
let _ = writeln!(
out,
"telemt_web_streams_total{{event=\"rejected\"}} {}",
totals.streams_rejected
);
let _ = writeln!(
out,
"# HELP telemt_web_carrier_bytes_total Process-owned WEB carrier payload bytes"
);
let _ = writeln!(out, "# TYPE telemt_web_carrier_bytes_total counter");
let _ = writeln!(
out,
"telemt_web_carrier_bytes_total{{direction=\"up\"}} {}",
totals.bytes_up
);
let _ = writeln!(
out,
"telemt_web_carrier_bytes_total{{direction=\"down\"}} {}",
totals.bytes_down
);
}
fn operator_state_token(state: OperatorLifecycleState) -> &'static str {
match state {
OperatorLifecycleState::Running => "running",
OperatorLifecycleState::Paused => "paused",
OperatorLifecycleState::Draining => "draining",
OperatorLifecycleState::ForceClosing => "force_closing",
OperatorLifecycleState::Drained => "drained",
}
}
const OPERATOR_STATES: [&str; 6] = [
"unavailable",
"running",
"paused",
"draining",
"force_closing",
"drained",
];
const fn flag(value: bool) -> u8 {
if value { 1 } else { 0 }
}
#[cfg(test)]
mod tests {
use crate::config::ProxyConfig;
use crate::web::control::WebRuntimeControl;
#[test]
fn renderer_emits_complete_zeroed_fixed_counter_sets() {
let control = WebRuntimeControl::new();
let publication = control.subscribe().borrow().clone();
let mut output = String::new();
super::render(&mut output, &publication, &ProxyConfig::default());
assert_eq!(
output.matches("telemt_web_rejections_total{").count(),
crate::web::telemetry::WebRejectionReason::ALL.len()
);
assert_eq!(
output
.matches("telemt_web_http_connection_overload_total{")
.count(),
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
);
assert_eq!(
output
.matches("telemt_web_decoy_upstream_requests_total{")
.count(),
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
);
assert!(output.contains("telemt_web_ingress_lifecycle_state{state=\"starting\"} 1"));
}
}
+23
View File
@@ -7,6 +7,7 @@ use std::time::Instant;
use tokio::sync::watch;
use super::manager::WebProcessRuntime;
use super::telemetry::WebTelemetry;
/// Process-owned WEB ingress lifecycle state.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -26,6 +27,16 @@ pub(crate) enum WebRuntimeLifecycle {
}
impl WebRuntimeLifecycle {
/// Complete fixed lifecycle set used by one-hot metrics.
pub(crate) const ALL: [Self; 6] = [
Self::Starting,
Self::NoWebListener,
Self::Running,
Self::Draining,
Self::Drained,
Self::DeadlineExceeded,
];
/// Returns the stable API token for this lifecycle state.
pub(crate) const fn as_str(self) -> &'static str {
match self {
@@ -52,28 +63,34 @@ pub(crate) struct WebRuntimePublication {
pub(crate) listeners: Arc<[SocketAddr]>,
/// Weak runtime access that never extends data-plane ownership.
pub(crate) runtime: Weak<WebProcessRuntime>,
/// Process-owned counters that remain readable after runtime release.
pub(crate) telemetry: Arc<WebTelemetry>,
}
/// Single-writer process lifecycle publisher for WEB ingress.
#[derive(Clone)]
pub(crate) struct WebRuntimeControl {
epoch: Arc<AtomicU64>,
telemetry: Arc<WebTelemetry>,
tx: watch::Sender<WebRuntimePublication>,
}
impl WebRuntimeControl {
/// Creates the process channel in the pre-listener `starting` state.
pub(crate) fn new() -> Self {
let telemetry = WebTelemetry::new();
let publication = WebRuntimePublication {
epoch: 1,
lifecycle: WebRuntimeLifecycle::Starting,
since: Instant::now(),
listeners: Arc::from([]),
runtime: Weak::new(),
telemetry: Arc::clone(&telemetry),
};
let (tx, _rx) = watch::channel(publication);
Self {
epoch: Arc::new(AtomicU64::new(1)),
telemetry,
tx,
}
}
@@ -83,6 +100,11 @@ impl WebRuntimeControl {
self.tx.subscribe()
}
/// Returns the shared process-owned WEB telemetry handle.
pub(crate) fn telemetry(&self) -> Arc<WebTelemetry> {
Arc::clone(&self.telemetry)
}
/// Publishes one lifecycle transition and optional weak runtime reference.
pub(crate) fn publish(
&self,
@@ -97,6 +119,7 @@ impl WebRuntimeControl {
since: Instant::now(),
listeners,
runtime,
telemetry: Arc::clone(&self.telemetry),
});
}
}
+60 -9
View File
@@ -15,6 +15,7 @@ use super::{
};
use crate::config::{WebRuntimeDecoy, WebRuntimeVhost};
use crate::web::manager::WebProcessRuntime;
use crate::web::telemetry::WebDecoyUpstreamOutcome;
/// Serves the configured ordinary site after optionally removing carrier material.
/// Transport-sanitized static fallbacks remain uncacheable after query removal.
@@ -195,16 +196,18 @@ async fn proxy_to_upstream(
.map(|value| value.as_str())
.unwrap_or("/");
let Ok(uri) = path_and_query.parse::<Uri>() else {
return bad_gateway();
return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
};
*request.uri_mut() = uri;
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => stream,
_ => return bad_gateway(),
let stream = match connect_upstream(addr, header_timeout).await {
Ok(stream) => stream,
Err(outcome) => return decoy_failure(runtime, outcome),
};
drop(_deadline_lease);
let max_header_bytes = runtime
@@ -217,12 +220,19 @@ async fn proxy_to_upstream(
builder.max_buf_size(max_header_bytes);
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let (mut sender, connection) =
match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
Ok(Ok(parts)) => parts,
_ => return bad_gateway(),
Ok(Err(_)) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeError);
}
Err(_) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeTimeout);
}
};
drop(_deadline_lease);
runtime.spawn_auxiliary(async move {
@@ -230,14 +240,24 @@ async fn proxy_to_upstream(
});
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
Ok(lease) => lease,
Err(()) => return bad_gateway(),
Err(()) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
}
};
let mut response =
match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
Ok(Ok(response)) => response,
_ => return bad_gateway(),
Ok(Err(_)) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
}
Err(_) => {
return decoy_failure(runtime, WebDecoyUpstreamOutcome::ResponseHeadTimeout);
}
};
drop(_deadline_lease);
runtime
.telemetry()
.record_decoy(WebDecoyUpstreamOutcome::Success);
remove_hop_by_hop(response.headers_mut());
response.map(|body| {
body.map_err(|error| -> BoxError { Box::new(error) })
@@ -245,6 +265,25 @@ async fn proxy_to_upstream(
})
}
async fn connect_upstream(
addr: SocketAddr,
timeout: Duration,
) -> Result<TcpStream, WebDecoyUpstreamOutcome> {
match tokio::time::timeout(timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(error)) if error.kind() == std::io::ErrorKind::ConnectionRefused => {
Err(WebDecoyUpstreamOutcome::ConnectRefused)
}
Ok(Err(_)) => Err(WebDecoyUpstreamOutcome::ConnectError),
Err(_) => Err(WebDecoyUpstreamOutcome::ConnectTimeout),
}
}
fn decoy_failure(runtime: &WebProcessRuntime, outcome: WebDecoyUpstreamOutcome) -> HttpResponse {
runtime.telemetry().record_decoy(outcome);
bad_gateway()
}
fn lease_deadline(
deadline: Option<&super::activity::RequestDeadlineHandle>,
timeout: Duration,
@@ -315,4 +354,16 @@ mod tests {
assert!(resolve_static_path("/../index.html", &site).is_none());
assert!(resolve_static_path("//index.html", &site).is_none());
}
#[tokio::test]
async fn closed_loopback_origin_is_classified_as_connect_refused() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
drop(listener);
assert_eq!(
connect_upstream(addr, Duration::from_secs(1)).await.err(),
Some(WebDecoyUpstreamOutcome::ConnectRefused)
);
}
}
+29 -28
View File
@@ -46,10 +46,7 @@ async fn live_runtime() -> (
Arc<crate::maestro::generation::RuntimeGeneration>,
TcpListener,
) {
let generation = test_runtime_generation(
1,
runtime_config([71; 32], WebCarrier::Https),
);
let generation = test_runtime_generation(1, runtime_config([71; 32], WebCarrier::Https));
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
(runtime, generation, listener)
@@ -100,12 +97,7 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
let hello = frame::encode(FrameType::Hello, 0, &[1]);
runtime.pause_operator().await.unwrap();
let paused_create = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let paused_create = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (paused_headers, _) = split_response(&paused_create);
assert!(paused_headers.starts_with(b"HTTP/1.1 503"));
assert_eq!(response_header(paused_headers, "retry-after"), "1");
@@ -131,26 +123,20 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
let decoy = request(&listener, &runtime, decoy).await;
let (decoy_headers, decoy_body) = split_response(&decoy);
assert!(decoy_headers.starts_with(b"HTTP/1.1 200"));
assert!(!decoy_body.windows(11).any(|window| window == b"bootstrap=\""));
assert!(
!decoy_body
.windows(11)
.any(|window| window == b"bootstrap=\"")
);
runtime.resume_operator().await.unwrap();
let created = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let created = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (created_headers, _) = split_response(&created);
assert!(created_headers.starts_with(b"HTTP/1.1 200"));
let token = response_header(created_headers, "x-session-token").to_string();
runtime.pause_operator().await.unwrap();
let replay = request(
&listener,
&runtime,
create_request(&bootstrap, &hello),
)
.await;
let replay = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
let (replay_headers, _) = split_response(&replay);
assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
assert_eq!(response_header(replay_headers, "x-session-token"), token);
@@ -230,7 +216,10 @@ async fn paused_replacement_preserves_old_session_and_attempt_replay() {
let replay = request(&listener, &runtime, first_request).await;
let (replay_headers, _) = split_response(&replay);
assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
assert_eq!(response_header(replay_headers, "x-session-token"), first_token);
assert_eq!(
response_header(replay_headers, "x-session-token"),
first_token
);
runtime.resume_operator().await.unwrap();
let replacement = request(&listener, &runtime, replacement_request).await;
@@ -249,7 +238,10 @@ async fn client_delete_during_drain_completes_naturally() {
let (runtime, generation, listener) = live_runtime().await;
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
let delete = format!(
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
@@ -281,8 +273,14 @@ async fn deadline_force_closes_all_sessions_and_requires_explicit_resume() {
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
assert_eq!(serde_json::to_value(&accepted).unwrap()["state"], "draining");
let accepted = runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
assert_eq!(
serde_json::to_value(&accepted).unwrap()["state"],
"draining"
);
assert!(matches!(
runtime.drain_operator(Duration::from_secs(30)).await,
Err(OperatorLifecycleError::OperationInProgress)
@@ -323,7 +321,10 @@ async fn resume_before_deadline_cancels_drain_without_closing_existing_session()
let bootstrap = issue_bootstrap(&runtime);
let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
let still_draining = runtime.pause_operator().await.unwrap();
assert_eq!(
serde_json::to_value(&still_draining).unwrap()["state"],
+3 -3
View File
@@ -190,9 +190,9 @@ pub(super) async fn handle_session(
}
Err(
error @ (ManagerError::Limit
| ManagerError::Backpressure
| ManagerError::Concurrent
| ManagerError::AdmissionPaused),
| ManagerError::Backpressure
| ManagerError::Concurrent
| ManagerError::AdmissionPaused),
) => {
runtime.trace().record_profile_lifecycle(
client_ip,
+10 -2
View File
@@ -46,7 +46,7 @@ pub(super) async fn reserve_data(
cancellation: &CancellationToken,
timeout: Duration,
) -> Result<WebSocketBudgetLease, ()> {
tokio::time::timeout(timeout, async {
match tokio::time::timeout(timeout, async {
loop {
if cancellation.is_cancelled() {
return Err(());
@@ -63,7 +63,15 @@ pub(super) async fn reserve_data(
}
})
.await
.map_err(|_| ())?
{
Ok(result) => result,
Err(_) => {
runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::WebSocketBytesCapacity,
);
Err(())
}
}
}
pub(super) async fn process_multiplex(
+61 -23
View File
@@ -1,7 +1,7 @@
use std::future::Future;
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::AtomicU64;
use std::time::Duration;
use arc_swap::ArcSwap;
@@ -12,6 +12,7 @@ use tokio_util::task::TaskTracker;
use crate::config::{WebCarrier, WebLimitsConfig};
use crate::maestro::generation::RuntimeGeneration;
use crate::web::telemetry::{WebRejectionReason, WebTelemetry};
use crate::web::trace::WebTraceStore;
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
@@ -37,7 +38,7 @@ pub(crate) use lifecycle::WebShutdownOutcome;
// Reversible operator admission stays independent from terminal process shutdown.
mod operator_lifecycle;
pub(crate) use operator_lifecycle::{
OperatorLifecycleError, OperatorLifecycleStatus,
OperatorLifecycleError, OperatorLifecycleState, OperatorLifecycleStatus,
};
// Queue and WebSocket allocations share one process-owned data-plane budget.
mod budget;
@@ -48,6 +49,9 @@ mod status;
pub(crate) use status::{
SessionDetail, SessionFilter, SessionListRequest, SessionRefError, WebRuntimeStatus,
};
// Fixed-cardinality capacity snapshots serve API and Prometheus observability.
mod observability;
pub(crate) use observability::{WebCapacityResourceStatus, WebCapacitySnapshot};
// Asynchronous bounded close operations isolate mutation lifecycle from HTTP requests.
mod control;
pub(crate) use budget::WebSocketBudgetLease;
@@ -152,6 +156,7 @@ pub(crate) struct WebProcessRuntime {
stream_admission: Mutex<StreamAdmissionState>,
learning: Mutex<learning::CarrierLearning>,
http_connections: Arc<Semaphore>,
http_overload_connections: Arc<Semaphore>,
http_handlers: Arc<Semaphore>,
lane_polls: Arc<Semaphore>,
lane_aux_polls: Arc<Semaphore>,
@@ -169,13 +174,7 @@ pub(crate) struct WebProcessRuntime {
next_control_operation_id: AtomicU64,
shutdown: CancellationToken,
tasks: TaskTracker,
sessions_created: AtomicU64,
sessions_closed: AtomicU64,
streams_opened: AtomicU64,
streams_rejected: AtomicU64,
bytes_up: AtomicU64,
bytes_down: AtomicU64,
limit_hits: AtomicU64,
telemetry: Arc<WebTelemetry>,
}
impl WebProcessRuntime {
@@ -184,13 +183,14 @@ impl WebProcessRuntime {
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
let config = active_runtime.load().config();
let trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
Self::start_with_trace(active_runtime, trace)
Self::start_with_trace(active_runtime, trace, WebTelemetry::new())
}
/// Starts one process-scoped manager with a shared API-visible trace store.
pub(crate) fn start_with_trace(
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
trace: Arc<WebTraceStore>,
telemetry: Arc<WebTelemetry>,
) -> Arc<Self> {
let initial_generation = active_runtime.load_full();
let config = initial_generation.config();
@@ -209,8 +209,7 @@ impl WebProcessRuntime {
.saturating_sub(limits.websocket_http_connection_reserve);
let lane_poll_limit = limits.max_http_handlers / 2;
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
let runtime_instance: Arc<str> =
Arc::from(format!("{:032x}", rand::random::<u128>()));
let runtime_instance: Arc<str> = Arc::from(format!("{:032x}", rand::random::<u128>()));
let runtime = Arc::new(Self {
operator_lifecycle: operator_lifecycle::OperatorLifecycle::new(Arc::clone(
&runtime_instance,
@@ -219,6 +218,9 @@ impl WebProcessRuntime {
active_runtime,
trace,
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
http_overload_connections: Arc::new(Semaphore::new(
limits.max_http_overload_connections,
)),
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
lane_polls: Arc::new(Semaphore::new(lane_poll_limit)),
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
@@ -239,13 +241,7 @@ impl WebProcessRuntime {
learning: Mutex::new(carrier_learning),
shutdown: CancellationToken::new(),
tasks: TaskTracker::new(),
sessions_created: AtomicU64::new(0),
sessions_closed: AtomicU64::new(0),
streams_opened: AtomicU64::new(0),
streams_rejected: AtomicU64::new(0),
bytes_up: AtomicU64::new(0),
bytes_down: AtomicU64::new(0),
limit_hits: AtomicU64::new(0),
telemetry,
});
let weak = Arc::downgrade(&runtime);
let shutdown = runtime.shutdown.clone();
@@ -287,6 +283,16 @@ impl WebProcessRuntime {
&self.trace
}
/// Returns the process-owned operational telemetry handle.
pub(crate) fn telemetry(&self) -> &Arc<WebTelemetry> {
&self.telemetry
}
/// Returns whether terminal process shutdown has started.
pub(crate) fn is_shutdown(&self) -> bool {
self.shutdown.is_cancelled()
}
/// Reserves one accepted HTTP connection.
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
@@ -296,11 +302,28 @@ impl WebProcessRuntime {
permit
}
/// Waits for one accepted HTTP connection slot after bounded overload admission.
pub(crate) async fn acquire_http_connection(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.http_connections)
.acquire_owned()
.await
.ok()
}
/// Reserves one accepted socket outside ordinary HTTP connection capacity.
pub(crate) fn try_http_overload_connection(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.http_overload_connections)
.try_acquire_owned()
.ok()
}
/// Reserves one concurrently executing HTTP request handler.
pub(crate) fn try_http_handler(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.http_handlers).try_acquire_owned().ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::HttpHandlerCapacity);
}
permit
}
@@ -315,6 +338,11 @@ impl WebProcessRuntime {
let permit = Arc::clone(slots).try_acquire_owned().ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry.record_rejection(if auxiliary {
WebRejectionReason::LaneAuxPollCapacity
} else {
WebRejectionReason::LanePollCapacity
});
}
permit
}
@@ -323,7 +351,7 @@ impl WebProcessRuntime {
pub(crate) fn try_stream_handshake(&self) -> Option<OwnedSemaphorePermit> {
let permit = Arc::clone(&self.stream_handshakes).try_acquire_owned().ok();
if permit.is_none() {
self.record_stream_rejected();
self.record_stream_rejected_reason(WebRejectionReason::StreamHandshakeCapacity);
}
permit
}
@@ -359,10 +387,14 @@ impl WebProcessRuntime {
) -> Option<(OwnedSemaphorePermit, OwnedSemaphorePermit)> {
let Some(bytes) = u32::try_from(bytes).ok() else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
return None;
};
let Some(reader) = Arc::clone(&self.body_readers).try_acquire_owned().ok() else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyReaderCapacity);
return None;
};
let Some(body) = Arc::clone(&self.body_bytes)
@@ -370,6 +402,8 @@ impl WebProcessRuntime {
.ok()
else {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
return None;
};
Some((reader, body))
@@ -383,6 +417,8 @@ impl WebProcessRuntime {
.ok();
if permit.is_none() {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BodyBytesCapacity);
}
permit
}
@@ -401,6 +437,8 @@ impl WebProcessRuntime {
.try_reserve_queue(owner, bytes, items, control, downlink)
{
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::QueueGlobalCapacity);
return false;
}
true
@@ -473,15 +511,15 @@ impl WebProcessRuntime {
/// Accounts one successfully committed carrier uplink body.
pub(crate) fn record_up(&self, bytes: usize) {
self.bytes_up.fetch_add(bytes as u64, Ordering::Relaxed);
self.telemetry.record_up(bytes);
}
/// Accounts one emitted carrier downlink body.
pub(crate) fn record_down(&self, bytes: usize) {
self.bytes_down.fetch_add(bytes as u64, Ordering::Relaxed);
self.telemetry.record_down(bytes);
}
fn record_limit_hit(&self) {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_limit_hit();
}
}
+24 -15
View File
@@ -1,9 +1,9 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{allocate_stream_port, allow_rate, decrement_map, release_stream_port};
use super::{ProfileKey, WebProcessRuntime};
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Reserves one process-wide and per-profile live logical-stream slot.
@@ -17,14 +17,16 @@ impl WebProcessRuntime {
let _operator_admission = match self.try_operator_admission() {
Ok(admission) => admission,
Err(error) => {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
return Err(error);
}
};
let now = Instant::now();
let mut state = self.stream_admission.lock();
if state.closed {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(super::ManagerError::Closed);
}
if state.streams_live >= self.limits.max_streams_global
@@ -34,25 +36,26 @@ impl WebProcessRuntime {
.copied()
.unwrap_or(0)
>= max_streams
|| !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
)
{
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamCapacity);
return Err(super::ManagerError::Limit);
}
if !allow_rate(
&mut state.stream_rate,
now,
self.limits.new_streams_per_minute,
self.limits.new_streams_burst,
) {
self.record_stream_rejected_reason(WebRejectionReason::StreamRate);
return Err(super::ManagerError::Limit);
}
let Some(peer_port) = allocate_stream_port(&mut state, client_ip, public_addr) else {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_stream_rejected_reason(WebRejectionReason::StreamTupleExhausted);
return Err(super::ManagerError::Limit);
};
state.streams_live += 1;
*state.streams_per_profile.entry(profile_key).or_insert(0) += 1;
self.streams_opened.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_opened();
Ok(peer_port)
}
@@ -76,9 +79,15 @@ impl WebProcessRuntime {
/// Records a logical stream rejected outside manager quota acquisition.
pub(crate) fn record_stream_rejected(&self) {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_stream_rejected();
self.record_limit_hit();
}
/// Records one logical stream rejection with its operational cause.
pub(crate) fn record_stream_rejected_reason(&self, reason: WebRejectionReason) {
self.record_stream_rejected();
self.telemetry.record_rejection(reason);
}
}
#[cfg(test)]
+41 -19
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -14,6 +13,7 @@ use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRun
use crate::config::WebRuntimeProfile;
use crate::maestro::generation::RuntimeGeneration;
use crate::web::session::WebSession;
use crate::web::telemetry::WebRejectionReason;
impl WebProcessRuntime {
/// Issues a one-use bootstrap credential for an active compatible profile.
@@ -63,7 +63,14 @@ impl WebProcessRuntime {
.as_ref()
.and_then(|runtime| matching_profile(runtime, &profile))
.ok_or(ManagerError::Authentication)?;
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
if !config.web.enabled {
self.telemetry
.record_rejection(WebRejectionReason::ConfigDisabled);
return Err(ManagerError::Closed);
}
if !generation.proxy_shared.is_user_enabled(&profile.user) {
self.telemetry
.record_rejection(WebRejectionReason::UserDisabled);
return Err(ManagerError::Closed);
}
let _operator_admission = self.try_operator_admission()?;
@@ -71,32 +78,47 @@ impl WebProcessRuntime {
let mut state = self.state.lock();
remove_expired_locked(&mut state, now);
state.apply_issuance_policy(generation.id, config.web.enabled);
if state.closed
|| !state.issuance_enabled
|| state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
|| !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
)
if state.closed || !state.issuance_enabled {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Limit);
}
if state
.bootstraps_per_ip
.get(&client_ip)
.copied()
.unwrap_or(0)
>= self.limits.max_bootstraps_per_ip
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
if !allow_rate(
&mut state.bootstrap_rate,
now,
self.limits.new_bootstraps_per_minute,
self.limits.new_bootstraps_burst,
) {
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapRate);
return Err(ManagerError::Limit);
}
if state.bootstraps.len() >= self.limits.max_bootstraps_global
&& !evict_oldest_unused_bootstrap(&mut state)
{
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
}
let Some((token, hash)) = new_unique_token(generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(WebRejectionReason::BootstrapCapacity);
return Err(ManagerError::Limit);
};
let trace_session_id = self.trace.next_session_id();
+17 -16
View File
@@ -1,5 +1,4 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use tokio::time::Instant as TokioInstant;
@@ -79,7 +78,7 @@ impl WebProcessRuntime {
for bootstrap_hash in bootstrap_hashes {
remove_bootstrap_locked(&mut state, bootstrap_hash);
}
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_closed();
drop(state);
self.notify_operator_work_changed();
}
@@ -93,6 +92,7 @@ impl WebProcessRuntime {
self.close_websockets();
self.data_budget.close();
self.http_connections.close();
self.http_overload_connections.close();
self.http_handlers.close();
self.lane_polls.close();
self.lane_aux_polls.close();
@@ -234,28 +234,29 @@ impl WebShutdownDrain {
.saturating_duration_since(self.started)
.as_millis()
.min(u128::from(u64::MAX)) as u64;
let aggregates = self.runtime.telemetry.aggregates();
match outcome {
WebShutdownOutcome::Drained => info!(
target: "telemt::web",
shutdown_drained = true,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime stopped"
),
WebShutdownOutcome::DeadlineExceeded => warn!(
@@ -263,22 +264,22 @@ impl WebShutdownDrain {
shutdown_drained = false,
shutdown_budget_ms = budget_ms,
shutdown_elapsed_ms = elapsed_ms,
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
sessions_created = aggregates.sessions_created,
sessions_closed = aggregates.sessions_closed,
sessions_live,
sessions_pending,
session_tasks_live,
auxiliary_tasks_live,
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
streams_opened = aggregates.streams_opened,
streams_rejected = aggregates.streams_rejected,
streams_live,
pending_bytes = budget.queue_bytes,
pending_items = budget.queue_items,
websocket_bytes = budget.websocket_bytes,
data_high_water_bytes = budget.high_water_bytes,
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
bytes_up = aggregates.bytes_up,
bytes_down = aggregates.bytes_down,
limit_hits = aggregates.limit_hits,
"WEB runtime shutdown deadline exceeded"
),
}
+187
View File
@@ -0,0 +1,187 @@
use serde::Serialize;
use super::WebProcessRuntime;
/// Current usage of one fixed process-wide WEB resource.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacityResourceStatus {
/// Stable closed-set resource token.
pub(crate) resource: &'static str,
/// Stable unit used by API consumers and metric-family routing.
pub(crate) unit: &'static str,
/// Currently retained capacity.
pub(crate) used: usize,
/// Unretained portion of the immutable ceiling before class-specific reserves.
pub(crate) available: usize,
/// Immutable process-wide ceiling.
pub(crate) limit: usize,
/// Whether terminal shutdown closed this allocation authority.
pub(crate) closed: bool,
}
/// Non-blocking fixed-cardinality capacity snapshot.
#[derive(Clone, Serialize)]
pub(crate) struct WebCapacitySnapshot {
/// Exact fixed process-wide resources available without dynamic labels.
pub(crate) resources: Vec<WebCapacityResourceStatus>,
/// Resources with no immediately available capacity.
pub(crate) saturated_resources: Vec<&'static str>,
/// Snapshot planes omitted because their short lock was contended.
pub(crate) partial: Vec<&'static str>,
}
impl WebProcessRuntime {
/// Captures fixed global resource usage without blocking the data plane.
pub(crate) fn capacity_snapshot(&self) -> WebCapacitySnapshot {
let websocket_capacity = self
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let mut resources = vec![
semaphore_status(
"http_connections",
"slots",
&self.http_connections,
self.limits.max_http_connections,
),
semaphore_status(
"http_overload_connections",
"slots",
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
semaphore_status(
"http_handlers",
"slots",
&self.http_handlers,
self.limits.max_http_handlers,
),
semaphore_status(
"lane_polls",
"slots",
&self.lane_polls,
self.limits.max_http_handlers / 2,
),
semaphore_status(
"lane_aux_polls",
"slots",
&self.lane_aux_polls,
(self.limits.max_http_handlers / 4).max(1),
),
semaphore_status(
"body_readers",
"slots",
&self.body_readers,
self.limits.max_body_readers,
),
semaphore_status(
"body_bytes",
"bytes",
&self.body_bytes,
self.limits.max_body_bytes_global,
),
semaphore_status(
"stream_handshakes",
"slots",
&self.stream_handshakes,
self.limits.max_stream_handshakes,
),
semaphore_status(
"websocket_connections",
"slots",
&self.websocket_connections,
websocket_capacity,
),
];
let mut partial = Vec::new();
if let Some(budget) = self.data_budget.try_snapshot() {
resources.extend([
bounded_status(
"pending_bytes",
"bytes",
budget.queue_bytes.saturating_add(budget.websocket_bytes),
self.limits.pending_bytes_global,
budget.closed,
),
bounded_status(
"queue_items",
"items",
budget.queue_items,
self.limits.pending_items_global,
budget.closed,
),
bounded_status(
"websocket_bytes",
"bytes",
budget.websocket_bytes,
self.limits.websocket_bytes_global,
budget.closed,
),
]);
} else {
partial.push("budget");
}
let saturated_resources = resources
.iter()
.filter(|status| status.available == 0 && !status.closed)
.map(|status| status.resource)
.collect();
WebCapacitySnapshot {
resources,
saturated_resources,
partial,
}
}
}
fn semaphore_status(
resource: &'static str,
unit: &'static str,
semaphore: &tokio::sync::Semaphore,
limit: usize,
) -> WebCapacityResourceStatus {
let available = semaphore.available_permits().min(limit);
WebCapacityResourceStatus {
resource,
unit,
used: limit.saturating_sub(available),
available,
limit,
closed: semaphore.is_closed(),
}
}
fn bounded_status(
resource: &'static str,
unit: &'static str,
used: usize,
limit: usize,
closed: bool,
) -> WebCapacityResourceStatus {
let used = used.min(limit);
WebCapacityResourceStatus {
resource,
unit,
used,
available: limit.saturating_sub(used),
limit,
closed,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use tokio::sync::Semaphore;
#[test]
fn semaphore_status_reports_usage_and_available_capacity() {
let semaphore = Arc::new(Semaphore::new(3));
let _permit = semaphore.clone().try_acquire_owned().unwrap();
let status = super::semaphore_status("test", "slots", &semaphore, 3);
assert_eq!(status.used, 1);
assert_eq!(status.available, 2);
assert_eq!(status.limit, 3);
}
}
+38 -46
View File
@@ -16,6 +16,9 @@ pub(crate) use status::{
OperatorDrainOutcome, OperatorDrainState, OperatorDrainStatus, OperatorLifecycleState,
OperatorLifecycleStatus,
};
// Mutable and published lifecycle state storage.
mod state;
use state::{ActiveDrain, OperatorLifecycleInner, OperatorSnapshot, WorkCounts};
const OPERATOR_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
const OPERATOR_REGISTRATION_COUNT: usize = OPERATOR_ADMISSION_CLOSED - 1;
@@ -30,19 +33,6 @@ pub(crate) enum OperatorLifecycleError {
OperationInProgress,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct WorkCounts {
sessions: usize,
streams: usize,
websockets: usize,
}
impl WorkCounts {
fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
struct OperatorAdmission {
state: AtomicUsize,
registrations_drained: Notify,
@@ -114,29 +104,6 @@ impl Drop for OperatorRegistration<'_> {
}
}
struct ActiveDrain {
sequence: u64,
cancellation: CancellationToken,
}
struct OperatorLifecycleInner {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
active: Option<ActiveDrain>,
drain: Option<OperatorDrainStatus>,
}
#[derive(Clone)]
struct OperatorSnapshot {
state: OperatorLifecycleState,
epoch: u64,
since: Instant,
terminal: bool,
drain: Option<OperatorDrainStatus>,
}
pub(super) struct OperatorLifecycle {
runtime_instance: Arc<str>,
admission: OperatorAdmission,
@@ -187,8 +154,8 @@ impl OperatorLifecycle {
pub(super) fn status(&self, config_enabled: bool) -> OperatorLifecycleStatus {
let snapshot = self.published.load();
let admission_open = !snapshot.terminal
&& snapshot.state == OperatorLifecycleState::Running;
let admission_open =
!snapshot.terminal && snapshot.state == OperatorLifecycleState::Running;
OperatorLifecycleStatus {
state: snapshot.state,
epoch: snapshot.epoch,
@@ -203,6 +170,30 @@ impl OperatorLifecycle {
self.published.load().terminal
}
fn rejection_reason(&self) -> crate::web::telemetry::WebRejectionReason {
let inner = self.inner.lock();
if inner.terminal {
return crate::web::telemetry::WebRejectionReason::RuntimeClosed;
}
match inner.state {
OperatorLifecycleState::Paused => {
crate::web::telemetry::WebRejectionReason::OperatorPaused
}
OperatorLifecycleState::Draining => {
crate::web::telemetry::WebRejectionReason::OperatorDraining
}
OperatorLifecycleState::ForceClosing => {
crate::web::telemetry::WebRejectionReason::OperatorForceClosing
}
OperatorLifecycleState::Drained => {
crate::web::telemetry::WebRejectionReason::OperatorDrained
}
OperatorLifecycleState::Running => {
crate::web::telemetry::WebRejectionReason::RuntimeClosed
}
}
}
fn publish_locked(&self, inner: &OperatorLifecycleInner) {
self.published.store(Arc::new(OperatorSnapshot {
state: inner.state,
@@ -213,11 +204,7 @@ impl OperatorLifecycle {
}));
}
fn transition_locked(
&self,
inner: &mut OperatorLifecycleInner,
state: OperatorLifecycleState,
) {
fn transition_locked(&self, inner: &mut OperatorLifecycleInner, state: OperatorLifecycleState) {
if inner.state != state {
inner.state = state;
inner.epoch = inner.epoch.saturating_add(1);
@@ -460,9 +447,14 @@ impl WebProcessRuntime {
pub(super) fn try_operator_admission(
&self,
) -> Result<OperatorRegistration<'_>, super::ManagerError> {
self.operator_lifecycle
.try_register()
.ok_or(super::ManagerError::AdmissionPaused)
match self.operator_lifecycle.try_register() {
Some(registration) => Ok(registration),
None => {
self.telemetry
.record_rejection(self.operator_lifecycle.rejection_reason());
Err(super::ManagerError::AdmissionPaused)
}
}
}
pub(super) fn notify_operator_work_changed(&self) {
@@ -0,0 +1,62 @@
use std::time::Instant;
use tokio_util::sync::CancellationToken;
use super::{OperatorDrainStatus, OperatorLifecycleState};
/// Exact live-work counts sampled while an operator drain is active.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) struct WorkCounts {
/// Live WEB sessions.
pub(super) sessions: usize,
/// Live logical streams.
pub(super) streams: usize,
/// Live session-owned WebSockets.
pub(super) websockets: usize,
}
impl WorkCounts {
/// Returns whether every tracked work class reached zero.
pub(super) fn is_zero(self) -> bool {
self.sessions == 0 && self.streams == 0 && self.websockets == 0
}
}
/// Single active drain identity and cancellation authority.
pub(super) struct ActiveDrain {
/// Process-local operation sequence.
pub(super) sequence: u64,
/// Cancellation token owned by resume or terminal shutdown.
pub(super) cancellation: CancellationToken,
}
/// Mutex-protected lifecycle state used for atomic transitions.
pub(super) struct OperatorLifecycleInner {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Currently active drain, if any.
pub(super) active: Option<ActiveDrain>,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
/// Lock-free read snapshot published after lifecycle mutations.
#[derive(Clone)]
pub(super) struct OperatorSnapshot {
/// Current reversible lifecycle state.
pub(super) state: OperatorLifecycleState,
/// Monotonic transition epoch.
pub(super) epoch: u64,
/// Monotonic transition timestamp.
pub(super) since: Instant,
/// Whether terminal process shutdown closed the state machine.
pub(super) terminal: bool,
/// Active or retained latest drain status.
pub(super) drain: Option<OperatorDrainStatus>,
}
+4 -1
View File
@@ -106,7 +106,10 @@ async fn pause_fence_leaves_no_late_admission_commits_under_scheduler_pressure()
#[tokio::test]
async fn empty_drain_completes_gracefully_and_stays_closed_until_resume() {
let (runtime, generation) = test_runtime();
let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
let accepted = runtime
.drain_operator(Duration::from_secs(30))
.await
.unwrap();
assert_eq!(accepted.state, OperatorLifecycleState::Draining);
let completed = tokio::time::timeout(Duration::from_secs(1), async {
+25 -15
View File
@@ -1,10 +1,10 @@
use std::net::IpAddr;
use std::sync::atomic::Ordering;
use std::time::Instant;
use super::state::{ManagerState, allow_rate};
use super::{ProfileKey, WebProcessRuntime};
use crate::config::WebRuntimeProfile;
use crate::web::telemetry::WebRejectionReason;
/// Applies process, address, profile, and rate ceilings to one initial session.
pub(super) fn admit_initial(
@@ -15,23 +15,33 @@ pub(super) fn admit_initial(
profile_key: ProfileKey,
profile: &WebRuntimeProfile,
) -> bool {
let admitted = state.sessions.len() < runtime.limits.max_sessions_global
&& state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
< runtime.limits.max_sessions_per_ip
&& state
if state.sessions.len() >= runtime.limits.max_sessions_global
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
>= runtime.limits.max_sessions_per_ip
|| state
.sessions_per_profile
.get(&profile_key)
.copied()
.unwrap_or(0)
< profile.max_sessions
&& allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
);
if !admitted {
runtime.limit_hits.fetch_add(1, Ordering::Relaxed);
>= profile.max_sessions
{
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionCapacity);
return false;
}
admitted
if !allow_rate(
&mut state.session_rate,
now,
runtime.limits.new_sessions_per_minute,
runtime.limits.new_sessions_burst,
) {
runtime.record_limit_hit();
runtime
.telemetry
.record_rejection(WebRejectionReason::SessionRate);
return false;
}
true
}
+4 -3
View File
@@ -1,6 +1,5 @@
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use sha2::{Digest, Sha256};
@@ -278,7 +277,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Limit);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
return Err(ManagerError::Limit);
};
let carrier_deadline_at = carrier_request
@@ -341,7 +342,7 @@ impl WebProcessRuntime {
)
};
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
let identity = session.trace_identity();
let result = CreateResult {
token: session_token,
@@ -48,7 +48,9 @@ impl WebProcessRuntime {
return Err(ManagerError::Closed);
}
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
self.record_limit_hit();
self.telemetry
.record_rejection(crate::web::telemetry::WebRejectionReason::SessionCapacity);
drop(state);
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
return Err(ManagerError::Limit);
@@ -112,8 +114,8 @@ impl WebProcessRuntime {
{
*slot = Some(replacement.old_session.carrier());
}
self.sessions_created.fetch_add(1, Ordering::Relaxed);
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
self.telemetry.record_session_created();
self.telemetry.record_session_closed();
let result = CreateResult {
token: session_token,
carrier: replacement.carrier,
+15 -8
View File
@@ -1,7 +1,6 @@
use std::net::IpAddr;
use std::ops::Bound::{Excluded, Unbounded};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;
use serde::Serialize;
@@ -298,6 +297,7 @@ impl WebProcessRuntime {
.limits
.max_http_connections
.saturating_sub(self.limits.websocket_http_connection_reserve);
let aggregates = self.telemetry.aggregates();
WebRuntimeStatus {
runtime_instance: self.runtime_instance().to_string(),
generation_id,
@@ -313,6 +313,13 @@ impl WebProcessRuntime {
"http_connections",
permits(&self.http_connections, self.limits.max_http_connections),
),
(
"http_overload_connections",
permits(
&self.http_overload_connections,
self.limits.max_http_overload_connections,
),
),
(
"http_handlers",
permits(&self.http_handlers, self.limits.max_http_handlers),
@@ -346,13 +353,13 @@ impl WebProcessRuntime {
),
],
auxiliary_tasks: self.tasks.len(),
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
session_incarnations_created: aggregates.sessions_created,
session_incarnations_closed: aggregates.sessions_closed,
streams_opened: aggregates.streams_opened,
streams_rejected: aggregates.streams_rejected,
bytes_up: aggregates.bytes_up,
bytes_down: aggregates.bytes_down,
limit_hits: aggregates.limit_hits,
partial,
}
}
+40 -14
View File
@@ -8,6 +8,7 @@ use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
use crate::web::telemetry::WebRejectionReason;
// Deterministic victim ordering remains isolated from registry mutation.
mod policy;
@@ -208,7 +209,7 @@ pub(super) async fn admit(
return Err(ManagerError::Closed);
}
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
match try_admit(
let capacity_reason = match try_admit(
runtime,
owner,
session_id,
@@ -220,12 +221,23 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => return Ok(connection),
Err(TryAdmitError::Conflict) => return Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => return Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {}
}
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
return Err(ManagerError::Concurrent);
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
return Err(ManagerError::Closed);
}
Err(TryAdmitError::Capacity(reason)) => reason,
};
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None, true) else {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(capacity_reason);
return Err(ManagerError::Limit);
};
let released = victim.released.cancelled();
@@ -249,17 +261,28 @@ pub(super) async fn admit(
&parent_cancellation,
) {
Ok(connection) => Ok(connection),
Err(TryAdmitError::Conflict) => Err(ManagerError::Concurrent),
Err(TryAdmitError::Closed) => Err(ManagerError::Closed),
Err(TryAdmitError::Capacity) => {
Err(TryAdmitError::Conflict) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::Concurrent);
Err(ManagerError::Concurrent)
}
Err(TryAdmitError::Closed) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
Err(ManagerError::Closed)
}
Err(TryAdmitError::Capacity(reason)) => {
runtime.record_limit_hit();
runtime.telemetry().record_rejection(reason);
Err(ManagerError::Limit)
}
}
}
enum TryAdmitError {
Capacity,
Capacity(WebRejectionReason),
Conflict,
Closed,
}
@@ -292,10 +315,13 @@ fn try_admit(
}
let slot = Arc::clone(&runtime.websocket_connections)
.try_acquire_owned()
.map_err(|_| TryAdmitError::Capacity)?;
let base_budget = runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let base_budget =
runtime
.try_websocket_base_budget(owner, base_bytes)
.ok_or(TryAdmitError::Capacity(
WebRejectionReason::WebSocketBytesCapacity,
))?;
if parent_cancellation.is_cancelled() {
return Err(TryAdmitError::Closed);
}
@@ -304,7 +330,7 @@ fn try_admit(
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_add(1)
})
.map_err(|_| TryAdmitError::Capacity)?;
.map_err(|_| TryAdmitError::Capacity(WebRejectionReason::WebSocketConnectionCapacity))?;
let now = runtime.websocket_tick();
let entry = Arc::new(WebSocketEntry {
id,
+2
View File
@@ -14,5 +14,7 @@ pub(crate) mod manager;
pub(crate) mod session;
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
pub(crate) mod stream;
/// Process-owned fixed-cardinality WEB operational telemetry.
pub(crate) mod telemetry;
/// Process-owned bounded WEB debugging records and capture lifecycle.
pub(crate) mod trace;
+9 -1
View File
@@ -31,6 +31,9 @@ impl WebSession {
};
let generation = manager.active_generation();
if !*generation.admission_rx.borrow() {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationAdmissionClosed,
);
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream.id),
@@ -70,6 +73,9 @@ impl WebSession {
}
};
if let Err(future) = generation.try_spawn_session(future) {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationScopeClosed,
);
retain_rejected.store(retain_reservation_on_reject, Ordering::Release);
self.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
@@ -257,7 +263,9 @@ async fn run_stream(
return;
};
let Ok(_connection_permit) = connection_permits.try_acquire_owned() else {
manager.record_stream_rejected();
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::GenerationConnectionCapacity,
);
session.trace_lifecycle(
crate::web::trace::TraceLifecycleEvent::StreamRejected,
Some(stream_identity.id),
+5
View File
@@ -183,6 +183,11 @@ impl WebSession {
&& data_items <= item_limit - items
};
if !fits {
if let Some(manager) = self.manager.upgrade() {
manager.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::QueueSessionCapacity,
);
}
return false;
}
let Some(manager) = self.manager.upgrade() else {
+4 -1
View File
@@ -284,10 +284,13 @@ impl WebSession {
}
fn reserve_stream_locked(&self, state: &mut SessionState) -> Option<u16> {
let manager = self.manager.upgrade()?;
if state.active_peer_ports.len() >= self.profile.max_streams_per_session {
manager.record_stream_rejected_reason(
crate::web::telemetry::WebRejectionReason::StreamSessionCapacity,
);
return None;
}
let manager = self.manager.upgrade()?;
let peer_port = manager
.try_acquire_stream(
self.profile_key,
+541
View File
@@ -0,0 +1,541 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::Instant;
use serde::Serialize;
/// Stable operational rejection reason recorded at the decision point.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebRejectionReason {
/// Ordinary accepted HTTP connection slots were exhausted.
HttpConnectionCapacity,
/// Concurrent HTTP request handler slots were exhausted.
HttpHandlerCapacity,
/// Primary long-poll slots were exhausted.
LanePollCapacity,
/// Auxiliary lane-open wait slots were exhausted.
LaneAuxPollCapacity,
/// Concurrent collected-body reader slots were exhausted.
BodyReaderCapacity,
/// Process-wide collected-body bytes were exhausted.
BodyBytesCapacity,
/// Bootstrap credential quota or identifier space was exhausted.
BootstrapCapacity,
/// Bootstrap issuance rate policy rejected the operation.
BootstrapRate,
/// Live session quota or identifier space was exhausted.
SessionCapacity,
/// Session creation rate policy rejected the operation.
SessionRate,
/// One session reached its logical-stream ceiling.
StreamSessionCapacity,
/// Global or profile logical-stream capacity was exhausted.
StreamCapacity,
/// Logical-stream creation rate policy rejected the operation.
StreamRate,
/// No synthetic source port remained for the exact relay tuple.
StreamTupleExhausted,
/// Concurrent inner-handshake slots were exhausted.
StreamHandshakeCapacity,
/// The active relay generation had no connection permit.
GenerationConnectionCapacity,
/// One session reached its queued data ceiling.
QueueSessionCapacity,
/// The process-wide queued data ceiling was exhausted.
QueueGlobalCapacity,
/// Process-wide WebSocket connection slots were exhausted.
WebSocketConnectionCapacity,
/// Shared WebSocket byte capacity was exhausted.
WebSocketBytesCapacity,
/// Bounded WebSocket replacement capacity was exhausted.
WebSocketEvictionCapacity,
/// Effective WEB configuration disabled new work.
ConfigDisabled,
/// Effective user policy disabled new work.
UserDisabled,
/// Operator admission was paused.
OperatorPaused,
/// Operator admission was gracefully draining.
OperatorDraining,
/// Operator drain had committed forced close signals.
OperatorForceClosing,
/// Operator drain had reached confirmed zero.
OperatorDrained,
/// The active relay generation closed new admission.
GenerationAdmissionClosed,
/// The active relay generation could not own a new task.
GenerationScopeClosed,
/// Terminal runtime closure rejected the operation.
RuntimeClosed,
/// A bounded operation could not acquire or complete its deadline.
Deadline,
/// Concurrent ownership rejected the operation.
Concurrent,
}
impl WebRejectionReason {
/// Complete fixed rejection set in stable metric order.
pub(crate) const ALL: [Self; 32] = [
Self::HttpConnectionCapacity,
Self::HttpHandlerCapacity,
Self::LanePollCapacity,
Self::LaneAuxPollCapacity,
Self::BodyReaderCapacity,
Self::BodyBytesCapacity,
Self::BootstrapCapacity,
Self::BootstrapRate,
Self::SessionCapacity,
Self::SessionRate,
Self::StreamSessionCapacity,
Self::StreamCapacity,
Self::StreamRate,
Self::StreamTupleExhausted,
Self::StreamHandshakeCapacity,
Self::GenerationConnectionCapacity,
Self::QueueSessionCapacity,
Self::QueueGlobalCapacity,
Self::WebSocketConnectionCapacity,
Self::WebSocketBytesCapacity,
Self::WebSocketEvictionCapacity,
Self::ConfigDisabled,
Self::UserDisabled,
Self::OperatorPaused,
Self::OperatorDraining,
Self::OperatorForceClosing,
Self::OperatorDrained,
Self::GenerationAdmissionClosed,
Self::GenerationScopeClosed,
Self::RuntimeClosed,
Self::Deadline,
Self::Concurrent,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::HttpConnectionCapacity => "http_connection_capacity",
Self::HttpHandlerCapacity => "http_handler_capacity",
Self::LanePollCapacity => "lane_poll_capacity",
Self::LaneAuxPollCapacity => "lane_aux_poll_capacity",
Self::BodyReaderCapacity => "body_reader_capacity",
Self::BodyBytesCapacity => "body_bytes_capacity",
Self::BootstrapCapacity => "bootstrap_capacity",
Self::BootstrapRate => "bootstrap_rate",
Self::SessionCapacity => "session_capacity",
Self::SessionRate => "session_rate",
Self::StreamSessionCapacity => "stream_session_capacity",
Self::StreamCapacity => "stream_capacity",
Self::StreamRate => "stream_rate",
Self::StreamTupleExhausted => "stream_tuple_exhausted",
Self::StreamHandshakeCapacity => "stream_handshake_capacity",
Self::GenerationConnectionCapacity => "generation_connection_capacity",
Self::QueueSessionCapacity => "queue_session_capacity",
Self::QueueGlobalCapacity => "queue_global_capacity",
Self::WebSocketConnectionCapacity => "websocket_connection_capacity",
Self::WebSocketBytesCapacity => "websocket_bytes_capacity",
Self::WebSocketEvictionCapacity => "websocket_eviction_capacity",
Self::ConfigDisabled => "config_disabled",
Self::UserDisabled => "user_disabled",
Self::OperatorPaused => "operator_paused",
Self::OperatorDraining => "operator_draining",
Self::OperatorForceClosing => "operator_force_closing",
Self::OperatorDrained => "operator_drained",
Self::GenerationAdmissionClosed => "generation_admission_closed",
Self::GenerationScopeClosed => "generation_scope_closed",
Self::RuntimeClosed => "runtime_closed",
Self::Deadline => "deadline",
Self::Concurrent => "concurrent",
}
}
}
/// Terminal result for one accepted socket that found normal HTTP capacity exhausted.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebHttpConnectionOverloadOutcome {
/// Legacy policy closed the accepted socket immediately.
Dropped,
/// Bounded waiting acquired ordinary connection capacity.
WaitAdmitted,
/// Bounded waiting expired and emitted the retryable response.
WaitTimeout503,
/// Immediate response policy emitted the retryable response.
Responded503,
/// The bounded overload socket pool was already full.
OverflowCapacityDrop,
/// Writing or closing the retryable response failed.
ResponseErrorDrop,
/// Listener or process shutdown cancelled overload handling.
ShutdownDrop,
}
impl WebHttpConnectionOverloadOutcome {
/// Complete fixed accepted-socket outcome set in stable metric order.
pub(crate) const ALL: [Self; 7] = [
Self::Dropped,
Self::WaitAdmitted,
Self::WaitTimeout503,
Self::Responded503,
Self::OverflowCapacityDrop,
Self::ResponseErrorDrop,
Self::ShutdownDrop,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Dropped => "dropped",
Self::WaitAdmitted => "wait_admitted",
Self::WaitTimeout503 => "wait_timeout_503",
Self::Responded503 => "responded_503",
Self::OverflowCapacityDrop => "overflow_capacity_drop",
Self::ResponseErrorDrop => "response_error_drop",
Self::ShutdownDrop => "shutdown_drop",
}
}
}
/// Passive outcome for one plain-HTTP decoy upstream request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(usize)]
pub(crate) enum WebDecoyUpstreamOutcome {
/// The internal decoy origin produced a response head.
Success,
/// The request activity plane could not grant a deadline lease.
DeadlineExhausted,
/// The configured origin actively refused the TCP connection.
ConnectRefused,
/// The bounded TCP connect phase timed out.
ConnectTimeout,
/// The TCP connect failed for another I/O reason.
ConnectError,
/// The bounded HTTP client handshake timed out.
HttpHandshakeTimeout,
/// The HTTP client handshake failed.
HttpHandshakeError,
/// The origin did not produce a response head before the deadline.
ResponseHeadTimeout,
/// URI preparation or the HTTP request failed.
RequestError,
}
impl WebDecoyUpstreamOutcome {
/// Complete fixed decoy-origin outcome set in stable metric order.
pub(crate) const ALL: [Self; 9] = [
Self::Success,
Self::DeadlineExhausted,
Self::ConnectRefused,
Self::ConnectTimeout,
Self::ConnectError,
Self::HttpHandshakeTimeout,
Self::HttpHandshakeError,
Self::ResponseHeadTimeout,
Self::RequestError,
];
/// Returns the stable API and Prometheus label token.
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Success => "success",
Self::DeadlineExhausted => "deadline_exhausted",
Self::ConnectRefused => "connect_refused",
Self::ConnectTimeout => "connect_timeout",
Self::ConnectError => "connect_error",
Self::HttpHandshakeTimeout => "http_handshake_timeout",
Self::HttpHandshakeError => "http_handshake_error",
Self::ResponseHeadTimeout => "response_head_timeout",
Self::RequestError => "request_error",
}
}
}
/// API-safe typed rejection counter.
#[derive(Clone, Serialize)]
pub(crate) struct WebRejectionCounter {
/// Stable closed-set rejection token.
pub(crate) reason: &'static str,
/// Monotonic process-lifetime count.
pub(crate) total: u64,
}
/// API-safe typed outcome counter.
#[derive(Clone, Serialize)]
pub(crate) struct WebOutcomeCounter {
/// Stable closed-set outcome token.
pub(crate) outcome: &'static str,
/// Monotonic process-lifetime count.
pub(crate) total: u64,
}
/// Existing aggregate WEB totals retained with their original semantics.
#[derive(Clone, Copy)]
pub(crate) struct WebAggregateSnapshot {
/// Created session incarnations.
pub(crate) sessions_created: u64,
/// Closed session incarnations.
pub(crate) sessions_closed: u64,
/// Admitted logical streams.
pub(crate) streams_opened: u64,
/// Rejected logical streams covered by the legacy aggregate.
pub(crate) streams_rejected: u64,
/// Accepted carrier uplink payload bytes.
pub(crate) bytes_up: u64,
/// Emitted carrier downlink payload bytes.
pub(crate) bytes_down: u64,
/// Legacy aggregate capacity-hit count.
pub(crate) limit_hits: u64,
}
/// Process-owned operational counters shared by ingress, API, and metrics.
pub(crate) struct WebTelemetry {
started: Instant,
live_acceptors: AtomicUsize,
accepted: AtomicU64,
accept_errors: AtomicU64,
rejections: [AtomicU64; WebRejectionReason::ALL.len()],
overload_outcomes: [AtomicU64; WebHttpConnectionOverloadOutcome::ALL.len()],
decoy_outcomes: [AtomicU64; WebDecoyUpstreamOutcome::ALL.len()],
last_decoy_outcome: AtomicUsize,
last_decoy_elapsed_ms: AtomicU64,
sessions_created: AtomicU64,
sessions_closed: AtomicU64,
streams_opened: AtomicU64,
streams_rejected: AtomicU64,
bytes_up: AtomicU64,
bytes_down: AtomicU64,
limit_hits: AtomicU64,
}
impl WebTelemetry {
/// Creates zeroed process-lifetime WEB telemetry.
pub(crate) fn new() -> Arc<Self> {
Arc::new(Self {
started: Instant::now(),
live_acceptors: AtomicUsize::new(0),
accepted: AtomicU64::new(0),
accept_errors: AtomicU64::new(0),
rejections: std::array::from_fn(|_| AtomicU64::new(0)),
overload_outcomes: std::array::from_fn(|_| AtomicU64::new(0)),
decoy_outcomes: std::array::from_fn(|_| AtomicU64::new(0)),
last_decoy_outcome: AtomicUsize::new(usize::MAX),
last_decoy_elapsed_ms: AtomicU64::new(0),
sessions_created: AtomicU64::new(0),
sessions_closed: AtomicU64::new(0),
streams_opened: AtomicU64::new(0),
streams_rejected: AtomicU64::new(0),
bytes_up: AtomicU64::new(0),
bytes_down: AtomicU64::new(0),
limit_hits: AtomicU64::new(0),
})
}
/// Registers one live accept loop before its task can be cancelled unpolled.
pub(crate) fn acceptor_guard(self: &Arc<Self>) -> WebAcceptorGuard {
self.live_acceptors.fetch_add(1, Ordering::AcqRel);
WebAcceptorGuard {
telemetry: Arc::clone(self),
}
}
/// Returns currently registered WEB accept loops.
pub(crate) fn live_acceptors(&self) -> usize {
self.live_acceptors.load(Ordering::Acquire)
}
/// Records one successfully accepted WEB socket.
pub(crate) fn record_accept(&self) {
self.accepted.fetch_add(1, Ordering::Relaxed);
}
/// Returns successfully accepted WEB sockets.
pub(crate) fn accepted(&self) -> u64 {
self.accepted.load(Ordering::Relaxed)
}
/// Records one WEB listener `accept` error.
pub(crate) fn record_accept_error(&self) {
self.accept_errors.fetch_add(1, Ordering::Relaxed);
}
/// Returns WEB listener `accept` errors.
pub(crate) fn accept_errors(&self) -> u64 {
self.accept_errors.load(Ordering::Relaxed)
}
/// Records one operational rejection at its admission decision point.
pub(crate) fn record_rejection(&self, reason: WebRejectionReason) {
self.rejections[reason as usize].fetch_add(1, Ordering::Relaxed);
}
/// Returns one fixed rejection counter.
pub(crate) fn rejection_total(&self, reason: WebRejectionReason) -> u64 {
self.rejections[reason as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed rejection set for API serialization.
pub(crate) fn rejection_counters(&self) -> Vec<WebRejectionCounter> {
WebRejectionReason::ALL
.into_iter()
.map(|reason| WebRejectionCounter {
reason: reason.as_str(),
total: self.rejection_total(reason),
})
.collect()
}
/// Records one terminal accepted-socket overload outcome.
pub(crate) fn record_overload(&self, outcome: WebHttpConnectionOverloadOutcome) {
self.overload_outcomes[outcome as usize].fetch_add(1, Ordering::Relaxed);
}
/// Returns one fixed accepted-socket overload counter.
pub(crate) fn overload_total(&self, outcome: WebHttpConnectionOverloadOutcome) -> u64 {
self.overload_outcomes[outcome as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed overload outcome set for API serialization.
pub(crate) fn overload_counters(&self) -> Vec<WebOutcomeCounter> {
WebHttpConnectionOverloadOutcome::ALL
.into_iter()
.map(|outcome| WebOutcomeCounter {
outcome: outcome.as_str(),
total: self.overload_total(outcome),
})
.collect()
}
/// Records one internal plain-HTTP decoy origin outcome.
pub(crate) fn record_decoy(&self, outcome: WebDecoyUpstreamOutcome) {
self.decoy_outcomes[outcome as usize].fetch_add(1, Ordering::Relaxed);
let elapsed_ms = self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
self.last_decoy_elapsed_ms
.store(elapsed_ms.saturating_add(1), Ordering::Relaxed);
self.last_decoy_outcome
.store(outcome as usize, Ordering::Release);
}
/// Returns one fixed internal decoy origin counter.
pub(crate) fn decoy_total(&self, outcome: WebDecoyUpstreamOutcome) -> u64 {
self.decoy_outcomes[outcome as usize].load(Ordering::Relaxed)
}
/// Captures the complete fixed decoy outcome set for API serialization.
pub(crate) fn decoy_counters(&self) -> Vec<WebOutcomeCounter> {
WebDecoyUpstreamOutcome::ALL
.into_iter()
.map(|outcome| WebOutcomeCounter {
outcome: outcome.as_str(),
total: self.decoy_total(outcome),
})
.collect()
}
/// Returns the last decoy outcome and its monotonic age in milliseconds.
pub(crate) fn last_decoy(&self) -> Option<(&'static str, u64)> {
let raw = self.last_decoy_outcome.load(Ordering::Acquire);
let outcome = WebDecoyUpstreamOutcome::ALL.get(raw).copied()?;
let recorded = self.last_decoy_elapsed_ms.load(Ordering::Relaxed);
let now = self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
Some((
outcome.as_str(),
now.saturating_sub(recorded.saturating_sub(1)),
))
}
/// Records one created session incarnation.
pub(crate) fn record_session_created(&self) {
self.sessions_created.fetch_add(1, Ordering::Relaxed);
}
/// Records one closed session incarnation.
pub(crate) fn record_session_closed(&self) {
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
}
/// Records one admitted logical stream.
pub(crate) fn record_stream_opened(&self) {
self.streams_opened.fetch_add(1, Ordering::Relaxed);
}
/// Records one logical stream in the legacy rejection aggregate.
pub(crate) fn record_stream_rejected(&self) {
self.streams_rejected.fetch_add(1, Ordering::Relaxed);
}
/// Adds accepted carrier uplink payload bytes.
pub(crate) fn record_up(&self, bytes: usize) {
self.bytes_up.fetch_add(bytes as u64, Ordering::Relaxed);
}
/// Adds emitted carrier downlink payload bytes.
pub(crate) fn record_down(&self, bytes: usize) {
self.bytes_down.fetch_add(bytes as u64, Ordering::Relaxed);
}
/// Records one legacy aggregate capacity hit.
pub(crate) fn record_limit_hit(&self) {
self.limit_hits.fetch_add(1, Ordering::Relaxed);
}
/// Captures legacy process-owned aggregate totals.
pub(crate) fn aggregates(&self) -> WebAggregateSnapshot {
WebAggregateSnapshot {
sessions_created: self.sessions_created.load(Ordering::Relaxed),
sessions_closed: self.sessions_closed.load(Ordering::Relaxed),
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
bytes_up: self.bytes_up.load(Ordering::Relaxed),
bytes_down: self.bytes_down.load(Ordering::Relaxed),
limit_hits: self.limit_hits.load(Ordering::Relaxed),
}
}
}
/// Cancellation-safe live-acceptor registration.
pub(crate) struct WebAcceptorGuard {
telemetry: Arc<WebTelemetry>,
}
impl Drop for WebAcceptorGuard {
fn drop(&mut self) {
self.telemetry.live_acceptors.fetch_sub(1, Ordering::AcqRel);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fixed_counter_sets_and_acceptor_guard_are_exact() {
let telemetry = WebTelemetry::new();
let guard = telemetry.acceptor_guard();
assert_eq!(telemetry.live_acceptors(), 1);
telemetry.record_rejection(WebRejectionReason::HttpConnectionCapacity);
telemetry.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
telemetry.record_decoy(WebDecoyUpstreamOutcome::ConnectRefused);
assert_eq!(
telemetry.rejection_counters().len(),
WebRejectionReason::ALL.len()
);
assert_eq!(
telemetry.overload_counters().len(),
WebHttpConnectionOverloadOutcome::ALL.len()
);
assert_eq!(
telemetry.decoy_counters().len(),
WebDecoyUpstreamOutcome::ALL.len()
);
assert_eq!(
telemetry.rejection_total(WebRejectionReason::HttpConnectionCapacity),
1
);
assert_eq!(
telemetry.last_decoy().map(|value| value.0),
Some("connect_refused")
);
drop(guard);
assert_eq!(telemetry.live_acceptors(), 0);
}
}