API for WEB: bounded lifecycle and overload observability added

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-22 10:38:26 +03:00 · 2026-08-29 16:19:21 +03:00
parent 012dc07a98
commit 084834f5ec
45 changed files with 2440 additions and 298 deletions
@@ -6,6 +6,7 @@
 //! - `bind` prepares and activates sockets without partial startup binding.
 //! - `accept` runs cancellation-aware TCP accept loops.
 //! - `control` coordinates reversible listener transitions and shutdown.
+//! - `web_overload` handles accepted WEB sockets outside ordinary capacity.

 mod accept;
 mod bind;
@@ -13,6 +14,7 @@ mod control;
 mod plan;
 #[cfg(unix)]
 mod unix;
+mod web_overload;

 pub(crate) use bind::bind_listeners;
 pub(crate) use control::{ListenerManager, PreparedListenerTransition};
@@ -13,9 +13,11 @@ use crate::config::{ListenerTransport, RstOnCloseMode};
 use crate::proxy::ClientHandler;
 use crate::transport::socket::set_linger_zero;
 use crate::web::manager::WebProcessRuntime;
+use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};

 use super::bind::BoundTcpListener;
 use super::plan::ListenerBindSpec;
+use super::web_overload;
 use crate::maestro::generation::RuntimeGeneration;
 use crate::maestro::helpers::{
    expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
@@ -190,6 +192,7 @@ async fn run_accept_loop(
    web_runtime: Option<Arc<WebProcessRuntime>>,
    connections: TaskTracker,
    cancellation: CancellationToken,
+    _web_acceptor_guard: Option<WebAcceptorGuard>,
 ) {
    loop {
        let accepted = tokio::select! {
@@ -204,8 +207,45 @@ async fn run_accept_loop(
                        error!(addr = %spec.addr, "WEB listener has no process runtime");
                        return;
                    };
+                    web_runtime.telemetry().record_accept();
                    let Some(connection_permit) = web_runtime.try_http_connection() else {
-                        drop(stream);
+                        let config = web_runtime.active_generation().config();
+                        let action = config.web.http_connection_capacity_action;
+                        let phase_timeout =
+                            Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
+                        drop(config);
+                        if action == crate::config::WebHttpConnectionCapacityAction::Drop {
+                            web_runtime.telemetry().record_rejection(
+                                crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
+                            );
+                            web_runtime
+                                .telemetry()
+                                .record_overload(WebHttpConnectionOverloadOutcome::Dropped);
+                            drop(stream);
+                            continue;
+                        }
+                        let Some(overload_permit) = web_runtime.try_http_overload_connection()
+                        else {
+                            web_runtime.telemetry().record_rejection(
+                                crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
+                            );
+                            web_runtime.telemetry().record_overload(
+                                WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
+                            );
+                            drop(stream);
+                            continue;
+                        };
+                        connections.spawn(web_overload::serve(
+                            stream,
+                            peer_addr,
+                            spec.web_client_ip_source,
+                            Arc::clone(&spec.web_trusted_proxy_cidrs),
+                            Arc::clone(web_runtime),
+                            cancellation.clone(),
+                            overload_permit,
+                            action,
+                            phase_timeout,
+                        ));
                        continue;
                    };
                    connections.spawn(crate::web::http::serve_connection(
@@ -245,6 +285,9 @@ async fn run_accept_loop(
                }
            }
            Err(error_value) => {
+                if let Some(web_runtime) = &web_runtime {
+                    web_runtime.telemetry().record_accept_error();
+                }
                error!(addr = %spec.addr, error = %error_value, "TCP accept error");
                tokio::select! {
                    biased;
@@ -262,8 +305,16 @@ impl ListenerSlot {
        active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
        web_runtime: Option<Arc<WebProcessRuntime>>,
    ) -> Self {
+        let web_runtime = if bound.spec.transport == ListenerTransport::Web {
+            web_runtime
+        } else {
+            None
+        };
        let cancellation = CancellationToken::new();
        let connections = TaskTracker::new();
+        let web_acceptor_guard = web_runtime
+            .as_ref()
+            .map(|runtime| runtime.telemetry().acceptor_guard());
        let task = tokio::spawn(run_accept_loop(
            bound.listener.clone(),
            bound.spec.clone(),
@@ -271,6 +322,7 @@ impl ListenerSlot {
            web_runtime.clone(),
            connections.clone(),
            cancellation.clone(),
+            web_acceptor_guard,
        ));
        Self {
            spec: bound.spec,
@@ -364,6 +416,10 @@ impl ListenerSlot {
        self.active_runtime = active_runtime.clone();
        self.cancellation = CancellationToken::new();
        self.connections = TaskTracker::new();
+        let web_acceptor_guard = self
+            .web_runtime
+            .as_ref()
+            .map(|runtime| runtime.telemetry().acceptor_guard());
        self.task = Some(tokio::spawn(run_accept_loop(
            self.listener.clone(),
            self.spec.clone(),
@@ -371,6 +427,7 @@ impl ListenerSlot {
            self.web_runtime.clone(),
            self.connections.clone(),
            self.cancellation.clone(),
+            web_acceptor_guard,
        )));
    }
 }
@@ -58,8 +58,13 @@ impl ListenerManager {
            .map(|listener| listener.spec.addr)
            .collect();
        let has_web = !web_listeners.is_empty();
-        let web_runtime =
-            has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
+        let web_runtime = has_web.then(|| {
+            WebProcessRuntime::start_with_trace(
+                active_runtime.clone(),
+                trace,
+                web_control.telemetry(),
+            )
+        });
        let mut slots = BTreeMap::new();
        for listener in bound.listeners {
            let addr = listener.spec.addr;
@@ -461,4 +466,30 @@ mod tests {
        manager.shutdown().await.unwrap();
        runtime.stop_sessions().await;
    }
+
+    #[tokio::test]
+    async fn acceptor_liveness_counts_only_web_listeners() {
+        let runtime = test_runtime_generation(1, ProxyConfig::default());
+        let active_runtime = Arc::new(ArcSwap::from(runtime.clone()));
+        let (native_listener, _native_addr) = bound_listener().await;
+        let (mut web_listener, _web_addr) = bound_listener().await;
+        web_listener.spec.transport = ListenerTransport::Web;
+        let bound = BoundListeners {
+            listeners: vec![native_listener, web_listener],
+            #[cfg(unix)]
+            unix_listener: None,
+        };
+        let trace = WebTraceStore::new(
+            runtime.config().web.debug.clone(),
+            &runtime.config().web.limits,
+        );
+        let control = WebRuntimeControl::new();
+        let receiver = control.subscribe();
+        let mut manager = ListenerManager::start(bound, active_runtime, trace, control);
+
+        assert_eq!(receiver.borrow().telemetry.live_acceptors(), 1);
+
+        manager.shutdown().await.unwrap();
+        runtime.stop_sessions().await;
+    }
 }
@@ -0,0 +1,277 @@
+use std::net::SocketAddr;
+use std::sync::Arc;
+use std::time::Duration;
+
+use ipnetwork::IpNetwork;
+use tokio::io::AsyncWriteExt;
+use tokio::net::TcpStream;
+use tokio::sync::OwnedSemaphorePermit;
+use tokio_util::sync::CancellationToken;
+
+use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
+use crate::web::manager::WebProcessRuntime;
+use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
+
+pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
+
+/// Handles one accepted WEB socket outside ordinary connection capacity.
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn serve(
+    stream: TcpStream,
+    peer: SocketAddr,
+    client_ip_source: WebClientIpSource,
+    trusted_proxy_cidrs: Arc<[IpNetwork]>,
+    runtime: Arc<WebProcessRuntime>,
+    cancellation: CancellationToken,
+    overload_permit: OwnedSemaphorePermit,
+    action: WebHttpConnectionCapacityAction,
+    phase_timeout: Duration,
+) {
+    match action {
+        WebHttpConnectionCapacityAction::Drop => unreachable!("drop is handled before spawn"),
+        WebHttpConnectionCapacityAction::Respond => {
+            let outcome = respond(stream, &cancellation, phase_timeout).await;
+            record_final_capacity_rejection(&runtime, outcome);
+            runtime.telemetry().record_overload(outcome);
+        }
+        WebHttpConnectionCapacityAction::Wait => {
+            let connection_permit = tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => {
+                    runtime
+                        .telemetry()
+                        .record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
+                    return;
+                }
+                permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
+                    permit.ok().flatten()
+                }
+            };
+            let Some(connection_permit) = connection_permit else {
+                if runtime.is_shutdown() {
+                    runtime
+                        .telemetry()
+                        .record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
+                    return;
+                }
+                let outcome = match respond(stream, &cancellation, phase_timeout).await {
+                    WebHttpConnectionOverloadOutcome::Responded503 => {
+                        WebHttpConnectionOverloadOutcome::WaitTimeout503
+                    }
+                    other => other,
+                };
+                record_final_capacity_rejection(&runtime, outcome);
+                runtime.telemetry().record_overload(outcome);
+                return;
+            };
+            runtime
+                .telemetry()
+                .record_overload(WebHttpConnectionOverloadOutcome::WaitAdmitted);
+            drop(overload_permit);
+            crate::web::http::serve_connection(
+                stream,
+                peer,
+                client_ip_source,
+                trusted_proxy_cidrs,
+                runtime,
+                cancellation,
+                connection_permit,
+            )
+            .await;
+        }
+    }
+}
+
+fn record_final_capacity_rejection(
+    runtime: &WebProcessRuntime,
+    outcome: WebHttpConnectionOverloadOutcome,
+) {
+    if matches!(
+        outcome,
+        WebHttpConnectionOverloadOutcome::Responded503
+            | WebHttpConnectionOverloadOutcome::WaitTimeout503
+            | WebHttpConnectionOverloadOutcome::ResponseErrorDrop
+    ) {
+        runtime
+            .telemetry()
+            .record_rejection(WebRejectionReason::HttpConnectionCapacity);
+    }
+}
+
+async fn respond(
+    stream: TcpStream,
+    cancellation: &CancellationToken,
+    phase_timeout: Duration,
+) -> WebHttpConnectionOverloadOutcome {
+    tokio::select! {
+        biased;
+        _ = cancellation.cancelled() => WebHttpConnectionOverloadOutcome::ShutdownDrop,
+        written = write_service_unavailable(stream, phase_timeout) => {
+            if written {
+                WebHttpConnectionOverloadOutcome::Responded503
+            } else {
+                WebHttpConnectionOverloadOutcome::ResponseErrorDrop
+            }
+        }
+    }
+}
+
+async fn write_service_unavailable(mut stream: TcpStream, deadline: Duration) -> bool {
+    tokio::time::timeout(deadline, async {
+        stream.write_all(SERVICE_UNAVAILABLE_RESPONSE).await?;
+        stream.shutdown().await
+    })
+    .await
+    .is_ok_and(|result| result.is_ok())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use arc_swap::ArcSwap;
+    use tokio::io::AsyncReadExt;
+    use tokio::net::{TcpListener, TcpStream};
+    use tokio_util::sync::CancellationToken;
+
+    use crate::config::{ProxyConfig, WebClientIpSource, WebHttpConnectionCapacityAction};
+    use crate::maestro::generation::test_runtime_generation;
+    use crate::web::manager::WebProcessRuntime;
+    use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
+
+    async fn tcp_pair() -> (TcpStream, TcpStream) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        let client = TcpStream::connect(addr);
+        let server = listener.accept();
+        let (client, server) = tokio::join!(client, server);
+        (server.unwrap().0, client.unwrap())
+    }
+
+    #[tokio::test]
+    async fn overload_response_is_exact_retryable_http() {
+        let (server, mut client) = tcp_pair().await;
+        assert!(super::write_service_unavailable(server, Duration::from_secs(1)).await);
+
+        let mut bytes = Vec::new();
+        client.read_to_end(&mut bytes).await.unwrap();
+        assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
+    }
+
+    #[tokio::test]
+    async fn wait_timeout_is_one_rejection_and_one_retryable_response() {
+        let (runtime, generation) = runtime();
+        let held = runtime.try_http_connection().unwrap();
+        let overload = runtime.try_http_overload_connection().unwrap();
+        let (server, mut client) = tcp_pair().await;
+        let peer = server.peer_addr().unwrap();
+
+        super::serve(
+            server,
+            peer,
+            WebClientIpSource::XForwardedFor,
+            trusted_loopback(),
+            Arc::clone(&runtime),
+            CancellationToken::new(),
+            overload,
+            WebHttpConnectionCapacityAction::Wait,
+            Duration::from_millis(10),
+        )
+        .await;
+        drop(held);
+
+        let mut bytes = Vec::new();
+        client.read_to_end(&mut bytes).await.unwrap();
+        assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
+        assert_eq!(
+            runtime
+                .telemetry()
+                .overload_total(WebHttpConnectionOverloadOutcome::WaitTimeout503,),
+            1
+        );
+        assert_eq!(
+            runtime
+                .telemetry()
+                .rejection_total(WebRejectionReason::HttpConnectionCapacity),
+            1
+        );
+        stop(runtime, generation).await;
+    }
+
+    #[tokio::test]
+    async fn admitted_wait_is_not_counted_as_a_rejection() {
+        let (runtime, generation) = runtime();
+        let held = runtime.try_http_connection().unwrap();
+        let overload = runtime.try_http_overload_connection().unwrap();
+        let (server, _client) = tcp_pair().await;
+        let peer = server.peer_addr().unwrap();
+        let cancellation = CancellationToken::new();
+        let task = tokio::spawn(super::serve(
+            server,
+            peer,
+            WebClientIpSource::XForwardedFor,
+            trusted_loopback(),
+            Arc::clone(&runtime),
+            cancellation.clone(),
+            overload,
+            WebHttpConnectionCapacityAction::Wait,
+            Duration::from_secs(1),
+        ));
+        tokio::task::yield_now().await;
+        drop(held);
+        for _ in 0..100 {
+            if runtime
+                .telemetry()
+                .overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted)
+                == 1
+            {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+        cancellation.cancel();
+        tokio::time::timeout(Duration::from_secs(1), task)
+            .await
+            .unwrap()
+            .unwrap();
+
+        assert_eq!(
+            runtime
+                .telemetry()
+                .overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted),
+            1
+        );
+        assert_eq!(
+            runtime
+                .telemetry()
+                .rejection_total(WebRejectionReason::HttpConnectionCapacity),
+            0
+        );
+        stop(runtime, generation).await;
+    }
+
+    fn runtime() -> (
+        Arc<WebProcessRuntime>,
+        Arc<crate::maestro::generation::RuntimeGeneration>,
+    ) {
+        let mut config = ProxyConfig::default();
+        config.web.limits.max_http_connections = 1;
+        let generation = test_runtime_generation(1, config);
+        let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
+        (runtime, generation)
+    }
+
+    fn trusted_loopback() -> Arc<[ipnetwork::IpNetwork]> {
+        Arc::from(["127.0.0.1/32".parse().unwrap()])
+    }
+
+    async fn stop(
+        runtime: Arc<WebProcessRuntime>,
+        generation: Arc<crate::maestro::generation::RuntimeGeneration>,
+    ) {
+        runtime.shutdown().await;
+        generation.stop_sessions().await;
+        generation.stop_background_tasks().await;
+    }
+}
@@ -315,6 +315,7 @@ pub(super) async fn run_telemt_core(
        &runtime.config,
        &startup_tracker,
        active_runtime.clone(),
+        web_runtime_control.subscribe(),
    )
    .await;

@@ -45,6 +45,9 @@ pub(crate) async fn prepare_runtime(
    quota_store: Arc<QuotaStore>,
    runtime_log_filter: RuntimeLogFilter,
 ) -> Result<PreparedRuntime, String> {
+    config
+        .validate_web_decoy_listener_separation()
+        .map_err(|error| error.to_string())?;
    let started_at_epoch_secs = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
@@ -230,3 +230,34 @@ fn synlimited_endpoint_move_remains_restart_only() {
    assert_eq!(resolved.effective.server.listeners[0].port, Some(443));
    assert!(!resolved.runtime_changed);
 }
+
+#[test]
+fn deferred_listener_identity_cannot_create_an_effective_decoy_loop() {
+    let mut old = ProxyConfig::default();
+    old.server.listeners = vec![test_listener(18080)];
+    old.server.listeners[0].transport = crate::config::ListenerTransport::Web;
+    let mut desired = old.clone();
+    desired.server.listeners[0].port = Some(18081);
+    desired.server.listen_backlog = desired.server.listen_backlog.saturating_add(1);
+    desired.web.vhosts = vec![
+        serde_json::from_value(serde_json::json!({
+            "host": "proxy.example",
+            "public_addr": "203.0.113.10:443",
+            "decoy": {
+                "mode": "http_upstream",
+                "upstream": "http://127.0.0.1:18080"
+            },
+            "profiles": []
+        }))
+        .unwrap(),
+    ];
+
+    assert!(desired.validate_web_decoy_listener_separation().is_ok());
+    let resolved = resolve_reload_config(&old, &desired);
+    assert!(
+        resolved
+            .effective
+            .validate_web_decoy_listener_separation()
+            .is_err()
+    );
+}
@@ -405,6 +405,7 @@ pub(crate) async fn spawn_metrics_if_configured(
    config: &Arc<ProxyConfig>,
    startup_tracker: &Arc<StartupTracker>,
    active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
+    web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
 ) {
    // metrics_listen takes precedence; fall back to metrics_port for backward compat.
    let metrics_target: Option<(u16, Option<String>)> =
@@ -437,7 +438,7 @@ pub(crate) async fn spawn_metrics_if_configured(
        let active_runtime = active_runtime.clone();
        let listen_backlog = config.server.listen_backlog;
        tokio::spawn(async move {
-            metrics::serve(port, listen, listen_backlog, active_runtime).await;
+            metrics::serve(port, listen, listen_backlog, active_runtime, web_runtime_rx).await;
        });
        startup_tracker
            .complete_component(