Runtime Ownership hardened

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/api/mod.rs

@@ -1,5 +1,6 @@
 #![allow(clippy::too_many_arguments)]
 
+use std::collections::BTreeSet;
 use std::io::{Error as IoError, ErrorKind};
 use std::net::{IpAddr, SocketAddr};
 use std::path::PathBuf;
@@ -22,10 +23,12 @@ use tracing::{debug, info, warn};
 
 use crate::config::ApiGrayAction;
 use crate::ip_tracker::UserIpTracker;
+use crate::maestro::control_plane::ProcessControlPlane;
 use crate::maestro::generation::{RuntimeGeneration, RuntimeWatchState};
 use crate::maestro::reload::{ReloadAccepted, ReloadControl, ReloadRequest, ReloadSubmitError};
 use crate::proxy::route_mode::RouteRuntimeController;
 use crate::proxy::shared_state::ProxySharedState;
+use crate::quota_state::QuotaStateOwner;
 use crate::startup::StartupTracker;
 use crate::stats::Stats;
 use crate::transport::UpstreamManager;
@@ -112,7 +115,7 @@ pub(super) struct ApiShared {
     pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
     pub(super) upstream_manager: Arc<UpstreamManager>,
     pub(super) config_path: PathBuf,
-    pub(super) quota_state_path: PathBuf,
+    pub(super) quota_state: Arc<QuotaStateOwner>,
     pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
     pub(super) mutation_lock: Arc<Mutex<()>>,
     pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
@@ -147,7 +150,7 @@ impl ApiShared {
             me_pool: runtime.me_pool_runtime.clone(),
             upstream_manager: runtime.upstream_manager.clone(),
             config_path: self.config_path.clone(),
-            quota_state_path: self.quota_state_path.clone(),
+            quota_state: self.quota_state.clone(),
             detected_ips_rx: self.detected_ips_rx.clone(),
             mutation_lock: self.mutation_lock.clone(),
             minimal_cache: self.minimal_cache.clone(),
@@ -282,8 +285,9 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
     }
 }
 
-pub async fn serve(
-    listen: SocketAddr,
+/// Serves the API on a process-owned listener and task scope.
+pub(crate) async fn serve(
+    listener: TcpListener,
     stats: Arc<Stats>,
     ip_tracker: Arc<UserIpTracker>,
     me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
@@ -291,7 +295,7 @@ pub async fn serve(
     proxy_shared: Arc<ProxySharedState>,
     upstream_manager: Arc<UpstreamManager>,
     config_path: PathBuf,
-    quota_state_path: PathBuf,
+    quota_state: Arc<QuotaStateOwner>,
     detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
     process_started_at_epoch_secs: u64,
     startup_tracker: Arc<StartupTracker>,
@@ -300,6 +304,7 @@ pub async fn serve(
     mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
     web_trace: Arc<WebTraceStore>,
     web_runtime_rx: watch::Receiver<WebRuntimePublication>,
+    control_plane: ProcessControlPlane,
 ) {
     let active_runtime = loop {
         if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
@@ -321,19 +326,9 @@ pub async fn serve(
     };
     let config_rx = initial_watch_state.config_rx.clone();
     let admission_rx = initial_watch_state.admission_rx.clone();
-    let listener = match TcpListener::bind(listen).await {
-        Ok(listener) => listener,
-        Err(error) => {
-            warn!(
-                error = %error,
-                listen = %listen,
-                "Failed to bind API listener"
-            );
-            return;
-        }
-    };
+    let listen = listener.local_addr().ok();
 
-    info!("API endpoint: http://{}/v1/* and /web-status", listen);
+    info!(listen = ?listen, "API endpoint ready at /v1/* and /web-status");
 
     let runtime_state = Arc::new(ApiRuntimeState {
         process_started_at_epoch_secs,
@@ -348,7 +343,7 @@ pub async fn serve(
         me_pool,
         upstream_manager,
         config_path,
-        quota_state_path,
+        quota_state,
         detected_ips_rx,
         mutation_lock: Arc::new(Mutex::new(())),
         minimal_cache: Arc::new(Mutex::new(None)),
@@ -373,6 +368,7 @@ pub async fn serve(
         runtime_watch_rx,
         runtime_state.clone(),
         shared.runtime_events.clone(),
+        &control_plane,
     );
 
     let connection_permits = Arc::new(Semaphore::new(API_MAX_CONTROL_CONNECTIONS));
@@ -399,7 +395,7 @@ pub async fn serve(
         };
 
         let shared_conn = shared.clone();
-        tokio::spawn(async move {
+        let _ = control_plane.spawn(async move {
             let _connection_permit = connection_permit;
             let svc = service_fn(move |req: Request<Incoming>| {
                 let shared_req = shared_conn.clone();
@@ -994,6 +990,7 @@ async fn handle(
                         ));
                     }
                     let expected_revision = parse_if_match(req.headers());
+                    let _mutation_guard = shared.mutation_lock.lock().await;
                     let disk_cfg = load_config_from_disk(&shared.config_path).await?;
                     ensure_expected_revision(&shared.config_path, expected_revision.as_deref())
                         .await?;
@@ -1003,12 +1000,16 @@ async fn handle(
                             ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
                         ));
                     }
-                    let snapshot = match crate::quota_state::reset_user_quota(
-                        &shared.quota_state_path,
-                        shared.stats.as_ref(),
-                        user,
-                    )
-                    .await
+                    let configured_users = disk_cfg
+                        .access
+                        .users
+                        .keys()
+                        .cloned()
+                        .collect::<BTreeSet<_>>();
+                    let snapshot = match shared
+                        .quota_state
+                        .reset_user(&configured_users, user)
+                        .await
                     {
                         Ok(snapshot) => snapshot,
                         Err(error) => {

src/api/model.rs

@@ -211,6 +211,7 @@ pub(super) struct ZeroMiddleProxyData {
     pub(super) reconnect_success_total: u64,
     pub(super) handshake_reject_total: u64,
     pub(super) handshake_error_codes: Vec<ZeroCodeCount>,
+    pub(super) handshake_error_code_overflow_total: u64,
     pub(super) reader_eof_total: u64,
     pub(super) idle_close_by_peer_total: u64,
     pub(super) route_drop_no_conn_total: u64,
@@ -388,8 +389,11 @@ pub(super) struct MinimalDcPathData {
 pub(super) struct MinimalMeRuntimeData {
     pub(super) active_generation: u64,
     pub(super) warm_generation: u64,
+    pub(super) warm_generations: Vec<u64>,
     pub(super) pending_hardswap_generation: u64,
     pub(super) pending_hardswap_age_secs: Option<u64>,
+    pub(super) reinit_inflight: usize,
+    pub(super) reinit_max_concurrency_effective: usize,
     pub(super) hardswap_enabled: bool,
     pub(super) floor_mode: &'static str,
     pub(super) adaptive_floor_idle_secs: u64,

src/api/runtime_min.rs

@@ -21,8 +21,11 @@ pub(super) struct SecurityWhitelistData {
 pub(super) struct RuntimeMePoolStateGenerationData {
     pub(super) active_generation: u64,
     pub(super) warm_generation: u64,
+    pub(super) warm_generations: Vec<u64>,
     pub(super) pending_hardswap_generation: u64,
     pub(super) pending_hardswap_age_secs: Option<u64>,
+    pub(super) reinit_inflight: usize,
+    pub(super) reinit_max_concurrency_effective: usize,
     pub(super) draining_generations: Vec<u64>,
 }
 
@@ -67,6 +70,8 @@ pub(super) struct RuntimeMePoolStateRefillDcData {
 pub(super) struct RuntimeMePoolStateRefillData {
     pub(super) inflight_endpoints_total: usize,
     pub(super) inflight_dc_total: usize,
+    pub(super) running_dc_total: usize,
+    pub(super) pending_dc_total: usize,
     pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
 }
 
@@ -291,8 +296,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
         };
     };
 
-    let status = pool.api_status_snapshot().await;
-    let runtime = pool.api_runtime_snapshot().await;
+    let (status, runtime) = pool.api_coherent_snapshots().await;
     let refill = pool.api_refill_snapshot().await;
 
     let mut draining_generations = BTreeSet::<u64>::new();
@@ -329,8 +333,11 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
             generations: RuntimeMePoolStateGenerationData {
                 active_generation: runtime.active_generation,
                 warm_generation: runtime.warm_generation,
+                warm_generations: runtime.warm_generations,
                 pending_hardswap_generation: runtime.pending_hardswap_generation,
                 pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
+                reinit_inflight: runtime.reinit_inflight,
+                reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
                 draining_generations: draining_generations.into_iter().collect(),
             },
             hardswap: RuntimeMePoolStateHardswapData {
@@ -356,6 +363,8 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
             refill: RuntimeMePoolStateRefillData {
                 inflight_endpoints_total: refill.inflight_endpoints_total,
                 inflight_dc_total: refill.inflight_dc_total,
+                running_dc_total: refill.running_dc_total,
+                pending_dc_total: refill.pending_dc_total,
                 by_dc: refill
                     .by_dc
                     .into_iter()

src/api/runtime_stats.rs

@@ -84,6 +84,8 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
             reconnect_success_total: stats.get_me_reconnect_success(),
             handshake_reject_total: stats.get_me_handshake_reject_total(),
             handshake_error_codes,
+            handshake_error_code_overflow_total: stats
+                .get_me_handshake_error_code_overflow_total(),
             reader_eof_total: stats.get_me_reader_eof_total(),
             idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
             route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
@@ -342,8 +344,7 @@ async fn get_minimal_payload_cached(
     }
 
     let pool = shared.me_pool.read().await.clone()?;
-    let status = pool.api_status_snapshot().await;
-    let runtime = pool.api_runtime_snapshot().await;
+    let (status, runtime) = pool.api_coherent_snapshots().await;
     let generated_at_epoch_secs = status.generated_at_epoch_secs;
 
     let me_writers = MeWritersData {
@@ -425,8 +426,11 @@ async fn get_minimal_payload_cached(
     let me_runtime = MinimalMeRuntimeData {
         active_generation: runtime.active_generation,
         warm_generation: runtime.warm_generation,
+        warm_generations: runtime.warm_generations,
         pending_hardswap_generation: runtime.pending_hardswap_generation,
         pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
+        reinit_inflight: runtime.reinit_inflight,
+        reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
         hardswap_enabled: runtime.hardswap_enabled,
         floor_mode: runtime.floor_mode,
         adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs,

src/api/runtime_watch.rs

@@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::sync::watch;
 
 use crate::maestro::generation::RuntimeWatchState;
+use crate::maestro::control_plane::ProcessControlPlane;
 
 use super::ApiRuntimeState;
 use super::events::ApiEventStore;
@@ -13,22 +14,29 @@ pub(super) fn spawn_runtime_watchers(
     runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
     runtime_state: Arc<ApiRuntimeState>,
     runtime_events: Arc<ApiEventStore>,
+    control_plane: &ProcessControlPlane,
 ) {
-    let _config_watcher = spawn_config_watcher(
+    spawn_config_watcher(
         runtime_watch_rx.clone(),
         runtime_state.clone(),
         runtime_events.clone(),
+        control_plane,
+    );
+    spawn_admission_watcher(
+        runtime_watch_rx,
+        runtime_state,
+        runtime_events,
+        control_plane,
     );
-    let _admission_watcher =
-        spawn_admission_watcher(runtime_watch_rx, runtime_state, runtime_events);
 }
 
 fn spawn_config_watcher(
     mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
     runtime_state: Arc<ApiRuntimeState>,
     runtime_events: Arc<ApiEventStore>,
-) -> tokio::task::JoinHandle<()> {
-    tokio::spawn(async move {
+    control_plane: &ProcessControlPlane,
+) {
+    let _ = control_plane.spawn(async move {
         let Some(mut current) = runtime_watch_rx.borrow().clone() else {
             return;
         };
@@ -78,15 +86,16 @@ fn spawn_config_watcher(
                 }
             }
         }
-    })
+    });
 }
 
 fn spawn_admission_watcher(
     mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
     runtime_state: Arc<ApiRuntimeState>,
     runtime_events: Arc<ApiEventStore>,
-) -> tokio::task::JoinHandle<()> {
-    tokio::spawn(async move {
+    control_plane: &ProcessControlPlane,
+) {
+    let _ = control_plane.spawn(async move {
         let Some(mut current) = runtime_watch_rx.borrow().clone() else {
             return;
         };
@@ -124,7 +133,7 @@ fn spawn_admission_watcher(
                 }
             }
         }
-    })
+    });
 }
 
 fn active_generation_id(
@@ -246,7 +255,13 @@ mod tests {
         let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
         let runtime_state = runtime_state();
         let events = Arc::new(ApiEventStore::new(16));
-        spawn_runtime_watchers(runtime_watch_rx, runtime_state.clone(), events.clone());
+        let control_plane = ProcessControlPlane::new();
+        spawn_runtime_watchers(
+            runtime_watch_rx,
+            runtime_state.clone(),
+            events.clone(),
+            &control_plane,
+        );
         tokio::task::yield_now().await;
 
         assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
@@ -283,6 +298,7 @@ mod tests {
                 .count(),
             3
         );
+        assert!(control_plane.shutdown(Duration::from_secs(1)).await);
     }
 
     #[tokio::test]
@@ -291,7 +307,13 @@ mod tests {
         let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
         let runtime_state = runtime_state();
         let events = Arc::new(ApiEventStore::new(16));
-        let watcher = spawn_config_watcher(runtime_watch_rx, runtime_state.clone(), events.clone());
+        let control_plane = ProcessControlPlane::new();
+        spawn_config_watcher(
+            runtime_watch_rx,
+            runtime_state.clone(),
+            events.clone(),
+            &control_plane,
+        );
         drop(initial_config_tx);
         tokio::task::yield_now().await;
 
@@ -302,10 +324,7 @@ mod tests {
         wait_for_count(&runtime_state, 2).await;
 
         drop(runtime_watch_tx);
-        tokio::time::timeout(Duration::from_secs(1), watcher)
-            .await
-            .unwrap()
-            .unwrap();
+        assert!(control_plane.shutdown(Duration::from_secs(1)).await);
         assert_eq!(
             events
                 .snapshot(16)

src/api/users/lifecycle.rs

@@ -1,4 +1,5 @@
 use super::*;
+use tracing::warn;
 
 pub(in crate::api) async fn rotate_secret(
     user: &str,
@@ -108,6 +109,18 @@ pub(in crate::api) async fn delete_user(
         .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
     let revision =
         save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
+    let configured_users = cfg.access.users.keys().cloned().collect();
+    if let Err(error) = shared
+        .quota_state
+        .remove_user(&configured_users, user)
+        .await
+    {
+        warn!(
+            user,
+            error = %error,
+            "Deleted user quota checkpoint cleanup will be reconciled on restart"
+        );
+    }
     drop(_guard);
     shared.ip_tracker.remove_user_limit(user).await;
     shared.ip_tracker.clear_user_ips(user).await;

src/api/web_runtime/observability.rs

@@ -18,6 +18,7 @@ pub(super) struct WebIngressStatus {
 }
 
 impl WebIngressStatus {
+    /// Builds a process-ingress snapshot without probing external TLS termination.
     pub(super) fn new(publication: &WebRuntimePublication, runtime_available: bool) -> Self {
         let configured_listeners = publication.listeners.len();
         let live_acceptors = publication.telemetry.live_acceptors();
@@ -64,6 +65,7 @@ pub(super) struct WebCapacityStatus {
 }
 
 impl WebCapacityStatus {
+    /// Builds a bounded capacity snapshot from non-blocking runtime observations.
     pub(super) fn new(
         publication: &WebRuntimePublication,
         runtime: Option<&WebProcessRuntime>,
@@ -104,6 +106,7 @@ pub(super) struct WebDecoyUpstreamStatus {
 }
 
 impl WebDecoyUpstreamStatus {
+    /// Builds the fixed internal decoy-origin outcome snapshot.
     pub(super) fn new(publication: &WebRuntimePublication) -> Self {
         let last = publication.telemetry.last_decoy();
         Self {