Runtime Ownership hardened

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-08-30 08:38:03 +03:00
parent 1bb6b0bdda
commit 281f63f940
91 changed files with 3972 additions and 1239 deletions
-4
View File
@@ -253,10 +253,6 @@ pub(super) async fn bootstrap(
}
}
if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) {
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
std::process::exit(1);
}
set_maestro_colors_enabled(!config.general.disable_colors);
startup_tracker
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
+217
View File
@@ -0,0 +1,217 @@
use std::future::Future;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
const CONTROL_TASK_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
const CONTROL_TASK_REGISTRATION_COUNT: usize = CONTROL_TASK_ADMISSION_CLOSED - 1;
struct ControlTaskAdmission {
state: AtomicUsize,
registrations_drained: Notify,
}
struct ControlTaskRegistration<'a> {
admission: &'a ControlTaskAdmission,
}
impl ControlTaskAdmission {
fn new() -> Self {
Self {
state: AtomicUsize::new(0),
registrations_drained: Notify::new(),
}
}
fn try_register(&self) -> Option<ControlTaskRegistration<'_>> {
let mut state = self.state.load(Ordering::Acquire);
loop {
if state & CONTROL_TASK_ADMISSION_CLOSED != 0
|| state & CONTROL_TASK_REGISTRATION_COUNT == CONTROL_TASK_REGISTRATION_COUNT
{
return None;
}
match self.state.compare_exchange_weak(
state,
state + 1,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Some(ControlTaskRegistration { admission: self }),
Err(observed) => state = observed,
}
}
}
fn close(&self) {
self.state
.fetch_or(CONTROL_TASK_ADMISSION_CLOSED, Ordering::AcqRel);
}
async fn wait_for_registrations(&self) {
loop {
let notified = self.registrations_drained.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.state.load(Ordering::Acquire) & CONTROL_TASK_REGISTRATION_COUNT == 0 {
return;
}
notified.await;
}
}
}
impl Drop for ControlTaskRegistration<'_> {
fn drop(&mut self) {
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
if previous & CONTROL_TASK_REGISTRATION_COUNT == 1 {
self.admission.registrations_drained.notify_waiters();
}
}
}
struct ProcessControlPlaneInner {
admission: ControlTaskAdmission,
cancellation: CancellationToken,
tasks: TaskTracker,
shutdown_completed: AtomicBool,
}
/// Process-owned cancellation and join scope for API, metrics, and signal tasks.
#[derive(Clone)]
pub(crate) struct ProcessControlPlane {
inner: Arc<ProcessControlPlaneInner>,
}
impl ProcessControlPlane {
/// Creates an open process control-plane scope.
pub(crate) fn new() -> Self {
Self {
inner: Arc::new(ProcessControlPlaneInner {
admission: ControlTaskAdmission::new(),
cancellation: CancellationToken::new(),
tasks: TaskTracker::new(),
shutdown_completed: AtomicBool::new(false),
}),
}
}
/// Registers a cancellable process control-plane task before it can be unpolled.
pub(crate) fn spawn<F>(&self, future: F) -> Result<(), F>
where
F: Future<Output = ()> + Send + 'static,
{
let Some(registration) = self.inner.admission.try_register() else {
return Err(future);
};
let cancellation = self.inner.cancellation.clone();
self.inner.tasks.spawn(async move {
tokio::select! {
biased;
_ = cancellation.cancelled() => {}
_ = future => {}
}
});
drop(registration);
Ok(())
}
/// Closes task admission, cancels all owned work, and joins it within the deadline.
pub(crate) async fn shutdown(&self, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
self.inner.admission.close();
self.inner.cancellation.cancel();
self.inner.tasks.close();
if self.inner.shutdown_completed.load(Ordering::Acquire) {
return true;
}
let registrations_stopped = tokio::time::timeout_at(
deadline,
self.inner.admission.wait_for_registrations(),
)
.await
.is_ok();
let tasks_stopped = tokio::time::timeout_at(deadline, self.inner.tasks.wait())
.await
.is_ok();
let outcome = registrations_stopped && tasks_stopped;
if outcome {
self.inner.shutdown_completed.store(true, Ordering::Release);
}
outcome
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use super::*;
#[tokio::test]
async fn shutdown_cancels_owned_tasks_and_rejects_late_registration() {
struct DropSignal(Arc<AtomicBool>);
impl Drop for DropSignal {
fn drop(&mut self) {
self.0.store(true, Ordering::Release);
}
}
let scope = ProcessControlPlane::new();
let dropped = Arc::new(AtomicBool::new(false));
let drop_signal = DropSignal(dropped.clone());
assert!(
scope
.spawn(async move {
let _drop_signal = drop_signal;
std::future::pending::<()>().await;
})
.is_ok()
);
assert!(scope.shutdown(Duration::from_secs(1)).await);
assert!(dropped.load(Ordering::Acquire));
assert!(scope.spawn(async {}).is_err());
}
#[tokio::test]
async fn concurrent_shutdown_callers_wait_for_completion() {
let scope = ProcessControlPlane::new();
let registration = scope.inner.admission.try_register().unwrap();
let first_scope = scope.clone();
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(1)).await });
tokio::task::yield_now().await;
let second_scope = scope.clone();
let second = tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
tokio::task::yield_now().await;
assert!(!first.is_finished());
assert!(!second.is_finished());
drop(registration);
assert!(first.await.unwrap());
assert!(second.await.unwrap());
}
#[tokio::test]
async fn cancelled_shutdown_caller_cannot_orphan_the_control_plane() {
let scope = ProcessControlPlane::new();
let registration = scope.inner.admission.try_register().unwrap();
let first_scope = scope.clone();
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
tokio::task::yield_now().await;
first.abort();
assert!(first.await.unwrap_err().is_cancelled());
assert!(scope.spawn(async {}).is_err());
drop(registration);
assert!(scope.shutdown(Duration::from_secs(1)).await);
}
}
+66 -10
View File
@@ -3,7 +3,7 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::sync::{RwLock, Semaphore, watch};
use tokio::sync::{Notify, RwLock, Semaphore, watch};
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -29,6 +29,7 @@ const SESSION_REGISTRATION_COUNT: usize = SESSION_ADMISSION_CLOSED - 1;
struct SessionAdmission {
state: AtomicUsize,
registrations_drained: Notify,
}
struct SessionRegistration<'a> {
@@ -39,6 +40,7 @@ impl SessionAdmission {
fn new() -> Self {
Self {
state: AtomicUsize::new(0),
registrations_drained: Notify::new(),
}
}
@@ -73,15 +75,24 @@ impl SessionAdmission {
}
async fn wait_for_registrations(&self) {
while self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT != 0 {
tokio::task::yield_now().await;
loop {
let notified = self.registrations_drained.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT == 0 {
return;
}
notified.await;
}
}
}
impl Drop for SessionRegistration<'_> {
fn drop(&mut self) {
self.admission.state.fetch_sub(1, Ordering::Release);
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
if previous & SESSION_REGISTRATION_COUNT == 1 {
self.admission.registrations_drained.notify_waiters();
}
}
}
@@ -98,6 +109,7 @@ pub(crate) struct RuntimeWatchState {
pub(crate) struct RuntimeTaskScope {
tracker: TaskTracker,
cancel: CancellationToken,
admission: Arc<SessionAdmission>,
}
impl RuntimeTaskScope {
@@ -106,6 +118,7 @@ impl RuntimeTaskScope {
Self {
tracker: TaskTracker::new(),
cancel: CancellationToken::new(),
admission: Arc::new(SessionAdmission::new()),
}
}
@@ -114,9 +127,13 @@ impl RuntimeTaskScope {
where
F: Future<Output = ()> + Send + 'static,
{
let Some(_registration) = self.admission.try_register() else {
return;
};
let cancel = self.cancel.clone();
self.tracker.spawn(async move {
tokio::select! {
biased;
_ = cancel.cancelled() => {}
_ = future => {}
}
@@ -130,6 +147,8 @@ impl RuntimeTaskScope {
/// Cancels the scope and waits within the bounded background-task budget.
pub(crate) async fn stop(&self) {
self.admission.close();
self.admission.wait_for_registrations().await;
self.cancel.cancel();
self.tracker.close();
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
@@ -263,6 +282,7 @@ impl RuntimeGeneration {
let cancel = self.session_cancel.clone();
self.sessions.spawn(async move {
tokio::select! {
biased;
_ = cancel.cancelled() => {}
_ = future => {}
}
@@ -275,11 +295,6 @@ impl RuntimeGeneration {
self.session_admission.close();
}
/// Reopens admission after a candidate activation rolls back.
pub(crate) fn resume_accepting_sessions(&self) {
self.session_admission.reopen();
}
/// Waits for registered sessions and cancels them when the deadline expires.
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
self.stop_accepting_sessions();
@@ -308,6 +323,27 @@ impl RuntimeGeneration {
pub(crate) async fn stop_background_tasks(&self) {
self.background_tasks.stop().await;
}
/// Terminally stops the generation's Middle-End task and writer scope.
pub(crate) async fn stop_middle_end(&self, timeout: Duration) -> bool {
let Some(pool) = self.current_me_pool().await else {
return true;
};
pool.shutdown_until(timeout).await
}
}
impl Drop for RuntimeGeneration {
fn drop(&mut self) {
if let Some(pool) = self.me_pool.as_ref() {
pool.begin_shutdown();
}
if let Ok(pool) = self.me_pool_runtime.try_read()
&& let Some(pool) = pool.as_ref()
{
pool.begin_shutdown();
}
}
}
#[cfg(test)]
@@ -393,11 +429,31 @@ mod tests {
#[tokio::test]
async fn runtime_task_scope_joins_cancelled_background_task() {
struct DropSignal(Arc<AtomicUsize>);
impl Drop for DropSignal {
fn drop(&mut self) {
self.0.fetch_add(1, Ordering::AcqRel);
}
}
let scope = RuntimeTaskScope::new();
scope.spawn(std::future::pending());
let dropped = Arc::new(AtomicUsize::new(0));
let drop_signal = DropSignal(dropped.clone());
scope.spawn(async move {
let _drop_signal = drop_signal;
std::future::pending::<()>().await;
});
tokio::time::timeout(Duration::from_secs(1), scope.stop())
.await
.unwrap();
assert_eq!(dropped.load(Ordering::Acquire), 1);
let late_drop_signal = DropSignal(dropped.clone());
scope.spawn(async move {
let _late_drop_signal = late_drop_signal;
});
assert_eq!(dropped.load(Ordering::Acquire), 2);
}
#[tokio::test]
+71 -34
View File
@@ -12,7 +12,7 @@ use tracing::{debug, error, info, warn};
use crate::config::{ListenerTransport, RstOnCloseMode};
use crate::proxy::ClientHandler;
use crate::transport::socket::set_linger_zero;
use crate::web::manager::WebProcessRuntime;
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};
use super::bind::BoundTcpListener;
@@ -208,45 +208,82 @@ async fn run_accept_loop(
return;
};
web_runtime.telemetry().record_accept();
let Some(connection_permit) = web_runtime.try_http_connection() else {
let config = web_runtime.active_generation().config();
let action = config.web.http_connection_capacity_action;
let phase_timeout =
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
drop(config);
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
if cancellation.is_cancelled() {
drop(stream);
continue;
}
if web_runtime.is_shutdown() {
web_runtime
.telemetry()
.record_rejection(
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
);
drop(stream);
continue;
}
let connection_permit = match web_runtime.try_http_connection() {
Ok(permit) => permit,
Err(HttpConnectionAdmissionError::Closed) => {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
);
web_runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
drop(stream);
continue;
}
let Some(overload_permit) = web_runtime.try_http_overload_connection()
else {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
Err(HttpConnectionAdmissionError::AtCapacity) => {
let config = web_runtime.active_generation().config();
let action = config.web.http_connection_capacity_action;
let phase_timeout = Duration::from_millis(
config.web.timeouts.http_overload_timeout_ms,
);
web_runtime.telemetry().record_overload(
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
);
drop(stream);
drop(config);
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
);
web_runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
drop(stream);
continue;
}
let overload_permit =
match web_runtime.try_http_overload_connection() {
Ok(permit) => permit,
Err(HttpConnectionAdmissionError::Closed) => {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
);
web_runtime.telemetry().record_overload(
WebHttpConnectionOverloadOutcome::ShutdownDrop,
);
drop(stream);
continue;
}
Err(HttpConnectionAdmissionError::AtCapacity) => {
web_runtime.telemetry().record_rejection(
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
);
web_runtime.telemetry().record_overload(
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
);
drop(stream);
continue;
}
};
connections.spawn(web_overload::serve(
stream,
peer_addr,
spec.web_client_ip_source,
Arc::clone(&spec.web_trusted_proxy_cidrs),
Arc::clone(web_runtime),
cancellation.clone(),
overload_permit,
action,
phase_timeout,
));
continue;
};
connections.spawn(web_overload::serve(
stream,
peer_addr,
spec.web_client_ip_source,
Arc::clone(&spec.web_trusted_proxy_cidrs),
Arc::clone(web_runtime),
cancellation.clone(),
overload_permit,
action,
phase_timeout,
));
continue;
}
};
connections.spawn(crate::web::http::serve_connection(
stream,
+20 -13
View File
@@ -9,9 +9,10 @@ use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
use crate::web::manager::WebProcessRuntime;
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
/// Exact bounded retryable response emitted before HTTP request parsing.
pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
/// Handles one accepted WEB socket outside ordinary connection capacity.
@@ -44,25 +45,31 @@ pub(super) async fn serve(
return;
}
permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
permit.ok().flatten()
permit
}
};
let Some(connection_permit) = connection_permit else {
if runtime.is_shutdown() {
let connection_permit = match connection_permit {
Ok(Ok(permit)) => permit,
Ok(Err(HttpConnectionAdmissionError::Closed)) => {
runtime
.telemetry()
.record_rejection(WebRejectionReason::RuntimeClosed);
runtime
.telemetry()
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
return;
}
let outcome = match respond(stream, &cancellation, phase_timeout).await {
WebHttpConnectionOverloadOutcome::Responded503 => {
WebHttpConnectionOverloadOutcome::WaitTimeout503
}
other => other,
};
record_final_capacity_rejection(&runtime, outcome);
runtime.telemetry().record_overload(outcome);
return;
Ok(Err(HttpConnectionAdmissionError::AtCapacity)) | Err(_) => {
let outcome = match respond(stream, &cancellation, phase_timeout).await {
WebHttpConnectionOverloadOutcome::Responded503 => {
WebHttpConnectionOverloadOutcome::WaitTimeout503
}
other => other,
};
record_final_capacity_rejection(&runtime, outcome);
runtime.telemetry().record_overload(outcome);
return;
}
};
runtime
.telemetry()
+1
View File
@@ -345,6 +345,7 @@ pub(crate) async fn initialize_me_pool(
config.general.me_route_blocking_send_timeout_ms,
config.general.me_route_inline_recovery_attempts,
config.general.me_route_inline_recovery_wait_ms,
(config.server.max_connections as usize).saturating_add(128),
);
startup_tracker
.complete_component(
+2
View File
@@ -6,6 +6,7 @@
// - admission: conditional-cast gate and route mode switching.
// - bootstrap: configuration and tracing initialization.
// - connectivity: startup ME/DC connectivity diagnostics.
// - control_plane: process-owned API, metrics, and signal task lifecycle.
// - generation: runtime generation state and task ownership.
// - helpers: CLI and shared startup/runtime helper routines.
// - listeners: TCP/Unix listener planning, binding, and lifecycle control.
@@ -21,6 +22,7 @@
mod admission;
mod bootstrap;
mod connectivity;
pub(crate) mod control_plane;
pub(crate) mod generation;
mod helpers;
mod listeners;
+65 -18
View File
@@ -1,9 +1,10 @@
use std::collections::BTreeSet;
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use arc_swap::ArcSwap;
use tokio::sync::{RwLock, watch};
use tracing::{error, info, warn};
use tracing::{error, info};
use crate::api;
use crate::ip_tracker::UserIpTracker;
@@ -15,14 +16,15 @@ use crate::startup::{COMPONENT_API_BOOTSTRAP, COMPONENT_NETWORK_PROBE};
use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{QuotaStore, Stats};
use crate::synlimit_control;
use crate::tls_front::cache::TlsFullCertBudget;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool;
use crate::web::control::WebRuntimeControl;
use crate::web::trace::WebTraceStore;
use super::{
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
shutdown, tls_bootstrap,
bootstrap, control_plane, generation, listeners, reload, reload_supervisor, runtime_startup,
runtime_tasks, shutdown, tls_bootstrap,
};
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
@@ -45,10 +47,22 @@ pub(super) async fn run_telemt_core(
let quota_store = Arc::new(QuotaStore::default());
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
let tls_full_cert_budget = Arc::new(TlsFullCertBudget::new());
let process_control_plane = control_plane::ProcessControlPlane::new();
let runtime_task_scope = generation::RuntimeTaskScope::new();
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
let quota_state_path = config.general.quota_state_path.clone();
crate::quota_state::load_quota_state(&quota_state_path, stats.as_ref()).await;
let quota_state = crate::quota_state::QuotaStateOwner::new(
quota_state_path,
quota_store.clone(),
);
let configured_quota_users = config
.access
.users
.keys()
.cloned()
.collect::<BTreeSet<_>>();
quota_state.load(&configured_quota_users).await;
let upstream_manager = Arc::new(
UpstreamManager::new(
@@ -136,15 +150,29 @@ pub(super) async fn run_telemt_core(
let listen = match config.server.api.listen.parse::<SocketAddr>() {
Ok(listen) => listen,
Err(error) => {
warn!(
error = %error,
listen = %config.server.api.listen,
"Invalid server.api.listen; API is disabled"
let message = format!(
"invalid server.api.listen \"{}\": {}",
config.server.api.listen, error
);
SocketAddr::from(([127, 0, 0, 1], 0))
startup_tracker
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.clone()))
.await;
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, message).into());
}
};
if listen.port() != 0 {
let api_listener = match tokio::net::TcpListener::bind(listen).await {
Ok(listener) => listener,
Err(error) => {
startup_tracker
.fail_component(
COMPONENT_API_BOOTSTRAP,
Some(format!("API listener bind failed on {listen}: {error}")),
)
.await;
return Err(error.into());
}
};
let stats_api = stats.clone();
let ip_tracker_api = ip_tracker.clone();
let me_pool_api = api_me_pool.clone();
@@ -152,7 +180,7 @@ pub(super) async fn run_telemt_core(
let route_runtime_api = route_runtime.clone();
let proxy_shared_api = shared_state.clone();
let config_path_api = config_path.clone();
let quota_state_path_api = quota_state_path.clone();
let quota_state_api = quota_state.clone();
let startup_tracker_api = startup_tracker.clone();
let detected_ips_rx_api = detected_ips_rx.clone();
let reload_control_api = reload_control.clone();
@@ -160,9 +188,11 @@ pub(super) async fn run_telemt_core(
let runtime_watch_rx_api = runtime_watch_rx.clone();
let web_trace_api = web_trace.clone();
let web_runtime_rx_api = web_runtime_control.subscribe();
tokio::spawn(async move {
let api_control_plane = process_control_plane.clone();
let api_task_control_plane = process_control_plane.clone();
let api_task = async move {
api::serve(
listen,
api_listener,
stats_api,
ip_tracker_api,
me_pool_api,
@@ -170,7 +200,7 @@ pub(super) async fn run_telemt_core(
proxy_shared_api,
upstream_manager_api,
config_path_api,
quota_state_path_api,
quota_state_api,
detected_ips_rx_api,
process_started_at_epoch_secs,
startup_tracker_api,
@@ -179,13 +209,21 @@ pub(super) async fn run_telemt_core(
runtime_watch_rx_api,
web_trace_api,
web_runtime_rx_api,
api_task_control_plane,
)
.await;
});
};
if api_control_plane.spawn(api_task).is_err() {
let message = "process control-plane task admission closed during API startup";
startup_tracker
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.to_string()))
.await;
return Err(std::io::Error::other(message).into());
}
startup_tracker
.complete_component(
COMPONENT_API_BOOTSTRAP,
Some(format!("api task spawned on {}", listen)),
Some(format!("API listener bound and supervised on {}", listen)),
)
.await;
} else {
@@ -219,6 +257,7 @@ pub(super) async fn run_telemt_core(
upstream_manager.clone(),
&startup_tracker,
runtime_task_scope.clone(),
tls_full_cert_budget.clone(),
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
)
.await?;
@@ -316,8 +355,10 @@ pub(super) async fn run_telemt_core(
&startup_tracker,
active_runtime.clone(),
web_runtime_control.subscribe(),
tls_full_cert_budget.clone(),
process_control_plane.clone(),
)
.await;
.await?;
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
active_runtime_tx.send_replace(Some(active_runtime.clone()));
@@ -335,6 +376,7 @@ pub(super) async fn run_telemt_core(
reload_commands,
config_path,
quota_store,
tls_full_cert_budget,
detected_ips_tx,
runtime_log_filter,
runtime_watch_tx,
@@ -342,12 +384,17 @@ pub(super) async fn run_telemt_core(
web_trace,
);
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
shutdown::spawn_signal_handlers(
active_runtime.clone(),
process_started_at,
process_control_plane.clone(),
);
shutdown::wait_for_shutdown(
process_started_at,
active_runtime,
quota_state_path,
quota_state,
reload_supervisor,
process_control_plane,
)
.await;
+11 -34
View File
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use crate::stats::QuotaStore;
use crate::tls_front::cache::TlsFullCertBudget;
use crate::web::trace::WebTraceStore;
use super::generation::{RuntimeGeneration, RuntimeWatchState};
@@ -26,6 +27,7 @@ pub(crate) struct ReloadSupervisor {
commands: ReloadCommandReceiver,
config_path: PathBuf,
quota_store: Arc<QuotaStore>,
tls_full_cert_budget: Arc<TlsFullCertBudget>,
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
runtime_log_filter: RuntimeLogFilter,
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
@@ -81,12 +83,7 @@ fn revision_gate_action(
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
generation.stop_background_tasks().await;
let Some(pool) = generation.current_me_pool().await else {
return false;
};
tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
.await
.is_err()
!generation.stop_middle_end(Duration::from_secs(5)).await
}
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
@@ -103,6 +100,7 @@ impl ReloadSupervisor {
commands: ReloadCommandReceiver,
config_path: PathBuf,
quota_store: Arc<QuotaStore>,
tls_full_cert_budget: Arc<TlsFullCertBudget>,
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
runtime_log_filter: RuntimeLogFilter,
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
@@ -116,6 +114,7 @@ impl ReloadSupervisor {
commands,
config_path,
quota_store,
tls_full_cert_budget,
detected_ips_tx,
runtime_log_filter,
runtime_watch_tx,
@@ -171,6 +170,7 @@ impl ReloadSupervisor {
&self.config_path,
self.quota_store.clone(),
self.runtime_log_filter.clone(),
self.tls_full_cert_budget.clone(),
)
.await
{
@@ -207,25 +207,18 @@ impl ReloadSupervisor {
prepared,
listener_transition,
revision_action,
|entries| {
crate::network::dns_overrides::install_entries(entries)
.map_err(|error| error.to_string())
},
)
.await;
}
#[cfg(test)]
async fn activate_prepared<InstallDns>(
async fn activate_prepared(
&self,
command: ReloadCommand,
old_runtime: Arc<RuntimeGeneration>,
prepared: PreparedRuntime,
revision_action: RevisionGateAction,
install_dns: InstallDns,
) where
InstallDns: FnOnce(&[String]) -> Result<(), String>,
{
) {
let listener_transition = match self
.listener_manager
.lock()
@@ -245,22 +238,18 @@ impl ReloadSupervisor {
prepared,
listener_transition,
revision_action,
install_dns,
)
.await;
}
async fn activate_prepared_with_transition<InstallDns>(
async fn activate_prepared_with_transition(
&self,
command: ReloadCommand,
old_runtime: Arc<RuntimeGeneration>,
prepared: PreparedRuntime,
listener_transition: Option<PreparedListenerTransition>,
revision_action: RevisionGateAction,
install_dns: InstallDns,
) where
InstallDns: FnOnce(&[String]) -> Result<(), String>,
{
) {
match revision_action {
RevisionGateAction::Proceed => {}
RevisionGateAction::Warn(warning) => {
@@ -283,18 +272,6 @@ impl ReloadSupervisor {
detected_ips,
config_watcher_activation,
} = prepared;
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
let message = format!("runtime DNS activation failed: {}", error);
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
old_runtime.resume_accepting_sessions();
let _ = cleanup_candidate(&new_runtime).await;
self.runtime_log_filter
.apply_reload(&old_runtime.config().general.log_level);
self.control.rolled_back(command.reload_id, message).await;
return;
}
self.control.add_warning(command.reload_id, message).await;
}
let pending_listener_transition = if let Some(listener_transition) = listener_transition {
match self
.listener_manager
@@ -367,7 +344,7 @@ impl ReloadSupervisor {
if stop_background_and_middle_end(&replaced).await {
let warning = format!(
"generation {} Middle-End close broadcast timed out",
"generation {} Middle-End lifecycle shutdown timed out",
replaced.id
);
warn!(reload_id = command.reload_id, warning = %warning);
+2 -41
View File
@@ -53,6 +53,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
commands,
config_path: PathBuf::new(),
quota_store: Arc::new(QuotaStore::default()),
tls_full_cert_budget: Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
detected_ips_tx,
runtime_log_filter: runtime_log_filter(),
runtime_watch_tx,
@@ -131,7 +132,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
fixture.old_runtime.clone(),
prepared_runtime(fixture.new_runtime),
RevisionGateAction::Rollback("revision changed".to_string()),
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
)
.await;
@@ -154,44 +154,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
fixture.old_runtime.stop_sessions().await;
}
#[tokio::test]
async fn dns_failure_policy_controls_rollback_or_keep_new() {
for policy in [ReloadFailurePolicy::Rollback, ReloadFailurePolicy::KeepNew] {
let fixture = fixture(ReloadRequest {
failure_policy: policy,
..ReloadRequest::default()
})
.await;
fixture
.supervisor
.activate_prepared(
fixture.command,
fixture.old_runtime.clone(),
prepared_runtime(fixture.new_runtime.clone()),
RevisionGateAction::Proceed,
|_| Err("invalid DNS entry".to_string()),
)
.await;
let status = fixture.control.status(1).await.unwrap();
match policy {
ReloadFailurePolicy::Rollback => {
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
assert_eq!(status.state, ReloadPhase::RolledBack);
assert!(fixture.old_runtime.spawn_session(async {}));
fixture.old_runtime.stop_sessions().await;
}
ReloadFailurePolicy::KeepNew => {
assert_eq!(fixture.supervisor.active_runtime.load().id, 2);
assert_eq!(status.state, ReloadPhase::Succeeded);
assert_eq!(status.warnings.len(), 1);
assert!(!fixture.old_runtime.spawn_session(async {}));
fixture.new_runtime.stop_sessions().await;
}
}
}
}
#[tokio::test]
async fn drain_publishes_new_generation_before_old_sessions_finish() {
let mut fixture = fixture(ReloadRequest {
@@ -220,7 +182,6 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
old_runtime,
prepared_runtime(new_runtime),
RevisionGateAction::Proceed,
|_| Ok(()),
)
.await;
});
@@ -273,7 +234,6 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
old_runtime,
prepared_runtime(new_runtime),
RevisionGateAction::Proceed,
|_| Ok(()),
)
.await;
});
@@ -304,6 +264,7 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
commands,
PathBuf::new(),
Arc::new(QuotaStore::default()),
Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
detected_ips_tx,
runtime_log_filter(),
runtime_watch_tx,
+3
View File
@@ -19,6 +19,7 @@ use crate::stats::beobachten::BeobachtenStore;
use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{QuotaStore, ReplayChecker, Stats};
use crate::stream::BufferPool;
use crate::tls_front::cache::TlsFullCertBudget;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool;
@@ -44,6 +45,7 @@ pub(crate) async fn prepare_runtime(
config_path: &Path,
quota_store: Arc<QuotaStore>,
runtime_log_filter: RuntimeLogFilter,
tls_full_cert_budget: Arc<TlsFullCertBudget>,
) -> Result<PreparedRuntime, String> {
config
.validate_web_decoy_listener_separation()
@@ -121,6 +123,7 @@ pub(crate) async fn prepare_runtime(
upstream_manager.clone(),
&startup_tracker,
task_scope.clone(),
tls_full_cert_budget,
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
)
.await
+31 -12
View File
@@ -26,6 +26,7 @@ use crate::stats::{ReplayChecker, Stats};
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
use super::control_plane::ProcessControlPlane;
use super::generation::RuntimeGeneration;
use super::generation::RuntimeTaskScope;
use super::helpers::write_beobachten_snapshot;
@@ -158,6 +159,7 @@ pub(crate) async fn spawn_runtime_tasks(
detected_ip_v4,
detected_ip_v6,
task_scope.cancellation_token(),
Some(upstream_manager.dns_resolver()),
config_watcher_activation,
);
task_scope.spawn(config_watcher_task);
@@ -168,7 +170,6 @@ pub(crate) async fn spawn_runtime_tasks(
)
.await;
let stats_policy = stats.clone();
let upstream_policy = upstream_manager.clone();
let mut config_rx_policy = config_rx.clone();
task_scope.spawn(async move {
loop {
@@ -178,9 +179,6 @@ pub(crate) async fn spawn_runtime_tasks(
let cfg = config_rx_policy.borrow_and_update().clone();
stats_policy
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
if let Err(error) = upstream_policy.update_dns_overrides(&cfg.network.dns_overrides) {
warn!(error = %error, "Failed to update generation DNS overrides");
}
if let Some(pool) = &me_pool_for_policy {
pool.update_runtime_transport_policy(
cfg.general.me_socks_kdf_policy,
@@ -406,7 +404,9 @@ pub(crate) async fn spawn_metrics_if_configured(
startup_tracker: &Arc<StartupTracker>,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
) {
tls_full_cert_budget: Arc<crate::tls_front::cache::TlsFullCertBudget>,
control_plane: ProcessControlPlane,
) -> std::io::Result<()> {
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
let metrics_target: Option<(u16, Option<String>)> =
if let Some(ref listen) = config.server.metrics_listen {
@@ -414,12 +414,15 @@ pub(crate) async fn spawn_metrics_if_configured(
Ok(addr) => Some((addr.port(), Some(listen.clone()))),
Err(e) => {
startup_tracker
.skip_component(
.fail_component(
COMPONENT_METRICS_START,
Some(format!("invalid metrics_listen \"{}\": {}", listen, e)),
)
.await;
None
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("invalid metrics_listen \"{}\": {}", listen, e),
));
}
}
} else {
@@ -435,15 +438,30 @@ pub(crate) async fn spawn_metrics_if_configured(
Some(format!("spawn metrics endpoint on {}", label)),
)
.await;
let active_runtime = active_runtime.clone();
let listen_backlog = config.server.listen_backlog;
tokio::spawn(async move {
metrics::serve(port, listen, listen_backlog, active_runtime, web_runtime_rx).await;
});
let bound = match metrics::bind(port, listen, listen_backlog) {
Ok(bound) => bound,
Err(error) => {
startup_tracker
.fail_component(
COMPONENT_METRICS_START,
Some(format!("metrics listener bind failed: {error}")),
)
.await;
return Err(error);
}
};
metrics::serve(
bound,
active_runtime,
web_runtime_rx,
tls_full_cert_budget,
control_plane,
);
startup_tracker
.complete_component(
COMPONENT_METRICS_START,
Some("metrics task spawned".to_string()),
Some("metrics listeners bound and supervised".to_string()),
)
.await;
} else if config.server.metrics_listen.is_none() {
@@ -454,6 +472,7 @@ pub(crate) async fn spawn_metrics_if_configured(
)
.await;
}
Ok(())
}
pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc<StartupTracker>) {
+33 -22
View File
@@ -8,7 +8,7 @@
//!
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
use std::path::PathBuf;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -19,11 +19,13 @@ use tokio::signal;
use tokio::signal::unix::{SignalKind, signal};
use tracing::{info, warn};
use super::control_plane::ProcessControlPlane;
use super::generation::RuntimeGeneration;
use super::helpers::{format_uptime, unit_label};
use super::reload_supervisor::ReloadSupervisorHandle;
use crate::stats::Stats;
use crate::synlimit_control;
use crate::quota_state::QuotaStateOwner;
/// Signal that triggered shutdown.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -50,16 +52,18 @@ impl std::fmt::Display for ShutdownSignal {
pub(crate) async fn wait_for_shutdown(
process_started_at: Instant,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
quota_state_path: PathBuf,
quota_state: Arc<QuotaStateOwner>,
reload_supervisor: ReloadSupervisorHandle,
process_control_plane: ProcessControlPlane,
) {
let signal = wait_for_shutdown_signal().await;
perform_shutdown(
signal,
process_started_at,
active_runtime,
quota_state_path,
quota_state,
reload_supervisor,
process_control_plane,
)
.await;
}
@@ -89,8 +93,9 @@ async fn perform_shutdown(
signal: ShutdownSignal,
process_started_at: Instant,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
quota_state_path: PathBuf,
quota_state: Arc<QuotaStateOwner>,
reload_supervisor: ReloadSupervisorHandle,
process_control_plane: ProcessControlPlane,
) {
let shutdown_started_at = Instant::now();
info!(signal = %signal, "Received shutdown signal");
@@ -115,37 +120,41 @@ async fn perform_shutdown(
// Graceful ME pool shutdown
runtime.stop_sessions().await;
runtime.stop_background_tasks().await;
if let Some(pool) = runtime.current_me_pool().await {
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
.await
{
Ok(total) => {
info!(
close_conn_sent = total,
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
);
}
Err(_) => {
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
}
}
if runtime.stop_middle_end(Duration::from_secs(5)).await {
info!("ME shutdown: pool lifecycle completed");
} else {
warn!("ME shutdown: pool lifecycle deadline expired");
}
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
}
match crate::quota_state::save_quota_state(&quota_state_path, stats).await {
if !process_control_plane
.shutdown(Duration::from_secs(5))
.await
{
warn!("Process control-plane task shutdown deadline expired");
}
let configured_quota_users = runtime
.config()
.access
.users
.keys()
.cloned()
.collect::<BTreeSet<_>>();
match quota_state.save(&configured_quota_users).await {
Ok(()) => {
info!(
path = %quota_state_path.display(),
path = %quota_state.path().display(),
"Persisted per-user quota state"
);
}
Err(error) => {
warn!(
error = %error,
path = %quota_state_path.display(),
path = %quota_state.path().display(),
"Failed to persist per-user quota state"
);
}
@@ -205,8 +214,9 @@ fn dump_stats(stats: &Stats, process_started_at: Instant) {
pub(crate) fn spawn_signal_handlers(
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
process_started_at: Instant,
process_control_plane: ProcessControlPlane,
) {
tokio::spawn(async move {
let _ = process_control_plane.spawn(async move {
let mut sigusr1 =
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
let mut sigusr2 =
@@ -231,6 +241,7 @@ pub(crate) fn spawn_signal_handlers(
pub(crate) fn spawn_signal_handlers(
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
_process_started_at: Instant,
_process_control_plane: ProcessControlPlane,
) {
// No SIGUSR1/SIGUSR2 on non-Unix
}
+7 -1
View File
@@ -8,6 +8,7 @@ use crate::config::ProxyConfig;
use crate::error::{ProxyError, Result};
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
use crate::tls_front::TlsFrontCache;
use crate::tls_front::cache::TlsFullCertBudget;
use crate::tls_front::fetcher::TlsFetchStrategy;
use crate::transport::UpstreamManager;
@@ -109,6 +110,7 @@ pub(crate) async fn bootstrap_tls_front(
upstream_manager: Arc<UpstreamManager>,
startup_tracker: &Arc<StartupTracker>,
task_scope: RuntimeTaskScope,
full_cert_budget: Arc<TlsFullCertBudget>,
policy: TlsBootstrapPolicy,
) -> Result<Option<Arc<TlsFrontCache>>> {
startup_tracker
@@ -128,10 +130,11 @@ pub(crate) async fn bootstrap_tls_front(
return Ok(None);
}
let cache = Arc::new(TlsFrontCache::new(
let cache = Arc::new(TlsFrontCache::new_with_full_cert_budget(
tls_domains,
config.censorship.fake_cert_len,
&config.censorship.tls_front_dir,
full_cert_budget,
));
cache.load_from_disk().await;
@@ -301,6 +304,7 @@ mod tests {
upstream_manager(&config),
&tracker,
scope.clone(),
Arc::new(TlsFullCertBudget::new()),
TlsBootstrapPolicy::RequireReady,
)
.await;
@@ -336,6 +340,7 @@ mod tests {
upstream_manager(&config),
&tracker,
scope.clone(),
Arc::new(TlsFullCertBudget::new()),
TlsBootstrapPolicy::RequireReady,
)
.await
@@ -364,6 +369,7 @@ mod tests {
upstream_manager(&config),
&tracker,
scope.clone(),
Arc::new(TlsFullCertBudget::new()),
TlsBootstrapPolicy::BestEffort,
)
.await