mirror of
https://github.com/telemt/telemt.git
synced 2026-09-21 18:18:24 +03:00
Runtime Ownership hardened
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -253,10 +253,6 @@ pub(super) async fn bootstrap(
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) {
|
||||
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
set_maestro_colors_enabled(!config.general.disable_colors);
|
||||
startup_tracker
|
||||
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
const CONTROL_TASK_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
|
||||
const CONTROL_TASK_REGISTRATION_COUNT: usize = CONTROL_TASK_ADMISSION_CLOSED - 1;
|
||||
|
||||
struct ControlTaskAdmission {
|
||||
state: AtomicUsize,
|
||||
registrations_drained: Notify,
|
||||
}
|
||||
|
||||
struct ControlTaskRegistration<'a> {
|
||||
admission: &'a ControlTaskAdmission,
|
||||
}
|
||||
|
||||
impl ControlTaskAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
registrations_drained: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn try_register(&self) -> Option<ControlTaskRegistration<'_>> {
|
||||
let mut state = self.state.load(Ordering::Acquire);
|
||||
loop {
|
||||
if state & CONTROL_TASK_ADMISSION_CLOSED != 0
|
||||
|| state & CONTROL_TASK_REGISTRATION_COUNT == CONTROL_TASK_REGISTRATION_COUNT
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match self.state.compare_exchange_weak(
|
||||
state,
|
||||
state + 1,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return Some(ControlTaskRegistration { admission: self }),
|
||||
Err(observed) => state = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.state
|
||||
.fetch_or(CONTROL_TASK_ADMISSION_CLOSED, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
async fn wait_for_registrations(&self) {
|
||||
loop {
|
||||
let notified = self.registrations_drained.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.state.load(Ordering::Acquire) & CONTROL_TASK_REGISTRATION_COUNT == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ControlTaskRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||
if previous & CONTROL_TASK_REGISTRATION_COUNT == 1 {
|
||||
self.admission.registrations_drained.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ProcessControlPlaneInner {
|
||||
admission: ControlTaskAdmission,
|
||||
cancellation: CancellationToken,
|
||||
tasks: TaskTracker,
|
||||
shutdown_completed: AtomicBool,
|
||||
}
|
||||
|
||||
/// Process-owned cancellation and join scope for API, metrics, and signal tasks.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ProcessControlPlane {
|
||||
inner: Arc<ProcessControlPlaneInner>,
|
||||
}
|
||||
|
||||
impl ProcessControlPlane {
|
||||
/// Creates an open process control-plane scope.
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(ProcessControlPlaneInner {
|
||||
admission: ControlTaskAdmission::new(),
|
||||
cancellation: CancellationToken::new(),
|
||||
tasks: TaskTracker::new(),
|
||||
shutdown_completed: AtomicBool::new(false),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers a cancellable process control-plane task before it can be unpolled.
|
||||
pub(crate) fn spawn<F>(&self, future: F) -> Result<(), F>
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(registration) = self.inner.admission.try_register() else {
|
||||
return Err(future);
|
||||
};
|
||||
let cancellation = self.inner.cancellation.clone();
|
||||
self.inner.tasks.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
drop(registration);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Closes task admission, cancels all owned work, and joins it within the deadline.
|
||||
pub(crate) async fn shutdown(&self, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
self.inner.admission.close();
|
||||
self.inner.cancellation.cancel();
|
||||
self.inner.tasks.close();
|
||||
if self.inner.shutdown_completed.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
let registrations_stopped = tokio::time::timeout_at(
|
||||
deadline,
|
||||
self.inner.admission.wait_for_registrations(),
|
||||
)
|
||||
.await
|
||||
.is_ok();
|
||||
let tasks_stopped = tokio::time::timeout_at(deadline, self.inner.tasks.wait())
|
||||
.await
|
||||
.is_ok();
|
||||
let outcome = registrations_stopped && tasks_stopped;
|
||||
if outcome {
|
||||
self.inner.shutdown_completed.store(true, Ordering::Release);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cancels_owned_tasks_and_rejects_late_registration() {
|
||||
struct DropSignal(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
let scope = ProcessControlPlane::new();
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let drop_signal = DropSignal(dropped.clone());
|
||||
assert!(
|
||||
scope
|
||||
.spawn(async move {
|
||||
let _drop_signal = drop_signal;
|
||||
std::future::pending::<()>().await;
|
||||
})
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
assert!(scope.spawn(async {}).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_shutdown_callers_wait_for_completion() {
|
||||
let scope = ProcessControlPlane::new();
|
||||
let registration = scope.inner.admission.try_register().unwrap();
|
||||
let first_scope = scope.clone();
|
||||
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(1)).await });
|
||||
tokio::task::yield_now().await;
|
||||
let second_scope = scope.clone();
|
||||
let second = tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!first.is_finished());
|
||||
assert!(!second.is_finished());
|
||||
drop(registration);
|
||||
|
||||
assert!(first.await.unwrap());
|
||||
assert!(second.await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_shutdown_caller_cannot_orphan_the_control_plane() {
|
||||
let scope = ProcessControlPlane::new();
|
||||
let registration = scope.inner.admission.try_register().unwrap();
|
||||
let first_scope = scope.clone();
|
||||
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
assert!(scope.spawn(async {}).is_err());
|
||||
drop(registration);
|
||||
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
}
|
||||
}
|
||||
+66
-10
@@ -3,7 +3,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tokio::sync::{Notify, RwLock, Semaphore, watch};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
@@ -29,6 +29,7 @@ const SESSION_REGISTRATION_COUNT: usize = SESSION_ADMISSION_CLOSED - 1;
|
||||
|
||||
struct SessionAdmission {
|
||||
state: AtomicUsize,
|
||||
registrations_drained: Notify,
|
||||
}
|
||||
|
||||
struct SessionRegistration<'a> {
|
||||
@@ -39,6 +40,7 @@ impl SessionAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
registrations_drained: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,15 +75,24 @@ impl SessionAdmission {
|
||||
}
|
||||
|
||||
async fn wait_for_registrations(&self) {
|
||||
while self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
loop {
|
||||
let notified = self.registrations_drained.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SessionRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.admission.state.fetch_sub(1, Ordering::Release);
|
||||
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||
if previous & SESSION_REGISTRATION_COUNT == 1 {
|
||||
self.admission.registrations_drained.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +109,7 @@ pub(crate) struct RuntimeWatchState {
|
||||
pub(crate) struct RuntimeTaskScope {
|
||||
tracker: TaskTracker,
|
||||
cancel: CancellationToken,
|
||||
admission: Arc<SessionAdmission>,
|
||||
}
|
||||
|
||||
impl RuntimeTaskScope {
|
||||
@@ -106,6 +118,7 @@ impl RuntimeTaskScope {
|
||||
Self {
|
||||
tracker: TaskTracker::new(),
|
||||
cancel: CancellationToken::new(),
|
||||
admission: Arc::new(SessionAdmission::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,9 +127,13 @@ impl RuntimeTaskScope {
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(_registration) = self.admission.try_register() else {
|
||||
return;
|
||||
};
|
||||
let cancel = self.cancel.clone();
|
||||
self.tracker.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
@@ -130,6 +147,8 @@ impl RuntimeTaskScope {
|
||||
|
||||
/// Cancels the scope and waits within the bounded background-task budget.
|
||||
pub(crate) async fn stop(&self) {
|
||||
self.admission.close();
|
||||
self.admission.wait_for_registrations().await;
|
||||
self.cancel.cancel();
|
||||
self.tracker.close();
|
||||
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
|
||||
@@ -263,6 +282,7 @@ impl RuntimeGeneration {
|
||||
let cancel = self.session_cancel.clone();
|
||||
self.sessions.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
@@ -275,11 +295,6 @@ impl RuntimeGeneration {
|
||||
self.session_admission.close();
|
||||
}
|
||||
|
||||
/// Reopens admission after a candidate activation rolls back.
|
||||
pub(crate) fn resume_accepting_sessions(&self) {
|
||||
self.session_admission.reopen();
|
||||
}
|
||||
|
||||
/// Waits for registered sessions and cancels them when the deadline expires.
|
||||
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
|
||||
self.stop_accepting_sessions();
|
||||
@@ -308,6 +323,27 @@ impl RuntimeGeneration {
|
||||
pub(crate) async fn stop_background_tasks(&self) {
|
||||
self.background_tasks.stop().await;
|
||||
}
|
||||
|
||||
/// Terminally stops the generation's Middle-End task and writer scope.
|
||||
pub(crate) async fn stop_middle_end(&self, timeout: Duration) -> bool {
|
||||
let Some(pool) = self.current_me_pool().await else {
|
||||
return true;
|
||||
};
|
||||
pool.shutdown_until(timeout).await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RuntimeGeneration {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pool) = self.me_pool.as_ref() {
|
||||
pool.begin_shutdown();
|
||||
}
|
||||
if let Ok(pool) = self.me_pool_runtime.try_read()
|
||||
&& let Some(pool) = pool.as_ref()
|
||||
{
|
||||
pool.begin_shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -393,11 +429,31 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_task_scope_joins_cancelled_background_task() {
|
||||
struct DropSignal(Arc<AtomicUsize>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
let scope = RuntimeTaskScope::new();
|
||||
scope.spawn(std::future::pending());
|
||||
let dropped = Arc::new(AtomicUsize::new(0));
|
||||
let drop_signal = DropSignal(dropped.clone());
|
||||
scope.spawn(async move {
|
||||
let _drop_signal = drop_signal;
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(dropped.load(Ordering::Acquire), 1);
|
||||
|
||||
let late_drop_signal = DropSignal(dropped.clone());
|
||||
scope.spawn(async move {
|
||||
let _late_drop_signal = late_drop_signal;
|
||||
});
|
||||
assert_eq!(dropped.load(Ordering::Acquire), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -12,7 +12,7 @@ use tracing::{debug, error, info, warn};
|
||||
use crate::config::{ListenerTransport, RstOnCloseMode};
|
||||
use crate::proxy::ClientHandler;
|
||||
use crate::transport::socket::set_linger_zero;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||
use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};
|
||||
|
||||
use super::bind::BoundTcpListener;
|
||||
@@ -208,45 +208,82 @@ async fn run_accept_loop(
|
||||
return;
|
||||
};
|
||||
web_runtime.telemetry().record_accept();
|
||||
let Some(connection_permit) = web_runtime.try_http_connection() else {
|
||||
let config = web_runtime.active_generation().config();
|
||||
let action = config.web.http_connection_capacity_action;
|
||||
let phase_timeout =
|
||||
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
|
||||
drop(config);
|
||||
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
if cancellation.is_cancelled() {
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
if web_runtime.is_shutdown() {
|
||||
web_runtime
|
||||
.telemetry()
|
||||
.record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let connection_permit = match web_runtime.try_http_connection() {
|
||||
Ok(permit) => permit,
|
||||
Err(HttpConnectionAdmissionError::Closed) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
web_runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let Some(overload_permit) = web_runtime.try_http_overload_connection()
|
||||
else {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||
let config = web_runtime.active_generation().config();
|
||||
let action = config.web.http_connection_capacity_action;
|
||||
let phase_timeout = Duration::from_millis(
|
||||
config.web.timeouts.http_overload_timeout_ms,
|
||||
);
|
||||
web_runtime.telemetry().record_overload(
|
||||
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
|
||||
);
|
||||
drop(stream);
|
||||
drop(config);
|
||||
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
);
|
||||
web_runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let overload_permit =
|
||||
match web_runtime.try_http_overload_connection() {
|
||||
Ok(permit) => permit,
|
||||
Err(HttpConnectionAdmissionError::Closed) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
web_runtime.telemetry().record_overload(
|
||||
WebHttpConnectionOverloadOutcome::ShutdownDrop,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
);
|
||||
web_runtime.telemetry().record_overload(
|
||||
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
connections.spawn(web_overload::serve(
|
||||
stream,
|
||||
peer_addr,
|
||||
spec.web_client_ip_source,
|
||||
Arc::clone(&spec.web_trusted_proxy_cidrs),
|
||||
Arc::clone(web_runtime),
|
||||
cancellation.clone(),
|
||||
overload_permit,
|
||||
action,
|
||||
phase_timeout,
|
||||
));
|
||||
continue;
|
||||
};
|
||||
connections.spawn(web_overload::serve(
|
||||
stream,
|
||||
peer_addr,
|
||||
spec.web_client_ip_source,
|
||||
Arc::clone(&spec.web_trusted_proxy_cidrs),
|
||||
Arc::clone(web_runtime),
|
||||
cancellation.clone(),
|
||||
overload_permit,
|
||||
action,
|
||||
phase_timeout,
|
||||
));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
connections.spawn(crate::web::http::serve_connection(
|
||||
stream,
|
||||
|
||||
@@ -9,9 +9,10 @@ use tokio::sync::OwnedSemaphorePermit;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
|
||||
|
||||
/// Exact bounded retryable response emitted before HTTP request parsing.
|
||||
pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
|
||||
|
||||
/// Handles one accepted WEB socket outside ordinary connection capacity.
|
||||
@@ -44,25 +45,31 @@ pub(super) async fn serve(
|
||||
return;
|
||||
}
|
||||
permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
|
||||
permit.ok().flatten()
|
||||
permit
|
||||
}
|
||||
};
|
||||
let Some(connection_permit) = connection_permit else {
|
||||
if runtime.is_shutdown() {
|
||||
let connection_permit = match connection_permit {
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(HttpConnectionAdmissionError::Closed)) => {
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_rejection(WebRejectionReason::RuntimeClosed);
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
|
||||
return;
|
||||
}
|
||||
let outcome = match respond(stream, &cancellation, phase_timeout).await {
|
||||
WebHttpConnectionOverloadOutcome::Responded503 => {
|
||||
WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
record_final_capacity_rejection(&runtime, outcome);
|
||||
runtime.telemetry().record_overload(outcome);
|
||||
return;
|
||||
Ok(Err(HttpConnectionAdmissionError::AtCapacity)) | Err(_) => {
|
||||
let outcome = match respond(stream, &cancellation, phase_timeout).await {
|
||||
WebHttpConnectionOverloadOutcome::Responded503 => {
|
||||
WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
record_final_capacity_rejection(&runtime, outcome);
|
||||
runtime.telemetry().record_overload(outcome);
|
||||
return;
|
||||
}
|
||||
};
|
||||
runtime
|
||||
.telemetry()
|
||||
|
||||
@@ -345,6 +345,7 @@ pub(crate) async fn initialize_me_pool(
|
||||
config.general.me_route_blocking_send_timeout_ms,
|
||||
config.general.me_route_inline_recovery_attempts,
|
||||
config.general.me_route_inline_recovery_wait_ms,
|
||||
(config.server.max_connections as usize).saturating_add(128),
|
||||
);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
// - admission: conditional-cast gate and route mode switching.
|
||||
// - bootstrap: configuration and tracing initialization.
|
||||
// - connectivity: startup ME/DC connectivity diagnostics.
|
||||
// - control_plane: process-owned API, metrics, and signal task lifecycle.
|
||||
// - generation: runtime generation state and task ownership.
|
||||
// - helpers: CLI and shared startup/runtime helper routines.
|
||||
// - listeners: TCP/Unix listener planning, binding, and lifecycle control.
|
||||
@@ -21,6 +22,7 @@
|
||||
mod admission;
|
||||
mod bootstrap;
|
||||
mod connectivity;
|
||||
pub(crate) mod control_plane;
|
||||
pub(crate) mod generation;
|
||||
mod helpers;
|
||||
mod listeners;
|
||||
|
||||
+65
-18
@@ -1,9 +1,10 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::{RwLock, watch};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::api;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
@@ -15,14 +16,15 @@ use crate::startup::{COMPONENT_API_BOOTSTRAP, COMPONENT_NETWORK_PROBE};
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{QuotaStore, Stats};
|
||||
use crate::synlimit_control;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::{
|
||||
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
||||
shutdown, tls_bootstrap,
|
||||
bootstrap, control_plane, generation, listeners, reload, reload_supervisor, runtime_startup,
|
||||
runtime_tasks, shutdown, tls_bootstrap,
|
||||
};
|
||||
|
||||
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
||||
@@ -45,10 +47,22 @@ pub(super) async fn run_telemt_core(
|
||||
|
||||
let quota_store = Arc::new(QuotaStore::default());
|
||||
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
|
||||
let tls_full_cert_budget = Arc::new(TlsFullCertBudget::new());
|
||||
let process_control_plane = control_plane::ProcessControlPlane::new();
|
||||
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||
let quota_state_path = config.general.quota_state_path.clone();
|
||||
crate::quota_state::load_quota_state("a_state_path, stats.as_ref()).await;
|
||||
let quota_state = crate::quota_state::QuotaStateOwner::new(
|
||||
quota_state_path,
|
||||
quota_store.clone(),
|
||||
);
|
||||
let configured_quota_users = config
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
quota_state.load(&configured_quota_users).await;
|
||||
|
||||
let upstream_manager = Arc::new(
|
||||
UpstreamManager::new(
|
||||
@@ -136,15 +150,29 @@ pub(super) async fn run_telemt_core(
|
||||
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
||||
Ok(listen) => listen,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
listen = %config.server.api.listen,
|
||||
"Invalid server.api.listen; API is disabled"
|
||||
let message = format!(
|
||||
"invalid server.api.listen \"{}\": {}",
|
||||
config.server.api.listen, error
|
||||
);
|
||||
SocketAddr::from(([127, 0, 0, 1], 0))
|
||||
startup_tracker
|
||||
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.clone()))
|
||||
.await;
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, message).into());
|
||||
}
|
||||
};
|
||||
if listen.port() != 0 {
|
||||
let api_listener = match tokio::net::TcpListener::bind(listen).await {
|
||||
Ok(listener) => listener,
|
||||
Err(error) => {
|
||||
startup_tracker
|
||||
.fail_component(
|
||||
COMPONENT_API_BOOTSTRAP,
|
||||
Some(format!("API listener bind failed on {listen}: {error}")),
|
||||
)
|
||||
.await;
|
||||
return Err(error.into());
|
||||
}
|
||||
};
|
||||
let stats_api = stats.clone();
|
||||
let ip_tracker_api = ip_tracker.clone();
|
||||
let me_pool_api = api_me_pool.clone();
|
||||
@@ -152,7 +180,7 @@ pub(super) async fn run_telemt_core(
|
||||
let route_runtime_api = route_runtime.clone();
|
||||
let proxy_shared_api = shared_state.clone();
|
||||
let config_path_api = config_path.clone();
|
||||
let quota_state_path_api = quota_state_path.clone();
|
||||
let quota_state_api = quota_state.clone();
|
||||
let startup_tracker_api = startup_tracker.clone();
|
||||
let detected_ips_rx_api = detected_ips_rx.clone();
|
||||
let reload_control_api = reload_control.clone();
|
||||
@@ -160,9 +188,11 @@ pub(super) async fn run_telemt_core(
|
||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||
let web_trace_api = web_trace.clone();
|
||||
let web_runtime_rx_api = web_runtime_control.subscribe();
|
||||
tokio::spawn(async move {
|
||||
let api_control_plane = process_control_plane.clone();
|
||||
let api_task_control_plane = process_control_plane.clone();
|
||||
let api_task = async move {
|
||||
api::serve(
|
||||
listen,
|
||||
api_listener,
|
||||
stats_api,
|
||||
ip_tracker_api,
|
||||
me_pool_api,
|
||||
@@ -170,7 +200,7 @@ pub(super) async fn run_telemt_core(
|
||||
proxy_shared_api,
|
||||
upstream_manager_api,
|
||||
config_path_api,
|
||||
quota_state_path_api,
|
||||
quota_state_api,
|
||||
detected_ips_rx_api,
|
||||
process_started_at_epoch_secs,
|
||||
startup_tracker_api,
|
||||
@@ -179,13 +209,21 @@ pub(super) async fn run_telemt_core(
|
||||
runtime_watch_rx_api,
|
||||
web_trace_api,
|
||||
web_runtime_rx_api,
|
||||
api_task_control_plane,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
};
|
||||
if api_control_plane.spawn(api_task).is_err() {
|
||||
let message = "process control-plane task admission closed during API startup";
|
||||
startup_tracker
|
||||
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.to_string()))
|
||||
.await;
|
||||
return Err(std::io::Error::other(message).into());
|
||||
}
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_API_BOOTSTRAP,
|
||||
Some(format!("api task spawned on {}", listen)),
|
||||
Some(format!("API listener bound and supervised on {}", listen)),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
@@ -219,6 +257,7 @@ pub(super) async fn run_telemt_core(
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
runtime_task_scope.clone(),
|
||||
tls_full_cert_budget.clone(),
|
||||
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await?;
|
||||
@@ -316,8 +355,10 @@ pub(super) async fn run_telemt_core(
|
||||
&startup_tracker,
|
||||
active_runtime.clone(),
|
||||
web_runtime_control.subscribe(),
|
||||
tls_full_cert_budget.clone(),
|
||||
process_control_plane.clone(),
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
|
||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||
@@ -335,6 +376,7 @@ pub(super) async fn run_telemt_core(
|
||||
reload_commands,
|
||||
config_path,
|
||||
quota_store,
|
||||
tls_full_cert_budget,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
@@ -342,12 +384,17 @@ pub(super) async fn run_telemt_core(
|
||||
web_trace,
|
||||
);
|
||||
|
||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
||||
shutdown::spawn_signal_handlers(
|
||||
active_runtime.clone(),
|
||||
process_started_at,
|
||||
process_control_plane.clone(),
|
||||
);
|
||||
shutdown::wait_for_shutdown(
|
||||
process_started_at,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
quota_state,
|
||||
reload_supervisor,
|
||||
process_control_plane,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::stats::QuotaStore;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
@@ -26,6 +27,7 @@ pub(crate) struct ReloadSupervisor {
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
@@ -81,12 +83,7 @@ fn revision_gate_action(
|
||||
|
||||
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
|
||||
generation.stop_background_tasks().await;
|
||||
let Some(pool) = generation.current_me_pool().await else {
|
||||
return false;
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
.is_err()
|
||||
!generation.stop_middle_end(Duration::from_secs(5)).await
|
||||
}
|
||||
|
||||
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
|
||||
@@ -103,6 +100,7 @@ impl ReloadSupervisor {
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
@@ -116,6 +114,7 @@ impl ReloadSupervisor {
|
||||
commands,
|
||||
config_path,
|
||||
quota_store,
|
||||
tls_full_cert_budget,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
@@ -171,6 +170,7 @@ impl ReloadSupervisor {
|
||||
&self.config_path,
|
||||
self.quota_store.clone(),
|
||||
self.runtime_log_filter.clone(),
|
||||
self.tls_full_cert_budget.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -207,25 +207,18 @@ impl ReloadSupervisor {
|
||||
prepared,
|
||||
listener_transition,
|
||||
revision_action,
|
||||
|entries| {
|
||||
crate::network::dns_overrides::install_entries(entries)
|
||||
.map_err(|error| error.to_string())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn activate_prepared<InstallDns>(
|
||||
async fn activate_prepared(
|
||||
&self,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
prepared: PreparedRuntime,
|
||||
revision_action: RevisionGateAction,
|
||||
install_dns: InstallDns,
|
||||
) where
|
||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
||||
{
|
||||
) {
|
||||
let listener_transition = match self
|
||||
.listener_manager
|
||||
.lock()
|
||||
@@ -245,22 +238,18 @@ impl ReloadSupervisor {
|
||||
prepared,
|
||||
listener_transition,
|
||||
revision_action,
|
||||
install_dns,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn activate_prepared_with_transition<InstallDns>(
|
||||
async fn activate_prepared_with_transition(
|
||||
&self,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
prepared: PreparedRuntime,
|
||||
listener_transition: Option<PreparedListenerTransition>,
|
||||
revision_action: RevisionGateAction,
|
||||
install_dns: InstallDns,
|
||||
) where
|
||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
||||
{
|
||||
) {
|
||||
match revision_action {
|
||||
RevisionGateAction::Proceed => {}
|
||||
RevisionGateAction::Warn(warning) => {
|
||||
@@ -283,18 +272,6 @@ impl ReloadSupervisor {
|
||||
detected_ips,
|
||||
config_watcher_activation,
|
||||
} = prepared;
|
||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
||||
let message = format!("runtime DNS activation failed: {}", error);
|
||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
||||
old_runtime.resume_accepting_sessions();
|
||||
let _ = cleanup_candidate(&new_runtime).await;
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&old_runtime.config().general.log_level);
|
||||
self.control.rolled_back(command.reload_id, message).await;
|
||||
return;
|
||||
}
|
||||
self.control.add_warning(command.reload_id, message).await;
|
||||
}
|
||||
let pending_listener_transition = if let Some(listener_transition) = listener_transition {
|
||||
match self
|
||||
.listener_manager
|
||||
@@ -367,7 +344,7 @@ impl ReloadSupervisor {
|
||||
|
||||
if stop_background_and_middle_end(&replaced).await {
|
||||
let warning = format!(
|
||||
"generation {} Middle-End close broadcast timed out",
|
||||
"generation {} Middle-End lifecycle shutdown timed out",
|
||||
replaced.id
|
||||
);
|
||||
warn!(reload_id = command.reload_id, warning = %warning);
|
||||
|
||||
@@ -53,6 +53,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
commands,
|
||||
config_path: PathBuf::new(),
|
||||
quota_store: Arc::new(QuotaStore::default()),
|
||||
tls_full_cert_budget: Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter: runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
@@ -131,7 +132,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
fixture.old_runtime.clone(),
|
||||
prepared_runtime(fixture.new_runtime),
|
||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -154,44 +154,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
||||
for policy in [ReloadFailurePolicy::Rollback, ReloadFailurePolicy::KeepNew] {
|
||||
let fixture = fixture(ReloadRequest {
|
||||
failure_policy: policy,
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
fixture
|
||||
.supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
prepared_runtime(fixture.new_runtime.clone()),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Err("invalid DNS entry".to_string()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
match policy {
|
||||
ReloadFailurePolicy::Rollback => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
|
||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
ReloadFailurePolicy::KeepNew => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 2);
|
||||
assert_eq!(status.state, ReloadPhase::Succeeded);
|
||||
assert_eq!(status.warnings.len(), 1);
|
||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
let mut fixture = fixture(ReloadRequest {
|
||||
@@ -220,7 +182,6 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
old_runtime,
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -273,7 +234,6 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
||||
old_runtime,
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -304,6 +264,7 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
||||
commands,
|
||||
PathBuf::new(),
|
||||
Arc::new(QuotaStore::default()),
|
||||
Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
@@ -44,6 +45,7 @@ pub(crate) async fn prepare_runtime(
|
||||
config_path: &Path,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
) -> Result<PreparedRuntime, String> {
|
||||
config
|
||||
.validate_web_decoy_listener_separation()
|
||||
@@ -121,6 +123,7 @@ pub(crate) async fn prepare_runtime(
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
task_scope.clone(),
|
||||
tls_full_cert_budget,
|
||||
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
||||
|
||||
use super::control_plane::ProcessControlPlane;
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::write_beobachten_snapshot;
|
||||
@@ -158,6 +159,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
task_scope.cancellation_token(),
|
||||
Some(upstream_manager.dns_resolver()),
|
||||
config_watcher_activation,
|
||||
);
|
||||
task_scope.spawn(config_watcher_task);
|
||||
@@ -168,7 +170,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
)
|
||||
.await;
|
||||
let stats_policy = stats.clone();
|
||||
let upstream_policy = upstream_manager.clone();
|
||||
let mut config_rx_policy = config_rx.clone();
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
@@ -178,9 +179,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
let cfg = config_rx_policy.borrow_and_update().clone();
|
||||
stats_policy
|
||||
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
|
||||
if let Err(error) = upstream_policy.update_dns_overrides(&cfg.network.dns_overrides) {
|
||||
warn!(error = %error, "Failed to update generation DNS overrides");
|
||||
}
|
||||
if let Some(pool) = &me_pool_for_policy {
|
||||
pool.update_runtime_transport_policy(
|
||||
cfg.general.me_socks_kdf_policy,
|
||||
@@ -406,7 +404,9 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
|
||||
) {
|
||||
tls_full_cert_budget: Arc<crate::tls_front::cache::TlsFullCertBudget>,
|
||||
control_plane: ProcessControlPlane,
|
||||
) -> std::io::Result<()> {
|
||||
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
||||
let metrics_target: Option<(u16, Option<String>)> =
|
||||
if let Some(ref listen) = config.server.metrics_listen {
|
||||
@@ -414,12 +414,15 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
Ok(addr) => Some((addr.port(), Some(listen.clone()))),
|
||||
Err(e) => {
|
||||
startup_tracker
|
||||
.skip_component(
|
||||
.fail_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some(format!("invalid metrics_listen \"{}\": {}", listen, e)),
|
||||
)
|
||||
.await;
|
||||
None
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid metrics_listen \"{}\": {}", listen, e),
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -435,15 +438,30 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
Some(format!("spawn metrics endpoint on {}", label)),
|
||||
)
|
||||
.await;
|
||||
let active_runtime = active_runtime.clone();
|
||||
let listen_backlog = config.server.listen_backlog;
|
||||
tokio::spawn(async move {
|
||||
metrics::serve(port, listen, listen_backlog, active_runtime, web_runtime_rx).await;
|
||||
});
|
||||
let bound = match metrics::bind(port, listen, listen_backlog) {
|
||||
Ok(bound) => bound,
|
||||
Err(error) => {
|
||||
startup_tracker
|
||||
.fail_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some(format!("metrics listener bind failed: {error}")),
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
metrics::serve(
|
||||
bound,
|
||||
active_runtime,
|
||||
web_runtime_rx,
|
||||
tls_full_cert_budget,
|
||||
control_plane,
|
||||
);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some("metrics task spawned".to_string()),
|
||||
Some("metrics listeners bound and supervised".to_string()),
|
||||
)
|
||||
.await;
|
||||
} else if config.server.metrics_listen.is_none() {
|
||||
@@ -454,6 +472,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc<StartupTracker>) {
|
||||
|
||||
+33
-22
@@ -8,7 +8,7 @@
|
||||
//!
|
||||
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -19,11 +19,13 @@ use tokio::signal;
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::control_plane::ProcessControlPlane;
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::helpers::{format_uptime, unit_label};
|
||||
use super::reload_supervisor::ReloadSupervisorHandle;
|
||||
use crate::stats::Stats;
|
||||
use crate::synlimit_control;
|
||||
use crate::quota_state::QuotaStateOwner;
|
||||
|
||||
/// Signal that triggered shutdown.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -50,16 +52,18 @@ impl std::fmt::Display for ShutdownSignal {
|
||||
pub(crate) async fn wait_for_shutdown(
|
||||
process_started_at: Instant,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
quota_state: Arc<QuotaStateOwner>,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
let signal = wait_for_shutdown_signal().await;
|
||||
perform_shutdown(
|
||||
signal,
|
||||
process_started_at,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
quota_state,
|
||||
reload_supervisor,
|
||||
process_control_plane,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -89,8 +93,9 @@ async fn perform_shutdown(
|
||||
signal: ShutdownSignal,
|
||||
process_started_at: Instant,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
quota_state: Arc<QuotaStateOwner>,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
let shutdown_started_at = Instant::now();
|
||||
info!(signal = %signal, "Received shutdown signal");
|
||||
@@ -115,37 +120,41 @@ async fn perform_shutdown(
|
||||
// Graceful ME pool shutdown
|
||||
runtime.stop_sessions().await;
|
||||
runtime.stop_background_tasks().await;
|
||||
if let Some(pool) = runtime.current_me_pool().await {
|
||||
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
{
|
||||
Ok(total) => {
|
||||
info!(
|
||||
close_conn_sent = total,
|
||||
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
|
||||
}
|
||||
}
|
||||
if runtime.stop_middle_end(Duration::from_secs(5)).await {
|
||||
info!("ME shutdown: pool lifecycle completed");
|
||||
} else {
|
||||
warn!("ME shutdown: pool lifecycle deadline expired");
|
||||
}
|
||||
|
||||
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
|
||||
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
||||
}
|
||||
|
||||
match crate::quota_state::save_quota_state("a_state_path, stats).await {
|
||||
if !process_control_plane
|
||||
.shutdown(Duration::from_secs(5))
|
||||
.await
|
||||
{
|
||||
warn!("Process control-plane task shutdown deadline expired");
|
||||
}
|
||||
|
||||
let configured_quota_users = runtime
|
||||
.config()
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
match quota_state.save(&configured_quota_users).await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
path = %quota_state_path.display(),
|
||||
path = %quota_state.path().display(),
|
||||
"Persisted per-user quota state"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
path = %quota_state_path.display(),
|
||||
path = %quota_state.path().display(),
|
||||
"Failed to persist per-user quota state"
|
||||
);
|
||||
}
|
||||
@@ -205,8 +214,9 @@ fn dump_stats(stats: &Stats, process_started_at: Instant) {
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
process_started_at: Instant,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let _ = process_control_plane.spawn(async move {
|
||||
let mut sigusr1 =
|
||||
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
||||
let mut sigusr2 =
|
||||
@@ -231,6 +241,7 @@ pub(crate) fn spawn_signal_handlers(
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
_process_started_at: Instant,
|
||||
_process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
// No SIGUSR1/SIGUSR2 on non-Unix
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::config::ProxyConfig;
|
||||
use crate::error::{ProxyError, Result};
|
||||
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::tls_front::fetcher::TlsFetchStrategy;
|
||||
use crate::transport::UpstreamManager;
|
||||
|
||||
@@ -109,6 +110,7 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
policy: TlsBootstrapPolicy,
|
||||
) -> Result<Option<Arc<TlsFrontCache>>> {
|
||||
startup_tracker
|
||||
@@ -128,10 +130,11 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let cache = Arc::new(TlsFrontCache::new(
|
||||
let cache = Arc::new(TlsFrontCache::new_with_full_cert_budget(
|
||||
tls_domains,
|
||||
config.censorship.fake_cert_len,
|
||||
&config.censorship.tls_front_dir,
|
||||
full_cert_budget,
|
||||
));
|
||||
cache.load_from_disk().await;
|
||||
|
||||
@@ -301,6 +304,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await;
|
||||
@@ -336,6 +340,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
@@ -364,6 +369,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user