Maestro: add in-process runtime generation reload

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-07-17 23:56:00 +03:00
parent 1f9c82c924
commit 991d5b2c38
26 changed files with 2207 additions and 652 deletions
+26 -84
View File
@@ -4,12 +4,12 @@ use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use arc_swap::ArcSwap;
use http_body_util::Full;
use hyper::body::Bytes;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Request, Response, StatusCode};
use ipnetwork::IpNetwork;
use tokio::net::TcpListener;
use tokio::sync::Semaphore;
use tokio::time::timeout;
@@ -17,6 +17,7 @@ use tracing::{debug, info, warn};
use crate::config::ProxyConfig;
use crate::ip_tracker::UserIpTracker;
use crate::maestro::generation::RuntimeGeneration;
use crate::proxy::shared_state::ProxySharedState;
use crate::stats::Stats;
use crate::stats::beobachten::BeobachtenStore;
@@ -36,16 +37,8 @@ pub async fn serve(
port: u16,
listen: Option<String>,
listen_backlog: u32,
stats: Arc<Stats>,
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Vec<IpNetwork>,
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
) {
let whitelist = Arc::new(whitelist);
// If `metrics_listen` is set, bind on that single address only.
if let Some(ref listen_addr) = listen {
let addr: SocketAddr = match listen_addr.parse() {
@@ -61,17 +54,7 @@ pub async fn serve(
match bind_metrics_listener(addr, ipv6_only, listen_backlog) {
Ok(listener) => {
info!("Metrics endpoint: http://{}/metrics and /beobachten", addr);
serve_listener(
listener,
stats,
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
.await;
serve_listener(listener, active_runtime).await;
}
Err(e) => {
warn!(error = %e, "Failed to bind metrics on {}", addr);
@@ -117,50 +100,14 @@ pub async fn serve(
warn!("Metrics listener is unavailable on both IPv4 and IPv6");
}
(Some(listener), None) | (None, Some(listener)) => {
serve_listener(
listener,
stats,
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
.await;
serve_listener(listener, active_runtime).await;
}
(Some(listener4), Some(listener6)) => {
let stats_v6 = stats.clone();
let beobachten_v6 = beobachten.clone();
let shared_state_v6 = shared_state.clone();
let ip_tracker_v6 = ip_tracker.clone();
let tls_cache_v6 = tls_cache.clone();
let config_rx_v6 = config_rx.clone();
let whitelist_v6 = whitelist.clone();
let active_runtime_v6 = active_runtime.clone();
tokio::spawn(async move {
serve_listener(
listener6,
stats_v6,
beobachten_v6,
shared_state_v6,
ip_tracker_v6,
tls_cache_v6,
config_rx_v6,
whitelist_v6,
)
.await;
serve_listener(listener6, active_runtime_v6).await;
});
serve_listener(
listener4,
stats,
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
.await;
serve_listener(listener4, active_runtime).await;
}
}
}
@@ -180,16 +127,7 @@ fn bind_metrics_listener(
TcpListener::from_std(socket.into())
}
async fn serve_listener(
listener: TcpListener,
stats: Arc<Stats>,
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Arc<Vec<IpNetwork>>,
) {
async fn serve_listener(listener: TcpListener, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
loop {
@@ -201,7 +139,15 @@ async fn serve_listener(
}
};
if !whitelist.is_empty() && !whitelist.iter().any(|net| net.contains(peer.ip())) {
let runtime = active_runtime.load_full();
let config = runtime.config();
if !config.server.metrics_whitelist.is_empty()
&& !config
.server
.metrics_whitelist
.iter()
.any(|net| net.contains(peer.ip()))
{
debug!(peer = %peer, "Metrics request denied by whitelist");
continue;
}
@@ -218,21 +164,17 @@ async fn serve_listener(
}
};
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config_rx_conn = config_rx.clone();
let active_runtime = active_runtime.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req| {
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config = config_rx_conn.borrow().clone();
let runtime = active_runtime.load_full();
let stats = runtime.stats.clone();
let beobachten = runtime.beobachten.clone();
let shared_state = runtime.proxy_shared.clone();
let ip_tracker = runtime.ip_tracker.clone();
let tls_cache = runtime.tls_cache.clone();
let config = runtime.config();
async move {
handle(
req,