mirror of
https://github.com/telemt/telemt.git
synced 2026-09-05 18:16:06 +03:00
WEB Knobs in API
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -87,6 +87,64 @@ async fn read_managed_config_strips_access() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() {
|
||||||
|
let (path, _directory) = temp_config(concat!(
|
||||||
|
"[web]\nenabled = false\ncarrier = \"https\"\n",
|
||||||
|
"[web.debug]\nenabled = true\ndefault_window_secs = 180\n",
|
||||||
|
"[access.users]\nbob = \"00000000000000000000000000000000\"\n",
|
||||||
|
));
|
||||||
|
|
||||||
|
let (value, _revision) = read_managed_config(&path).await.unwrap();
|
||||||
|
let table = value.as_table().unwrap();
|
||||||
|
|
||||||
|
assert!(table.contains_key("web"));
|
||||||
|
assert!(table["web"].get("debug").is_some());
|
||||||
|
assert!(table["web"].get("runtime").is_none());
|
||||||
|
assert!(!table.contains_key("access"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let debug_patch: Json = serde_json::json!({
|
||||||
|
"web": {"debug": {"enabled": true, "capture_headers": false}}
|
||||||
|
});
|
||||||
|
let debug = apply_patch_to_path(&path, &debug_patch, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!debug.process_restart_required);
|
||||||
|
assert!(debug.changed.iter().any(|section| section == "web"));
|
||||||
|
|
||||||
|
let limits_patch: Json = serde_json::json!({
|
||||||
|
"web": {"limits": {"max_http_connections": 2049}}
|
||||||
|
});
|
||||||
|
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(limits.process_restart_required);
|
||||||
|
assert!(
|
||||||
|
limits
|
||||||
|
.deferred_process_fields
|
||||||
|
.iter()
|
||||||
|
.any(|field| field == "web.limits")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn invalid_web_patch_does_not_modify_the_source() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let original = tokio::fs::read_to_string(&path).await.unwrap();
|
||||||
|
let patch: Json = serde_json::json!({
|
||||||
|
"web": {"debug": {"default_window_secs": 181, "max_window_secs": 180}}
|
||||||
|
});
|
||||||
|
|
||||||
|
let error = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||||
|
|
||||||
|
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), original);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn read_managed_config_returns_only_editable_sections() {
|
async fn read_managed_config_returns_only_editable_sections() {
|
||||||
// Full server (api/port) and network must not leak. Listeners-only server
|
// Full server (api/port) and network must not leak. Listeners-only server
|
||||||
|
|||||||
@@ -262,12 +262,12 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
|
|||||||
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn load_config_for_reload(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
pub(super) async fn load_config_for_reload(
|
||||||
let config_path = config_path.to_path_buf();
|
config_path: &Path,
|
||||||
tokio::task::spawn_blocking(move || ProxyConfig::load(config_path))
|
) -> Result<(ProxyConfig, String), ApiFailure> {
|
||||||
.await
|
let loaded = load_config_snapshot(config_path, true).await?;
|
||||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {}", error)))?
|
let revision = compute_snapshot_revision(&loaded);
|
||||||
.map_err(|error| ApiFailure::bad_request(format!("invalid runtime config: {}", error)))
|
Ok((loaded.config, revision))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
@@ -307,6 +307,7 @@ pub(super) const EDITABLE_SECTIONS: &[&str] = &[
|
|||||||
"censorship",
|
"censorship",
|
||||||
"upstreams",
|
"upstreams",
|
||||||
"dc_overrides",
|
"dc_overrides",
|
||||||
|
"web",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Nested fields under `[server]` that may be read/patched via the config API.
|
/// Nested fields under `[server]` that may be read/patched via the config API.
|
||||||
|
|||||||
+41
-4
@@ -30,6 +30,7 @@ use crate::startup::StartupTracker;
|
|||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::web::control::WebRuntimePublication;
|
||||||
use crate::web::trace::WebTraceStore;
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
mod config_edit;
|
mod config_edit;
|
||||||
@@ -48,6 +49,8 @@ mod runtime_stats;
|
|||||||
mod runtime_watch;
|
mod runtime_watch;
|
||||||
mod runtime_zero;
|
mod runtime_zero;
|
||||||
mod users;
|
mod users;
|
||||||
|
// WEB runtime status and bounded controls remain separate from general API DTOs.
|
||||||
|
mod web_runtime;
|
||||||
mod web_status;
|
mod web_status;
|
||||||
|
|
||||||
use config_store::{
|
use config_store::{
|
||||||
@@ -125,6 +128,7 @@ pub(super) struct ApiShared {
|
|||||||
pub(super) reload_control: ReloadControl,
|
pub(super) reload_control: ReloadControl,
|
||||||
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
pub(super) web_trace: Arc<WebTraceStore>,
|
pub(super) web_trace: Arc<WebTraceStore>,
|
||||||
|
pub(super) web_runtime_rx: watch::Receiver<WebRuntimePublication>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiShared {
|
impl ApiShared {
|
||||||
@@ -159,6 +163,7 @@ impl ApiShared {
|
|||||||
reload_control: self.reload_control.clone(),
|
reload_control: self.reload_control.clone(),
|
||||||
active_runtime: self.active_runtime.clone(),
|
active_runtime: self.active_runtime.clone(),
|
||||||
web_trace: self.web_trace.clone(),
|
web_trace: self.web_trace.clone(),
|
||||||
|
web_runtime_rx: self.web_runtime_rx.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,9 +201,15 @@ async fn submit_reload_from_disk(
|
|||||||
request: ReloadRequest,
|
request: ReloadRequest,
|
||||||
) -> Result<(ReloadAccepted, String), ApiFailure> {
|
) -> Result<(ReloadAccepted, String), ApiFailure> {
|
||||||
let _guard = mutation_lock.lock().await;
|
let _guard = mutation_lock.lock().await;
|
||||||
ensure_expected_revision(config_path, expected_revision).await?;
|
let (config, revision) = load_config_for_reload(config_path).await?;
|
||||||
let revision = current_revision(config_path).await?;
|
if expected_revision.is_some_and(|expected| expected != revision) {
|
||||||
let config = Arc::new(load_config_for_reload(config_path).await?);
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"revision_conflict",
|
||||||
|
"Config revision mismatch",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let config = Arc::new(config);
|
||||||
let accepted = reload_control
|
let accepted = reload_control
|
||||||
.submit(config, revision.clone(), request)
|
.submit(config, revision.clone(), request)
|
||||||
.await
|
.await
|
||||||
@@ -218,6 +229,9 @@ async fn submit_reload_from_disk(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||||
|
if let Some(allow) = web_runtime::allowed_methods(path) {
|
||||||
|
return Some(allow);
|
||||||
|
}
|
||||||
match path {
|
match path {
|
||||||
"/v1/health"
|
"/v1/health"
|
||||||
| "/v1/health/ready"
|
| "/v1/health/ready"
|
||||||
@@ -285,6 +299,7 @@ pub async fn serve(
|
|||||||
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
||||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||||
web_trace: Arc<WebTraceStore>,
|
web_trace: Arc<WebTraceStore>,
|
||||||
|
web_runtime_rx: watch::Receiver<WebRuntimePublication>,
|
||||||
) {
|
) {
|
||||||
let active_runtime = loop {
|
let active_runtime = loop {
|
||||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||||
@@ -351,6 +366,7 @@ pub async fn serve(
|
|||||||
reload_control,
|
reload_control,
|
||||||
active_runtime,
|
active_runtime,
|
||||||
web_trace,
|
web_trace,
|
||||||
|
web_runtime_rx,
|
||||||
});
|
});
|
||||||
|
|
||||||
spawn_runtime_watchers(
|
spawn_runtime_watchers(
|
||||||
@@ -498,9 +514,30 @@ async fn handle(
|
|||||||
let body_limit = api_cfg.request_body_limit_bytes;
|
let body_limit = api_cfg.request_body_limit_bytes;
|
||||||
|
|
||||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||||
|
if web_runtime::is_route(normalized_path) {
|
||||||
|
let web_mutation = method == Method::POST;
|
||||||
|
let result = web_runtime::handle(
|
||||||
|
method,
|
||||||
|
normalized_path,
|
||||||
|
query.as_deref(),
|
||||||
|
req,
|
||||||
|
shared.as_ref(),
|
||||||
|
cfg.as_ref(),
|
||||||
|
request_id,
|
||||||
|
body_limit,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if web_mutation && let Err(error) = &result {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.control.failed",
|
||||||
|
format!("path={} code={}", normalized_path, error.code),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
match (method.as_str(), normalized_path) {
|
match (method.as_str(), normalized_path) {
|
||||||
("GET", "/web-status") => {
|
("GET", "/web-status") => {
|
||||||
Ok(web_status::render(query.as_deref(), &shared.web_trace, &cfg.web.debug).await)
|
Ok(web_status::render(query.as_deref(), &shared.web_trace).await)
|
||||||
}
|
}
|
||||||
("GET", "/v1/health") => {
|
("GET", "/v1/health") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
|||||||
@@ -0,0 +1,444 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use http_body_util::Full;
|
||||||
|
use hyper::body::{Bytes, Incoming};
|
||||||
|
use hyper::header::CONTENT_TYPE;
|
||||||
|
use hyper::{Method, Request, Response, StatusCode};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::config_store::current_revision;
|
||||||
|
use super::http_utils::{read_json, success_response};
|
||||||
|
use super::model::ApiFailure;
|
||||||
|
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||||
|
use crate::web::manager::{ControlError, SessionDetail, WebProcessRuntime};
|
||||||
|
|
||||||
|
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
|
||||||
|
mod request;
|
||||||
|
use request::{
|
||||||
|
CloseRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
|
||||||
|
valid_runtime_instance,
|
||||||
|
};
|
||||||
|
|
||||||
|
const STATUS_PATH: &str = "/v1/runtime/web/status";
|
||||||
|
const SESSIONS_PATH: &str = "/v1/runtime/web/sessions";
|
||||||
|
const CLOSE_PATH: &str = "/v1/runtime/web/sessions/close";
|
||||||
|
const DEBUG_CLEAR_PATH: &str = "/v1/runtime/web/debug/clear";
|
||||||
|
const LEARNING_RESET_PATH: &str = "/v1/runtime/web/carrier-learning/reset";
|
||||||
|
const SESSION_DETAIL_PREFIX: &str = "/v1/runtime/web/sessions/";
|
||||||
|
const OPERATION_PREFIX: &str = "/v1/runtime/web/operations/";
|
||||||
|
const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
|
||||||
|
|
||||||
|
/// Returns the exact allowed method set for a WEB runtime route.
|
||||||
|
pub(super) fn allowed_methods(path: &str) -> Option<&'static str> {
|
||||||
|
match path {
|
||||||
|
STATUS_PATH | SESSIONS_PATH => Some(ALLOW_GET),
|
||||||
|
CLOSE_PATH | DEBUG_CLEAR_PATH | LEARNING_RESET_PATH => Some(ALLOW_POST),
|
||||||
|
_ if detail_ref(path).is_some() || operation_ref(path).is_some() => Some(ALLOW_GET),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the normalized API path belongs to WEB runtime control.
|
||||||
|
pub(super) fn is_route(path: &str) -> bool {
|
||||||
|
allowed_methods(path).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dispatches one authenticated WEB runtime status or control request.
|
||||||
|
pub(super) async fn handle(
|
||||||
|
method: Method,
|
||||||
|
path: &str,
|
||||||
|
query: Option<&str>,
|
||||||
|
request: Request<Incoming>,
|
||||||
|
shared: &ApiShared,
|
||||||
|
config: &ProxyConfig,
|
||||||
|
_request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
match (method.as_str(), path) {
|
||||||
|
("GET", STATUS_PATH) => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
let runtime = publication.runtime.upgrade();
|
||||||
|
let data = WebStatusData::new(publication, runtime.as_deref(), config.web.enabled);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", SESSIONS_PATH) => {
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let request = parse_session_query(&runtime, query)?;
|
||||||
|
let page = runtime.list_sessions(request);
|
||||||
|
Ok(success_response(StatusCode::OK, page, revision))
|
||||||
|
}
|
||||||
|
("GET", _) if detail_ref(path).is_some() => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let session_ref = detail_ref(path).expect("route guard checked detail reference");
|
||||||
|
let trace_session_id = parse_session_ref(&runtime, session_ref)?;
|
||||||
|
match runtime.session_detail(trace_session_id) {
|
||||||
|
SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
|
||||||
|
SessionDetail::Gone { attempt } => Ok(success_response(
|
||||||
|
StatusCode::GONE,
|
||||||
|
GoneSessionData {
|
||||||
|
session_ref: session_ref.to_string(),
|
||||||
|
state: "closed",
|
||||||
|
attempt,
|
||||||
|
},
|
||||||
|
revision,
|
||||||
|
)),
|
||||||
|
SessionDetail::Busy => Err(snapshot_busy()),
|
||||||
|
SessionDetail::NotFound => Err(ApiFailure::new(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"web_session_not_found",
|
||||||
|
"WEB session was not found",
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
("GET", _) if operation_ref(path).is_some() => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let operation_id = operation_ref(path).expect("route guard checked operation id");
|
||||||
|
let status = runtime
|
||||||
|
.control_operation(operation_id)
|
||||||
|
.map_err(control_failure)?;
|
||||||
|
Ok(success_response(StatusCode::OK, status, revision))
|
||||||
|
}
|
||||||
|
("POST", CLOSE_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<CloseRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let selector = request.selector.resolve(&runtime)?;
|
||||||
|
let status = runtime
|
||||||
|
.start_close_operation(&request.runtime_instance, selector)
|
||||||
|
.map_err(control_failure)?;
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.sessions.close.accepted",
|
||||||
|
format!(
|
||||||
|
"operation_id={} requested={}",
|
||||||
|
status.operation_id, status.requested
|
||||||
|
),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::ACCEPTED, status, revision))
|
||||||
|
}
|
||||||
|
("POST", DEBUG_CLEAR_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let outcome = runtime.clear_debug().map_err(control_failure)?;
|
||||||
|
let data = DebugClearData {
|
||||||
|
runtime_instance: runtime.runtime_instance().to_string(),
|
||||||
|
records_cleared: outcome.records_cleared,
|
||||||
|
leased_bytes: outcome.leased_bytes,
|
||||||
|
epoch: outcome.epoch,
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.debug.clear.ok",
|
||||||
|
format!("records={} epoch={}", data.records_cleared, data.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("POST", LEARNING_RESET_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let outcome = runtime
|
||||||
|
.reset_carrier_learning()
|
||||||
|
.map_err(|_| runtime_unavailable(WebRuntimeLifecycle::Draining))?;
|
||||||
|
let data = LearningResetData {
|
||||||
|
runtime_instance: runtime.runtime_instance().to_string(),
|
||||||
|
entries_cleared: outcome.entries_cleared,
|
||||||
|
epoch: outcome.epoch,
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.carrier_learning.reset.ok",
|
||||||
|
format!("entries={} epoch={}", data.entries_cleared, data.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
_ => Err(ApiFailure::method_not_allowed(
|
||||||
|
allowed_methods(path).unwrap_or(ALLOW_GET),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct WebStatusData {
|
||||||
|
lifecycle: &'static str,
|
||||||
|
lifecycle_epoch: u64,
|
||||||
|
lifecycle_age_ms: u64,
|
||||||
|
available: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
listeners: Vec<String>,
|
||||||
|
effective_config_enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
runtime: Option<crate::web::manager::WebRuntimeStatus>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebStatusData {
|
||||||
|
fn new(
|
||||||
|
publication: WebRuntimePublication,
|
||||||
|
runtime: Option<&WebProcessRuntime>,
|
||||||
|
effective_config_enabled: bool,
|
||||||
|
) -> Self {
|
||||||
|
let available = runtime.is_some()
|
||||||
|
&& matches!(
|
||||||
|
publication.lifecycle,
|
||||||
|
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||||
|
);
|
||||||
|
let reason = if available {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(match publication.lifecycle {
|
||||||
|
WebRuntimeLifecycle::Starting => "starting",
|
||||||
|
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
|
||||||
|
WebRuntimeLifecycle::Running => "runtime_released",
|
||||||
|
WebRuntimeLifecycle::Draining => "runtime_released",
|
||||||
|
WebRuntimeLifecycle::Drained => "drained",
|
||||||
|
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||||
|
})
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
lifecycle: publication.lifecycle.as_str(),
|
||||||
|
lifecycle_epoch: publication.epoch,
|
||||||
|
lifecycle_age_ms: millis(Instant::now().saturating_duration_since(publication.since)),
|
||||||
|
available,
|
||||||
|
reason,
|
||||||
|
listeners: publication
|
||||||
|
.listeners
|
||||||
|
.iter()
|
||||||
|
.map(ToString::to_string)
|
||||||
|
.collect(),
|
||||||
|
effective_config_enabled,
|
||||||
|
runtime: runtime.map(WebProcessRuntime::try_status),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct GoneSessionData {
|
||||||
|
session_ref: String,
|
||||||
|
state: &'static str,
|
||||||
|
attempt: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct DebugClearData {
|
||||||
|
runtime_instance: String,
|
||||||
|
records_cleared: usize,
|
||||||
|
leased_bytes: usize,
|
||||||
|
epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct LearningResetData {
|
||||||
|
runtime_instance: String,
|
||||||
|
entries_cleared: usize,
|
||||||
|
epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn readable_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
if !matches!(
|
||||||
|
publication.lifecycle,
|
||||||
|
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||||
|
) {
|
||||||
|
return Err(runtime_unavailable(publication.lifecycle));
|
||||||
|
}
|
||||||
|
publication
|
||||||
|
.runtime
|
||||||
|
.upgrade()
|
||||||
|
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn control_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
if publication.lifecycle != WebRuntimeLifecycle::Running {
|
||||||
|
return Err(runtime_unavailable(publication.lifecycle));
|
||||||
|
}
|
||||||
|
publication
|
||||||
|
.runtime
|
||||||
|
.upgrade()
|
||||||
|
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_unavailable(lifecycle: WebRuntimeLifecycle) -> ApiFailure {
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"web_runtime_unavailable",
|
||||||
|
format!("WEB runtime is unavailable: {}", lifecycle.as_str()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_mutable(config: &ProxyConfig) -> Result<(), ApiFailure> {
|
||||||
|
if config.server.api.read_only {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_json_content_type<B>(request: &Request<B>) -> Result<(), ApiFailure> {
|
||||||
|
let mut values = request.headers().get_all(CONTENT_TYPE).iter();
|
||||||
|
let exact = values
|
||||||
|
.next()
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.is_some_and(|value| value == "application/json")
|
||||||
|
&& values.next().is_none();
|
||||||
|
if !exact {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||||
|
"unsupported_media_type",
|
||||||
|
"Content-Type must be exactly application/json",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_runtime_instance(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
runtime_instance: &str,
|
||||||
|
) -> Result<(), ApiFailure> {
|
||||||
|
if !valid_runtime_instance(runtime_instance) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"runtime_instance must be 32 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if runtime.runtime_instance() != runtime_instance {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB runtime instance no longer matches",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn control_failure(error: ControlError) -> ApiFailure {
|
||||||
|
match error {
|
||||||
|
ControlError::StaleInstance => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB runtime instance no longer matches",
|
||||||
|
),
|
||||||
|
ControlError::InvalidSelector | ControlError::InvalidOperation => {
|
||||||
|
ApiFailure::bad_request("Invalid WEB control request")
|
||||||
|
}
|
||||||
|
ControlError::IssuanceEnabled => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_issuance_enabled",
|
||||||
|
"Close-all requires effective WEB issuance to be disabled",
|
||||||
|
),
|
||||||
|
ControlError::OperationInProgress => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_operation_in_progress",
|
||||||
|
"Another WEB close operation is active",
|
||||||
|
),
|
||||||
|
ControlError::OperationNotFound => ApiFailure::new(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"web_operation_not_found",
|
||||||
|
"WEB control operation was not found",
|
||||||
|
),
|
||||||
|
ControlError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot_busy() -> ApiFailure {
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"web_snapshot_busy",
|
||||||
|
"WEB runtime snapshot is temporarily busy",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject_query(query: Option<&str>) -> Result<(), ApiFailure> {
|
||||||
|
if query.is_some_and(|query| !query.is_empty()) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"This endpoint does not accept query parameters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn detail_ref(path: &str) -> Option<&str> {
|
||||||
|
path.strip_prefix(SESSION_DETAIL_PREFIX)
|
||||||
|
.filter(|value| !value.is_empty() && !value.contains('/') && *value != "close")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation_ref(path: &str) -> Option<&str> {
|
||||||
|
path.strip_prefix(OPERATION_PREFIX)
|
||||||
|
.filter(|value| !value.is_empty() && !value.contains('/'))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn millis(duration: std::time::Duration) -> u64 {
|
||||||
|
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use hyper::header::HeaderValue;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn route_table_keeps_status_read_only_and_controls_post_only() {
|
||||||
|
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
|
||||||
|
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
|
||||||
|
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(
|
||||||
|
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
|
||||||
|
Some(ALLOW_GET)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn control_content_type_is_exact_and_single() {
|
||||||
|
let exact = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
assert!(require_json_content_type(&exact).is_ok());
|
||||||
|
|
||||||
|
let parameterized = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json; charset=utf-8")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
assert!(require_json_content_type(¶meterized).is_err());
|
||||||
|
|
||||||
|
let mut duplicated = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
duplicated
|
||||||
|
.headers_mut()
|
||||||
|
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||||
|
assert!(require_json_content_type(&duplicated).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
|
||||||
|
use hyper::StatusCode;
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use crate::config::WebCarrier;
|
||||||
|
use crate::web::manager::{
|
||||||
|
CloseOperationSelector, SessionFilter, SessionListRequest, SessionRefError, WebProcessRuntime,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::super::model::ApiFailure;
|
||||||
|
|
||||||
|
const DEFAULT_SESSION_LIMIT: usize = 50;
|
||||||
|
const MAX_SESSION_LIMIT: usize = 200;
|
||||||
|
|
||||||
|
/// Exact process-instance fence for one runtime mutation.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(super) struct RuntimeInstanceRequest {
|
||||||
|
/// Random process identifier copied from WEB runtime status.
|
||||||
|
pub(super) runtime_instance: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One process-fenced asynchronous close request.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(super) struct CloseRequest {
|
||||||
|
/// Random process identifier copied from WEB runtime status.
|
||||||
|
pub(super) runtime_instance: String,
|
||||||
|
/// Exact point-in-time close selector.
|
||||||
|
pub(super) selector: CloseSelectorRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strict tagged selector accepted by the WEB close endpoint.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub(super) enum CloseSelectorRequest {
|
||||||
|
/// Closes an explicit bounded set of logical sessions.
|
||||||
|
Refs {
|
||||||
|
/// Unique current-instance opaque session references.
|
||||||
|
session_refs: Vec<String>,
|
||||||
|
},
|
||||||
|
/// Closes the point-in-time sessions matching every supplied field.
|
||||||
|
Filter {
|
||||||
|
#[serde(default)]
|
||||||
|
session_ref: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
ip: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
host: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
user: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
user_agent_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
key_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
carrier: Option<WebCarrier>,
|
||||||
|
#[serde(default)]
|
||||||
|
state: Option<String>,
|
||||||
|
},
|
||||||
|
/// Closes every point-in-time session below the submission high-water mark.
|
||||||
|
All {},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CloseSelectorRequest {
|
||||||
|
/// Validates external identifiers and resolves them to manager-owned values.
|
||||||
|
pub(super) fn resolve(
|
||||||
|
self,
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||||
|
match self {
|
||||||
|
Self::Refs { session_refs } => resolve_refs(runtime, session_refs),
|
||||||
|
Self::Filter {
|
||||||
|
session_ref,
|
||||||
|
ip,
|
||||||
|
host,
|
||||||
|
user,
|
||||||
|
user_agent_id,
|
||||||
|
key_id,
|
||||||
|
carrier,
|
||||||
|
state,
|
||||||
|
} => {
|
||||||
|
validate_filter_strings(
|
||||||
|
host.as_deref(),
|
||||||
|
user.as_deref(),
|
||||||
|
key_id.as_deref(),
|
||||||
|
state.as_deref(),
|
||||||
|
)?;
|
||||||
|
let trace_session_id = session_ref
|
||||||
|
.as_deref()
|
||||||
|
.map(|value| parse_session_ref(runtime, value))
|
||||||
|
.transpose()?;
|
||||||
|
let client_ip = ip.as_deref().map(parse_canonical_ip).transpose()?;
|
||||||
|
let filter = SessionFilter {
|
||||||
|
trace_session_id,
|
||||||
|
client_ip,
|
||||||
|
host,
|
||||||
|
user,
|
||||||
|
user_agent_id: user_agent_id
|
||||||
|
.as_deref()
|
||||||
|
.map(parse_user_agent_id)
|
||||||
|
.transpose()?,
|
||||||
|
key_id,
|
||||||
|
carrier,
|
||||||
|
state,
|
||||||
|
};
|
||||||
|
if filter.is_empty() {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"filter selector requires at least one filter",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(CloseOperationSelector::Filter(filter))
|
||||||
|
}
|
||||||
|
Self::All {} => Ok(CloseOperationSelector::All),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_refs(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
session_refs: Vec<String>,
|
||||||
|
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||||
|
if session_refs.is_empty() || session_refs.len() > 200 {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_refs must contain 1..200 references",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut resolved = Vec::with_capacity(session_refs.len());
|
||||||
|
let mut unique = BTreeSet::new();
|
||||||
|
for session_ref in session_refs {
|
||||||
|
let id = parse_session_ref(runtime, &session_ref)?;
|
||||||
|
if !unique.insert(id) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_refs must not contain duplicates",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
resolved.push(id);
|
||||||
|
}
|
||||||
|
Ok(CloseOperationSelector::Refs(resolved))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses one duplicate-free bounded session-list query.
|
||||||
|
pub(super) fn parse_session_query(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
raw: Option<&str>,
|
||||||
|
) -> Result<SessionListRequest, ApiFailure> {
|
||||||
|
let mut limit = DEFAULT_SESSION_LIMIT;
|
||||||
|
let mut cursor = None;
|
||||||
|
let mut filter = SessionFilter::default();
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||||
|
if !seen.insert(name.to_string()) {
|
||||||
|
return Err(ApiFailure::bad_request(format!("{} must not repeat", name)));
|
||||||
|
}
|
||||||
|
match name.as_ref() {
|
||||||
|
"limit" => {
|
||||||
|
limit = value
|
||||||
|
.parse::<usize>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| (1..=MAX_SESSION_LIMIT).contains(value))
|
||||||
|
.ok_or_else(|| ApiFailure::bad_request("limit must be within 1..200"))?;
|
||||||
|
}
|
||||||
|
"cursor" => cursor = Some(parse_session_ref(runtime, &value)?),
|
||||||
|
"session_ref" => {
|
||||||
|
let id = parse_session_ref(runtime, &value)?;
|
||||||
|
filter.trace_session_id = Some(id);
|
||||||
|
cursor = id.checked_sub(1);
|
||||||
|
limit = 1;
|
||||||
|
}
|
||||||
|
"ip" => {
|
||||||
|
filter.client_ip = Some(parse_canonical_ip(&value)?);
|
||||||
|
}
|
||||||
|
"host" => filter.host = Some(value.into_owned()),
|
||||||
|
"user" => filter.user = Some(value.into_owned()),
|
||||||
|
"user_agent_id" => filter.user_agent_id = Some(parse_user_agent_id(&value)?),
|
||||||
|
"key_id" => filter.key_id = Some(value.into_owned()),
|
||||||
|
"carrier" => filter.carrier = Some(parse_carrier(&value)?),
|
||||||
|
"state" => filter.state = Some(value.into_owned()),
|
||||||
|
_ => {
|
||||||
|
return Err(ApiFailure::bad_request(format!(
|
||||||
|
"unknown query field `{}`",
|
||||||
|
name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filter.trace_session_id.is_some() && (seen.contains("cursor") || seen.contains("limit")) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_ref must not be combined with cursor or limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
validate_filter_strings(
|
||||||
|
filter.host.as_deref(),
|
||||||
|
filter.user.as_deref(),
|
||||||
|
filter.key_id.as_deref(),
|
||||||
|
filter.state.as_deref(),
|
||||||
|
)?;
|
||||||
|
Ok(SessionListRequest {
|
||||||
|
limit,
|
||||||
|
cursor,
|
||||||
|
filter,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Maps one opaque session-reference failure to the stable API error contract.
|
||||||
|
pub(super) fn parse_session_ref(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
session_ref: &str,
|
||||||
|
) -> Result<u64, ApiFailure> {
|
||||||
|
runtime
|
||||||
|
.parse_session_ref(session_ref)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
SessionRefError::Invalid => ApiFailure::bad_request("Invalid WEB session reference"),
|
||||||
|
SessionRefError::StaleInstance => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB session reference belongs to another runtime instance",
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_carrier(value: &str) -> Result<WebCarrier, ApiFailure> {
|
||||||
|
WebCarrier::ALL
|
||||||
|
.into_iter()
|
||||||
|
.find(|carrier| carrier.as_str() == value)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ApiFailure::bad_request(
|
||||||
|
"carrier must be https, https-lanes, websocket, or websocket-lanes",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_filter_strings(
|
||||||
|
host: Option<&str>,
|
||||||
|
user: Option<&str>,
|
||||||
|
key_id: Option<&str>,
|
||||||
|
state: Option<&str>,
|
||||||
|
) -> Result<(), ApiFailure> {
|
||||||
|
if host.is_some_and(|value| value.is_empty() || value.len() > 253) {
|
||||||
|
return Err(ApiFailure::bad_request("host must contain 1..253 bytes"));
|
||||||
|
}
|
||||||
|
if user.is_some_and(|value| value.is_empty() || value.len() > 64) {
|
||||||
|
return Err(ApiFailure::bad_request("user must contain 1..64 bytes"));
|
||||||
|
}
|
||||||
|
if key_id.is_some_and(|value| !lower_hex(value, 16)) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"key_id must be 16 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if state.is_some_and(|value| {
|
||||||
|
!matches!(
|
||||||
|
value,
|
||||||
|
"provisional"
|
||||||
|
| "replacing"
|
||||||
|
| "committed"
|
||||||
|
| "healthy"
|
||||||
|
| "closing"
|
||||||
|
| "superseded"
|
||||||
|
| "closed"
|
||||||
|
)
|
||||||
|
}) {
|
||||||
|
return Err(ApiFailure::bad_request("Invalid WEB session state"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_user_agent_id(value: &str) -> Result<[u8; 16], ApiFailure> {
|
||||||
|
if !lower_hex(value, 32) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"user_agent_id must be 32 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut id = [0; 16];
|
||||||
|
hex::decode_to_slice(value, &mut id).map_err(|_| {
|
||||||
|
ApiFailure::bad_request("user_agent_id must be 32 lowercase hexadecimal characters")
|
||||||
|
})?;
|
||||||
|
Ok(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lower_hex(value: &str, length: usize) -> bool {
|
||||||
|
value.len() == length
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether a process instance uses its canonical lowercase form.
|
||||||
|
pub(super) fn valid_runtime_instance(value: &str) -> bool {
|
||||||
|
lower_hex(value, 32)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_canonical_ip(value: &str) -> Result<IpAddr, ApiFailure> {
|
||||||
|
let ip = value
|
||||||
|
.parse::<IpAddr>()
|
||||||
|
.map_err(|_| ApiFailure::bad_request("ip must be a canonical IP address"))?;
|
||||||
|
if ip.to_string() != value {
|
||||||
|
return Err(ApiFailure::bad_request("ip must use canonical formatting"));
|
||||||
|
}
|
||||||
|
Ok(ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn filter_identifiers_are_exact_lowercase_hex() {
|
||||||
|
assert!(lower_hex("0123456789abcdef", 16));
|
||||||
|
assert!(!lower_hex("0123456789ABCDEF", 16));
|
||||||
|
assert!(!lower_hex("0123", 16));
|
||||||
|
assert!(valid_runtime_instance("0123456789abcdef0123456789abcdef"));
|
||||||
|
assert!(!valid_runtime_instance("0123456789ABCDEF0123456789ABCDEF"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mutation_dtos_reject_unknown_fields() {
|
||||||
|
let runtime_instance = "0123456789abcdef0123456789abcdef";
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<RuntimeInstanceRequest>(serde_json::json!({
|
||||||
|
"runtime_instance": runtime_instance,
|
||||||
|
"extra": true,
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<CloseRequest>(serde_json::json!({
|
||||||
|
"runtime_instance": runtime_instance,
|
||||||
|
"selector": {"kind": "all", "extra": true},
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn filter_ips_require_canonical_text() {
|
||||||
|
assert!(parse_canonical_ip("2001:db8::1").is_ok());
|
||||||
|
assert!(parse_canonical_ip("2001:0db8::1").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn state_filter_accepts_every_emitted_session_state() {
|
||||||
|
for state in [
|
||||||
|
"provisional",
|
||||||
|
"replacing",
|
||||||
|
"committed",
|
||||||
|
"healthy",
|
||||||
|
"closing",
|
||||||
|
"superseded",
|
||||||
|
"closed",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
validate_filter_strings(None, None, None, Some(state)).is_ok(),
|
||||||
|
"state {state} must be accepted"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exact_session_query_rejects_pagination_fields_in_any_order() {
|
||||||
|
let generation = crate::maestro::generation::test_runtime_generation(
|
||||||
|
1,
|
||||||
|
crate::config::ProxyConfig::default(),
|
||||||
|
);
|
||||||
|
let runtime = WebProcessRuntime::start(std::sync::Arc::new(arc_swap::ArcSwap::from(
|
||||||
|
generation.clone(),
|
||||||
|
)));
|
||||||
|
let session_ref = runtime.session_ref(1);
|
||||||
|
let cursor = runtime.session_ref(2);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
parse_session_query(
|
||||||
|
&runtime,
|
||||||
|
Some(&format!("session_ref={session_ref}&cursor={cursor}")),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
parse_session_query(
|
||||||
|
&runtime,
|
||||||
|
Some(&format!("limit=2&session_ref={session_ref}")),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,10 +41,9 @@ impl AsRef<[u8]> for RenderedPage {
|
|||||||
pub(super) async fn render(
|
pub(super) async fn render(
|
||||||
raw_query: Option<&str>,
|
raw_query: Option<&str>,
|
||||||
store: &Arc<WebTraceStore>,
|
store: &Arc<WebTraceStore>,
|
||||||
policy: &WebDebugConfig,
|
|
||||||
) -> Response<Full<Bytes>> {
|
) -> Response<Full<Bytes>> {
|
||||||
store.apply_policy(policy);
|
let status = store.status();
|
||||||
let query = match parse_query(raw_query, policy) {
|
let query = match parse_query(raw_query, &status.policy) {
|
||||||
Ok(query) => query,
|
Ok(query) => query,
|
||||||
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
||||||
};
|
};
|
||||||
@@ -62,7 +61,6 @@ pub(super) async fn render(
|
|||||||
0
|
0
|
||||||
};
|
};
|
||||||
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||||
let status = store.status();
|
|
||||||
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||||
push_page_start(&mut html);
|
push_page_start(&mut html);
|
||||||
html.push_str("<h1>WEB status</h1>");
|
html.push_str("<h1>WEB status</h1>");
|
||||||
|
|||||||
@@ -46,7 +46,6 @@ async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
|||||||
let response = render(
|
let response = render(
|
||||||
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
||||||
&store,
|
&store,
|
||||||
&policy,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
@@ -69,18 +68,36 @@ async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
|||||||
let limits = crate::config::WebLimitsConfig::default();
|
let limits = crate::config::WebLimitsConfig::default();
|
||||||
let store = WebTraceStore::new(policy.clone(), &limits);
|
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||||
|
|
||||||
let first = render(None, &store, &policy).await;
|
let first = render(None, &store).await;
|
||||||
let second = render(None, &store, &policy).await;
|
let second = render(None, &store).await;
|
||||||
let busy = render(None, &store, &policy).await;
|
let busy = render(None, &store).await;
|
||||||
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
|
||||||
drop(first);
|
drop(first);
|
||||||
let admitted = render(None, &store, &policy).await;
|
let admitted = render(None, &store).await;
|
||||||
assert_eq!(admitted.status(), StatusCode::OK);
|
assert_eq!(admitted.status(), StatusCode::OK);
|
||||||
drop(second);
|
drop(second);
|
||||||
drop(admitted);
|
drop(admitted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_renderer_cannot_restore_an_old_debug_policy() {
|
||||||
|
let stale_policy = WebDebugConfig::default();
|
||||||
|
let active_policy = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
capture_headers: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let limits = crate::config::WebLimitsConfig::default();
|
||||||
|
let store = WebTraceStore::new(stale_policy.clone(), &limits);
|
||||||
|
store.apply_policy(2, &active_policy);
|
||||||
|
|
||||||
|
let response = render(None, &store).await;
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
assert_eq!(store.status().policy.as_ref(), &active_policy);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||||
let mut html = "\u{044f}".repeat(MAX_PAGE_BYTES);
|
let mut html = "\u{044f}".repeat(MAX_PAGE_BYTES);
|
||||||
|
|||||||
@@ -256,6 +256,46 @@ fn reload_applies_hot_change_on_first_observed_snapshot() {
|
|||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_watcher_waits_for_activation_and_reconciles_disk() {
|
||||||
|
let initial_tag = "10101010101010101010101010101010";
|
||||||
|
let disk_tag = "20202020202020202020202020202020";
|
||||||
|
let path = temp_config_path("telemt_hot_reload_activation_gate");
|
||||||
|
write_reload_config(&path, Some(initial_tag), None);
|
||||||
|
let initial = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||||
|
write_reload_config(&path, Some(disk_tag), None);
|
||||||
|
let cancellation = tokio_util::sync::CancellationToken::new();
|
||||||
|
let (activation_tx, activation_rx) = watch::channel(false);
|
||||||
|
let (mut config_rx, _log_rx, watcher) = spawn_config_watcher(
|
||||||
|
path.clone(),
|
||||||
|
initial,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
cancellation.clone(),
|
||||||
|
Some(activation_rx),
|
||||||
|
);
|
||||||
|
let watcher = tokio::spawn(watcher);
|
||||||
|
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(
|
||||||
|
config_rx.borrow().general.ad_tag.as_deref(),
|
||||||
|
Some(initial_tag)
|
||||||
|
);
|
||||||
|
activation_tx.send_replace(true);
|
||||||
|
tokio::time::timeout(Duration::from_secs(2), config_rx.changed())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
config_rx.borrow_and_update().general.ad_tag.as_deref(),
|
||||||
|
Some(disk_tag)
|
||||||
|
);
|
||||||
|
|
||||||
|
cancellation.cancel();
|
||||||
|
watcher.await.unwrap();
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
||||||
let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||||
|
|||||||
@@ -213,20 +213,47 @@ pub fn spawn_config_watcher(
|
|||||||
detected_ip_v4: Option<IpAddr>,
|
detected_ip_v4: Option<IpAddr>,
|
||||||
detected_ip_v6: Option<IpAddr>,
|
detected_ip_v6: Option<IpAddr>,
|
||||||
cancellation: tokio_util::sync::CancellationToken,
|
cancellation: tokio_util::sync::CancellationToken,
|
||||||
) -> (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) {
|
mut activation: Option<watch::Receiver<bool>>,
|
||||||
|
) -> (
|
||||||
|
watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
watch::Receiver<LogLevel>,
|
||||||
|
impl std::future::Future<Output = ()> + Send + 'static,
|
||||||
|
) {
|
||||||
let initial_level = initial.general.log_level.clone();
|
let initial_level = initial.general.log_level.clone();
|
||||||
let (config_tx, config_rx) = watch::channel(initial);
|
let (config_tx, config_rx) = watch::channel(initial);
|
||||||
let (log_tx, log_rx) = watch::channel(initial_level);
|
let (log_tx, log_rx) = watch::channel(initial_level);
|
||||||
|
|
||||||
let config_path = normalize_watch_path(&config_path);
|
let config_path = normalize_watch_path(&config_path);
|
||||||
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
let task = async move {
|
||||||
let initial_manifest = initial_loaded
|
if let Some(activation) = activation.as_mut() {
|
||||||
.as_ref()
|
loop {
|
||||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
if *activation.borrow_and_update() {
|
||||||
.unwrap_or_else(|| WatchManifest::from_source_files(std::slice::from_ref(&config_path)));
|
break;
|
||||||
let initial_snapshot_hash = initial_loaded.as_ref().map(|loaded| loaded.rendered_hash);
|
}
|
||||||
|
tokio::select! {
|
||||||
tokio::spawn(async move {
|
result = activation.changed() => {
|
||||||
|
if result.is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = cancellation.cancelled() => return,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
||||||
|
let initial_manifest = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
WatchManifest::from_source_files(std::slice::from_ref(&config_path))
|
||||||
|
});
|
||||||
|
let initial_matches_disk = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|loaded| config_equal(config_tx.borrow().as_ref(), &loaded.config));
|
||||||
|
let initial_snapshot_hash = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.filter(|_| initial_matches_disk)
|
||||||
|
.map(|loaded| loaded.rendered_hash);
|
||||||
let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4);
|
let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4);
|
||||||
let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default()));
|
let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default()));
|
||||||
let mut reload_state = ReloadState::new(initial_snapshot_hash);
|
let mut reload_state = ReloadState::new(initial_snapshot_hash);
|
||||||
@@ -304,6 +331,9 @@ pub fn spawn_config_watcher(
|
|||||||
if poll_watcher.is_some() {
|
if poll_watcher.is_some() {
|
||||||
info!("config watcher: poll watcher active (Docker/NFS safe)");
|
info!("config watcher: poll watcher active (Docker/NFS safe)");
|
||||||
}
|
}
|
||||||
|
if initial_loaded.is_some() && !initial_matches_disk {
|
||||||
|
let _ = notify_tx.try_send(());
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
let mut sighup = {
|
let mut sighup = {
|
||||||
@@ -364,7 +394,7 @@ pub fn spawn_config_watcher(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
};
|
||||||
|
|
||||||
(config_rx, log_rx)
|
(config_rx, log_rx, task)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -381,6 +381,11 @@ fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||||
|
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||||
|
));
|
||||||
|
}
|
||||||
if !config.access.users.contains_key(&profile.user) {
|
if !config.access.users.contains_key(&profile.user) {
|
||||||
return config_error(&format!(
|
return config_error(&format!(
|
||||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||||
|
|||||||
@@ -56,6 +56,17 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_profile_user_labels_are_bounded_for_runtime_status() {
|
||||||
|
let user = "a".repeat(65);
|
||||||
|
let invalid = WEB_CONFIG.replace("alice", &user);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&invalid)
|
||||||
|
.contains("web.vhosts[0].profiles[0].user must contain 1..64 bytes")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn web_carriers_missing_or_false_disable_negotiation() {
|
fn web_carriers_missing_or_false_disable_negotiation() {
|
||||||
let missing = load_config_from_temp_toml(WEB_CONFIG);
|
let missing = load_config_from_temp_toml(WEB_CONFIG);
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use super::bind::{BoundListeners, BoundTcpListener, PreparedTcpListener, prepare
|
|||||||
use super::plan::{ListenerBindSpec, listener_bind_plan};
|
use super::plan::{ListenerBindSpec, listener_bind_plan};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use super::unix::UnixAcceptHandle;
|
use super::unix::UnixAcceptHandle;
|
||||||
|
use crate::web::control::{WebRuntimeControl, WebRuntimeLifecycle};
|
||||||
use crate::web::manager::{WebProcessRuntime, WebShutdownOutcome};
|
use crate::web::manager::{WebProcessRuntime, WebShutdownOutcome};
|
||||||
use crate::web::trace::WebTraceStore;
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
@@ -22,6 +23,8 @@ pub(crate) struct ListenerManager {
|
|||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
slots: BTreeMap<SocketAddr, ListenerSlot>,
|
slots: BTreeMap<SocketAddr, ListenerSlot>,
|
||||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||||
|
web_control: WebRuntimeControl,
|
||||||
|
web_listeners: Arc<[SocketAddr]>,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix: Option<UnixAcceptHandle>,
|
unix: Option<UnixAcceptHandle>,
|
||||||
}
|
}
|
||||||
@@ -46,11 +49,15 @@ impl ListenerManager {
|
|||||||
bound: BoundListeners,
|
bound: BoundListeners,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
trace: Arc<WebTraceStore>,
|
trace: Arc<WebTraceStore>,
|
||||||
|
web_control: WebRuntimeControl,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let has_web = bound
|
let web_listeners: Arc<[SocketAddr]> = bound
|
||||||
.listeners
|
.listeners
|
||||||
.iter()
|
.iter()
|
||||||
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
.filter(|listener| listener.spec.transport == ListenerTransport::Web)
|
||||||
|
.map(|listener| listener.spec.addr)
|
||||||
|
.collect();
|
||||||
|
let has_web = !web_listeners.is_empty();
|
||||||
let web_runtime =
|
let web_runtime =
|
||||||
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
||||||
let mut slots = BTreeMap::new();
|
let mut slots = BTreeMap::new();
|
||||||
@@ -65,10 +72,23 @@ impl ListenerManager {
|
|||||||
let unix = bound
|
let unix = bound
|
||||||
.unix_listener
|
.unix_listener
|
||||||
.map(|listener| UnixAcceptHandle::start(listener, active_runtime.clone()));
|
.map(|listener| UnixAcceptHandle::start(listener, active_runtime.clone()));
|
||||||
|
web_control.publish(
|
||||||
|
if has_web {
|
||||||
|
WebRuntimeLifecycle::Running
|
||||||
|
} else {
|
||||||
|
WebRuntimeLifecycle::NoWebListener
|
||||||
|
},
|
||||||
|
Arc::clone(&web_listeners),
|
||||||
|
web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||||
|
);
|
||||||
Self {
|
Self {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
slots,
|
slots,
|
||||||
web_runtime,
|
web_runtime,
|
||||||
|
web_control,
|
||||||
|
web_listeners,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix,
|
unix,
|
||||||
}
|
}
|
||||||
@@ -76,10 +96,18 @@ impl ListenerManager {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn empty(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Self {
|
pub(crate) fn empty(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Self {
|
||||||
|
let web_control = WebRuntimeControl::new();
|
||||||
|
web_control.publish(
|
||||||
|
WebRuntimeLifecycle::NoWebListener,
|
||||||
|
Arc::from([]),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
Self {
|
Self {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
slots: BTreeMap::new(),
|
slots: BTreeMap::new(),
|
||||||
web_runtime: None,
|
web_runtime: None,
|
||||||
|
web_control,
|
||||||
|
web_listeners: Arc::from([]),
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix: None,
|
unix: None,
|
||||||
}
|
}
|
||||||
@@ -217,6 +245,13 @@ impl ListenerManager {
|
|||||||
|
|
||||||
/// Stops every accept task and applies one deadline to the complete WEB ingress.
|
/// Stops every accept task and applies one deadline to the complete WEB ingress.
|
||||||
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
||||||
|
self.web_control.publish(
|
||||||
|
WebRuntimeLifecycle::Draining,
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
self.web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||||
|
);
|
||||||
if self.web_runtime.is_none() {
|
if self.web_runtime.is_none() {
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
for slot in self.slots.values_mut() {
|
for slot in self.slots.values_mut() {
|
||||||
@@ -235,6 +270,11 @@ impl ListenerManager {
|
|||||||
{
|
{
|
||||||
self.unix = None;
|
self.unix = None;
|
||||||
}
|
}
|
||||||
|
self.web_control.publish(
|
||||||
|
WebRuntimeLifecycle::Drained,
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
return if errors.is_empty() {
|
return if errors.is_empty() {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
@@ -289,6 +329,15 @@ impl ListenerManager {
|
|||||||
{
|
{
|
||||||
self.unix = None;
|
self.unix = None;
|
||||||
}
|
}
|
||||||
|
self.web_control.publish(
|
||||||
|
if web_outcome == WebShutdownOutcome::DeadlineExceeded {
|
||||||
|
WebRuntimeLifecycle::DeadlineExceeded
|
||||||
|
} else {
|
||||||
|
WebRuntimeLifecycle::Drained
|
||||||
|
},
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
if errors.is_empty() {
|
if errors.is_empty() {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
@@ -367,7 +416,8 @@ mod tests {
|
|||||||
runtime.config().web.debug.clone(),
|
runtime.config().web.debug.clone(),
|
||||||
&runtime.config().web.limits,
|
&runtime.config().web.limits,
|
||||||
);
|
);
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
let mut manager =
|
||||||
|
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||||
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let blocked_addr = blocker.local_addr().unwrap();
|
let blocked_addr = blocker.local_addr().unwrap();
|
||||||
let mut desired = ProxyConfig::default();
|
let mut desired = ProxyConfig::default();
|
||||||
@@ -394,7 +444,8 @@ mod tests {
|
|||||||
runtime.config().web.debug.clone(),
|
runtime.config().web.debug.clone(),
|
||||||
&runtime.config().web.limits,
|
&runtime.config().web.limits,
|
||||||
);
|
);
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
let mut manager =
|
||||||
|
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||||
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let new_addr = reservation.local_addr().unwrap();
|
let new_addr = reservation.local_addr().unwrap();
|
||||||
drop(reservation);
|
drop(reservation);
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
|
|||||||
use crate::synlimit_control;
|
use crate::synlimit_control;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::web::control::WebRuntimeControl;
|
||||||
use crate::web::trace::WebTraceStore;
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
@@ -106,6 +107,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
config.access.cidr_rate_limits.clone(),
|
config.access.cidr_rate_limits.clone(),
|
||||||
);
|
);
|
||||||
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||||
|
let web_runtime_control = WebRuntimeControl::new();
|
||||||
|
|
||||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||||
@@ -157,6 +159,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||||
let web_trace_api = web_trace.clone();
|
let web_trace_api = web_trace.clone();
|
||||||
|
let web_runtime_rx_api = web_runtime_control.subscribe();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
api::serve(
|
api::serve(
|
||||||
listen,
|
listen,
|
||||||
@@ -175,6 +178,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
active_runtime_rx_api,
|
active_runtime_rx_api,
|
||||||
runtime_watch_rx_api,
|
runtime_watch_rx_api,
|
||||||
web_trace_api,
|
web_trace_api,
|
||||||
|
web_runtime_rx_api,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -318,8 +322,12 @@ pub(super) async fn run_telemt_core(
|
|||||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||||
|
|
||||||
let listener_manager =
|
let listener_manager = listeners::ListenerManager::start(
|
||||||
listeners::ListenerManager::start(bound, active_runtime.clone(), web_trace.clone());
|
bound,
|
||||||
|
active_runtime.clone(),
|
||||||
|
web_trace.clone(),
|
||||||
|
web_runtime_control,
|
||||||
|
);
|
||||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||||
active_runtime.clone(),
|
active_runtime.clone(),
|
||||||
reload_control,
|
reload_control,
|
||||||
|
|||||||
@@ -278,7 +278,11 @@ impl ReloadSupervisor {
|
|||||||
self.control
|
self.control
|
||||||
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
||||||
.await;
|
.await;
|
||||||
let new_runtime = prepared.generation;
|
let PreparedRuntime {
|
||||||
|
generation: new_runtime,
|
||||||
|
detected_ips,
|
||||||
|
config_watcher_activation,
|
||||||
|
} = prepared;
|
||||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
||||||
let message = format!("runtime DNS activation failed: {}", error);
|
let message = format!("runtime DNS activation failed: {}", error);
|
||||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
||||||
@@ -313,14 +317,16 @@ impl ReloadSupervisor {
|
|||||||
};
|
};
|
||||||
old_runtime.stop_accepting_sessions();
|
old_runtime.stop_accepting_sessions();
|
||||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||||
self.web_trace.apply_policy(&new_runtime.config().web.debug);
|
self.web_trace
|
||||||
|
.apply_policy(new_runtime.id, &new_runtime.config().web.debug);
|
||||||
|
config_watcher_activation.send_replace(true);
|
||||||
if let Some(pending) = pending_listener_transition {
|
if let Some(pending) = pending_listener_transition {
|
||||||
self.listener_manager
|
self.listener_manager
|
||||||
.lock()
|
.lock()
|
||||||
.await
|
.await
|
||||||
.finish_transition(pending);
|
.finish_transition(pending);
|
||||||
}
|
}
|
||||||
self.detected_ips_tx.send_replace(prepared.detected_ips);
|
self.detected_ips_tx.send_replace(detected_ips);
|
||||||
self.runtime_log_filter
|
self.runtime_log_filter
|
||||||
.apply_reload(&new_runtime.config().general.log_level);
|
.apply_reload(&new_runtime.config().general.log_level);
|
||||||
self.runtime_watch_tx
|
self.runtime_watch_tx
|
||||||
|
|||||||
@@ -21,6 +21,15 @@ fn runtime_log_filter() -> RuntimeLogFilter {
|
|||||||
RuntimeLogFilter::new(handle)
|
RuntimeLogFilter::new(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn prepared_runtime(generation: Arc<RuntimeGeneration>) -> PreparedRuntime {
|
||||||
|
let (config_watcher_activation, _activation_rx) = watch::channel(false);
|
||||||
|
PreparedRuntime {
|
||||||
|
generation,
|
||||||
|
detected_ips: (None, None),
|
||||||
|
config_watcher_activation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||||
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||||
let new_config = Arc::new(ProxyConfig::default());
|
let new_config = Arc::new(ProxyConfig::default());
|
||||||
@@ -120,10 +129,7 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
fixture.old_runtime.clone(),
|
fixture.old_runtime.clone(),
|
||||||
PreparedRuntime {
|
prepared_runtime(fixture.new_runtime),
|
||||||
generation: fixture.new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
||||||
)
|
)
|
||||||
@@ -161,10 +167,7 @@ async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
fixture.old_runtime.clone(),
|
fixture.old_runtime.clone(),
|
||||||
PreparedRuntime {
|
prepared_runtime(fixture.new_runtime.clone()),
|
||||||
generation: fixture.new_runtime.clone(),
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
RevisionGateAction::Proceed,
|
||||||
|_| Err("invalid DNS entry".to_string()),
|
|_| Err("invalid DNS entry".to_string()),
|
||||||
)
|
)
|
||||||
@@ -215,10 +218,7 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
old_runtime,
|
old_runtime,
|
||||||
PreparedRuntime {
|
prepared_runtime(new_runtime),
|
||||||
generation: new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
RevisionGateAction::Proceed,
|
||||||
|_| Ok(()),
|
|_| Ok(()),
|
||||||
)
|
)
|
||||||
@@ -271,10 +271,7 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
old_runtime,
|
old_runtime,
|
||||||
PreparedRuntime {
|
prepared_runtime(new_runtime),
|
||||||
generation: new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
RevisionGateAction::Proceed,
|
||||||
|_| Ok(()),
|
|_| Ok(()),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -28,9 +28,14 @@ use super::listeners::listener_rebind_supported;
|
|||||||
use super::runtime_tasks::RuntimeLogFilter;
|
use super::runtime_tasks::RuntimeLogFilter;
|
||||||
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
||||||
|
|
||||||
|
/// Fully prepared candidate runtime and its activation-gated config watcher.
|
||||||
pub(crate) struct PreparedRuntime {
|
pub(crate) struct PreparedRuntime {
|
||||||
|
/// Candidate generation ready for publication.
|
||||||
pub(crate) generation: Arc<RuntimeGeneration>,
|
pub(crate) generation: Arc<RuntimeGeneration>,
|
||||||
|
/// Detected public addresses associated with the candidate.
|
||||||
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
||||||
|
/// Gate opened only after the candidate becomes the active generation.
|
||||||
|
pub(crate) config_watcher_activation: watch::Sender<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn prepare_runtime(
|
pub(crate) async fn prepare_runtime(
|
||||||
@@ -171,6 +176,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
config.server.max_connections as usize
|
config.server.max_connections as usize
|
||||||
};
|
};
|
||||||
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
||||||
|
let (config_watcher_activation, config_watcher_activation_rx) = watch::channel(false);
|
||||||
let watches = runtime_tasks::spawn_runtime_tasks(
|
let watches = runtime_tasks::spawn_runtime_tasks(
|
||||||
&config,
|
&config,
|
||||||
config_path,
|
config_path,
|
||||||
@@ -190,6 +196,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
proxy_shared.clone(),
|
proxy_shared.clone(),
|
||||||
me_ready_tx.clone(),
|
me_ready_tx.clone(),
|
||||||
task_scope.clone(),
|
task_scope.clone(),
|
||||||
|
Some(config_watcher_activation_rx),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = watches.config_rx;
|
let config_rx = watches.config_rx;
|
||||||
@@ -295,6 +302,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
|
|
||||||
Ok(PreparedRuntime {
|
Ok(PreparedRuntime {
|
||||||
generation,
|
generation,
|
||||||
|
config_watcher_activation,
|
||||||
detected_ips: (
|
detected_ips: (
|
||||||
probe.detected_ipv4.map(IpAddr::V4),
|
probe.detected_ipv4.map(IpAddr::V4),
|
||||||
probe.detected_ipv6.map(IpAddr::V6),
|
probe.detected_ipv6.map(IpAddr::V6),
|
||||||
|
|||||||
@@ -247,6 +247,7 @@ pub(super) async fn prepare_runtime(
|
|||||||
shared_state.clone(),
|
shared_state.clone(),
|
||||||
me_ready_tx.clone(),
|
me_ready_tx.clone(),
|
||||||
runtime_task_scope.clone(),
|
runtime_task_scope.clone(),
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = runtime_watches.config_rx;
|
let config_rx = runtime_watches.config_rx;
|
||||||
|
|||||||
@@ -107,6 +107,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
me_ready_tx: watch::Sender<u64>,
|
me_ready_tx: watch::Sender<u64>,
|
||||||
task_scope: RuntimeTaskScope,
|
task_scope: RuntimeTaskScope,
|
||||||
|
config_watcher_activation: Option<watch::Receiver<bool>>,
|
||||||
) -> RuntimeWatches {
|
) -> RuntimeWatches {
|
||||||
let um_clone = upstream_manager.clone();
|
let um_clone = upstream_manager.clone();
|
||||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||||
@@ -151,14 +152,15 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
Some("spawn config hot-reload watcher".to_string()),
|
Some("spawn config hot-reload watcher".to_string()),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let (config_rx, log_level_rx): (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) =
|
let (config_rx, log_level_rx, config_watcher_task) = spawn_config_watcher(
|
||||||
spawn_config_watcher(
|
config_path.to_path_buf(),
|
||||||
config_path.to_path_buf(),
|
config.clone(),
|
||||||
config.clone(),
|
detected_ip_v4,
|
||||||
detected_ip_v4,
|
detected_ip_v6,
|
||||||
detected_ip_v6,
|
task_scope.cancellation_token(),
|
||||||
task_scope.cancellation_token(),
|
config_watcher_activation,
|
||||||
);
|
);
|
||||||
|
task_scope.spawn(config_watcher_task);
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
COMPONENT_CONFIG_WATCHER_START,
|
COMPONENT_CONFIG_WATCHER_START,
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::Weak;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use tokio::sync::watch;
|
||||||
|
|
||||||
|
use super::manager::WebProcessRuntime;
|
||||||
|
|
||||||
|
/// Process-owned WEB ingress lifecycle state.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum WebRuntimeLifecycle {
|
||||||
|
/// Listener orchestration has not completed.
|
||||||
|
Starting,
|
||||||
|
/// This process has no WEB listener.
|
||||||
|
NoWebListener,
|
||||||
|
/// WEB admission and request handling are active.
|
||||||
|
Running,
|
||||||
|
/// WEB admission is closed while owned work drains.
|
||||||
|
Draining,
|
||||||
|
/// All WEB ingress work drained within the deadline.
|
||||||
|
Drained,
|
||||||
|
/// The bounded shutdown deadline expired.
|
||||||
|
DeadlineExceeded,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebRuntimeLifecycle {
|
||||||
|
/// Returns the stable API token for this lifecycle state.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Starting => "starting",
|
||||||
|
Self::NoWebListener => "no_web_listener",
|
||||||
|
Self::Running => "running",
|
||||||
|
Self::Draining => "draining",
|
||||||
|
Self::Drained => "drained",
|
||||||
|
Self::DeadlineExceeded => "deadline_exceeded",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One immutable lifecycle publication consumed by the control plane.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct WebRuntimePublication {
|
||||||
|
/// Monotonic process-local lifecycle transition number.
|
||||||
|
pub(crate) epoch: u64,
|
||||||
|
/// Current lifecycle state.
|
||||||
|
pub(crate) lifecycle: WebRuntimeLifecycle,
|
||||||
|
/// Monotonic transition time used only for relative age.
|
||||||
|
pub(crate) since: Instant,
|
||||||
|
/// Actual WEB listener addresses frozen for this process.
|
||||||
|
pub(crate) listeners: Arc<[SocketAddr]>,
|
||||||
|
/// Weak runtime access that never extends data-plane ownership.
|
||||||
|
pub(crate) runtime: Weak<WebProcessRuntime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-writer process lifecycle publisher for WEB ingress.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct WebRuntimeControl {
|
||||||
|
epoch: Arc<AtomicU64>,
|
||||||
|
tx: watch::Sender<WebRuntimePublication>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebRuntimeControl {
|
||||||
|
/// Creates the process channel in the pre-listener `starting` state.
|
||||||
|
pub(crate) fn new() -> Self {
|
||||||
|
let publication = WebRuntimePublication {
|
||||||
|
epoch: 1,
|
||||||
|
lifecycle: WebRuntimeLifecycle::Starting,
|
||||||
|
since: Instant::now(),
|
||||||
|
listeners: Arc::from([]),
|
||||||
|
runtime: Weak::new(),
|
||||||
|
};
|
||||||
|
let (tx, _rx) = watch::channel(publication);
|
||||||
|
Self {
|
||||||
|
epoch: Arc::new(AtomicU64::new(1)),
|
||||||
|
tx,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Subscribes without transferring runtime ownership to the receiver.
|
||||||
|
pub(crate) fn subscribe(&self) -> watch::Receiver<WebRuntimePublication> {
|
||||||
|
self.tx.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publishes one lifecycle transition and optional weak runtime reference.
|
||||||
|
pub(crate) fn publish(
|
||||||
|
&self,
|
||||||
|
lifecycle: WebRuntimeLifecycle,
|
||||||
|
listeners: Arc<[SocketAddr]>,
|
||||||
|
runtime: Weak<WebProcessRuntime>,
|
||||||
|
) {
|
||||||
|
let epoch = self.epoch.fetch_add(1, Ordering::AcqRel).saturating_add(1);
|
||||||
|
self.tx.send_replace(WebRuntimePublication {
|
||||||
|
epoch,
|
||||||
|
lifecycle,
|
||||||
|
since: Instant::now(),
|
||||||
|
listeners,
|
||||||
|
runtime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WebRuntimeControl {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_is_monotonic_and_does_not_require_a_runtime_owner() {
|
||||||
|
let control = WebRuntimeControl::new();
|
||||||
|
let receiver = control.subscribe();
|
||||||
|
control.publish(
|
||||||
|
WebRuntimeLifecycle::NoWebListener,
|
||||||
|
Arc::from([]),
|
||||||
|
Weak::new(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let publication = receiver.borrow().clone();
|
||||||
|
assert_eq!(publication.epoch, 2);
|
||||||
|
assert_eq!(publication.lifecycle, WebRuntimeLifecycle::NoWebListener);
|
||||||
|
assert!(publication.runtime.upgrade().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn publication_keeps_only_weak_runtime_ownership() {
|
||||||
|
let generation = crate::maestro::generation::test_runtime_generation(
|
||||||
|
1,
|
||||||
|
crate::config::ProxyConfig::default(),
|
||||||
|
);
|
||||||
|
let runtime =
|
||||||
|
WebProcessRuntime::start(Arc::new(arc_swap::ArcSwap::from(generation.clone())));
|
||||||
|
let strong_before = Arc::strong_count(&runtime);
|
||||||
|
let control = WebRuntimeControl::new();
|
||||||
|
control.publish(
|
||||||
|
WebRuntimeLifecycle::Running,
|
||||||
|
Arc::from([]),
|
||||||
|
Arc::downgrade(&runtime),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(Arc::strong_count(&runtime), strong_before);
|
||||||
|
runtime.shutdown().await;
|
||||||
|
drop(runtime);
|
||||||
|
assert!(control.subscribe().borrow().runtime.upgrade().is_none());
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
+13
-2
@@ -17,6 +17,7 @@ use tokio::net::TcpStream;
|
|||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
||||||
|
use crate::maestro::generation::RuntimeGeneration;
|
||||||
use crate::web::bridge;
|
use crate::web::bridge;
|
||||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||||
|
|
||||||
@@ -201,6 +202,7 @@ async fn handle_request(
|
|||||||
client_ip_source,
|
client_ip_source,
|
||||||
trusted_proxy_cidrs,
|
trusted_proxy_cidrs,
|
||||||
runtime,
|
runtime,
|
||||||
|
generation,
|
||||||
vhost,
|
vhost,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -214,6 +216,7 @@ async fn handle_root(
|
|||||||
client_ip_source: WebClientIpSource,
|
client_ip_source: WebClientIpSource,
|
||||||
trusted_proxy_cidrs: &[IpNetwork],
|
trusted_proxy_cidrs: &[IpNetwork],
|
||||||
runtime: Arc<WebProcessRuntime>,
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
generation: Arc<RuntimeGeneration>,
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
let (candidate, canonical) = bridge_candidate(request.uri().query());
|
let (candidate, canonical) = bridge_candidate(request.uri().query());
|
||||||
@@ -229,7 +232,16 @@ async fn handle_root(
|
|||||||
trace.set_route(TraceRoute::Bridge);
|
trace.set_route(TraceRoute::Bridge);
|
||||||
trace.set_effective_ip(client_ip);
|
trace.set_effective_ip(client_ip);
|
||||||
}
|
}
|
||||||
let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
|
let user_agent = request
|
||||||
|
.headers()
|
||||||
|
.get(header::USER_AGENT)
|
||||||
|
.and_then(|value| value.to_str().ok());
|
||||||
|
let bootstrap = match runtime.issue_bootstrap_for_request(
|
||||||
|
&generation,
|
||||||
|
Arc::clone(&profile),
|
||||||
|
client_ip,
|
||||||
|
user_agent,
|
||||||
|
) {
|
||||||
Ok(bootstrap) => bootstrap,
|
Ok(bootstrap) => bootstrap,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
runtime.trace().record_profile_lifecycle(
|
runtime.trace().record_profile_lifecycle(
|
||||||
@@ -248,7 +260,6 @@ async fn handle_root(
|
|||||||
trace.bind_profile(&profile, bootstrap.trace_session_id);
|
trace.bind_profile(&profile, bootstrap.trace_session_id);
|
||||||
trace.register_redaction(bootstrap.token.as_bytes());
|
trace.register_redaction(bootstrap.token.as_bytes());
|
||||||
}
|
}
|
||||||
let generation = runtime.active_generation();
|
|
||||||
let config = generation.config();
|
let config = generation.config();
|
||||||
let page = bridge::render(
|
let page = bridge::render(
|
||||||
&vhost.host,
|
&vhost.host,
|
||||||
|
|||||||
@@ -63,18 +63,20 @@ impl ConnectionActivity {
|
|||||||
if state.failed {
|
if state.failed {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
let request_protected = state
|
let request_deadline = state
|
||||||
.request
|
.request
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|request| request.deadline)
|
.and_then(|request| request.deadline)
|
||||||
.is_some_and(|deadline| now <= deadline.deadline);
|
.map(|deadline| deadline.deadline);
|
||||||
let upgrade_protected = state
|
let upgrade_deadline = state
|
||||||
.upgrade
|
.upgrade
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|upgrade| now <= upgrade.deadline.deadline);
|
.map(|upgrade| upgrade.deadline.deadline);
|
||||||
!request_protected
|
let protected_until = request_deadline.into_iter().chain(upgrade_deadline).max();
|
||||||
&& !upgrade_protected
|
let idle_since = protected_until
|
||||||
&& now.saturating_duration_since(state.last_progress) >= idle
|
.filter(|deadline| *deadline > state.last_progress)
|
||||||
|
.unwrap_or(state.last_progress);
|
||||||
|
now.saturating_duration_since(idle_since) >= idle
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fail(&self) {
|
fn fail(&self) {
|
||||||
@@ -390,6 +392,20 @@ mod tests {
|
|||||||
assert!(!activity.should_close(Instant::now(), Duration::from_secs(1)));
|
assert!(!activity.should_close(Instant::now(), Duration::from_secs(1)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expired_operation_lease_gets_one_idle_interval_to_publish_its_result() {
|
||||||
|
let activity = ConnectionActivity::new();
|
||||||
|
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||||
|
let now = Instant::now();
|
||||||
|
let _lease = request
|
||||||
|
.deadline_handle()
|
||||||
|
.lease_until(now + Duration::from_secs(1))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(!activity.should_close(now + Duration::from_millis(1500), Duration::from_secs(1)));
|
||||||
|
assert!(activity.should_close(now + Duration::from_secs(2), Duration::from_secs(1)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stale_request_lease_cannot_clear_a_new_request_deadline() {
|
fn stale_request_lease_cannot_clear_a_new_request_deadline() {
|
||||||
let activity = ConnectionActivity::new();
|
let activity = ConnectionActivity::new();
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn runtime_index_lists_and_asynchronously_closes_by_opaque_reference() {
|
||||||
|
let capability = [19u8; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(active_runtime);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let root = format!(
|
||||||
|
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.19\r\nUser-Agent: Telemt-Control-Test/1\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let root_response = request(&listener, &runtime, root).await;
|
||||||
|
let (_, root_body) = split_response(&root_response);
|
||||||
|
let bootstrap = std::str::from_utf8(root_body)
|
||||||
|
.unwrap()
|
||||||
|
.split_once("bootstrap=\"")
|
||||||
|
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||||
|
.map(|(token, _)| token)
|
||||||
|
.unwrap();
|
||||||
|
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||||
|
let mut create = format!(
|
||||||
|
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.19\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
hello.len()
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
create.extend_from_slice(&hello);
|
||||||
|
let create_response = request(&listener, &runtime, create).await;
|
||||||
|
assert!(create_response.starts_with(b"HTTP/1.1 200"));
|
||||||
|
|
||||||
|
let page = runtime.list_sessions(SessionListRequest {
|
||||||
|
limit: 50,
|
||||||
|
cursor: None,
|
||||||
|
filter: SessionFilter::default(),
|
||||||
|
});
|
||||||
|
assert_eq!(page.sessions.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
page.sessions[0].user_agent.as_deref(),
|
||||||
|
Some("Telemt-Control-Test/1")
|
||||||
|
);
|
||||||
|
let session_ref = page.sessions[0].session_ref.clone();
|
||||||
|
let trace_session_id = runtime.parse_session_ref(&session_ref).unwrap();
|
||||||
|
let noncanonical_session_ref = format!("ws1.{}.000000000000000A", runtime.runtime_instance());
|
||||||
|
assert_eq!(
|
||||||
|
runtime.parse_session_ref(&noncanonical_session_ref),
|
||||||
|
Err(SessionRefError::Invalid)
|
||||||
|
);
|
||||||
|
let noncanonical_operation_id = format!("wo1.{}.000000000000000A", runtime.runtime_instance());
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.control_operation(&noncanonical_operation_id),
|
||||||
|
Err(ControlError::InvalidOperation)
|
||||||
|
));
|
||||||
|
let nonmatching = runtime
|
||||||
|
.start_close_operation(
|
||||||
|
runtime.runtime_instance(),
|
||||||
|
CloseOperationSelector::Filter(SessionFilter {
|
||||||
|
state: Some("healthy".to_string()),
|
||||||
|
..SessionFilter::default()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut nonmatching_status = None;
|
||||||
|
for _ in 0..32 {
|
||||||
|
let status = runtime
|
||||||
|
.control_operation(&nonmatching.operation_id)
|
||||||
|
.unwrap();
|
||||||
|
if serde_json::to_value(&status).unwrap()["state"] == "completed" {
|
||||||
|
nonmatching_status = Some(status);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
let nonmatching_status = nonmatching_status.expect("filtered close operation completed");
|
||||||
|
assert_eq!(nonmatching_status.matched, 0);
|
||||||
|
assert_eq!(nonmatching_status.close_signalled, 0);
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.session_detail(trace_session_id),
|
||||||
|
SessionDetail::Active(_)
|
||||||
|
));
|
||||||
|
let operation = runtime
|
||||||
|
.start_close_operation(
|
||||||
|
runtime.runtime_instance(),
|
||||||
|
CloseOperationSelector::Refs(vec![trace_session_id]),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for _ in 0..32 {
|
||||||
|
let status = runtime.control_operation(&operation.operation_id).unwrap();
|
||||||
|
if serde_json::to_value(&status).unwrap()["state"] == "completed" {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.session_detail(trace_session_id),
|
||||||
|
SessionDetail::Gone { .. }
|
||||||
|
));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
@@ -68,7 +68,7 @@ pub(super) async fn handle_session(
|
|||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
||||||
let Some((trace_session_id, profile, frozen_body_timeout)) =
|
let Some((trace_session_id, profile, body_timeout)) =
|
||||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||||
else {
|
else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
@@ -77,9 +77,6 @@ pub(super) async fn handle_session(
|
|||||||
trace.set_route(TraceRoute::Session);
|
trace.set_route(TraceRoute::Session);
|
||||||
trace.bind_profile(&profile, trace_session_id);
|
trace.bind_profile(&profile, trace_session_id);
|
||||||
}
|
}
|
||||||
let body_timeout = frozen_body_timeout.unwrap_or_else(|| {
|
|
||||||
Duration::from_secs(runtime.active_generation().config().web.timeouts.body_secs)
|
|
||||||
});
|
|
||||||
let CollectedBody {
|
let CollectedBody {
|
||||||
request,
|
request,
|
||||||
body,
|
body,
|
||||||
|
|||||||
@@ -75,6 +75,61 @@ async fn request_with_body_delay(
|
|||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn issued_bootstrap_timeouts_survive_reload_before_session_creation() {
|
||||||
|
let capability = [20u8; 32];
|
||||||
|
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
initial_config.web.timeouts.body_secs = 3;
|
||||||
|
initial_config.web.timeouts.long_poll_secs = 3;
|
||||||
|
initial_config.web.timeouts.bootstrap_lifetime_secs = 5;
|
||||||
|
let generation = test_runtime_generation(1, initial_config);
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let root = format!(
|
||||||
|
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let root_response = request(&listener, &runtime, root).await;
|
||||||
|
let (_, root_body) = split_response(&root_response);
|
||||||
|
let bootstrap = std::str::from_utf8(root_body)
|
||||||
|
.unwrap()
|
||||||
|
.split_once("bootstrap=\"")
|
||||||
|
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||||
|
.map(|(token, _)| token.to_string())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
replacement_config.web.timeouts.body_secs = 1;
|
||||||
|
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||||
|
replacement_config.web.timeouts.bootstrap_lifetime_secs = 1;
|
||||||
|
let replacement = test_runtime_generation(2, replacement_config);
|
||||||
|
active_runtime.store(Arc::clone(&replacement));
|
||||||
|
|
||||||
|
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||||
|
let create_head = format!(
|
||||||
|
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
hello.len()
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let create_response = request_with_body_delay(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
create_head,
|
||||||
|
&hello,
|
||||||
|
std::time::Duration::from_millis(1200),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(create_response.starts_with(b"HTTP/1.1 200"));
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
replacement.stop_sessions().await;
|
||||||
|
replacement.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||||
let capability = [21u8; 32];
|
let capability = [21u8; 32];
|
||||||
|
|||||||
+27
-1
@@ -15,7 +15,10 @@ use crate::config::{
|
|||||||
};
|
};
|
||||||
use crate::maestro::generation::test_runtime_generation;
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
use crate::web::frame::{self, FrameType};
|
use crate::web::frame::{self, FrameType};
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::manager::{
|
||||||
|
CloseOperationSelector, ControlError, SessionDetail, SessionFilter, SessionListRequest,
|
||||||
|
SessionRefError, WebProcessRuntime,
|
||||||
|
};
|
||||||
|
|
||||||
#[path = "legacy_tests.rs"]
|
#[path = "legacy_tests.rs"]
|
||||||
mod legacy_tests;
|
mod legacy_tests;
|
||||||
@@ -24,6 +27,9 @@ mod negotiation_tests;
|
|||||||
// Reload-stability tests for session-owned timeout policy.
|
// Reload-stability tests for session-owned timeout policy.
|
||||||
#[path = "session_policy_tests.rs"]
|
#[path = "session_policy_tests.rs"]
|
||||||
mod session_policy_tests;
|
mod session_policy_tests;
|
||||||
|
// Runtime control integration stays separate from carrier protocol scenarios.
|
||||||
|
#[path = "control_tests.rs"]
|
||||||
|
mod control_tests;
|
||||||
|
|
||||||
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
||||||
|
|
||||||
@@ -394,6 +400,26 @@ async fn unused_bootstrap_survives_equivalent_runtime_generation_swap() {
|
|||||||
replacement.stop_background_tasks().await;
|
replacement.stop_background_tasks().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bridge_bootstrap_uses_the_generation_that_selected_its_profile() {
|
||||||
|
let initial = test_runtime_generation(1, runtime_config([21; 32], WebCarrier::Https));
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&initial)));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||||
|
let profile = initial.config().web.runtime.as_ref().unwrap().profiles[0].clone();
|
||||||
|
let replacement = test_runtime_generation(2, runtime_config([22; 32], WebCarrier::HttpsLanes));
|
||||||
|
active_runtime.store(Arc::clone(&replacement));
|
||||||
|
|
||||||
|
let result =
|
||||||
|
runtime.issue_bootstrap_for_generation(&initial, profile, "192.0.2.10".parse().unwrap());
|
||||||
|
|
||||||
|
assert!(result.is_ok());
|
||||||
|
runtime.shutdown().await;
|
||||||
|
initial.stop_sessions().await;
|
||||||
|
initial.stop_background_tasks().await;
|
||||||
|
replacement.stop_sessions().await;
|
||||||
|
replacement.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn unused_bootstrap_is_rejected_after_profile_identity_change() {
|
async fn unused_bootstrap_is_rejected_after_profile_identity_change() {
|
||||||
let capability = [11u8; 32];
|
let capability = [11u8; 32];
|
||||||
|
|||||||
+28
-4
@@ -38,8 +38,16 @@ pub(crate) use lifecycle::WebShutdownOutcome;
|
|||||||
mod budget;
|
mod budget;
|
||||||
// WebSocket admission, replacement, and liveness are process-scoped.
|
// WebSocket admission, replacement, and liveness are process-scoped.
|
||||||
mod websocket;
|
mod websocket;
|
||||||
|
// Bounded read-only snapshots and opaque session references serve the API.
|
||||||
|
mod status;
|
||||||
|
pub(crate) use status::{
|
||||||
|
SessionDetail, SessionFilter, SessionListRequest, SessionRefError, WebRuntimeStatus,
|
||||||
|
};
|
||||||
|
// Asynchronous bounded close operations isolate mutation lifecycle from HTTP requests.
|
||||||
|
mod control;
|
||||||
pub(crate) use budget::WebSocketBudgetLease;
|
pub(crate) use budget::WebSocketBudgetLease;
|
||||||
use budget::{WebDataBudget, WebSocketBudgetClass};
|
use budget::{WebDataBudget, WebSocketBudgetClass};
|
||||||
|
pub(crate) use control::{CloseOperationSelector, ControlError};
|
||||||
pub(crate) use negotiation::{
|
pub(crate) use negotiation::{
|
||||||
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierLearningContext, CarrierRequest,
|
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierLearningContext, CarrierRequest,
|
||||||
};
|
};
|
||||||
@@ -128,6 +136,7 @@ pub(crate) struct BootstrapResult {
|
|||||||
|
|
||||||
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
||||||
pub(crate) struct WebProcessRuntime {
|
pub(crate) struct WebProcessRuntime {
|
||||||
|
runtime_instance: Arc<str>,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
trace: Arc<WebTraceStore>,
|
trace: Arc<WebTraceStore>,
|
||||||
limits: WebLimitsConfig,
|
limits: WebLimitsConfig,
|
||||||
@@ -147,6 +156,8 @@ pub(crate) struct WebProcessRuntime {
|
|||||||
websocket_clock: std::time::Instant,
|
websocket_clock: std::time::Instant,
|
||||||
websocket_notify: Arc<Notify>,
|
websocket_notify: Arc<Notify>,
|
||||||
data_budget: Arc<WebDataBudget>,
|
data_budget: Arc<WebDataBudget>,
|
||||||
|
control_operations: Mutex<control::ControlOperationRegistry>,
|
||||||
|
next_control_operation_id: AtomicU64,
|
||||||
shutdown: CancellationToken,
|
shutdown: CancellationToken,
|
||||||
tasks: TaskTracker,
|
tasks: TaskTracker,
|
||||||
sessions_created: AtomicU64,
|
sessions_created: AtomicU64,
|
||||||
@@ -172,7 +183,9 @@ impl WebProcessRuntime {
|
|||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
trace: Arc<WebTraceStore>,
|
trace: Arc<WebTraceStore>,
|
||||||
) -> Arc<Self> {
|
) -> Arc<Self> {
|
||||||
let config = active_runtime.load().config();
|
let initial_generation = active_runtime.load_full();
|
||||||
|
let config = initial_generation.config();
|
||||||
|
trace.apply_policy(initial_generation.id, &config.web.debug);
|
||||||
let limits = config.web.limits.clone();
|
let limits = config.web.limits.clone();
|
||||||
let learning_capacity = limits.max_carrier_learning_entries;
|
let learning_capacity = limits.max_carrier_learning_entries;
|
||||||
let mut carrier_learning = learning::CarrierLearning::new(learning_capacity);
|
let mut carrier_learning = learning::CarrierLearning::new(learning_capacity);
|
||||||
@@ -188,6 +201,7 @@ impl WebProcessRuntime {
|
|||||||
let lane_poll_limit = limits.max_http_handlers / 2;
|
let lane_poll_limit = limits.max_http_handlers / 2;
|
||||||
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
|
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
|
||||||
let runtime = Arc::new(Self {
|
let runtime = Arc::new(Self {
|
||||||
|
runtime_instance: Arc::from(format!("{:032x}", rand::random::<u128>())),
|
||||||
active_runtime,
|
active_runtime,
|
||||||
trace,
|
trace,
|
||||||
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
||||||
@@ -203,8 +217,10 @@ impl WebProcessRuntime {
|
|||||||
websocket_clock: std::time::Instant::now(),
|
websocket_clock: std::time::Instant::now(),
|
||||||
websocket_notify: Arc::new(Notify::new()),
|
websocket_notify: Arc::new(Notify::new()),
|
||||||
data_budget: WebDataBudget::new(limits.clone()),
|
data_budget: WebDataBudget::new(limits.clone()),
|
||||||
|
control_operations: Mutex::new(control::ControlOperationRegistry::default()),
|
||||||
|
next_control_operation_id: AtomicU64::new(1),
|
||||||
limits,
|
limits,
|
||||||
state: Mutex::new(ManagerState::default()),
|
state: Mutex::new(ManagerState::new(initial_generation.id, config.web.enabled)),
|
||||||
stream_admission: Mutex::new(StreamAdmissionState::default()),
|
stream_admission: Mutex::new(StreamAdmissionState::default()),
|
||||||
learning: Mutex::new(carrier_learning),
|
learning: Mutex::new(carrier_learning),
|
||||||
shutdown: CancellationToken::new(),
|
shutdown: CancellationToken::new(),
|
||||||
@@ -229,8 +245,11 @@ impl WebProcessRuntime {
|
|||||||
let Some(runtime) = weak.upgrade() else {
|
let Some(runtime) = weak.upgrade() else {
|
||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
let policy = runtime.active_generation().config().web.debug.clone();
|
let generation = runtime.active_generation();
|
||||||
runtime.trace.apply_policy(&policy);
|
let policy = generation.config().web.debug.clone();
|
||||||
|
runtime
|
||||||
|
.trace
|
||||||
|
.apply_policy(generation.id, &policy);
|
||||||
runtime.cleanup();
|
runtime.cleanup();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -244,6 +263,11 @@ impl WebProcessRuntime {
|
|||||||
self.active_runtime.load_full()
|
self.active_runtime.load_full()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the random process-instance fence used by control-plane references.
|
||||||
|
pub(crate) fn runtime_instance(&self) -> &str {
|
||||||
|
&self.runtime_instance
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns the process-owned WEB debug trace store.
|
/// Returns the process-owned WEB debug trace store.
|
||||||
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
|
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
|
||||||
&self.trace
|
&self.trace
|
||||||
|
|||||||
@@ -52,10 +52,18 @@ pub(crate) struct WebDataBudgetSnapshot {
|
|||||||
pub(crate) queue_bytes: usize,
|
pub(crate) queue_bytes: usize,
|
||||||
/// Total queue items currently retained.
|
/// Total queue items currently retained.
|
||||||
pub(crate) queue_items: usize,
|
pub(crate) queue_items: usize,
|
||||||
|
/// Control bytes included in the queue total.
|
||||||
|
pub(crate) queue_control_bytes: usize,
|
||||||
|
/// Control items included in the queue total.
|
||||||
|
pub(crate) queue_control_items: usize,
|
||||||
/// Total WebSocket bytes currently retained.
|
/// Total WebSocket bytes currently retained.
|
||||||
pub(crate) websocket_bytes: usize,
|
pub(crate) websocket_bytes: usize,
|
||||||
/// Largest combined byte usage observed since process start.
|
/// Largest combined byte usage observed since process start.
|
||||||
pub(crate) high_water_bytes: usize,
|
pub(crate) high_water_bytes: usize,
|
||||||
|
/// Distinct profile owners currently charged.
|
||||||
|
pub(crate) owners: usize,
|
||||||
|
/// Whether shutdown closed this allocation authority.
|
||||||
|
pub(crate) closed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bounded owner-usage view captured before WebSocket registry selection.
|
/// Bounded owner-usage view captured before WebSocket registry selection.
|
||||||
@@ -259,11 +267,29 @@ impl WebDataBudget {
|
|||||||
WebDataBudgetSnapshot {
|
WebDataBudgetSnapshot {
|
||||||
queue_bytes: state.queue_bytes,
|
queue_bytes: state.queue_bytes,
|
||||||
queue_items: state.queue_items,
|
queue_items: state.queue_items,
|
||||||
|
queue_control_bytes: state.queue_control_bytes,
|
||||||
|
queue_control_items: state.queue_control_items,
|
||||||
websocket_bytes: state.websocket_bytes,
|
websocket_bytes: state.websocket_bytes,
|
||||||
high_water_bytes: state.high_water_bytes,
|
high_water_bytes: state.high_water_bytes,
|
||||||
|
owners: state.owner_bytes.len(),
|
||||||
|
closed: state.closed,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn try_snapshot(&self) -> Option<WebDataBudgetSnapshot> {
|
||||||
|
let state = self.state.try_lock()?;
|
||||||
|
Some(WebDataBudgetSnapshot {
|
||||||
|
queue_bytes: state.queue_bytes,
|
||||||
|
queue_items: state.queue_items,
|
||||||
|
queue_control_bytes: state.queue_control_bytes,
|
||||||
|
queue_control_items: state.queue_control_items,
|
||||||
|
websocket_bytes: state.websocket_bytes,
|
||||||
|
high_water_bytes: state.high_water_bytes,
|
||||||
|
owners: state.owner_bytes.len(),
|
||||||
|
closed: state.closed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn close(&self) {
|
pub(super) fn close(&self) {
|
||||||
self.state.lock().closed = true;
|
self.state.lock().closed = true;
|
||||||
self.notify.notify_waiters();
|
self.notify.notify_waiters();
|
||||||
|
|||||||
@@ -163,6 +163,34 @@ pub(super) struct CarrierLearning {
|
|||||||
policy_started_at: Instant,
|
policy_started_at: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bounded control-plane summary of carrier-learning state.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub(crate) struct CarrierLearningStatus {
|
||||||
|
/// Whether outcome learning is active in the effective policy.
|
||||||
|
pub(crate) enabled: bool,
|
||||||
|
/// Effective evidence thresholds.
|
||||||
|
pub(crate) aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||||
|
/// Current evidence epoch, or none after counter exhaustion.
|
||||||
|
pub(crate) epoch: Option<u64>,
|
||||||
|
/// Retained evidence entries.
|
||||||
|
pub(crate) entries: usize,
|
||||||
|
/// Restart-owned evidence ceiling.
|
||||||
|
pub(crate) capacity: usize,
|
||||||
|
/// Effective evidence lifetime.
|
||||||
|
pub(crate) lifetime_secs: u64,
|
||||||
|
/// Monotonic age of the current policy epoch.
|
||||||
|
pub(crate) age_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of one epoch-fenced learning reset.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub(crate) struct CarrierLearningResetOutcome {
|
||||||
|
/// Evidence entries detached by the reset.
|
||||||
|
pub(crate) entries_cleared: usize,
|
||||||
|
/// New epoch fencing pre-reset outcomes.
|
||||||
|
pub(crate) epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl CarrierLearning {
|
impl CarrierLearning {
|
||||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||||
pub(super) fn new(capacity: usize) -> Self {
|
pub(super) fn new(capacity: usize) -> Self {
|
||||||
@@ -177,6 +205,23 @@ impl CarrierLearning {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn status(&self, now: Instant) -> CarrierLearningStatus {
|
||||||
|
let policy = self.policy.unwrap_or(LearningPolicy {
|
||||||
|
enabled: false,
|
||||||
|
aggressiveness: WebCarrierNegotiationAggressiveness::Conservative,
|
||||||
|
lifetime: Duration::ZERO,
|
||||||
|
});
|
||||||
|
CarrierLearningStatus {
|
||||||
|
enabled: policy.enabled,
|
||||||
|
aggressiveness: policy.aggressiveness,
|
||||||
|
epoch: self.epoch,
|
||||||
|
entries: self.entries.len(),
|
||||||
|
capacity: self.capacity,
|
||||||
|
lifetime_secs: policy.lifetime.as_secs(),
|
||||||
|
age_ms: millis(now.saturating_duration_since(self.policy_started_at)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Applies hot-reloaded learning policy and returns its outcome epoch.
|
/// Applies hot-reloaded learning policy and returns its outcome epoch.
|
||||||
pub(super) fn apply_policy(
|
pub(super) fn apply_policy(
|
||||||
&mut self,
|
&mut self,
|
||||||
@@ -424,6 +469,51 @@ impl CarrierLearning {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl super::WebProcessRuntime {
|
||||||
|
/// Captures learning state without waiting for a contended evidence lock.
|
||||||
|
pub(crate) fn try_carrier_learning_status(&self) -> Option<CarrierLearningStatus> {
|
||||||
|
self.learning
|
||||||
|
.try_lock()
|
||||||
|
.map(|learning| learning.status(Instant::now()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clears all evidence under a new epoch without changing the active policy.
|
||||||
|
pub(crate) fn reset_carrier_learning(
|
||||||
|
&self,
|
||||||
|
) -> Result<CarrierLearningResetOutcome, super::ManagerError> {
|
||||||
|
let control = self
|
||||||
|
.control_mutation_guard()
|
||||||
|
.map_err(|_| super::ManagerError::Closed)?;
|
||||||
|
let (outcome, retired_entries, retired_order) = {
|
||||||
|
let mut learning = self.learning.lock();
|
||||||
|
let epoch = learning
|
||||||
|
.epoch
|
||||||
|
.and_then(|epoch| epoch.checked_add(1))
|
||||||
|
.ok_or(super::ManagerError::Closed)?;
|
||||||
|
learning.epoch = Some(epoch);
|
||||||
|
learning.insertion_sequence = 1;
|
||||||
|
learning.policy_started_at = Instant::now();
|
||||||
|
let retired_entries = std::mem::take(&mut learning.entries);
|
||||||
|
let retired_order = std::mem::take(&mut learning.insertion_order);
|
||||||
|
(
|
||||||
|
CarrierLearningResetOutcome {
|
||||||
|
entries_cleared: retired_entries.len(),
|
||||||
|
epoch,
|
||||||
|
},
|
||||||
|
retired_entries,
|
||||||
|
retired_order,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
drop(control);
|
||||||
|
drop((retired_entries, retired_order));
|
||||||
|
Ok(outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn millis(duration: Duration) -> u64 {
|
||||||
|
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
|
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
|
||||||
configured
|
configured
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -220,3 +220,52 @@ fn fifo_metadata_stays_within_the_entry_capacity() {
|
|||||||
assert!(learning.insertion_order.len() <= 3);
|
assert!(learning.insertion_order.len() <= 3);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn explicit_reset_preserves_policy_and_rejects_old_epoch_outcomes() {
|
||||||
|
let generation = crate::maestro::generation::test_runtime_generation(
|
||||||
|
1,
|
||||||
|
crate::config::ProxyConfig::default(),
|
||||||
|
);
|
||||||
|
let runtime = crate::web::manager::WebProcessRuntime::start(std::sync::Arc::new(
|
||||||
|
arc_swap::ArcSwap::from(generation.clone()),
|
||||||
|
));
|
||||||
|
let now = Instant::now();
|
||||||
|
let old_epoch = {
|
||||||
|
let mut learning = runtime.learning.lock();
|
||||||
|
let epoch = learning
|
||||||
|
.apply_policy(
|
||||||
|
now,
|
||||||
|
true,
|
||||||
|
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||||
|
Duration::from_secs(10),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
learning.record_chain(now, epoch, context(7), &[], WebCarrier::Websocket);
|
||||||
|
epoch
|
||||||
|
};
|
||||||
|
|
||||||
|
let outcome = runtime.reset_carrier_learning().unwrap();
|
||||||
|
{
|
||||||
|
let mut learning = runtime.learning.lock();
|
||||||
|
learning.record_chain(
|
||||||
|
Instant::now(),
|
||||||
|
old_epoch,
|
||||||
|
context(7),
|
||||||
|
&[],
|
||||||
|
WebCarrier::Https,
|
||||||
|
);
|
||||||
|
let status = learning.status(Instant::now());
|
||||||
|
assert!(status.enabled);
|
||||||
|
assert_eq!(
|
||||||
|
status.aggressiveness,
|
||||||
|
WebCarrierNegotiationAggressiveness::Aggressive
|
||||||
|
);
|
||||||
|
assert_eq!(status.entries, 0);
|
||||||
|
assert_eq!(status.epoch, Some(outcome.epoch));
|
||||||
|
}
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,441 @@
|
|||||||
|
use std::collections::{BTreeSet, VecDeque};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::status::immutable_matches;
|
||||||
|
use super::{SessionFilter, WebProcessRuntime};
|
||||||
|
|
||||||
|
const OPERATION_REF_VERSION: &str = "wo1";
|
||||||
|
const OPERATION_RETENTION: usize = 32;
|
||||||
|
const CLOSE_CHUNK: usize = 128;
|
||||||
|
|
||||||
|
/// Validated bulk-close selector.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) enum CloseOperationSelector {
|
||||||
|
/// Exact logical session references resolved by the API.
|
||||||
|
Refs(Vec<u64>),
|
||||||
|
/// Point-in-time sessions matching a bounded filter.
|
||||||
|
Filter(SessionFilter),
|
||||||
|
/// Every logical session at or below the submission high-water mark.
|
||||||
|
All,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stable close-operation state.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub(crate) enum ControlOperationState {
|
||||||
|
/// Accepted but not yet executing.
|
||||||
|
Queued,
|
||||||
|
/// Scanning the bounded point-in-time registry.
|
||||||
|
Running,
|
||||||
|
/// Finished the complete bounded scan.
|
||||||
|
Completed,
|
||||||
|
/// Stopped because process shutdown began.
|
||||||
|
Cancelled,
|
||||||
|
/// Stopped on one sanitized internal failure.
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retained status for one asynchronous close operation.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(crate) struct ControlOperationStatus {
|
||||||
|
/// Opaque process-fenced operation reference.
|
||||||
|
pub(crate) operation_id: String,
|
||||||
|
/// Current lifecycle state.
|
||||||
|
pub(crate) state: ControlOperationState,
|
||||||
|
/// Highest logical session eligible for this point-in-time operation.
|
||||||
|
pub(crate) high_water_session_ref: Option<String>,
|
||||||
|
/// Exact submitted reference count, or zero for filter/all selectors.
|
||||||
|
pub(crate) requested: usize,
|
||||||
|
/// Registry candidates visited so far.
|
||||||
|
pub(crate) scanned: usize,
|
||||||
|
/// Candidates that matched the complete selector.
|
||||||
|
pub(crate) matched: usize,
|
||||||
|
/// Matching session incarnations sent a close signal.
|
||||||
|
pub(crate) close_signalled: usize,
|
||||||
|
/// Matching candidates replaced or closed before signalling.
|
||||||
|
pub(crate) conflicted: usize,
|
||||||
|
/// Wall-clock creation timestamp for operator correlation.
|
||||||
|
pub(crate) created_epoch_millis: u64,
|
||||||
|
/// Wall-clock timestamp of the latest status mutation.
|
||||||
|
pub(crate) updated_epoch_millis: u64,
|
||||||
|
/// Stable sanitized failure token.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(crate) failure: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runtime control validation failure.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum ControlError {
|
||||||
|
/// The supplied process-instance fence is stale.
|
||||||
|
StaleInstance,
|
||||||
|
/// The selector is empty or exceeds its bound.
|
||||||
|
InvalidSelector,
|
||||||
|
/// Close-all was requested before issuance stopped.
|
||||||
|
IssuanceEnabled,
|
||||||
|
/// The single operation slot is occupied.
|
||||||
|
OperationInProgress,
|
||||||
|
/// The operation reference is not canonical.
|
||||||
|
InvalidOperation,
|
||||||
|
/// The canonical operation is outside retained history.
|
||||||
|
OperationNotFound,
|
||||||
|
/// Process shutdown has closed the mutation gate.
|
||||||
|
Closed,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-slot execution gate with bounded terminal history.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(super) struct ControlOperationRegistry {
|
||||||
|
active: Option<u64>,
|
||||||
|
retained: VecDeque<(u64, ControlOperationStatus)>,
|
||||||
|
closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WorkCandidate {
|
||||||
|
trace_session_id: u64,
|
||||||
|
session: Arc<crate::web::session::WebSession>,
|
||||||
|
bootstrap_hash: super::TokenHash,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebProcessRuntime {
|
||||||
|
/// Acquires the process mutation gate without permitting post-shutdown work.
|
||||||
|
pub(super) fn control_mutation_guard(
|
||||||
|
&self,
|
||||||
|
) -> Result<parking_lot::MutexGuard<'_, ControlOperationRegistry>, ControlError> {
|
||||||
|
let operations = self.control_operations.lock();
|
||||||
|
if operations.closed || self.shutdown.is_cancelled() {
|
||||||
|
return Err(ControlError::Closed);
|
||||||
|
}
|
||||||
|
Ok(operations)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clears debug records under the process mutation gate.
|
||||||
|
pub(crate) fn clear_debug(&self) -> Result<crate::web::trace::TraceClearOutcome, ControlError> {
|
||||||
|
let _control = self.control_mutation_guard()?;
|
||||||
|
Ok(self.trace.clear())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Starts one bounded point-in-time close sweep.
|
||||||
|
pub(crate) fn start_close_operation(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
runtime_instance: &str,
|
||||||
|
selector: CloseOperationSelector,
|
||||||
|
) -> Result<ControlOperationStatus, ControlError> {
|
||||||
|
if runtime_instance != self.runtime_instance() {
|
||||||
|
return Err(ControlError::StaleInstance);
|
||||||
|
}
|
||||||
|
if matches!(&selector, CloseOperationSelector::Refs(refs) if refs.is_empty() || refs.len() > 200)
|
||||||
|
|| matches!(&selector, CloseOperationSelector::Filter(filter) if filter.is_empty())
|
||||||
|
{
|
||||||
|
return Err(ControlError::InvalidSelector);
|
||||||
|
}
|
||||||
|
let generation = self.active_generation();
|
||||||
|
let web_enabled = generation.config().web.enabled;
|
||||||
|
let (high_water, issuance_enabled) = {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
state.apply_issuance_policy(generation.id, web_enabled);
|
||||||
|
(
|
||||||
|
state.session_index.last_key_value().map(|(id, _)| *id),
|
||||||
|
state.issuance_enabled,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if matches!(selector, CloseOperationSelector::All) && issuance_enabled {
|
||||||
|
return Err(ControlError::IssuanceEnabled);
|
||||||
|
}
|
||||||
|
if self.shutdown.is_cancelled() {
|
||||||
|
return Err(ControlError::Closed);
|
||||||
|
}
|
||||||
|
let sequence = self
|
||||||
|
.next_control_operation_id
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let operation_id = self.operation_ref(sequence);
|
||||||
|
let now = crate::web::trace::store_epoch_millis();
|
||||||
|
let requested = match &selector {
|
||||||
|
CloseOperationSelector::Refs(refs) => refs.len(),
|
||||||
|
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => 0,
|
||||||
|
};
|
||||||
|
let status = ControlOperationStatus {
|
||||||
|
operation_id,
|
||||||
|
state: ControlOperationState::Queued,
|
||||||
|
high_water_session_ref: high_water.map(|id| self.session_ref(id)),
|
||||||
|
requested,
|
||||||
|
scanned: 0,
|
||||||
|
matched: 0,
|
||||||
|
close_signalled: 0,
|
||||||
|
conflicted: 0,
|
||||||
|
created_epoch_millis: now,
|
||||||
|
updated_epoch_millis: now,
|
||||||
|
failure: None,
|
||||||
|
};
|
||||||
|
let tracked = {
|
||||||
|
let mut operations = self.control_operations.lock();
|
||||||
|
if operations.closed || self.shutdown.is_cancelled() {
|
||||||
|
return Err(ControlError::Closed);
|
||||||
|
}
|
||||||
|
if operations.active.is_some() {
|
||||||
|
return Err(ControlError::OperationInProgress);
|
||||||
|
}
|
||||||
|
operations.active = Some(sequence);
|
||||||
|
operations.retained.push_back((sequence, status.clone()));
|
||||||
|
trim_operations(&mut operations);
|
||||||
|
let weak = Arc::downgrade(self);
|
||||||
|
self.tasks.track_future(async move {
|
||||||
|
if let Some(runtime) = weak.upgrade() {
|
||||||
|
runtime
|
||||||
|
.run_close_operation(sequence, high_water, selector)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
};
|
||||||
|
drop(tokio::spawn(tracked));
|
||||||
|
Ok(status)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns one retained operation status under the process-instance fence.
|
||||||
|
pub(crate) fn control_operation(
|
||||||
|
&self,
|
||||||
|
operation_id: &str,
|
||||||
|
) -> Result<ControlOperationStatus, ControlError> {
|
||||||
|
let sequence = self.parse_operation_ref(operation_id)?;
|
||||||
|
self.control_operations
|
||||||
|
.lock()
|
||||||
|
.retained
|
||||||
|
.iter()
|
||||||
|
.find_map(|(id, status)| (*id == sequence).then(|| status.clone()))
|
||||||
|
.ok_or(ControlError::OperationNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prevents new control work from racing process-runtime shutdown.
|
||||||
|
pub(super) fn close_control_submission_gate(&self) {
|
||||||
|
self.control_operations.lock().closed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_close_operation(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
sequence: u64,
|
||||||
|
high_water: Option<u64>,
|
||||||
|
selector: CloseOperationSelector,
|
||||||
|
) {
|
||||||
|
self.update_operation(sequence, |status| {
|
||||||
|
status.state = ControlOperationState::Running
|
||||||
|
});
|
||||||
|
let refs = match &selector {
|
||||||
|
CloseOperationSelector::Refs(refs) => {
|
||||||
|
Some(refs.iter().copied().collect::<BTreeSet<_>>())
|
||||||
|
}
|
||||||
|
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => None,
|
||||||
|
};
|
||||||
|
let filter = match &selector {
|
||||||
|
CloseOperationSelector::Filter(filter) => Some(filter),
|
||||||
|
CloseOperationSelector::Refs(_) | CloseOperationSelector::All => None,
|
||||||
|
};
|
||||||
|
let mut cursor = None;
|
||||||
|
loop {
|
||||||
|
if self.shutdown.is_cancelled() {
|
||||||
|
self.finish_operation(sequence, ControlOperationState::Cancelled, None);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut direct = Vec::new();
|
||||||
|
let mut state_filtered = Vec::new();
|
||||||
|
let (scanned, next_cursor, reached_end) = {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let mut scanned = 0usize;
|
||||||
|
let mut next_cursor = cursor;
|
||||||
|
let mut reached_end = true;
|
||||||
|
let ids = state
|
||||||
|
.session_index
|
||||||
|
.range((
|
||||||
|
cursor.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Excluded),
|
||||||
|
std::ops::Bound::Unbounded,
|
||||||
|
))
|
||||||
|
.take(CLOSE_CHUNK)
|
||||||
|
.map(|(id, _)| *id)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for trace_session_id in ids {
|
||||||
|
if high_water.is_some_and(|high_water| trace_session_id > high_water) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
scanned += 1;
|
||||||
|
next_cursor = Some(trace_session_id);
|
||||||
|
let Some(index) = state.session_index.get(&trace_session_id) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let selected = refs
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|refs| refs.contains(&trace_session_id));
|
||||||
|
let Some(session) = selected
|
||||||
|
.then(|| state.sessions.get(&index.session_hash).cloned())
|
||||||
|
.flatten()
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if filter.is_some_and(|filter| !immutable_matches(&session, index, filter)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let candidate = WorkCandidate {
|
||||||
|
trace_session_id,
|
||||||
|
session,
|
||||||
|
bootstrap_hash: index.bootstrap_hash,
|
||||||
|
};
|
||||||
|
if filter.and_then(|filter| filter.state.as_ref()).is_some() {
|
||||||
|
state_filtered.push(candidate);
|
||||||
|
} else {
|
||||||
|
mark_close_locked(&mut state, &candidate);
|
||||||
|
direct.push(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if scanned == CLOSE_CHUNK {
|
||||||
|
reached_end = false;
|
||||||
|
}
|
||||||
|
(scanned, next_cursor, reached_end)
|
||||||
|
};
|
||||||
|
self.update_operation(sequence, |status| {
|
||||||
|
status.scanned = status.scanned.saturating_add(scanned);
|
||||||
|
status.matched = status.matched.saturating_add(direct.len());
|
||||||
|
});
|
||||||
|
for candidate in direct {
|
||||||
|
candidate.session.close();
|
||||||
|
self.update_operation(sequence, |status| {
|
||||||
|
status.close_signalled = status.close_signalled.saturating_add(1)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(expected_state) = filter.and_then(|filter| filter.state.as_deref()) {
|
||||||
|
for candidate in state_filtered {
|
||||||
|
let matches_state = candidate
|
||||||
|
.session
|
||||||
|
.try_status(Instant::now())
|
||||||
|
.is_some_and(|status| status.state == expected_state);
|
||||||
|
if !matches_state {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let session = {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let current = state
|
||||||
|
.session_index
|
||||||
|
.get(&candidate.trace_session_id)
|
||||||
|
.and_then(|index| state.sessions.get(&index.session_hash))
|
||||||
|
.filter(|current| Arc::ptr_eq(current, &candidate.session))
|
||||||
|
.cloned();
|
||||||
|
if current.is_some() {
|
||||||
|
mark_close_locked(&mut state, &candidate);
|
||||||
|
}
|
||||||
|
current
|
||||||
|
};
|
||||||
|
if let Some(session) = session {
|
||||||
|
session.close();
|
||||||
|
self.update_operation(sequence, |status| {
|
||||||
|
status.matched = status.matched.saturating_add(1);
|
||||||
|
status.close_signalled = status.close_signalled.saturating_add(1);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
self.update_operation(sequence, |status| {
|
||||||
|
status.matched = status.matched.saturating_add(1);
|
||||||
|
status.conflicted = status.conflicted.saturating_add(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = next_cursor;
|
||||||
|
if reached_end || cursor.is_none() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
self.finish_operation(sequence, ControlOperationState::Completed, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_operation(&self, sequence: u64, update: impl FnOnce(&mut ControlOperationStatus)) {
|
||||||
|
let mut operations = self.control_operations.lock();
|
||||||
|
if let Some((_, status)) = operations
|
||||||
|
.retained
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(id, _)| *id == sequence)
|
||||||
|
{
|
||||||
|
update(status);
|
||||||
|
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish_operation(
|
||||||
|
&self,
|
||||||
|
sequence: u64,
|
||||||
|
state: ControlOperationState,
|
||||||
|
failure: Option<&'static str>,
|
||||||
|
) {
|
||||||
|
let mut operations = self.control_operations.lock();
|
||||||
|
if let Some((_, status)) = operations
|
||||||
|
.retained
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(id, _)| *id == sequence)
|
||||||
|
{
|
||||||
|
status.state = failure.map_or(state, |_| ControlOperationState::Failed);
|
||||||
|
status.failure = failure;
|
||||||
|
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
|
||||||
|
}
|
||||||
|
if operations.active == Some(sequence) {
|
||||||
|
operations.active = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation_ref(&self, sequence: u64) -> String {
|
||||||
|
format!(
|
||||||
|
"{OPERATION_REF_VERSION}.{}.{sequence:016x}",
|
||||||
|
self.runtime_instance()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_operation_ref(&self, value: &str) -> Result<u64, ControlError> {
|
||||||
|
let mut parts = value.split('.');
|
||||||
|
if parts.next() != Some(OPERATION_REF_VERSION) {
|
||||||
|
return Err(ControlError::InvalidOperation);
|
||||||
|
}
|
||||||
|
let instance = parts.next().ok_or(ControlError::InvalidOperation)?;
|
||||||
|
if instance.len() != 32
|
||||||
|
|| !instance
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
{
|
||||||
|
return Err(ControlError::InvalidOperation);
|
||||||
|
}
|
||||||
|
let sequence = parts
|
||||||
|
.next()
|
||||||
|
.filter(|value| {
|
||||||
|
value.len() == 16
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
})
|
||||||
|
.and_then(|value| u64::from_str_radix(value, 16).ok())
|
||||||
|
.filter(|value| *value != 0)
|
||||||
|
.ok_or(ControlError::InvalidOperation)?;
|
||||||
|
if parts.next().is_some() {
|
||||||
|
return Err(ControlError::InvalidOperation);
|
||||||
|
}
|
||||||
|
if instance != self.runtime_instance() {
|
||||||
|
return Err(ControlError::StaleInstance);
|
||||||
|
}
|
||||||
|
Ok(sequence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mark_close_locked(state: &mut super::state::ManagerState, candidate: &WorkCandidate) {
|
||||||
|
if let Some(bootstrap) = state.bootstraps.get_mut(&candidate.bootstrap_hash) {
|
||||||
|
bootstrap.close_requested = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trim_operations(operations: &mut ControlOperationRegistry) {
|
||||||
|
while operations.retained.len() > OPERATION_RETENTION {
|
||||||
|
if operations
|
||||||
|
.retained
|
||||||
|
.front()
|
||||||
|
.is_some_and(|(id, _)| Some(*id) == operations.active)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
operations.retained.pop_front();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ use std::sync::Arc;
|
|||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
use zeroize::Zeroizing;
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
use super::state::{
|
use super::state::{
|
||||||
@@ -11,16 +12,50 @@ use super::state::{
|
|||||||
};
|
};
|
||||||
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
|
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
|
||||||
use crate::config::WebRuntimeProfile;
|
use crate::config::WebRuntimeProfile;
|
||||||
|
use crate::maestro::generation::RuntimeGeneration;
|
||||||
use crate::web::session::WebSession;
|
use crate::web::session::WebSession;
|
||||||
|
|
||||||
impl WebProcessRuntime {
|
impl WebProcessRuntime {
|
||||||
/// Issues a one-use bootstrap credential for an active compatible profile.
|
/// Issues a one-use bootstrap credential for an active compatible profile.
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn issue_bootstrap(
|
pub(crate) fn issue_bootstrap(
|
||||||
&self,
|
&self,
|
||||||
profile: Arc<WebRuntimeProfile>,
|
profile: Arc<WebRuntimeProfile>,
|
||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
let generation = self.active_generation();
|
let generation = self.active_generation();
|
||||||
|
self.issue_bootstrap_inner(&generation, profile, client_ip, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Issues one bootstrap against the generation that selected the bridge profile.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn issue_bootstrap_for_generation(
|
||||||
|
&self,
|
||||||
|
generation: &Arc<RuntimeGeneration>,
|
||||||
|
profile: Arc<WebRuntimeProfile>,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
|
self.issue_bootstrap_inner(generation, profile, client_ip, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Issues one bridge bootstrap with bounded non-secret request metadata.
|
||||||
|
pub(crate) fn issue_bootstrap_for_request(
|
||||||
|
&self,
|
||||||
|
generation: &Arc<RuntimeGeneration>,
|
||||||
|
profile: Arc<WebRuntimeProfile>,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
user_agent: Option<&str>,
|
||||||
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
|
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn issue_bootstrap_inner(
|
||||||
|
&self,
|
||||||
|
generation: &Arc<RuntimeGeneration>,
|
||||||
|
profile: Arc<WebRuntimeProfile>,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
user_agent: Option<&str>,
|
||||||
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
let config = generation.config();
|
let config = generation.config();
|
||||||
let profile = config
|
let profile = config
|
||||||
.web
|
.web
|
||||||
@@ -34,7 +69,9 @@ impl WebProcessRuntime {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
remove_expired_locked(&mut state, now);
|
remove_expired_locked(&mut state, now);
|
||||||
|
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||||
if state.closed
|
if state.closed
|
||||||
|
|| !state.issuance_enabled
|
||||||
|| state
|
|| state
|
||||||
.bootstraps_per_ip
|
.bootstraps_per_ip
|
||||||
.get(&client_ip)
|
.get(&client_ip)
|
||||||
@@ -57,11 +94,12 @@ impl WebProcessRuntime {
|
|||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
return Err(ManagerError::Limit);
|
return Err(ManagerError::Limit);
|
||||||
}
|
}
|
||||||
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
let Some((token, hash)) = new_unique_token(generation, &state) else {
|
||||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
return Err(ManagerError::Limit);
|
return Err(ManagerError::Limit);
|
||||||
};
|
};
|
||||||
let trace_session_id = self.trace.next_session_id();
|
let trace_session_id = self.trace.next_session_id();
|
||||||
|
let (user_agent, user_agent_id) = bounded_user_agent(user_agent);
|
||||||
state.bootstraps.insert(
|
state.bootstraps.insert(
|
||||||
hash,
|
hash,
|
||||||
Bootstrap {
|
Bootstrap {
|
||||||
@@ -69,7 +107,10 @@ impl WebProcessRuntime {
|
|||||||
issued_at: now,
|
issued_at: now,
|
||||||
issuance_ip: client_ip,
|
issuance_ip: client_ip,
|
||||||
profile,
|
profile,
|
||||||
|
timeouts: config.web.timeouts.clone(),
|
||||||
trace_session_id,
|
trace_session_id,
|
||||||
|
user_agent,
|
||||||
|
user_agent_id,
|
||||||
body_digest: [0; TOKEN_BYTES],
|
body_digest: [0; TOKEN_BYTES],
|
||||||
session_token: Zeroizing::new(String::new()),
|
session_token: Zeroizing::new(String::new()),
|
||||||
session: None,
|
session: None,
|
||||||
@@ -110,12 +151,12 @@ impl WebProcessRuntime {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolves bootstrap trace identity and the frozen live-session body timeout.
|
/// Resolves bootstrap trace identity and its issuance-frozen body timeout.
|
||||||
pub(crate) fn bootstrap_trace_identity(
|
pub(crate) fn bootstrap_trace_identity(
|
||||||
&self,
|
&self,
|
||||||
hash: TokenHash,
|
hash: TokenHash,
|
||||||
host: &str,
|
host: &str,
|
||||||
) -> Option<(u64, Arc<WebRuntimeProfile>, Option<Duration>)> {
|
) -> Option<(u64, Arc<WebRuntimeProfile>, Duration)> {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
self.state
|
self.state
|
||||||
.lock()
|
.lock()
|
||||||
@@ -126,10 +167,10 @@ impl WebProcessRuntime {
|
|||||||
(
|
(
|
||||||
entry.trace_session_id,
|
entry.trace_session_id,
|
||||||
Arc::clone(&entry.profile),
|
Arc::clone(&entry.profile),
|
||||||
entry
|
entry.session.as_ref().map_or_else(
|
||||||
.session
|
|| Duration::from_secs(entry.timeouts.body_secs),
|
||||||
.as_ref()
|
|session| Duration::from_secs(session.timeouts().body_secs),
|
||||||
.map(|session| Duration::from_secs(session.timeouts().body_secs)),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -185,3 +226,30 @@ impl WebProcessRuntime {
|
|||||||
closed.then_some(()).ok_or(ManagerError::Authentication)
|
closed.then_some(()).ok_or(ManagerError::Authentication)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bounded_user_agent(value: Option<&str>) -> (Option<Arc<str>>, Option<[u8; 16]>) {
|
||||||
|
const DISPLAY_BYTES: usize = 256;
|
||||||
|
const HASH_CONTEXT: &[u8] = b"telemt-web-user-agent-v1\0";
|
||||||
|
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||||
|
return (None, None);
|
||||||
|
};
|
||||||
|
let mut digest = Sha256::new();
|
||||||
|
digest.update(HASH_CONTEXT);
|
||||||
|
digest.update(value.as_bytes());
|
||||||
|
let digest = digest.finalize();
|
||||||
|
let mut id = [0; 16];
|
||||||
|
id.copy_from_slice(&digest[..16]);
|
||||||
|
let mut display = String::with_capacity(value.len().min(DISPLAY_BYTES));
|
||||||
|
for character in value.chars() {
|
||||||
|
let character = if character.is_control() {
|
||||||
|
'\u{fffd}'
|
||||||
|
} else {
|
||||||
|
character
|
||||||
|
};
|
||||||
|
if display.len().saturating_add(character.len_utf8()) > DISPLAY_BYTES {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
display.push(character);
|
||||||
|
}
|
||||||
|
(Some(Arc::from(display)), Some(id))
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ use tokio::time::Instant as TokioInstant;
|
|||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use super::state::{
|
use super::state::{
|
||||||
decrement_map, remember_closed_token_locked, remove_bootstrap_locked, remove_expired_locked,
|
decrement_map, remember_closed_session_locked, remember_closed_token_locked,
|
||||||
|
remove_bootstrap_locked, remove_expired_locked,
|
||||||
};
|
};
|
||||||
use super::{ProfileKey, TokenHash, WebProcessRuntime};
|
use super::{ProfileKey, TokenHash, WebProcessRuntime};
|
||||||
|
|
||||||
@@ -37,9 +38,9 @@ impl WebProcessRuntime {
|
|||||||
closed_token_lifetime: Duration,
|
closed_token_lifetime: Duration,
|
||||||
) {
|
) {
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
if state.sessions.remove(&hash).is_none() {
|
let Some(session) = state.sessions.remove(&hash) else {
|
||||||
return;
|
return;
|
||||||
}
|
};
|
||||||
decrement_map(&mut state.sessions_per_ip, &client_ip);
|
decrement_map(&mut state.sessions_per_ip, &client_ip);
|
||||||
decrement_map(&mut state.sessions_per_profile, &profile_key);
|
decrement_map(&mut state.sessions_per_profile, &profile_key);
|
||||||
remember_closed_token_locked(
|
remember_closed_token_locked(
|
||||||
@@ -49,6 +50,21 @@ impl WebProcessRuntime {
|
|||||||
closed_token_lifetime,
|
closed_token_lifetime,
|
||||||
self.limits.max_sessions_global.saturating_mul(16),
|
self.limits.max_sessions_global.saturating_mul(16),
|
||||||
);
|
);
|
||||||
|
let trace_session_id = session.trace_session_id();
|
||||||
|
if state
|
||||||
|
.session_index
|
||||||
|
.get(&trace_session_id)
|
||||||
|
.is_some_and(|index| index.session_hash == hash)
|
||||||
|
{
|
||||||
|
state.session_index.remove(&trace_session_id);
|
||||||
|
remember_closed_session_locked(
|
||||||
|
&mut state,
|
||||||
|
trace_session_id,
|
||||||
|
session.carrier_attempt(),
|
||||||
|
closed_token_lifetime,
|
||||||
|
self.limits.max_sessions_global,
|
||||||
|
);
|
||||||
|
}
|
||||||
let bootstrap_hashes = state
|
let bootstrap_hashes = state
|
||||||
.bootstraps
|
.bootstraps
|
||||||
.iter()
|
.iter()
|
||||||
@@ -70,6 +86,7 @@ impl WebProcessRuntime {
|
|||||||
pub(crate) fn begin_shutdown(self: &std::sync::Arc<Self>) -> WebShutdownDrain {
|
pub(crate) fn begin_shutdown(self: &std::sync::Arc<Self>) -> WebShutdownDrain {
|
||||||
let started = TokioInstant::now();
|
let started = TokioInstant::now();
|
||||||
self.shutdown.cancel();
|
self.shutdown.cancel();
|
||||||
|
self.close_control_submission_gate();
|
||||||
self.close_websockets();
|
self.close_websockets();
|
||||||
self.data_budget.close();
|
self.data_budget.close();
|
||||||
self.http_connections.close();
|
self.http_connections.close();
|
||||||
@@ -141,6 +158,7 @@ impl WebProcessRuntime {
|
|||||||
drop(learning);
|
drop(learning);
|
||||||
let (sessions, expired_chains) = {
|
let (sessions, expired_chains) = {
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
|
state.apply_issuance_policy(generation.id, config.enabled);
|
||||||
let expired = state
|
let expired = state
|
||||||
.bootstraps
|
.bootstraps
|
||||||
.iter()
|
.iter()
|
||||||
@@ -293,6 +311,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::maestro::generation::test_runtime_generation;
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::manager::{CloseOperationSelector, ControlError};
|
||||||
|
|
||||||
struct DropProbe {
|
struct DropProbe {
|
||||||
polls: Arc<AtomicUsize>,
|
polls: Arc<AtomicUsize>,
|
||||||
@@ -375,6 +394,44 @@ mod tests {
|
|||||||
generation.stop_background_tasks().await;
|
generation.stop_background_tasks().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn initial_trace_policy_is_attributed_to_active_generation() {
|
||||||
|
let (runtime, generation) = runtime();
|
||||||
|
|
||||||
|
assert_eq!(runtime.trace().status().policy_generation, generation.id);
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn shutdown_closes_the_control_submission_gate() {
|
||||||
|
let (runtime, generation) = runtime();
|
||||||
|
let drain = runtime.begin_shutdown();
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.start_close_operation(
|
||||||
|
runtime.runtime_instance(),
|
||||||
|
CloseOperationSelector::Refs(vec![1]),
|
||||||
|
),
|
||||||
|
Err(ControlError::Closed)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.reset_carrier_learning(),
|
||||||
|
Err(crate::web::manager::ManagerError::Closed)
|
||||||
|
));
|
||||||
|
assert!(matches!(runtime.clear_debug(), Err(ControlError::Closed)));
|
||||||
|
assert_eq!(
|
||||||
|
drain
|
||||||
|
.wait_until(TokioInstant::now() + Duration::from_secs(1))
|
||||||
|
.await,
|
||||||
|
WebShutdownOutcome::Drained
|
||||||
|
);
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn expired_deadline_still_closes_every_runtime_gate() {
|
async fn expired_deadline_still_closes_every_runtime_gate() {
|
||||||
let (runtime, generation) = runtime();
|
let (runtime, generation) = runtime();
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ use zeroize::Zeroizing;
|
|||||||
use super::negotiation::carrier_attempt_deadline_index;
|
use super::negotiation::carrier_attempt_deadline_index;
|
||||||
use super::session_admission::admit_initial;
|
use super::session_admission::admit_initial;
|
||||||
use super::state::{
|
use super::state::{
|
||||||
CarrierChainPhase, decrement_map, matching_profile, new_unique_token, profile_key,
|
CarrierChainPhase, LiveSessionIndex, decrement_map, matching_profile, new_unique_token,
|
||||||
remember_closed_token_locked, remove_expired_locked,
|
profile_key, remember_closed_token_locked, remove_expired_locked,
|
||||||
};
|
};
|
||||||
use super::{
|
use super::{
|
||||||
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
|
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
|
||||||
@@ -56,6 +56,10 @@ impl WebProcessRuntime {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
remove_expired_locked(&mut state, now);
|
remove_expired_locked(&mut state, now);
|
||||||
|
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||||
|
if state.closed || !state.issuance_enabled {
|
||||||
|
return Err(ManagerError::Closed);
|
||||||
|
}
|
||||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||||
return Err(ManagerError::Authentication);
|
return Err(ManagerError::Authentication);
|
||||||
};
|
};
|
||||||
@@ -201,9 +205,7 @@ impl WebProcessRuntime {
|
|||||||
}
|
}
|
||||||
let trace_session_id = entry.trace_session_id;
|
let trace_session_id = entry.trace_session_id;
|
||||||
let issued_profile = Arc::clone(&entry.profile);
|
let issued_profile = Arc::clone(&entry.profile);
|
||||||
if state.closed || !config.web.enabled {
|
let issued_timeouts = entry.timeouts.clone();
|
||||||
return Err(ManagerError::Closed);
|
|
||||||
}
|
|
||||||
let profile = config
|
let profile = config
|
||||||
.web
|
.web
|
||||||
.runtime
|
.runtime
|
||||||
@@ -303,33 +305,39 @@ impl WebProcessRuntime {
|
|||||||
learning_context,
|
learning_context,
|
||||||
carrier_request.is_automatic(),
|
carrier_request.is_automatic(),
|
||||||
self.limits.clone(),
|
self.limits.clone(),
|
||||||
config.web.timeouts.clone(),
|
issued_timeouts.clone(),
|
||||||
);
|
);
|
||||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||||
let entry = state
|
let (issuance_ip, candidate_count, user_agent, user_agent_id) = {
|
||||||
.bootstraps
|
let entry = state
|
||||||
.get_mut(&bootstrap_hash)
|
.bootstraps
|
||||||
.ok_or(ManagerError::Authentication)?;
|
.get_mut(&bootstrap_hash)
|
||||||
entry.used = true;
|
.ok_or(ManagerError::Authentication)?;
|
||||||
entry.body_digest = body_digest;
|
entry.used = true;
|
||||||
entry.session_token = Zeroizing::new(session_token.clone());
|
entry.body_digest = body_digest;
|
||||||
entry.session = Some(Arc::clone(&session));
|
entry.session_token = Zeroizing::new(session_token.clone());
|
||||||
entry.carrier_request = Some(carrier_request);
|
entry.session = Some(Arc::clone(&session));
|
||||||
entry.carrier_candidates = candidates.into();
|
entry.carrier_request = Some(carrier_request);
|
||||||
entry.carrier_scores = scores;
|
entry.carrier_candidates = candidates.into();
|
||||||
entry.carrier_attempt = 1;
|
entry.carrier_scores = scores;
|
||||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
entry.carrier_attempt = 1;
|
||||||
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||||
entry.carrier_deadline_at = carrier_deadline_at;
|
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
||||||
entry.carrier_failures = [None; 3];
|
entry.carrier_deadline_at = carrier_deadline_at;
|
||||||
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
entry.carrier_failures = [None; 3];
|
||||||
entry.expires_at = now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs);
|
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
||||||
entry.session_client_ip = Some(client_ip);
|
entry.expires_at = now + Duration::from_secs(issued_timeouts.bootstrap_lifetime_secs);
|
||||||
entry.session_ip_learning_eligible = ip_learning_eligible;
|
entry.session_client_ip = Some(client_ip);
|
||||||
let issuance_ip = entry.issuance_ip;
|
entry.session_ip_learning_eligible = ip_learning_eligible;
|
||||||
let candidate_count = u8::try_from(entry.carrier_candidates.len()).unwrap_or(4);
|
(
|
||||||
|
entry.issuance_ip,
|
||||||
|
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
|
||||||
|
entry.user_agent.clone(),
|
||||||
|
entry.user_agent_id,
|
||||||
|
)
|
||||||
|
};
|
||||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||||
let identity = session.trace_identity();
|
let identity = session.trace_identity();
|
||||||
@@ -345,6 +353,16 @@ impl WebProcessRuntime {
|
|||||||
.is_automatic()
|
.is_automatic()
|
||||||
.then_some(CarrierChainPhase::Provisional.as_str()),
|
.then_some(CarrierChainPhase::Provisional.as_str()),
|
||||||
};
|
};
|
||||||
|
state.session_index.insert(
|
||||||
|
trace_session_id,
|
||||||
|
LiveSessionIndex {
|
||||||
|
session_hash,
|
||||||
|
bootstrap_hash,
|
||||||
|
attempt: 1,
|
||||||
|
user_agent,
|
||||||
|
user_agent_id,
|
||||||
|
},
|
||||||
|
);
|
||||||
drop(state);
|
drop(state);
|
||||||
self.trace.record_carrier_lifecycle(
|
self.trace.record_carrier_lifecycle(
|
||||||
client_ip,
|
client_ip,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ impl WebProcessRuntime {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
remove_expired_locked(&mut state, now);
|
remove_expired_locked(&mut state, now);
|
||||||
|
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||||
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
||||||
entry.carrier_transitioning
|
entry.carrier_transitioning
|
||||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||||
@@ -37,7 +38,7 @@ impl WebProcessRuntime {
|
|||||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
|
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
|
||||||
if !valid
|
if !valid
|
||||||
|| state.closed
|
|| state.closed
|
||||||
|| !config.web.enabled
|
|| !state.issuance_enabled
|
||||||
|| !generation
|
|| !generation
|
||||||
.proxy_shared
|
.proxy_shared
|
||||||
.is_user_enabled(&replacement.profile.user)
|
.is_user_enabled(&replacement.profile.user)
|
||||||
@@ -121,6 +122,13 @@ impl WebProcessRuntime {
|
|||||||
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||||
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
||||||
};
|
};
|
||||||
|
if let Some(index) = state.session_index.get_mut(&replacement.trace_session_id)
|
||||||
|
&& index.session_hash == old_hash
|
||||||
|
{
|
||||||
|
index.session_hash = session_hash;
|
||||||
|
index.bootstrap_hash = bootstrap_hash;
|
||||||
|
index.attempt = replacement.attempt;
|
||||||
|
}
|
||||||
let identity = session.trace_identity();
|
let identity = session.trace_identity();
|
||||||
let old_identity = replacement.old_session.trace_identity();
|
let old_identity = replacement.old_session.trace_identity();
|
||||||
drop(state);
|
drop(state);
|
||||||
|
|||||||
+113
-3
@@ -1,4 +1,4 @@
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
|
||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
@@ -8,7 +8,7 @@ use sha2::{Digest, Sha256};
|
|||||||
use zeroize::Zeroizing;
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
|
use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
|
||||||
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile};
|
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile, WebTimeoutsConfig};
|
||||||
use crate::maestro::generation::RuntimeGeneration;
|
use crate::maestro::generation::RuntimeGeneration;
|
||||||
use crate::web::session::WebSession;
|
use crate::web::session::WebSession;
|
||||||
|
|
||||||
@@ -41,8 +41,14 @@ pub(super) struct Bootstrap {
|
|||||||
pub(super) issuance_ip: IpAddr,
|
pub(super) issuance_ip: IpAddr,
|
||||||
/// Immutable profile selected during capability validation.
|
/// Immutable profile selected during capability validation.
|
||||||
pub(super) profile: Arc<WebRuntimeProfile>,
|
pub(super) profile: Arc<WebRuntimeProfile>,
|
||||||
|
/// Request and session deadlines frozen with the generated bridge.
|
||||||
|
pub(super) timeouts: WebTimeoutsConfig,
|
||||||
/// Process-unique non-secret identifier shared by bootstrap and session traces.
|
/// Process-unique non-secret identifier shared by bootstrap and session traces.
|
||||||
pub(super) trace_session_id: u64,
|
pub(super) trace_session_id: u64,
|
||||||
|
/// Bounded display form of the issuing User-Agent.
|
||||||
|
pub(super) user_agent: Option<Arc<str>>,
|
||||||
|
/// Opaque non-secret identifier used for exact User-Agent filtering.
|
||||||
|
pub(super) user_agent_id: Option<[u8; 16]>,
|
||||||
/// Digest of the accepted HELLO body for idempotent retry matching.
|
/// Digest of the accepted HELLO body for idempotent retry matching.
|
||||||
pub(super) body_digest: TokenHash,
|
pub(super) body_digest: TokenHash,
|
||||||
/// Zeroizing copy returned only for an exact session-creation retry.
|
/// Zeroizing copy returned only for an exact session-creation retry.
|
||||||
@@ -87,6 +93,28 @@ pub(super) struct ClosedToken {
|
|||||||
pub(super) host: String,
|
pub(super) host: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Current logical-session owner stored without exposing bearer credentials.
|
||||||
|
pub(super) struct LiveSessionIndex {
|
||||||
|
/// Current bearer hash used only for internal pointer revalidation.
|
||||||
|
pub(super) session_hash: TokenHash,
|
||||||
|
/// Bootstrap chain that owns carrier replacement and close intent.
|
||||||
|
pub(super) bootstrap_hash: TokenHash,
|
||||||
|
/// Current carrier incarnation in the logical trace session.
|
||||||
|
pub(super) attempt: u8,
|
||||||
|
/// Bounded display form of the issuing User-Agent.
|
||||||
|
pub(super) user_agent: Option<Arc<str>>,
|
||||||
|
/// Opaque non-secret identifier used for exact User-Agent filtering.
|
||||||
|
pub(super) user_agent_id: Option<[u8; 16]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded logical-session tombstone used for exact detail semantics.
|
||||||
|
pub(super) struct ClosedSession {
|
||||||
|
/// Tombstone expiry deadline.
|
||||||
|
pub(super) expires_at: Instant,
|
||||||
|
/// Last carrier incarnation closed for this logical session.
|
||||||
|
pub(super) attempt: u8,
|
||||||
|
}
|
||||||
|
|
||||||
/// Token-bucket state for one process-wide creation class.
|
/// Token-bucket state for one process-wide creation class.
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub(super) struct RateState {
|
pub(super) struct RateState {
|
||||||
@@ -114,7 +142,6 @@ pub(super) struct StreamAdmissionState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Process-wide WEB registries and quota accounting protected by one short lock.
|
/// Process-wide WEB registries and quota accounting protected by one short lock.
|
||||||
#[derive(Default)]
|
|
||||||
pub(super) struct ManagerState {
|
pub(super) struct ManagerState {
|
||||||
/// Bootstrap credentials indexed by their SHA-256 token hash.
|
/// Bootstrap credentials indexed by their SHA-256 token hash.
|
||||||
pub(super) bootstraps: HashMap<TokenHash, Bootstrap>,
|
pub(super) bootstraps: HashMap<TokenHash, Bootstrap>,
|
||||||
@@ -122,6 +149,12 @@ pub(super) struct ManagerState {
|
|||||||
pub(super) bootstraps_per_ip: HashMap<IpAddr, usize>,
|
pub(super) bootstraps_per_ip: HashMap<IpAddr, usize>,
|
||||||
/// Live sessions indexed by bearer-token hash.
|
/// Live sessions indexed by bearer-token hash.
|
||||||
pub(super) sessions: HashMap<TokenHash, Arc<WebSession>>,
|
pub(super) sessions: HashMap<TokenHash, Arc<WebSession>>,
|
||||||
|
/// Stable ordered logical-session lookup independent from bearer hashes.
|
||||||
|
pub(super) session_index: BTreeMap<u64, LiveSessionIndex>,
|
||||||
|
/// Recently closed logical sessions retained for exact detail responses.
|
||||||
|
pub(super) closed_sessions: HashMap<u64, ClosedSession>,
|
||||||
|
/// Insertion order for bounded logical-session tombstones.
|
||||||
|
pub(super) closed_session_order: VecDeque<u64>,
|
||||||
/// Recently closed token hashes retained for idempotent DELETE semantics.
|
/// Recently closed token hashes retained for idempotent DELETE semantics.
|
||||||
pub(super) closed_tokens: HashMap<TokenHash, ClosedToken>,
|
pub(super) closed_tokens: HashMap<TokenHash, ClosedToken>,
|
||||||
/// Live session counts by forwarded client address.
|
/// Live session counts by forwarded client address.
|
||||||
@@ -132,10 +165,48 @@ pub(super) struct ManagerState {
|
|||||||
pub(super) bootstrap_rate: RateState,
|
pub(super) bootstrap_rate: RateState,
|
||||||
/// Session creation rate limiter.
|
/// Session creation rate limiter.
|
||||||
pub(super) session_rate: RateState,
|
pub(super) session_rate: RateState,
|
||||||
|
/// Generation-fenced issuance gate mirrored from the effective WEB policy.
|
||||||
|
pub(super) issuance_enabled: bool,
|
||||||
|
/// Generation that last authored `issuance_enabled`.
|
||||||
|
pub(super) issuance_generation: u64,
|
||||||
/// Process shutdown admission latch.
|
/// Process shutdown admission latch.
|
||||||
pub(super) closed: bool,
|
pub(super) closed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ManagerState {
|
||||||
|
pub(super) fn new(issuance_generation: u64, issuance_enabled: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
bootstraps: HashMap::new(),
|
||||||
|
bootstraps_per_ip: HashMap::new(),
|
||||||
|
sessions: HashMap::new(),
|
||||||
|
session_index: BTreeMap::new(),
|
||||||
|
closed_sessions: HashMap::new(),
|
||||||
|
closed_session_order: VecDeque::new(),
|
||||||
|
closed_tokens: HashMap::new(),
|
||||||
|
sessions_per_ip: HashMap::new(),
|
||||||
|
sessions_per_profile: HashMap::new(),
|
||||||
|
bootstrap_rate: RateState::default(),
|
||||||
|
session_rate: RateState::default(),
|
||||||
|
issuance_enabled,
|
||||||
|
issuance_generation,
|
||||||
|
closed: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn apply_issuance_policy(&mut self, generation: u64, enabled: bool) {
|
||||||
|
if generation >= self.issuance_generation {
|
||||||
|
self.issuance_generation = generation;
|
||||||
|
self.issuance_enabled = enabled;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ManagerState {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(0, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Generates one collision-checked credential and its stable hash key.
|
/// Generates one collision-checked credential and its stable hash key.
|
||||||
pub(super) fn new_unique_token(
|
pub(super) fn new_unique_token(
|
||||||
generation: &RuntimeGeneration,
|
generation: &RuntimeGeneration,
|
||||||
@@ -241,6 +312,16 @@ pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) {
|
|||||||
state
|
state
|
||||||
.closed_tokens
|
.closed_tokens
|
||||||
.retain(|_, closed| now <= closed.expires_at);
|
.retain(|_, closed| now <= closed.expires_at);
|
||||||
|
while state
|
||||||
|
.closed_session_order
|
||||||
|
.front()
|
||||||
|
.and_then(|trace_session_id| state.closed_sessions.get(trace_session_id))
|
||||||
|
.is_some_and(|closed| now > closed.expires_at)
|
||||||
|
{
|
||||||
|
if let Some(trace_session_id) = state.closed_session_order.pop_front() {
|
||||||
|
state.closed_sessions.remove(&trace_session_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes one bootstrap and releases its per-address issuance quota when unused.
|
/// Removes one bootstrap and releases its per-address issuance quota when unused.
|
||||||
@@ -281,6 +362,35 @@ pub(super) fn remember_closed_token_locked(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retains one bounded logical-session marker without storing its bearer identity.
|
||||||
|
pub(super) fn remember_closed_session_locked(
|
||||||
|
state: &mut ManagerState,
|
||||||
|
trace_session_id: u64,
|
||||||
|
attempt: u8,
|
||||||
|
lifetime: Duration,
|
||||||
|
capacity: usize,
|
||||||
|
) {
|
||||||
|
if state
|
||||||
|
.closed_sessions
|
||||||
|
.insert(
|
||||||
|
trace_session_id,
|
||||||
|
ClosedSession {
|
||||||
|
expires_at: Instant::now() + lifetime,
|
||||||
|
attempt,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
state.closed_session_order.push_back(trace_session_id);
|
||||||
|
}
|
||||||
|
while state.closed_sessions.len() > capacity {
|
||||||
|
let Some(oldest) = state.closed_session_order.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
state.closed_sessions.remove(&oldest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Decrements one counted owner and removes its map entry at zero.
|
/// Decrements one counted owner and removes its map entry at zero.
|
||||||
pub(super) fn decrement_map<K, Q>(values: &mut HashMap<K, usize>, key: &Q)
|
pub(super) fn decrement_map<K, Q>(values: &mut HashMap<K, usize>, key: &Q)
|
||||||
where
|
where
|
||||||
|
|||||||
@@ -0,0 +1,560 @@
|
|||||||
|
use std::net::IpAddr;
|
||||||
|
use std::ops::Bound::{Excluded, Unbounded};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::WebProcessRuntime;
|
||||||
|
use crate::config::{
|
||||||
|
WebCarrier, WebCarrierNegotiationAggressiveness, WebDebugConfig, WebLimitsConfig,
|
||||||
|
};
|
||||||
|
use crate::web::session::WebSessionStatus;
|
||||||
|
|
||||||
|
const SESSION_REF_VERSION: &str = "ws1";
|
||||||
|
const MAX_SESSION_SCAN: usize = 1000;
|
||||||
|
|
||||||
|
/// Current usage of one process-owned semaphore.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct PermitStatus {
|
||||||
|
used: usize,
|
||||||
|
available: usize,
|
||||||
|
capacity: usize,
|
||||||
|
closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Short-lock manager registry counts.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct ManagerStatus {
|
||||||
|
issuance_enabled: bool,
|
||||||
|
issuance_generation: u64,
|
||||||
|
shutdown: bool,
|
||||||
|
bootstraps: usize,
|
||||||
|
sessions: usize,
|
||||||
|
closed_tokens: usize,
|
||||||
|
closed_sessions: usize,
|
||||||
|
client_ips: usize,
|
||||||
|
profiles: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Logical-stream admission counters.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct StreamStatus {
|
||||||
|
live: usize,
|
||||||
|
profiles: usize,
|
||||||
|
closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared queue and WebSocket byte-budget counters.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct BudgetStatus {
|
||||||
|
queue_bytes: usize,
|
||||||
|
queue_items: usize,
|
||||||
|
control_bytes: usize,
|
||||||
|
control_items: usize,
|
||||||
|
websocket_bytes: usize,
|
||||||
|
high_water_bytes: usize,
|
||||||
|
owners: usize,
|
||||||
|
closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process WebSocket registry counters.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct WebSocketStatus {
|
||||||
|
entries: usize,
|
||||||
|
claims: usize,
|
||||||
|
evictions_in_flight: usize,
|
||||||
|
closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process-local carrier-learning summary.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct LearningStatus {
|
||||||
|
enabled: bool,
|
||||||
|
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||||
|
epoch: Option<u64>,
|
||||||
|
entries: usize,
|
||||||
|
capacity: usize,
|
||||||
|
lifetime_secs: u64,
|
||||||
|
age_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Effective trace policy and bounded ring counters.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct DebugStatus {
|
||||||
|
policy: WebDebugConfig,
|
||||||
|
policy_generation: u64,
|
||||||
|
epoch: u64,
|
||||||
|
records: usize,
|
||||||
|
records_capacity: usize,
|
||||||
|
used_bytes: usize,
|
||||||
|
bytes_capacity: usize,
|
||||||
|
contention_drops: u64,
|
||||||
|
evictions: u64,
|
||||||
|
byte_truncations: u64,
|
||||||
|
earliest_seq: Option<u64>,
|
||||||
|
latest_seq: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One non-blocking multi-plane WEB runtime snapshot.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(crate) struct WebRuntimeStatus {
|
||||||
|
runtime_instance: String,
|
||||||
|
generation_id: u64,
|
||||||
|
limits: WebLimitsConfig,
|
||||||
|
manager: Option<ManagerStatus>,
|
||||||
|
streams: Option<StreamStatus>,
|
||||||
|
budget: Option<BudgetStatus>,
|
||||||
|
websockets: Option<WebSocketStatus>,
|
||||||
|
learning: Option<LearningStatus>,
|
||||||
|
debug: Option<DebugStatus>,
|
||||||
|
permits: Vec<(&'static str, PermitStatus)>,
|
||||||
|
auxiliary_tasks: usize,
|
||||||
|
session_incarnations_created: u64,
|
||||||
|
session_incarnations_closed: u64,
|
||||||
|
streams_opened: u64,
|
||||||
|
streams_rejected: u64,
|
||||||
|
bytes_up: u64,
|
||||||
|
bytes_down: u64,
|
||||||
|
limit_hits: u64,
|
||||||
|
partial: Vec<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strict bounded filters for session enumeration and bulk close.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
pub(crate) struct SessionFilter {
|
||||||
|
/// Exact process-local logical session identifier.
|
||||||
|
pub(crate) trace_session_id: Option<u64>,
|
||||||
|
/// Exact forwarded client address.
|
||||||
|
pub(crate) client_ip: Option<IpAddr>,
|
||||||
|
/// Exact canonical virtual host.
|
||||||
|
pub(crate) host: Option<String>,
|
||||||
|
/// Exact configured user label.
|
||||||
|
pub(crate) user: Option<String>,
|
||||||
|
/// Exact non-secret User-Agent identifier.
|
||||||
|
pub(crate) user_agent_id: Option<[u8; 16]>,
|
||||||
|
/// Exact non-secret profile-key fingerprint.
|
||||||
|
pub(crate) key_id: Option<String>,
|
||||||
|
/// Exact current carrier.
|
||||||
|
pub(crate) carrier: Option<WebCarrier>,
|
||||||
|
/// Exact point-in-time lifecycle token.
|
||||||
|
pub(crate) state: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SessionFilter {
|
||||||
|
/// Returns whether the selector would match every live session.
|
||||||
|
pub(crate) fn is_empty(&self) -> bool {
|
||||||
|
self.trace_session_id.is_none()
|
||||||
|
&& self.client_ip.is_none()
|
||||||
|
&& self.host.is_none()
|
||||||
|
&& self.user.is_none()
|
||||||
|
&& self.user_agent_id.is_none()
|
||||||
|
&& self.key_id.is_none()
|
||||||
|
&& self.carrier.is_none()
|
||||||
|
&& self.state.is_none()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validated bounded list request.
|
||||||
|
pub(crate) struct SessionListRequest {
|
||||||
|
/// Maximum returned rows.
|
||||||
|
pub(crate) limit: usize,
|
||||||
|
/// Exclusive ordered logical-session cursor.
|
||||||
|
pub(crate) cursor: Option<u64>,
|
||||||
|
/// Exact bounded filters.
|
||||||
|
pub(crate) filter: SessionFilter,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One API-safe session row with optional User-Agent metadata.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(crate) struct SessionRow {
|
||||||
|
/// Opaque process-fenced logical-session reference.
|
||||||
|
pub(crate) session_ref: String,
|
||||||
|
/// Bounded sanitized User-Agent display value.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(crate) user_agent: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
user_agent_id: Option<String>,
|
||||||
|
#[serde(flatten)]
|
||||||
|
status: WebSessionStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded session page and continuation metadata.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(crate) struct SessionPage {
|
||||||
|
/// Ordered live-session rows captured without blocking.
|
||||||
|
pub(crate) sessions: Vec<SessionRow>,
|
||||||
|
next_cursor: Option<String>,
|
||||||
|
scanned: usize,
|
||||||
|
scan_truncated: bool,
|
||||||
|
partial_sessions: usize,
|
||||||
|
partial: Vec<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exact detail lookup outcome.
|
||||||
|
pub(crate) enum SessionDetail {
|
||||||
|
/// One exact live-session snapshot.
|
||||||
|
Active(Box<SessionRow>),
|
||||||
|
/// One bounded retained closed-session tombstone.
|
||||||
|
Gone { attempt: u8 },
|
||||||
|
/// A required short lock was contended.
|
||||||
|
Busy,
|
||||||
|
/// Neither a live session nor a retained tombstone exists.
|
||||||
|
NotFound,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Candidate {
|
||||||
|
trace_session_id: u64,
|
||||||
|
session: Arc<crate::web::session::WebSession>,
|
||||||
|
user_agent: Option<Arc<str>>,
|
||||||
|
user_agent_id: Option<[u8; 16]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebProcessRuntime {
|
||||||
|
/// Captures every independent plane without blocking on a contended lock.
|
||||||
|
pub(crate) fn try_status(&self) -> WebRuntimeStatus {
|
||||||
|
let generation_id = self.active_generation().id;
|
||||||
|
let mut partial = Vec::new();
|
||||||
|
let manager = self.state.try_lock().map(|state| ManagerStatus {
|
||||||
|
issuance_enabled: state.issuance_enabled,
|
||||||
|
issuance_generation: state.issuance_generation,
|
||||||
|
shutdown: state.closed,
|
||||||
|
bootstraps: state.bootstraps.len(),
|
||||||
|
sessions: state.sessions.len(),
|
||||||
|
closed_tokens: state.closed_tokens.len(),
|
||||||
|
closed_sessions: state.closed_sessions.len(),
|
||||||
|
client_ips: state.sessions_per_ip.len(),
|
||||||
|
profiles: state.sessions_per_profile.len(),
|
||||||
|
});
|
||||||
|
if manager.is_none() {
|
||||||
|
partial.push("manager");
|
||||||
|
}
|
||||||
|
let streams = self.stream_admission.try_lock().map(|state| StreamStatus {
|
||||||
|
live: state.streams_live,
|
||||||
|
profiles: state.streams_per_profile.len(),
|
||||||
|
closed: state.closed,
|
||||||
|
});
|
||||||
|
if streams.is_none() {
|
||||||
|
partial.push("streams");
|
||||||
|
}
|
||||||
|
let budget = self.data_budget.try_snapshot().map(|status| BudgetStatus {
|
||||||
|
queue_bytes: status.queue_bytes,
|
||||||
|
queue_items: status.queue_items,
|
||||||
|
control_bytes: status.queue_control_bytes,
|
||||||
|
control_items: status.queue_control_items,
|
||||||
|
websocket_bytes: status.websocket_bytes,
|
||||||
|
high_water_bytes: status.high_water_bytes,
|
||||||
|
owners: status.owners,
|
||||||
|
closed: status.closed,
|
||||||
|
});
|
||||||
|
if budget.is_none() {
|
||||||
|
partial.push("budget");
|
||||||
|
}
|
||||||
|
let websockets = self.websockets.try_lock().map(|registry| {
|
||||||
|
let status = registry.status();
|
||||||
|
WebSocketStatus {
|
||||||
|
entries: status.entries,
|
||||||
|
claims: status.claims,
|
||||||
|
evictions_in_flight: status.evictions_in_flight,
|
||||||
|
closed: status.closed,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if websockets.is_none() {
|
||||||
|
partial.push("websockets");
|
||||||
|
}
|
||||||
|
let learning = self
|
||||||
|
.try_carrier_learning_status()
|
||||||
|
.map(|status| LearningStatus {
|
||||||
|
enabled: status.enabled,
|
||||||
|
aggressiveness: status.aggressiveness,
|
||||||
|
epoch: status.epoch,
|
||||||
|
entries: status.entries,
|
||||||
|
capacity: status.capacity,
|
||||||
|
lifetime_secs: status.lifetime_secs,
|
||||||
|
age_ms: status.age_ms,
|
||||||
|
});
|
||||||
|
if learning.is_none() {
|
||||||
|
partial.push("learning");
|
||||||
|
}
|
||||||
|
let debug = self.trace.try_status().map(|status| DebugStatus {
|
||||||
|
policy: status.policy.as_ref().clone(),
|
||||||
|
policy_generation: status.policy_generation,
|
||||||
|
epoch: status.epoch,
|
||||||
|
records: status.records,
|
||||||
|
records_capacity: status.records_capacity,
|
||||||
|
used_bytes: status.used_bytes,
|
||||||
|
bytes_capacity: status.bytes_capacity,
|
||||||
|
contention_drops: status.contention_drops,
|
||||||
|
evictions: status.evictions,
|
||||||
|
byte_truncations: status.byte_truncations,
|
||||||
|
earliest_seq: status.earliest_seq,
|
||||||
|
latest_seq: status.latest_seq,
|
||||||
|
});
|
||||||
|
if debug.is_none() {
|
||||||
|
partial.push("debug");
|
||||||
|
}
|
||||||
|
let websocket_capacity = self
|
||||||
|
.limits
|
||||||
|
.max_http_connections
|
||||||
|
.saturating_sub(self.limits.websocket_http_connection_reserve);
|
||||||
|
WebRuntimeStatus {
|
||||||
|
runtime_instance: self.runtime_instance().to_string(),
|
||||||
|
generation_id,
|
||||||
|
limits: self.limits.clone(),
|
||||||
|
manager,
|
||||||
|
streams,
|
||||||
|
budget,
|
||||||
|
websockets,
|
||||||
|
learning,
|
||||||
|
debug,
|
||||||
|
permits: vec![
|
||||||
|
(
|
||||||
|
"http_connections",
|
||||||
|
permits(&self.http_connections, self.limits.max_http_connections),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"http_handlers",
|
||||||
|
permits(&self.http_handlers, self.limits.max_http_handlers),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"lane_polls",
|
||||||
|
permits(&self.lane_polls, self.limits.max_http_handlers / 2),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"lane_aux_polls",
|
||||||
|
permits(
|
||||||
|
&self.lane_aux_polls,
|
||||||
|
(self.limits.max_http_handlers / 4).max(1),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"body_readers",
|
||||||
|
permits(&self.body_readers, self.limits.max_body_readers),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"body_bytes",
|
||||||
|
permits(&self.body_bytes, self.limits.max_body_bytes_global),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"stream_handshakes",
|
||||||
|
permits(&self.stream_handshakes, self.limits.max_stream_handshakes),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"websocket_connections",
|
||||||
|
permits(&self.websocket_connections, websocket_capacity),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
auxiliary_tasks: self.tasks.len(),
|
||||||
|
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
|
||||||
|
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
|
||||||
|
streams_opened: self.streams_opened.load(Ordering::Relaxed),
|
||||||
|
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
|
||||||
|
bytes_up: self.bytes_up.load(Ordering::Relaxed),
|
||||||
|
bytes_down: self.bytes_down.load(Ordering::Relaxed),
|
||||||
|
limit_hits: self.limit_hits.load(Ordering::Relaxed),
|
||||||
|
partial,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Formats an opaque process-fenced session reference.
|
||||||
|
pub(crate) fn session_ref(&self, trace_session_id: u64) -> String {
|
||||||
|
format!(
|
||||||
|
"{SESSION_REF_VERSION}.{}.{trace_session_id:016x}",
|
||||||
|
self.runtime_instance()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses an exact reference and distinguishes stale process instances.
|
||||||
|
pub(crate) fn parse_session_ref(&self, value: &str) -> Result<u64, SessionRefError> {
|
||||||
|
let mut parts = value.split('.');
|
||||||
|
let version = parts.next();
|
||||||
|
let instance = parts.next();
|
||||||
|
let id = parts.next();
|
||||||
|
if version != Some(SESSION_REF_VERSION) || parts.next().is_some() {
|
||||||
|
return Err(SessionRefError::Invalid);
|
||||||
|
}
|
||||||
|
if !instance.is_some_and(|instance| {
|
||||||
|
instance.len() == 32
|
||||||
|
&& instance
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}) {
|
||||||
|
return Err(SessionRefError::Invalid);
|
||||||
|
}
|
||||||
|
let id = id
|
||||||
|
.filter(|id| {
|
||||||
|
id.len() == 16
|
||||||
|
&& id
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
})
|
||||||
|
.and_then(|id| u64::from_str_radix(id, 16).ok())
|
||||||
|
.filter(|id| *id != 0)
|
||||||
|
.ok_or(SessionRefError::Invalid)?;
|
||||||
|
if instance != Some(self.runtime_instance()) {
|
||||||
|
return Err(SessionRefError::StaleInstance);
|
||||||
|
}
|
||||||
|
Ok(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lists live sessions under an ordered bounded scan.
|
||||||
|
pub(crate) fn list_sessions(&self, request: SessionListRequest) -> SessionPage {
|
||||||
|
let Some(state) = self.state.try_lock() else {
|
||||||
|
return SessionPage {
|
||||||
|
sessions: Vec::new(),
|
||||||
|
next_cursor: request.cursor.map(|id| self.session_ref(id)),
|
||||||
|
scanned: 0,
|
||||||
|
scan_truncated: false,
|
||||||
|
partial_sessions: 0,
|
||||||
|
partial: vec!["manager"],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
let mut candidates = Vec::with_capacity(request.limit);
|
||||||
|
let mut scanned = 0usize;
|
||||||
|
let mut last_scanned = request.cursor;
|
||||||
|
for (&trace_session_id, index) in state
|
||||||
|
.session_index
|
||||||
|
.range((request.cursor.map_or(Unbounded, Excluded), Unbounded))
|
||||||
|
{
|
||||||
|
if scanned >= MAX_SESSION_SCAN || candidates.len() >= request.limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
scanned += 1;
|
||||||
|
last_scanned = Some(trace_session_id);
|
||||||
|
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !immutable_matches(&session, index, &request.filter) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
candidates.push(Candidate {
|
||||||
|
trace_session_id,
|
||||||
|
session,
|
||||||
|
user_agent: index.user_agent.clone(),
|
||||||
|
user_agent_id: index.user_agent_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
drop(state);
|
||||||
|
let mut rows = Vec::with_capacity(candidates.len());
|
||||||
|
let mut partial_sessions = 0usize;
|
||||||
|
let now = Instant::now();
|
||||||
|
for candidate in candidates {
|
||||||
|
let Some(status) = candidate.session.try_status(now) else {
|
||||||
|
partial_sessions += 1;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if request
|
||||||
|
.filter
|
||||||
|
.state
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|expected| expected != status.state)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rows.push(self.row(candidate, status));
|
||||||
|
}
|
||||||
|
let scan_truncated = scanned >= MAX_SESSION_SCAN;
|
||||||
|
SessionPage {
|
||||||
|
sessions: rows,
|
||||||
|
next_cursor: (scan_truncated || scanned >= request.limit)
|
||||||
|
.then(|| last_scanned.map(|id| self.session_ref(id)))
|
||||||
|
.flatten(),
|
||||||
|
scanned,
|
||||||
|
scan_truncated,
|
||||||
|
partial_sessions,
|
||||||
|
partial: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves one active or recently closed logical session.
|
||||||
|
pub(crate) fn session_detail(&self, trace_session_id: u64) -> SessionDetail {
|
||||||
|
let Some(state) = self.state.try_lock() else {
|
||||||
|
return SessionDetail::Busy;
|
||||||
|
};
|
||||||
|
if let Some(index) = state.session_index.get(&trace_session_id) {
|
||||||
|
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
|
||||||
|
return SessionDetail::Busy;
|
||||||
|
};
|
||||||
|
let candidate = Candidate {
|
||||||
|
trace_session_id,
|
||||||
|
session,
|
||||||
|
user_agent: index.user_agent.clone(),
|
||||||
|
user_agent_id: index.user_agent_id,
|
||||||
|
};
|
||||||
|
drop(state);
|
||||||
|
return candidate
|
||||||
|
.session
|
||||||
|
.try_status(Instant::now())
|
||||||
|
.map(|status| SessionDetail::Active(Box::new(self.row(candidate, status))))
|
||||||
|
.unwrap_or(SessionDetail::Busy);
|
||||||
|
}
|
||||||
|
let closed = state
|
||||||
|
.closed_sessions
|
||||||
|
.get(&trace_session_id)
|
||||||
|
.map(|closed| closed.attempt);
|
||||||
|
closed.map_or(SessionDetail::NotFound, |attempt| SessionDetail::Gone {
|
||||||
|
attempt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row(&self, candidate: Candidate, status: WebSessionStatus) -> SessionRow {
|
||||||
|
SessionRow {
|
||||||
|
session_ref: self.session_ref(candidate.trace_session_id),
|
||||||
|
user_agent: candidate.user_agent.map(|value| value.to_string()),
|
||||||
|
user_agent_id: candidate.user_agent_id.map(hex::encode),
|
||||||
|
status,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opaque session-reference validation failure.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) enum SessionRefError {
|
||||||
|
/// The reference does not use the canonical versioned shape.
|
||||||
|
Invalid,
|
||||||
|
/// The reference belongs to another process runtime.
|
||||||
|
StaleInstance,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tests immutable candidate fields before any optional state-lock read.
|
||||||
|
pub(super) fn immutable_matches(
|
||||||
|
session: &crate::web::session::WebSession,
|
||||||
|
index: &super::state::LiveSessionIndex,
|
||||||
|
filter: &SessionFilter,
|
||||||
|
) -> bool {
|
||||||
|
filter
|
||||||
|
.trace_session_id
|
||||||
|
.is_none_or(|value| value == session.trace_session_id())
|
||||||
|
&& filter
|
||||||
|
.client_ip
|
||||||
|
.is_none_or(|value| value == session.client_ip())
|
||||||
|
&& filter
|
||||||
|
.host
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| value == session.profile_host())
|
||||||
|
&& filter
|
||||||
|
.user
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| value == session.profile_user())
|
||||||
|
&& filter
|
||||||
|
.key_id
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| session.key_id() == value)
|
||||||
|
&& filter
|
||||||
|
.carrier
|
||||||
|
.is_none_or(|value| value == session.carrier())
|
||||||
|
&& filter
|
||||||
|
.user_agent_id
|
||||||
|
.is_none_or(|value| index.user_agent_id == Some(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn permits(semaphore: &Arc<tokio::sync::Semaphore>, capacity: usize) -> PermitStatus {
|
||||||
|
let available = semaphore.available_permits().min(capacity);
|
||||||
|
PermitStatus {
|
||||||
|
used: capacity.saturating_sub(available),
|
||||||
|
available,
|
||||||
|
capacity,
|
||||||
|
closed: semaphore.is_closed(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -61,6 +61,26 @@ pub(super) struct WebSocketRegistry {
|
|||||||
closed: bool,
|
closed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Point-in-time WebSocket registry counters.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub(super) struct WebSocketRegistryStatus {
|
||||||
|
pub(super) entries: usize,
|
||||||
|
pub(super) claims: usize,
|
||||||
|
pub(super) evictions_in_flight: usize,
|
||||||
|
pub(super) closed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebSocketRegistry {
|
||||||
|
pub(super) fn status(&self) -> WebSocketRegistryStatus {
|
||||||
|
WebSocketRegistryStatus {
|
||||||
|
entries: self.entries.len(),
|
||||||
|
claims: self.claims.len(),
|
||||||
|
evictions_in_flight: self.evictions_in_flight,
|
||||||
|
closed: self.closed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Exact process-owned admission retained through the upgraded socket lifetime.
|
/// Exact process-owned admission retained through the upgraded socket lifetime.
|
||||||
pub(crate) struct WebSocketConnection {
|
pub(crate) struct WebSocketConnection {
|
||||||
runtime: std::sync::Weak<WebProcessRuntime>,
|
runtime: std::sync::Weak<WebProcessRuntime>,
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
/// Browser bridge generation for the serialized HTTPS carrier.
|
/// Browser bridge generation for the serialized HTTPS carrier.
|
||||||
pub(crate) mod bridge;
|
pub(crate) mod bridge;
|
||||||
|
/// Process lifecycle publication shared with the control plane.
|
||||||
|
pub(crate) mod control;
|
||||||
/// Shared binary frame codec and protocol constants.
|
/// Shared binary frame codec and protocol constants.
|
||||||
pub(crate) mod frame;
|
pub(crate) mod frame;
|
||||||
/// Plain HTTP ingress and decoy routing behind external TLS termination.
|
/// Plain HTTP ingress and decoy routing behind external TLS termination.
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ mod backend;
|
|||||||
mod downlink;
|
mod downlink;
|
||||||
// Response ownership keeps detached batches charged until the last body clone drops.
|
// Response ownership keeps detached batches charged until the last body clone drops.
|
||||||
mod resident;
|
mod resident;
|
||||||
|
// Read-only control-plane snapshots stay isolated from carrier operations.
|
||||||
|
mod status;
|
||||||
|
pub(crate) use status::WebSessionStatus;
|
||||||
// Lane carrier state isolates request sequencing and downlink replay per logical stream.
|
// Lane carrier state isolates request sequencing and downlink replay per logical stream.
|
||||||
mod lanes;
|
mod lanes;
|
||||||
// Lane batch staging transfers queue ownership without escaping process budgets.
|
// Lane batch staging transfers queue ownership without escaping process budgets.
|
||||||
@@ -205,6 +208,7 @@ pub(crate) struct WebSession {
|
|||||||
carrier_class: CarrierClientClass,
|
carrier_class: CarrierClientClass,
|
||||||
learning_context: Option<CarrierLearningContext>,
|
learning_context: Option<CarrierLearningContext>,
|
||||||
automatic_carrier: bool,
|
automatic_carrier: bool,
|
||||||
|
created_at: Instant,
|
||||||
limits: WebLimitsConfig,
|
limits: WebLimitsConfig,
|
||||||
timeouts: WebTimeoutsConfig,
|
timeouts: WebTimeoutsConfig,
|
||||||
state: Mutex<SessionState>,
|
state: Mutex<SessionState>,
|
||||||
@@ -268,6 +272,7 @@ impl WebSession {
|
|||||||
carrier_class,
|
carrier_class,
|
||||||
learning_context,
|
learning_context,
|
||||||
automatic_carrier,
|
automatic_carrier,
|
||||||
|
created_at: Instant::now(),
|
||||||
limits,
|
limits,
|
||||||
timeouts,
|
timeouts,
|
||||||
state: Mutex::new(SessionState {
|
state: Mutex::new(SessionState {
|
||||||
@@ -344,6 +349,11 @@ impl WebSession {
|
|||||||
self.trace_session_id
|
self.trace_session_id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the immutable carrier-attempt incarnation number.
|
||||||
|
pub(crate) fn carrier_attempt(&self) -> u8 {
|
||||||
|
self.carrier_attempt
|
||||||
|
}
|
||||||
|
|
||||||
/// Creates a child cancellation boundary for one owned carrier task.
|
/// Creates a child cancellation boundary for one owned carrier task.
|
||||||
pub(crate) fn carrier_cancellation(&self) -> CancellationToken {
|
pub(crate) fn carrier_cancellation(&self) -> CancellationToken {
|
||||||
self.cancel.child_token()
|
self.cancel.child_token()
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::{SessionNegotiationPhase, WebSession};
|
||||||
|
use crate::config::WebCarrier;
|
||||||
|
|
||||||
|
/// One bounded point-in-time session snapshot without bearer identity.
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(crate) struct WebSessionStatus {
|
||||||
|
/// Stable trace identifier within this process.
|
||||||
|
pub(crate) trace_session_id: u64,
|
||||||
|
/// Forwarded client address frozen at session creation.
|
||||||
|
pub(crate) client_ip: std::net::IpAddr,
|
||||||
|
/// Canonical WEB virtual host.
|
||||||
|
pub(crate) host: String,
|
||||||
|
/// Configured non-secret user label.
|
||||||
|
pub(crate) user: String,
|
||||||
|
/// Non-secret configured key fingerprint.
|
||||||
|
pub(crate) key_id: String,
|
||||||
|
/// Current carrier incarnation.
|
||||||
|
pub(crate) carrier: WebCarrier,
|
||||||
|
/// One-based carrier attempt.
|
||||||
|
pub(crate) attempt: u8,
|
||||||
|
/// Stable client classification token.
|
||||||
|
pub(crate) client_class: &'static str,
|
||||||
|
/// Whether server-side carrier negotiation owns this chain.
|
||||||
|
pub(crate) automatic: bool,
|
||||||
|
/// Current session lifecycle token.
|
||||||
|
pub(crate) state: &'static str,
|
||||||
|
/// Live logical streams.
|
||||||
|
pub(crate) streams: usize,
|
||||||
|
/// Stream relay tasks that have not exited.
|
||||||
|
pub(crate) tasks: usize,
|
||||||
|
/// Carrier lanes currently retained.
|
||||||
|
pub(crate) lanes: usize,
|
||||||
|
/// Lane OPEN polls currently waiting.
|
||||||
|
pub(crate) lane_open_waits: usize,
|
||||||
|
/// WebSocket lane slots reserved before ownership transfer.
|
||||||
|
pub(crate) websocket_lane_reservations: usize,
|
||||||
|
/// Whether the multiplexed WebSocket carrier is active.
|
||||||
|
pub(crate) websocket_active: bool,
|
||||||
|
/// Queued and response-resident bytes charged to this session.
|
||||||
|
pub(crate) pending_bytes: usize,
|
||||||
|
/// Queued and response-resident items charged to this session.
|
||||||
|
pub(crate) pending_items: usize,
|
||||||
|
/// Control bytes included in the pending total.
|
||||||
|
pub(crate) control_bytes: usize,
|
||||||
|
/// Control items included in the pending total.
|
||||||
|
pub(crate) control_items: usize,
|
||||||
|
/// Monotonic age since session creation.
|
||||||
|
pub(crate) age_ms: u64,
|
||||||
|
/// Monotonic age since the latest carrier activity.
|
||||||
|
pub(crate) idle_ms: u64,
|
||||||
|
/// Remaining automatic negotiation deadline.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(crate) negotiation_remaining_ms: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebSession {
|
||||||
|
/// Captures one short-lock session snapshot or reports lock contention.
|
||||||
|
pub(crate) fn try_status(&self, now: Instant) -> Option<WebSessionStatus> {
|
||||||
|
let state = self.state.try_lock()?;
|
||||||
|
let resident = self.resident.snapshot();
|
||||||
|
let state_name = if state.closed {
|
||||||
|
"closed"
|
||||||
|
} else if state.close_requested {
|
||||||
|
"closing"
|
||||||
|
} else if state.carrier_health_reported {
|
||||||
|
"healthy"
|
||||||
|
} else {
|
||||||
|
match state.negotiation_phase {
|
||||||
|
SessionNegotiationPhase::Uncommitted => "provisional",
|
||||||
|
SessionNegotiationPhase::Replacing => "replacing",
|
||||||
|
SessionNegotiationPhase::Committed => "committed",
|
||||||
|
SessionNegotiationPhase::Superseded => "superseded",
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Some(WebSessionStatus {
|
||||||
|
trace_session_id: self.trace_session_id,
|
||||||
|
client_ip: self.client_ip,
|
||||||
|
host: self.profile.host.clone(),
|
||||||
|
user: self.profile.user.clone(),
|
||||||
|
key_id: self.profile.key_fingerprint.clone(),
|
||||||
|
carrier: self.selected_carrier,
|
||||||
|
attempt: self.carrier_attempt,
|
||||||
|
client_class: self.carrier_class.as_str(),
|
||||||
|
automatic: self.automatic_carrier,
|
||||||
|
state: state_name,
|
||||||
|
streams: state.streams.len(),
|
||||||
|
tasks: self.tasks_live(),
|
||||||
|
lanes: state.carrier_lanes.len(),
|
||||||
|
lane_open_waits: state.lane_open_waits,
|
||||||
|
websocket_lane_reservations: state.websocket_lane_reservations.len(),
|
||||||
|
websocket_active: state.websocket_carrier_active,
|
||||||
|
pending_bytes: state.pending_bytes.saturating_add(resident.bytes()),
|
||||||
|
pending_items: state.pending_items.saturating_add(resident.items()),
|
||||||
|
control_bytes: state
|
||||||
|
.pending_control_bytes
|
||||||
|
.saturating_add(resident.control_bytes),
|
||||||
|
control_items: state
|
||||||
|
.pending_control_items
|
||||||
|
.saturating_add(resident.control_items),
|
||||||
|
age_ms: millis(now.saturating_duration_since(self.created_at)),
|
||||||
|
idle_ms: millis(now.saturating_duration_since(state.last_activity)),
|
||||||
|
negotiation_remaining_ms: self
|
||||||
|
.carrier_deadline_at
|
||||||
|
.map(|deadline| millis(deadline.saturating_duration_since(now))),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the forwarded client address frozen at session creation.
|
||||||
|
pub(crate) fn client_ip(&self) -> std::net::IpAddr {
|
||||||
|
self.client_ip
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the canonical profile host frozen into this session.
|
||||||
|
pub(crate) fn profile_host(&self) -> &str {
|
||||||
|
&self.profile.host
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the configured non-secret user label.
|
||||||
|
pub(crate) fn profile_user(&self) -> &str {
|
||||||
|
&self.profile.user
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the configured non-secret key fingerprint.
|
||||||
|
pub(crate) fn key_id(&self) -> &str {
|
||||||
|
&self.profile.key_fingerprint
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn millis(duration: std::time::Duration) -> u64 {
|
||||||
|
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
@@ -10,7 +10,9 @@ mod store;
|
|||||||
mod types;
|
mod types;
|
||||||
|
|
||||||
pub(crate) use exchange::HttpTraceExchange;
|
pub(crate) use exchange::HttpTraceExchange;
|
||||||
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
|
pub(crate) use store::{
|
||||||
|
StoredTraceRecord, TraceClearOutcome, WebTraceStore, epoch_millis as store_epoch_millis,
|
||||||
|
};
|
||||||
pub(crate) use types::{
|
pub(crate) use types::{
|
||||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
||||||
TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind, TraceRoute,
|
TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind, TraceRoute,
|
||||||
|
|||||||
+78
-72
@@ -41,6 +41,10 @@ impl Drop for StoredTraceRecord {
|
|||||||
pub(crate) struct TraceStoreStatus {
|
pub(crate) struct TraceStoreStatus {
|
||||||
/// Current debug policy.
|
/// Current debug policy.
|
||||||
pub(crate) policy: Arc<WebDebugConfig>,
|
pub(crate) policy: Arc<WebDebugConfig>,
|
||||||
|
/// Runtime generation that last authored the effective policy.
|
||||||
|
pub(crate) policy_generation: u64,
|
||||||
|
/// Epoch fencing in-flight records across policy changes and clears.
|
||||||
|
pub(crate) epoch: u64,
|
||||||
/// Retained record count.
|
/// Retained record count.
|
||||||
pub(crate) records: usize,
|
pub(crate) records: usize,
|
||||||
/// Configured record capacity.
|
/// Configured record capacity.
|
||||||
@@ -61,11 +65,22 @@ pub(crate) struct TraceStoreStatus {
|
|||||||
pub(crate) latest_seq: Option<u64>,
|
pub(crate) latest_seq: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Result of one constant-time logical trace clear.
|
||||||
|
pub(crate) struct TraceClearOutcome {
|
||||||
|
/// Records detached from the ring.
|
||||||
|
pub(crate) records_cleared: usize,
|
||||||
|
/// Bytes still retained by in-flight snapshots after detached records drop.
|
||||||
|
pub(crate) leased_bytes: usize,
|
||||||
|
/// New epoch rejecting commits started before the clear.
|
||||||
|
pub(crate) epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Process-owned bounded WEB debug trace store.
|
/// Process-owned bounded WEB debug trace store.
|
||||||
pub(crate) struct WebTraceStore {
|
pub(crate) struct WebTraceStore {
|
||||||
policy: ArcSwap<WebDebugConfig>,
|
policy: ArcSwap<WebDebugConfig>,
|
||||||
policy_update: Mutex<()>,
|
policy_update: Mutex<()>,
|
||||||
enabled: AtomicBool,
|
enabled: AtomicBool,
|
||||||
|
policy_generation: AtomicU64,
|
||||||
epoch: AtomicU64,
|
epoch: AtomicU64,
|
||||||
records_capacity: usize,
|
records_capacity: usize,
|
||||||
bytes_capacity: usize,
|
bytes_capacity: usize,
|
||||||
@@ -88,6 +103,7 @@ impl WebTraceStore {
|
|||||||
enabled: AtomicBool::new(policy.enabled),
|
enabled: AtomicBool::new(policy.enabled),
|
||||||
policy: ArcSwap::from_pointee(policy),
|
policy: ArcSwap::from_pointee(policy),
|
||||||
policy_update: Mutex::new(()),
|
policy_update: Mutex::new(()),
|
||||||
|
policy_generation: AtomicU64::new(0),
|
||||||
epoch: AtomicU64::new(1),
|
epoch: AtomicU64::new(1),
|
||||||
records_capacity: limits.debug_records_capacity,
|
records_capacity: limits.debug_records_capacity,
|
||||||
bytes_capacity: limits.debug_bytes_global,
|
bytes_capacity: limits.debug_bytes_global,
|
||||||
@@ -106,11 +122,16 @@ impl WebTraceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Applies one hot policy and clears incompatible retained records.
|
/// Applies one generation-authored policy and rejects stale generation writers.
|
||||||
pub(crate) fn apply_policy(&self, policy: &WebDebugConfig) {
|
pub(crate) fn apply_policy(&self, generation: u64, policy: &WebDebugConfig) {
|
||||||
let _policy_update = self.policy_update.lock();
|
let _policy_update = self.policy_update.lock();
|
||||||
|
let current_generation = self.policy_generation.load(Ordering::Acquire);
|
||||||
|
if generation < current_generation {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let current = self.policy.load_full();
|
let current = self.policy.load_full();
|
||||||
if current.as_ref() == policy {
|
if current.as_ref() == policy {
|
||||||
|
self.policy_generation.store(generation, Ordering::Release);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let capture_changed = current.enabled != policy.enabled
|
let capture_changed = current.enabled != policy.enabled
|
||||||
@@ -122,10 +143,40 @@ impl WebTraceStore {
|
|||||||
|| current.body_prefix_bytes != policy.body_prefix_bytes
|
|| current.body_prefix_bytes != policy.body_prefix_bytes
|
||||||
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
|
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
|
||||||
self.policy.store(Arc::new(policy.clone()));
|
self.policy.store(Arc::new(policy.clone()));
|
||||||
|
self.policy_generation.store(generation, Ordering::Release);
|
||||||
self.enabled.store(policy.enabled, Ordering::Release);
|
self.enabled.store(policy.enabled, Ordering::Release);
|
||||||
if capture_changed {
|
if capture_changed {
|
||||||
self.epoch.fetch_add(1, Ordering::AcqRel);
|
self.epoch.fetch_add(1, Ordering::AcqRel);
|
||||||
self.ring.lock().records.clear();
|
let detached = {
|
||||||
|
let mut ring = self.ring.lock();
|
||||||
|
std::mem::replace(
|
||||||
|
&mut ring.records,
|
||||||
|
VecDeque::with_capacity(self.records_capacity),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
drop(_policy_update);
|
||||||
|
drop(detached);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clears retained records while fencing all in-flight pre-clear commits.
|
||||||
|
pub(crate) fn clear(&self) -> TraceClearOutcome {
|
||||||
|
let _policy_update = self.policy_update.lock();
|
||||||
|
let epoch = self.epoch.fetch_add(1, Ordering::AcqRel).saturating_add(1);
|
||||||
|
let detached = {
|
||||||
|
let mut ring = self.ring.lock();
|
||||||
|
std::mem::replace(
|
||||||
|
&mut ring.records,
|
||||||
|
VecDeque::with_capacity(self.records_capacity),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let records_cleared = detached.len();
|
||||||
|
drop(_policy_update);
|
||||||
|
drop(detached);
|
||||||
|
TraceClearOutcome {
|
||||||
|
records_cleared,
|
||||||
|
leased_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||||
|
epoch,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,9 +348,12 @@ impl WebTraceStore {
|
|||||||
|
|
||||||
/// Returns current bounds, counters, and retained sequence range.
|
/// Returns current bounds, counters, and retained sequence range.
|
||||||
pub(crate) fn status(&self) -> TraceStoreStatus {
|
pub(crate) fn status(&self) -> TraceStoreStatus {
|
||||||
|
let _policy_update = self.policy_update.lock();
|
||||||
let ring = self.ring.lock();
|
let ring = self.ring.lock();
|
||||||
TraceStoreStatus {
|
TraceStoreStatus {
|
||||||
policy: self.policy.load_full(),
|
policy: self.policy.load_full(),
|
||||||
|
policy_generation: self.policy_generation.load(Ordering::Acquire),
|
||||||
|
epoch: self.epoch.load(Ordering::Acquire),
|
||||||
records: ring.records.len(),
|
records: ring.records.len(),
|
||||||
records_capacity: self.records_capacity,
|
records_capacity: self.records_capacity,
|
||||||
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||||
@@ -312,6 +366,26 @@ impl WebTraceStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns a non-blocking status snapshot or `None` on trace-store contention.
|
||||||
|
pub(crate) fn try_status(&self) -> Option<TraceStoreStatus> {
|
||||||
|
let _policy_update = self.policy_update.try_lock()?;
|
||||||
|
let ring = self.ring.try_lock()?;
|
||||||
|
Some(TraceStoreStatus {
|
||||||
|
policy: self.policy.load_full(),
|
||||||
|
policy_generation: self.policy_generation.load(Ordering::Acquire),
|
||||||
|
epoch: self.epoch.load(Ordering::Acquire),
|
||||||
|
records: ring.records.len(),
|
||||||
|
records_capacity: self.records_capacity,
|
||||||
|
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||||
|
bytes_capacity: self.bytes_capacity,
|
||||||
|
contention_drops: self.contention_drops.load(Ordering::Relaxed),
|
||||||
|
evictions: self.evictions.load(Ordering::Relaxed),
|
||||||
|
byte_truncations: self.byte_truncations.load(Ordering::Relaxed),
|
||||||
|
earliest_seq: ring.records.front().map(|record| record.record.seq),
|
||||||
|
latest_seq: ring.records.back().map(|record| record.record.seq),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Reserves one of two bounded concurrent status-page render slots.
|
/// Reserves one of two bounded concurrent status-page render slots.
|
||||||
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
|
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
|
||||||
Arc::clone(&self.renders).try_acquire_owned().ok()
|
Arc::clone(&self.renders).try_acquire_owned().ok()
|
||||||
@@ -427,72 +501,4 @@ pub(crate) fn epoch_millis() -> u64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests;
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
|
||||||
let policy = WebDebugConfig {
|
|
||||||
enabled: true,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let limits = WebLimitsConfig {
|
|
||||||
debug_records_capacity: records_capacity,
|
|
||||||
debug_bytes_global: bytes_capacity,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
WebTraceStore::new(policy, &limits)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
|
||||||
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
|
||||||
for _ in 0..3 {
|
|
||||||
store.record_lifecycle(
|
|
||||||
None,
|
|
||||||
Some("192.0.2.10".parse().unwrap()),
|
|
||||||
TraceIdentity::default(),
|
|
||||||
TraceLifecycleEvent::BridgeIssued,
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let snapshot = store.snapshot_matching(|_| true);
|
|
||||||
assert_eq!(
|
|
||||||
snapshot
|
|
||||||
.iter()
|
|
||||||
.map(|record| record.record.seq)
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
vec![3, 2]
|
|
||||||
);
|
|
||||||
assert_eq!(store.status().evictions, 1);
|
|
||||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
|
||||||
|
|
||||||
let policy = WebDebugConfig::default();
|
|
||||||
store.apply_policy(&policy);
|
|
||||||
assert_eq!(store.status().records, 0);
|
|
||||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
|
||||||
drop(snapshot);
|
|
||||||
assert_eq!(store.status().used_bytes, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
|
||||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
|
||||||
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
|
||||||
let exchange = store
|
|
||||||
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let changed = WebDebugConfig {
|
|
||||||
enabled: true,
|
|
||||||
capture_headers: false,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
store.apply_policy(&changed);
|
|
||||||
exchange.commit();
|
|
||||||
|
|
||||||
assert_eq!(store.status().records, 0);
|
|
||||||
assert_eq!(store.status().used_bytes, 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||||
|
let policy = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
debug_records_capacity: records_capacity,
|
||||||
|
debug_bytes_global: bytes_capacity,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
WebTraceStore::new(policy, &limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
||||||
|
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||||
|
for _ in 0..3 {
|
||||||
|
store.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.10".parse().unwrap()),
|
||||||
|
TraceIdentity::default(),
|
||||||
|
TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = store.snapshot_matching(|_| true);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot
|
||||||
|
.iter()
|
||||||
|
.map(|record| record.record.seq)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec![3, 2]
|
||||||
|
);
|
||||||
|
assert_eq!(store.status().evictions, 1);
|
||||||
|
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||||
|
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
store.apply_policy(2, &policy);
|
||||||
|
assert_eq!(store.status().records, 0);
|
||||||
|
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||||
|
drop(snapshot);
|
||||||
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
||||||
|
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||||
|
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||||
|
let exchange = store
|
||||||
|
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let changed = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
capture_headers: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
store.apply_policy(2, &changed);
|
||||||
|
exchange.commit();
|
||||||
|
|
||||||
|
assert_eq!(store.status().records, 0);
|
||||||
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stale_generation_cannot_restore_an_old_policy() {
|
||||||
|
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||||
|
let current = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
capture_headers: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
store.apply_policy(3, ¤t);
|
||||||
|
|
||||||
|
store.apply_policy(2, &WebDebugConfig::default());
|
||||||
|
|
||||||
|
let status = store.status();
|
||||||
|
assert_eq!(status.policy_generation, 3);
|
||||||
|
assert_eq!(status.policy.as_ref(), ¤t);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_clear_fences_inflight_commits_and_preserves_snapshot_leases() {
|
||||||
|
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||||
|
store.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.30".parse().unwrap()),
|
||||||
|
TraceIdentity::default(),
|
||||||
|
TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let snapshot = store.snapshot_matching(|_| true);
|
||||||
|
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||||
|
let exchange = store
|
||||||
|
.begin_http(&request, "192.0.2.30".parse().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let cleared = store.clear();
|
||||||
|
exchange.commit();
|
||||||
|
|
||||||
|
assert_eq!(cleared.records_cleared, 1);
|
||||||
|
assert_eq!(store.status().records, 0);
|
||||||
|
assert_eq!(store.status().used_bytes, BASE_RECORD_RESERVATION);
|
||||||
|
drop(snapshot);
|
||||||
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user