From 996274bed3148b7a3fca23323834b834853526b4 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:10:01 +0300 Subject: [PATCH] API: expose desired and effective state with runtime status --- src/api/config_edit.rs | 1 + src/api/config_edit/diagnostics.rs | 76 +--- src/api/config_edit/preflight.rs | 21 +- src/api/config_projection.rs | 350 ++++++++++++++++ src/api/config_state.rs | 372 ++++++++++++++++++ src/api/config_state/bounds_tests.rs | 198 ++++++++++ src/api/config_state/preparation_tests.rs | 143 +++++++ src/api/config_state/tests.rs | 352 +++++++++++++++++ src/api/config_store.rs | 16 +- src/api/discovery.rs | 2 + src/api/discovery/http_tests.rs | 1 + src/api/discovery/operations.json | 1 + src/api/discovery/schema.rs | 16 + src/api/handler/read_routes.rs | 3 + src/api/mod.rs | 6 + src/api/users/period_tests.rs | 1 + src/config/hot_reload/watcher.rs | 90 ++++- .../hot_reload/watcher/observation_tests.rs | 115 ++++++ src/config/hot_reload/watcher/preparation.rs | 64 ++- src/config/load.rs | 9 + src/config/load/decode.rs | 3 + src/config/mod.rs | 1 + src/config/observation.rs | 175 ++++++++ src/maestro/bootstrap.rs | 3 + src/maestro/orchestrator.rs | 1 + src/maestro/reload.rs | 22 ++ src/maestro/reload_observation.rs | 69 ++++ src/maestro/reload_supervisor.rs | 13 +- src/maestro/reload_supervisor_tests.rs | 43 ++ src/maestro/reload_tests.rs | 26 ++ src/maestro/runtime_build.rs | 39 +- src/maestro/runtime_startup.rs | 3 + 32 files changed, 2114 insertions(+), 121 deletions(-) create mode 100644 src/api/config_projection.rs create mode 100644 src/api/config_state.rs create mode 100644 src/api/config_state/bounds_tests.rs create mode 100644 src/api/config_state/preparation_tests.rs create mode 100644 src/api/config_state/tests.rs create mode 100644 src/config/hot_reload/watcher/observation_tests.rs create mode 100644 src/config/observation.rs create mode 100644 src/maestro/reload_observation.rs diff --git a/src/api/config_edit.rs b/src/api/config_edit.rs index bf96192..c432018 100644 --- a/src/api/config_edit.rs +++ b/src/api/config_edit.rs @@ -27,6 +27,7 @@ use std::sync::Arc; mod diagnostics; mod preflight; mod preparation; +pub(super) use preflight::{RuntimeBaseline, RuntimeFence}; pub(super) use preflight::{parse_runtime_match, validate_config}; use preparation::prepare_patch_to_path; diff --git a/src/api/config_edit/diagnostics.rs b/src/api/config_edit/diagnostics.rs index d4db75b..8aa6042 100644 --- a/src/api/config_edit/diagnostics.rs +++ b/src/api/config_edit/diagnostics.rs @@ -1,9 +1,7 @@ -use std::collections::BTreeSet; - use super::*; use crate::api::model::FieldIssue; -pub(super) const MAX_PATHS: usize = 4096; +pub(super) use crate::api::config_projection::{MAX_PATHS, changed_paths}; pub(super) fn issue( path: impl Into, @@ -114,78 +112,6 @@ pub(super) fn reject_unknown(patch: &Json) -> Result<(), ApiFailure> { Err(error) } -/// Paths only: no before/after values or dynamic keys are returned. Arrays use -/// indices, open maps use their documented placeholder to avoid leaking identities. -pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec { - fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet) { - if old == new { - return; - } - if out.len() > MAX_PATHS { - return; - } - let map_key = if path == "dc_overrides" { - Some("{dc}") - } else if path == "censorship.exclusive_mask" { - Some("{hostname}") - } else if path.starts_with("access.") { - Some("{entry}") - } else { - None - }; - match (old, new) { - (Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => { - let a = old.and_then(Json::as_object); - let b = new.and_then(Json::as_object); - let keys: BTreeSet<_> = a - .into_iter() - .flat_map(|v| v.keys()) - .chain(b.into_iter().flat_map(|v| v.keys())) - .collect(); - if keys.is_empty() { - out.insert(path.to_string()); - } - for key in keys { - let key_path = map_key.unwrap_or(key); - let next = if path.is_empty() { - key_path.to_string() - } else { - format!("{path}.{key_path}") - }; - walk( - a.and_then(|v| v.get(key)), - b.and_then(|v| v.get(key)), - &next, - out, - ); - } - } - (Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => { - let a = old - .and_then(Json::as_array) - .map(Vec::as_slice) - .unwrap_or_default(); - let b = new - .and_then(Json::as_array) - .map(Vec::as_slice) - .unwrap_or_default(); - if a.is_empty() && b.is_empty() { - out.insert(path.to_string()); - } - for index in 0..a.len().max(b.len()) { - walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out); - } - } - _ => { - out.insert(path.to_string()); - } - } - } - let mut paths = BTreeSet::new(); - walk(Some(old), Some(new), "", &mut paths); - paths.into_iter().collect() -} - #[derive(Debug, Serialize)] pub(in crate::api) struct NormalizationEffect { pub(super) path: String, diff --git a/src/api/config_edit/preflight.rs b/src/api/config_edit/preflight.rs index 3b5a161..329bfbf 100644 --- a/src/api/config_edit/preflight.rs +++ b/src/api/config_edit/preflight.rs @@ -10,21 +10,24 @@ use hyper::{HeaderMap, StatusCode}; use super::*; #[derive(Debug, Serialize)] -pub(super) struct RuntimeBaseline { - pub(super) active_generation: u64, - pub(super) config_epoch: u64, - pub(super) runtime_match: String, +pub(in crate::api) struct RuntimeBaseline { + pub(in crate::api) active_generation: u64, + pub(in crate::api) config_epoch: u64, + pub(in crate::api) runtime_match: String, } #[derive(Clone)] -pub(super) struct RuntimeFence { +pub(in crate::api) struct RuntimeFence { active: Arc>, generation: Arc, - pub(super) config: Arc, + pub(in crate::api) config: Arc, } impl RuntimeFence { - pub(super) fn capture(shared: &ApiShared, expected: Option<&str>) -> Result { + pub(in crate::api) fn capture( + shared: &ApiShared, + expected: Option<&str>, + ) -> Result { let generation = shared.active_runtime.load_full(); let config = generation.config(); let fence = Self { @@ -39,7 +42,7 @@ impl RuntimeFence { Ok(fence) } - pub(super) fn baseline(&self) -> RuntimeBaseline { + pub(in crate::api) fn baseline(&self) -> RuntimeBaseline { static PROCESS: OnceLock = OnceLock::new(); let process = PROCESS.get_or_init(|| { let mut bytes = [0_u8; 16]; @@ -63,7 +66,7 @@ impl RuntimeFence { && Arc::ptr_eq(&self.active.load_full(), &self.generation) } - pub(super) fn ensure_current(&self) -> Result<(), ApiFailure> { + pub(in crate::api) fn ensure_current(&self) -> Result<(), ApiFailure> { if self.matches() { Ok(()) } else { diff --git a/src/api/config_projection.rs b/src/api/config_projection.rs new file mode 100644 index 0000000..70e6d5d --- /dev/null +++ b/src/api/config_projection.rs @@ -0,0 +1,350 @@ +//! Shared value-free differences and conservative read-only config projections. + +use serde::Serialize; +use serde_json::Value as Json; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::sync::OnceLock; + +use super::model::ApiFailure; +use crate::config::{ProxyConfig, observation::ConfigProvenance}; + +pub(super) const MAX_PATHS: usize = 4096; + +pub(super) const SECTIONS: &[&str] = &[ + "general", + "network", + "server", + "timeouts", + "censorship", + "upstreams", + "dc_overrides", + "web", + "access", + "logging", + "root", +]; +const MAX_FIELDS: usize = 1024; + +pub(super) fn selected(path: &str, sections: &BTreeSet) -> bool { + let root = path.split(['.', '[']).next().unwrap_or_default(); + sections.contains(if SECTIONS.contains(&root) { + root + } else { + "root" + }) +} + +pub(super) fn retain_sections(value: &mut Json, sections: &BTreeSet) { + if let Some(object) = value.as_object_mut() { + object.retain(|key, _| selected(key, sections)); + } +} + +struct Rule { + public: bool, + schema: Json, + default: Option, +} +fn rules() -> Result<&'static BTreeMap, ApiFailure> { + static RULES: OnceLock, String>> = OnceLock::new(); + RULES + .get_or_init(|| { + super::discovery::projection_rules() + .map(|fields| { + fields + .into_iter() + .map(|(path, public, schema, default)| { + ( + path, + Rule { + public, + schema, + default, + }, + ) + }) + .collect() + }) + .map_err(|_| "Cannot load configuration projection policy".to_owned()) + }) + .as_ref() + .map_err(|error| ApiFailure::internal(error.clone())) +} + +#[derive(Debug, Clone, Serialize)] +pub(super) struct FieldProjection { + pub(super) path: String, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, + redacted: bool, + present: bool, + entries: Option, + differs: Option, + origin: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + inherited_from: Option<&'static str>, +} + +#[derive(Debug, Clone, Serialize)] +pub(super) struct Projection { + pub(super) fields: Vec, + pub(super) truncated: bool, + provenance: &'static str, + /// Input provenance only; this is deliberately not named applied_revision. + source_revision: Option, + source_count: Option, +} + +pub(super) fn project( + config: &ProxyConfig, + value: &Json, + other: Option<&Json>, + sections: &BTreeSet, + desired: bool, +) -> Result { + let rules = rules()?; + let (provenance, source_revision, source_count) = match &config.provenance { + ConfigProvenance::Source(source) => ( + "normalized_source", + Some(source.revision.clone()), + Some(source.source_count), + ), + ConfigProvenance::Mixed => ("mixed_publication", None, None), + ConfigProvenance::Unknown => ("unknown", None, None), + }; + let mut output = Projection { + fields: Vec::new(), + truncated: false, + provenance, + source_revision, + source_count, + }; + struct Walk<'a> { + config: &'a ProxyConfig, + rules: &'a BTreeMap, + desired: bool, + comparing: bool, + } + impl Walk<'_> { + fn visit( + &self, + value: &Json, + other: Option<&Json>, + path: &str, + canonical: &str, + output: &mut Projection, + ) { + if output.fields.len() >= MAX_FIELDS || path.len() > 256 { + output.truncated = true; + return; + } + let rule = self.rules.get(canonical); + let public = rule.is_some_and(|rule| rule.public); + let open_map = rule.is_some_and(|rule| { + rule.schema + .get("additionalProperties") + .is_some_and(|value| value != &Json::Bool(false)) + }) || path == "dc_overrides" + || path == "censorship.exclusive_mask"; + if public + && !open_map + && let Some(object) = value.as_object() + { + if !object.is_empty() { + for (key, child) in object { + self.visit( + child, + other.and_then(|value| value.get(key)), + &format!("{path}.{key}"), + &format!("{canonical}.{key}"), + output, + ); + } + return; + } + } + if public + && let Some(array) = value.as_array() + && rule.is_some_and(|rule| rule.schema["items"]["type"] == "object") + { + if !array.is_empty() { + for (index, child) in array.iter().enumerate() { + if let Some(object) = child.as_object() { + for (key, field) in object { + self.visit( + field, + other.and_then(|v| v.get(index)).and_then(|v| v.get(key)), + &format!("{path}[{index}].{key}"), + &format!("{canonical}[].{key}"), + output, + ); + } + } + } + return; + } + } + // Arbitrary strings and all dynamic identities are private, including + // URL userinfo in a field whose static metadata says "public". + let safe_scalar = |value: &Json, schema: &Json| { + value.is_null() + || value.is_boolean() + || value.is_number() + || schema + .get("enum") + .and_then(Json::as_array) + .is_some_and(|allowed| allowed.contains(value)) + }; + let runtime_derived = path == "censorship.fake_cert_len" + && matches!(&self.config.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len); + let generated = self.desired && runtime_derived; + let visible = public + && !open_map + && !generated + && rule.is_some_and(|rule| { + safe_scalar(value, &rule.schema) + || value.as_array().is_some_and(|array| { + array.len() <= 32 + && array + .iter() + .all(|item| safe_scalar(item, &rule.schema["items"])) + }) + }); + let (origin, inherited_from) = if runtime_derived { + ("runtime_derived", None) + } else if canonical == "upstreams[].prefer" && value.is_null() { + ("inherited", Some("network.prefer")) + } else { + ( + match &self.config.provenance { + ConfigProvenance::Source(source) + if source.explicit_paths.contains(path) => + { + "source" + } + ConfigProvenance::Source(source) if !source.complete => "unknown", + ConfigProvenance::Source(_) + if rule.and_then(|rule| rule.default.as_ref()) == Some(value) => + { + "type_default" + } + ConfigProvenance::Source(_) => "default_or_derived", + ConfigProvenance::Mixed => "mixed_unknown", + ConfigProvenance::Unknown => "unknown", + }, + None, + ) + }; + let present = !value.is_null() + && value.as_str().is_none_or(|v| !v.is_empty()) + && value.as_object().is_none_or(|v| !v.is_empty()) + && value.as_array().is_none_or(|v| !v.is_empty()); + output.fields.push(FieldProjection { + path: path.to_owned(), + value: visible.then(|| value.clone()), + redacted: !visible, + present, + entries: value + .as_object() + .map(|v| v.len()) + .or_else(|| value.as_array().map(Vec::len)), + differs: (self.comparing && !runtime_derived).then(|| other != Some(value)), + origin, + inherited_from, + }); + } + } + let walk = Walk { + config, + rules, + desired, + comparing: other.is_some(), + }; + if let Some(object) = value.as_object() { + for (key, child) in object { + if selected(key, sections) { + walk.visit( + child, + other.and_then(|value| value.get(key)), + key, + key, + &mut output, + ); + } + } + } + Ok(output) +} + +/// Paths only: no before/after values or dynamic keys are returned. Arrays use +/// indices, open maps use their documented placeholder to avoid leaking identities. +pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec { + fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet) { + if old == new { + return; + } + if out.len() > MAX_PATHS { + return; + } + let map_key = if path == "dc_overrides" { + Some("{dc}") + } else if path == "censorship.exclusive_mask" { + Some("{hostname}") + } else if path.starts_with("access.") { + Some("{entry}") + } else { + None + }; + match (old, new) { + (Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => { + let a = old.and_then(Json::as_object); + let b = new.and_then(Json::as_object); + let keys: BTreeSet<_> = a + .into_iter() + .flat_map(|v| v.keys()) + .chain(b.into_iter().flat_map(|v| v.keys())) + .collect(); + if keys.is_empty() { + out.insert(path.to_string()); + } + for key in keys { + let key_path = map_key.unwrap_or(key); + let next = if path.is_empty() { + key_path.to_string() + } else { + format!("{path}.{key_path}") + }; + walk( + a.and_then(|v| v.get(key)), + b.and_then(|v| v.get(key)), + &next, + out, + ); + } + } + (Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => { + let a = old + .and_then(Json::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let b = new + .and_then(Json::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + if a.is_empty() && b.is_empty() { + out.insert(path.to_string()); + } + for index in 0..a.len().max(b.len()) { + walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out); + } + } + _ => { + out.insert(path.to_string()); + } + } + } + let mut paths = BTreeSet::new(); + walk(Some(old), Some(new), "", &mut paths); + paths.into_iter().collect() +} diff --git a/src/api/config_state.rs b/src/api/config_state.rs new file mode 100644 index 0000000..e56dad3 --- /dev/null +++ b/src/api/config_state.rs @@ -0,0 +1,372 @@ +//! Bounded inspection of desired source and actual owner publications. + +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use http_body_util::Full; +use hyper::{Response, StatusCode, body::Bytes}; +use serde::Serialize; +use serde_json::Value as Json; + +use super::config_edit::{RuntimeBaseline, RuntimeFence}; +use super::model::{ApiFailure, FieldIssue}; +use super::{ApiShared, config_projection as projection}; +use crate::config::{ + ProxyConfig, + observation::{ConfigProvenance, WatchOutcome}, +}; +use crate::maestro::{reload::ReloadObservation, runtime_build::project_reload_config}; + +const MAX_RESPONSE_BYTES: usize = 512 * 1024; +const MAX_STATE_PATHS: usize = 512; + +#[derive(Debug, Serialize)] +struct Comparison { + scope: &'static str, + status: &'static str, + pending_paths: Vec, + watcher_pending_paths: Vec, + generation_pending_paths: Vec, + process_deferred_paths: Vec, + deferred_process_fields: Vec, + runtime_derived_paths: Vec, + truncated: bool, +} + +impl Default for Comparison { + fn default() -> Self { + Self { + scope: "selected_sections", + status: "unknown", + pending_paths: Vec::new(), + watcher_pending_paths: Vec::new(), + generation_pending_paths: Vec::new(), + process_deferred_paths: Vec::new(), + deferred_process_fields: Vec::new(), + runtime_derived_paths: Vec::new(), + truncated: false, + } + } +} + +#[derive(Debug, Serialize)] +pub(super) struct ConfigState { + scope: &'static str, + sections: BTreeSet, + runtime: RuntimeBaseline, + runtime_sample_current: bool, + pub(super) desired_revision: Option, + desired_status: &'static str, + desired_error: Option, + desired: Option, + effective: Option, + projection_error: Option, + comparison: Comparison, + watcher: Option, + watcher_matches_desired: Option, + reload: Option, + reload_observation: &'static str, + reload_matches_desired: Option, + subsystem_convergence: &'static str, +} + +fn issue(code: &'static str, message: &'static str) -> FieldIssue { + FieldIssue { + path: "$".into(), + code, + message: message.into(), + } +} + +fn sections(query: Option<&str>) -> Result, ApiFailure> { + let Some(query) = query.filter(|query| !query.is_empty()) else { + return Ok(projection::SECTIONS + .iter() + .map(|section| (*section).to_owned()) + .collect()); + }; + if query.len() > 256 { + return Err(ApiFailure::bad_request( + "Config state query exceeds 256 bytes", + )); + } + let mut parsed = url::form_urlencoded::parse(query.as_bytes()); + let Some((key, value)) = parsed.next() else { + return Err(ApiFailure::bad_request("Expected sections query")); + }; + if key != "sections" || parsed.next().is_some() { + return Err(ApiFailure::bad_request( + "Only one sections query parameter is supported", + )); + } + let mut sections = BTreeSet::new(); + for section in value.split(',') { + if !projection::SECTIONS.contains(§ion) || !sections.insert(section.to_owned()) { + return Err(ApiFailure::bad_request( + "Unknown, empty or repeated configuration section", + )); + } + } + Ok(sections) +} + +fn json(config: &ProxyConfig) -> Result { + serde_json::to_value(config) + .map_err(|_| ApiFailure::internal("Cannot project configuration snapshot")) +} + +struct Desired { + revision: Option, + status: &'static str, + error: Option, + projection: Option, + effective: Option, + comparison: Comparison, +} + +fn unavailable(revision: Option, status: &'static str, code: &'static str) -> Desired { + Desired { + revision, + status, + error: Some(issue( + code, + "Desired configuration is unavailable; effective publication remains readable", + )), + projection: None, + effective: None, + comparison: Comparison::default(), + } +} + +fn read_desired( + path: &std::path::Path, + active: &ProxyConfig, + sections: &BTreeSet, +) -> Result { + let graph = match ProxyConfig::read_source_graph(path) { + Ok(graph) => graph, + Err(_) => return Ok(unavailable(None, "unreadable", "desired_source_unreadable")), + }; + let revision = super::config_store::compute_source_revision(&graph); + let mut desired = match ProxyConfig::parse_captured_source(graph) { + Ok(parsed) => parsed.config, + Err(_) => { + return Ok(unavailable( + Some(revision), + "invalid", + "desired_config_invalid", + )); + } + }; + let mut derived = Vec::new(); + if matches!(&desired.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len) + { + desired.censorship.fake_cert_len = active.censorship.fake_cert_len; + if projection::selected("censorship.fake_cert_len", sections) { + derived.push("censorship.fake_cert_len".into()); + } + } + let mut active_json = json(active)?; + let mut desired_json = json(&desired)?; + let effective = + projection::project(active, &active_json, Some(&desired_json), sections, false)?; + let desired_projection = + projection::project(&desired, &desired_json, Some(&active_json), sections, true)?; + let hot = crate::config::hot_reload::overlay_hot_fields(active, &desired); + let projected = project_reload_config(active, &desired); + let mut hot_json = json(&hot)?; + let mut projected_json = json(&projected.effective)?; + for value in [ + &mut active_json, + &mut desired_json, + &mut hot_json, + &mut projected_json, + ] { + projection::retain_sections(value, sections); + } + let mut comparison = Comparison { + pending_paths: projection::changed_paths(&active_json, &desired_json), + watcher_pending_paths: projection::changed_paths(&active_json, &hot_json), + generation_pending_paths: projection::changed_paths(&hot_json, &projected_json), + process_deferred_paths: projection::changed_paths(&projected_json, &desired_json), + deferred_process_fields: projected + .deferred_process_fields + .into_iter() + .filter(|path| projection::selected(path, sections)) + .collect(), + runtime_derived_paths: derived, + ..Comparison::default() + }; + comparison.status = if comparison.pending_paths.is_empty() { + "published_match" + } else if comparison.watcher_pending_paths.is_empty() + && comparison.generation_pending_paths.is_empty() + { + "process_deferred" + } else { + "pending_publication" + }; + for paths in [ + &mut comparison.pending_paths, + &mut comparison.watcher_pending_paths, + &mut comparison.generation_pending_paths, + &mut comparison.process_deferred_paths, + ] { + comparison.truncated |= paths.len() > MAX_STATE_PATHS; + paths.truncate(MAX_STATE_PATHS); + } + Ok(Desired { + revision: Some(revision), + status: "valid", + error: None, + projection: Some(desired_projection), + effective: Some(effective), + comparison, + }) +} + +pub(super) async fn state( + shared: &ApiShared, + query: Option<&str>, +) -> Result { + let sections = sections(query)?; + let fence = RuntimeFence::capture(shared, None)?; + let mut data = ConfigState { + scope: "published_configuration_not_subsystem_acknowledgment", + sections: sections.clone(), + runtime: fence.baseline(), + runtime_sample_current: true, + desired_revision: None, + desired_status: "busy", + desired_error: None, + desired: None, + effective: None, + projection_error: None, + comparison: Comparison::default(), + watcher: None, + watcher_matches_desired: None, + reload: None, + reload_observation: "unknown", + reload_matches_desired: None, + subsystem_convergence: "unknown", + }; + if let Ok(permit) = shared.config_state_slots.clone().try_acquire_owned() { + let permit = Arc::new(permit); + let guard = permit.clone(); + let active = fence.config.clone(); + let scope = sections.clone(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let effective = tokio::task::spawn_blocking(move || { + let _guard = guard; + projection::project(&active, &json(&active)?, None, &scope, false) + }); + match tokio::time::timeout_at(deadline, effective).await { + Ok(Ok(Ok(effective))) => data.effective = Some(effective), + _ => { + data.projection_error = Some(issue( + "effective_projection_unavailable", + "Runtime identity is available but its projection could not complete", + )) + } + } + if data.effective.is_some() { + let active = fence.config.clone(); + let path = shared.config_path.clone(); + let desired = tokio::task::spawn_blocking(move || { + let _guard = permit; + read_desired(&path, &active, §ions) + }); + let desired = match tokio::time::timeout_at(deadline, desired).await { + Ok(Ok(Ok(desired))) => desired, + Ok(_) => unavailable(None, "unavailable", "desired_projection_failed"), + Err(_) => unavailable(None, "timeout", "desired_read_timeout"), + }; + data.desired_revision = desired.revision; + data.desired_status = desired.status; + data.desired_error = desired.error; + data.desired = desired.projection; + if desired.effective.is_some() { + data.effective = desired.effective; + } + data.comparison = desired.comparison; + } else { + data.desired_status = "unavailable"; + data.desired_error = Some(issue( + "state_projection_unavailable", + "Configuration projection could not complete within the read budget", + )); + } + } else { + data.desired_error = Some(issue( + "config_state_busy", + "Both configuration inspection slots are occupied", + )); + } + data.runtime_sample_current = fence.ensure_current().is_ok(); + data.watcher = fence + .config + .watcher_observation + .as_ref() + .and_then(|observer| observer.snapshot()) + .map(|outcome| (*outcome).clone()); + data.watcher_matches_desired = data.desired_revision.as_ref().and_then(|revision| { + data.watcher + .as_ref()? + .source_revision + .as_ref() + .map(|source| source == revision) + }); + match shared.reload_control.observation() { + Some(reload) => { + data.reload = reload; + data.reload_observation = "sampled"; + } + None => data.reload_observation = "busy", + } + data.reload_matches_desired = data.desired_revision.as_ref().and_then(|revision| { + data.reload + .as_ref() + .map(|reload| &reload.source_revision == revision) + }); + Ok(data) +} + +pub(super) async fn response( + shared: &ApiShared, + query: Option<&str>, +) -> Result>, ApiFailure> { + let mut data = state(shared, query).await?; + loop { + let body = serde_json::to_vec( + &serde_json::json!({"ok": true, "revision": data.desired_revision, "data": data}), + ) + .map_err(|_| ApiFailure::internal("Cannot encode configuration state"))?; + if body.len() <= MAX_RESPONSE_BYTES { + return Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json; charset=utf-8") + .body(Full::new(Bytes::from(body))) + .map_err(|_| ApiFailure::internal("Cannot build configuration state response")); + } + // Each list was already capped. Shrinking the two projections guarantees + // that oversized deployment shapes cannot exceed the response envelope. + let mut shrunk = false; + for projection in [&mut data.desired, &mut data.effective] + .into_iter() + .flatten() + { + shrunk |= !projection.fields.is_empty(); + projection.fields.truncate(projection.fields.len() / 2); + projection.truncated = true; + } + if !shrunk { + return Err(ApiFailure::internal( + "Configuration state metadata exceeded the response bound", + )); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/src/api/config_state/bounds_tests.rs b/src/api/config_state/bounds_tests.rs new file mode 100644 index 0000000..4d05e84 --- /dev/null +++ b/src/api/config_state/bounds_tests.rs @@ -0,0 +1,198 @@ +use super::*; +use http_body_util::BodyExt; + +#[tokio::test] +async fn busy_state_preserves_runtime_identity_and_does_not_queue_more_work() { + let (fixture, _sender) = fixture("", true).await; + let permits = fixture + .shared + .config_state_slots + .clone() + .acquire_many_owned(2) + .await + .unwrap(); + let data = inspect(&fixture, None).await; + assert_eq!(data["desired_status"], "busy"); + assert_eq!(data["desired_error"]["code"], "config_state_busy"); + assert_eq!(data["runtime"]["active_generation"], 1); + assert!(data["effective"].is_null()); + assert_eq!(data["comparison"]["status"], "unknown"); + drop(permits); + assert_eq!(inspect(&fixture, None).await["desired_status"], "valid"); +} + +#[tokio::test] +async fn large_deployment_bounds_fields_paths_bytes_and_dynamic_identities() { + let (fixture, _sender) = fixture("", true).await; + let mut text = + String::from("[access.users]\nprivate_identity='000102030405060708090a0b0c0d0e0f'\n"); + for _ in 0..1200 { + text.push_str("[[upstreams]]\ntype='socks5'\naddress='127.0.0.1:1080'\nusername='private_socks_user'\npassword='private_socks_password'\n"); + } + source(&fixture, &text, true); + let response = response(&fixture.shared, None).await.unwrap(); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert!(bytes.len() <= MAX_RESPONSE_BYTES); + let data: Json = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(data["data"]["desired_status"], "valid"); + assert_eq!(data["data"]["desired"]["truncated"], true); + assert!(data["data"]["desired"]["fields"].as_array().unwrap().len() <= 1024); + assert_eq!(data["data"]["comparison"]["truncated"], true); + assert!( + data["data"]["comparison"]["pending_paths"] + .as_array() + .unwrap() + .len() + <= MAX_STATE_PATHS + ); + let encoded = String::from_utf8(bytes.to_vec()).unwrap(); + for private in [ + "private_identity", + "private_socks_user", + "private_socks_password", + "000102030405060708090a0b0c0d0e0f", + "Bearer discovery-test", + ] { + assert!(!encoded.contains(private)); + } +} + +fn abandoned_state_retains_budget(timeout: bool) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + let (fixture, _sender) = fixture("", true).await; + let original = std::fs::read(&fixture.shared.config_path).unwrap(); + let active = fixture.shared.active_runtime.load_full(); + let (release, wait) = std::sync::mpsc::channel(); + let (started, ready) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + wait.recv().unwrap(); + }); + ready.await.unwrap(); + tokio::time::pause(); + let shared = fixture.shared.clone(); + let task = tokio::spawn(async move { state(&shared, None).await }); + while fixture.shared.config_state_slots.available_permits() == 2 { + tokio::task::yield_now().await; + } + if timeout { + fixture + .shared + .active_runtime + .store(test_runtime_generation(2, active.config().as_ref().clone())); + tokio::time::advance(Duration::from_secs(6)).await; + let data = serde_json::to_value(task.await.unwrap().unwrap()).unwrap(); + assert_eq!(data["runtime"]["active_generation"], 1); + assert_eq!(data["runtime_sample_current"], false); + assert_eq!(data["desired_status"], "unavailable"); + assert_eq!( + data["projection_error"]["code"], + "effective_projection_unavailable" + ); + } else { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + } + assert_eq!(fixture.shared.config_state_slots.available_permits(), 1); + tokio::time::resume(); + release.send(()).unwrap(); + blocker.await.unwrap(); + let _permits = tokio::time::timeout( + Duration::from_secs(5), + fixture + .shared + .config_state_slots + .clone() + .acquire_many_owned(2), + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + std::fs::read(&fixture.shared.config_path).unwrap(), + original + ); + assert!(fixture.shared.reload_control.status(1).await.is_none()); + }); +} + +#[test] +fn cancellation_retains_read_slot_until_detached_worker_finishes() { + abandoned_state_retains_budget(false); +} + +#[test] +fn timeout_retains_read_slot_and_reports_a_stale_runtime_sample() { + abandoned_state_retains_budget(true); +} + +#[test] +fn desired_worker_timeout_preserves_effective_and_retains_its_read_slot() { + use std::future::Future; + use std::task::Poll; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + let (fixture, _sender) = fixture("", true).await; + let (release_initial, wait_initial) = std::sync::mpsc::channel(); + let (initial_started, initial_ready) = tokio::sync::oneshot::channel(); + let initial_blocker = tokio::task::spawn_blocking(move || { + initial_started.send(()).unwrap(); + wait_initial.recv().unwrap(); + }); + initial_ready.await.unwrap(); + let mut read = Box::pin(state(&fixture.shared, None)); + // Queue effective projection, then occupy the only blocking thread once + // it completes. The desired-source worker will have to wait behind it. + std::future::poll_fn(|context| { + assert!(read.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + let (release, wait) = std::sync::mpsc::channel(); + let (started, ready) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + wait.recv().unwrap(); + }); + release_initial.send(()).unwrap(); + initial_blocker.await.unwrap(); + ready.await.unwrap(); + tokio::time::pause(); + std::future::poll_fn(|context| { + assert!(read.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_secs(6)).await; + let data = serde_json::to_value(read.await.unwrap()).unwrap(); + assert_eq!(data["desired_status"], "timeout"); + assert_eq!(data["desired_error"]["code"], "desired_read_timeout"); + assert!(!data["effective"].is_null()); + assert_eq!(data["comparison"]["status"], "unknown"); + assert_eq!(fixture.shared.config_state_slots.available_permits(), 1); + tokio::time::resume(); + release.send(()).unwrap(); + blocker.await.unwrap(); + let _permits = tokio::time::timeout( + Duration::from_secs(5), + fixture + .shared + .config_state_slots + .clone() + .acquire_many_owned(2), + ) + .await + .unwrap() + .unwrap(); + }); +} diff --git a/src/api/config_state/preparation_tests.rs b/src/api/config_state/preparation_tests.rs new file mode 100644 index 0000000..0861459 --- /dev/null +++ b/src/api/config_state/preparation_tests.rs @@ -0,0 +1,143 @@ +use super::*; + +const WEB: &str = r#" +[access.users] +alice = "000102030405060708090a0b0c0d0e0f" +[[server.listeners]] +ip = "127.0.0.1" +port = 18080 +transport = "web" +web_trusted_proxy_cidrs = ["127.0.0.1/32"] +[web] +enabled = true +[[web.vhosts]] +host = "proxy.example.com" +public_addr = "203.0.113.10:443" +[web.vhosts.decoy] +mode = "http_upstream" +upstream = "http://127.0.0.1:18081" +[[web.vhosts.profiles]] +user = "alice" +secret_mode = "dd" +"#; + +#[tokio::test] +async fn inspection_does_not_acquire_decoy_dns_or_static_files() { + let (fixture, _sender) = fixture(WEB, true).await; + source( + &fixture, + &WEB.replace( + "http://127.0.0.1:18081", + "http://unresolvable.invalid:18081", + ) + .replace( + "[[web.vhosts.profiles]]", + "resolve = 'startup'\n[[web.vhosts.profiles]]", + ), + true, + ); + let data = inspect(&fixture, Some("sections=web")).await; + assert_eq!(data["desired_status"], "valid"); + assert_eq!( + field(&data, "desired", "web.vhosts[0].decoy.upstream")["redacted"], + true + ); + assert_eq!(data["comparison"]["status"], "pending_publication"); + + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("index.html"), + "private synthetic asset", + ) + .unwrap(); + let static_source = WEB.replace( + "mode = \"http_upstream\"\nupstream = \"http://127.0.0.1:18081\"", + &format!( + "mode = 'static_directory'\ndirectory = '{}'\nindex = 'index.html'", + directory.path().display() + ), + ); + let (fixture, _sender) = self::fixture(&static_source, true).await; + directory.close().unwrap(); + assert!( + ProxyConfig::load_prepared(fixture.shared.config_path.clone()) + .await + .is_err() + ); + let data = inspect(&fixture, None).await; + assert_eq!(data["desired_status"], "valid"); + assert_eq!(data["comparison"]["status"], "published_match"); + assert_eq!(data["subsystem_convergence"], "unknown"); + assert!(!data.to_string().contains("private synthetic asset")); +} + +#[tokio::test] +async fn conditional_web_capacity_and_includes_preserve_source_evidence() { + let (fixture, _sender) = fixture("[web.limits]\ndebug_bytes_global=4096", true).await; + source( + &fixture, + "[web.limits]\ndebug_bytes_global=8192\n[web.debug]\nbody_prefix_bytes=6144", + true, + ); + let data = inspect(&fixture, Some("sections=web")).await; + assert_eq!(data["comparison"]["status"], "process_deferred"); + assert!( + data["comparison"]["process_deferred_paths"] + .as_array() + .unwrap() + .contains(&json!("web.debug.body_prefix_bytes")) + ); + assert!( + data["comparison"]["deferred_process_fields"] + .as_array() + .unwrap() + .contains(&json!("web.debug")) + ); + source(&fixture, "include = \"extra.toml\"", true); + std::fs::write( + fixture + .shared + .config_path + .parent() + .unwrap() + .join("extra.toml"), + "[general]\nlog_level='debug'", + ) + .unwrap(); + let data = inspect(&fixture, Some("sections=general")).await; + assert_eq!(data["desired"]["source_count"], 2); + assert_eq!( + field(&data, "desired", "general.log_level")["origin"], + "source" + ); + assert!(!data.to_string().contains("extra.toml")); +} + +#[test] +fn projection_fails_closed_for_arbitrary_strings_unknown_fields_and_private_maps() { + let config = ProxyConfig::default(); + let raw = json!({"general":{"proxy_secret_url":"https://private-user:private-pass@example.invalid/key", "unknown_field":"private-value"}, + "access":{"user_rate_limits":{"private-name":{"up_bps":12345}}}, + "dc_overrides":{"private-dc":{"address":"private-value"}}}); + let scope = sections(None).unwrap(); + let projection = projection::project(&config, &raw, Some(&json!({})), &scope, false).unwrap(); + let result = serde_json::to_value(projection).unwrap(); + for value in [ + "private-user", + "private-pass", + "example.invalid", + "private-value", + "private-name", + "private-dc", + "12345", + ] { + assert!(!result.to_string().contains(value)); + } + assert!( + result["fields"] + .as_array() + .unwrap() + .iter() + .all(|field| field["redacted"] == true && field.get("value").is_none()) + ); +} diff --git a/src/api/config_state/tests.rs b/src/api/config_state/tests.rs new file mode 100644 index 0000000..ced4410 --- /dev/null +++ b/src/api/config_state/tests.rs @@ -0,0 +1,352 @@ +use super::*; +use crate::api::discovery::http_tests::Fixture; +use crate::config::observation::WatchObservation; +use crate::maestro::generation::test_runtime_generation; +use crate::maestro::reload::{ReloadControl, ReloadPhase, ReloadRequest}; +use serde_json::json; +use tokio::sync::watch; + +#[path = "bounds_tests.rs"] +mod bounds; + +#[path = "preparation_tests.rs"] +mod preparation; + +fn source(fixture: &Fixture, text: &str, read_only: bool) { + std::fs::write(&fixture.shared.config_path, format!("{text}\n[server.api]\nenabled = true\nread_only = {read_only}\nauth_header = \"Bearer discovery-test\"\n")).unwrap(); +} + +async fn fixture(text: &str, read_only: bool) -> (Fixture, watch::Sender>) { + let fixture = Fixture::new(false); + source(&fixture, text, read_only); + let mut config = ProxyConfig::load_prepared(fixture.shared.config_path.clone()) + .await + .unwrap() + .config; + config.watcher_observation = Some(Arc::new(WatchObservation::default())); + let mut runtime = test_runtime_generation(1, config); + let (sender, receiver) = watch::channel(runtime.config()); + Arc::get_mut(&mut runtime).unwrap().config_rx = receiver; + fixture.shared.active_runtime.store(runtime); + (fixture, sender) +} + +async fn inspect(fixture: &Fixture, query: Option<&str>) -> Json { + serde_json::to_value(state(&fixture.shared, query).await.unwrap()).unwrap() +} + +fn field<'a>(data: &'a Json, projection: &str, path: &str) -> &'a Json { + data[projection]["fields"] + .as_array() + .unwrap() + .iter() + .find(|field| field["path"] == path) + .unwrap_or_else(|| panic!("missing {projection} field {path}")) +} + +#[tokio::test] +async fn state_http_auth_scope_queries_read_only_and_no_mutation() { + let (fixture, _sender) = fixture("", true).await; + let bytes = std::fs::read(&fixture.shared.config_path).unwrap(); + let active = fixture.shared.active_runtime.load_full(); + for (method, path, authorized, expected) in [ + ("GET", "/v1/config/state", false, 401), + ("POST", "/v1/config/state", true, 405), + ( + "GET", + "/v1/config/state?sections=general,general", + true, + 400, + ), + ("GET", "/v1/config/state?sections=", true, 400), + ("GET", "/v1/config/state?sections=unknown", true, 400), + ( + "GET", + "/v1/config/state?sections=general§ions=web", + true, + 400, + ), + ("GET", "/v1/config/state?debug=true", true, 400), + ] { + let (status, allow, _) = fixture.request(method, path, authorized).await; + assert_eq!(status, expected, "{method} {path}"); + if status == 405 { + assert_eq!(allow, "GET"); + } + } + assert!(sections(Some(&format!("sections={}", "a".repeat(256)))).is_err()); + let (status, _, response) = fixture + .request("GET", "/v1/config/state?sections=general,access", true) + .await; + assert_eq!(status, 200, "{response}"); + let data = &response["data"]; + assert_eq!(response["revision"], data["desired_revision"]); + assert_eq!(data["sections"], json!(["access", "general"])); + assert_eq!(data["comparison"]["status"], "published_match"); + assert_eq!(data["runtime"]["active_generation"], 1); + assert_eq!(data["runtime_sample_current"], true); + assert_eq!(data["subsystem_convergence"], "unknown"); + assert!( + data["effective"]["fields"] + .as_array() + .unwrap() + .iter() + .all(|field| { + let path = field["path"].as_str().unwrap(); + path.starts_with("general.") || path.starts_with("access.") + }) + ); + assert_eq!(std::fs::read(&fixture.shared.config_path).unwrap(), bytes); + assert!(Arc::ptr_eq( + &active, + &fixture.shared.active_runtime.load_full() + )); + assert!(fixture.shared.reload_control.status(1).await.is_none()); + assert!(fixture.shared.runtime_events.snapshot(64).events.is_empty()); + let denied = Fixture::new(true); + assert_eq!(denied.request("GET", "/v1/config/state", true).await.0, 403); + let caps = fixture.request("GET", "/v1/capabilities", true).await.2; + let operation = caps["data"]["operations"] + .as_array() + .unwrap() + .iter() + .find(|op| op["id"] == "config_state") + .unwrap(); + assert_eq!(operation["available"], true); + assert_eq!(operation["mutable"], false); +} + +#[tokio::test] +async fn invalid_or_unreadable_desired_preserves_effective_and_hides_errors() { + let (fixture, _sender) = fixture("[general]\nlog_level='debug'", true).await; + let epoch = fixture + .shared + .active_runtime + .load() + .config() + .runtime_config_epoch; + for invalid in [ + "[general]\nme_writer_pick_mode='synthetic-private-value'", + "[broken synthetic-private-value", + ] { + std::fs::write(&fixture.shared.config_path, invalid).unwrap(); + let (status, _, response) = fixture.request("GET", "/v1/config/state", true).await; + assert_eq!(status, 200); + let data = &response["data"]; + assert_eq!(data["desired_status"], "invalid"); + assert!(response["revision"].is_string()); + assert_eq!(data["desired_error"]["code"], "desired_config_invalid"); + assert_eq!(data["runtime"]["config_epoch"], epoch); + assert_eq!( + field(data, "effective", "general.log_level")["value"], + "debug" + ); + assert!(data["desired"].is_null()); + assert_eq!(data["comparison"]["status"], "unknown"); + assert!(!response.to_string().contains("synthetic-private-value")); + } + std::fs::remove_file(&fixture.shared.config_path).unwrap(); + let response = fixture.request("GET", "/v1/config/state", true).await.2; + assert!(response["revision"].is_null()); + assert_eq!(response["data"]["desired_status"], "unreadable"); + assert!(!response["data"]["effective"].is_null()); + assert!( + !response + .to_string() + .contains(fixture.shared.config_path.to_str().unwrap()) + ); +} + +#[tokio::test] +async fn secret_rotation_stays_pending_until_publication_and_never_claims_convergence() { + const OLD: &str = "000102030405060708090a0b0c0d0e0f"; + const NEW: &str = "f0e0d0c0b0a090807060504030201000"; + let (fixture, sender) = + fixture(&format!("[access.users]\nprivate_account='{OLD}'"), false).await; + let before = inspect(&fixture, None).await; + let (status, _, response) = fixture + .request_body( + "POST", + "/v1/users/private_account/rotate-secret", + true, + Some(&json!({"secret": NEW})), + &[], + ) + .await; + assert_eq!(status, 200, "{response}"); + let pending = inspect(&fixture, None).await; + assert_ne!(pending["desired_revision"], before["desired_revision"]); + assert_eq!(pending["runtime"], before["runtime"]); + assert_eq!(pending["comparison"]["status"], "pending_publication"); + assert!( + pending["comparison"]["watcher_pending_paths"] + .as_array() + .unwrap() + .contains(&json!("access.users.{entry}")) + ); + let users = field(&pending, "effective", "access.users"); + assert_eq!(users["differs"], true); + assert_eq!(users["present"], true); + assert_eq!(users["entries"], 1); + assert_eq!(users["redacted"], true); + assert!(users.get("value").is_none()); + for private in [OLD, NEW, "private_account", "Bearer discovery-test"] { + assert!(!pending.to_string().contains(private)); + } + let mut published = ProxyConfig::load_prepared(fixture.shared.config_path.clone()) + .await + .unwrap() + .config + .with_publication_epoch(); + published.watcher_observation = sender.borrow().watcher_observation.clone(); + published.watcher_observation.as_ref().unwrap().record( + "published", + published.provenance.revision().map(str::to_owned), + Some(published.runtime_config_epoch), + None, + ); + sender.send(Arc::new(published)).unwrap(); + let after = inspect(&fixture, None).await; + assert_eq!(after["comparison"]["status"], "published_match"); + assert_ne!( + after["runtime"]["config_epoch"], + before["runtime"]["config_epoch"] + ); + assert_eq!(after["watcher_matches_desired"], true); + assert_eq!(after["subsystem_convergence"], "unknown"); + assert_eq!(field(&after, "effective", "access.users")["differs"], false); +} + +#[tokio::test] +async fn hot_generation_and_process_changes_have_distinct_paths() { + let (fixture, _sender) = fixture("", true).await; + source( + &fixture, + "[general]\nlog_level='debug'\ndisable_colors=true\n[network]\nipv6=true", + true, + ); + let data = inspect(&fixture, None).await; + let comparison = &data["comparison"]; + assert!( + comparison["watcher_pending_paths"] + .as_array() + .unwrap() + .contains(&json!("general.log_level")) + ); + assert!( + comparison["generation_pending_paths"] + .as_array() + .unwrap() + .contains(&json!("network.ipv6")) + ); + assert!( + comparison["process_deferred_paths"] + .as_array() + .unwrap() + .contains(&json!("general.disable_colors")) + ); + source(&fixture, "[general]\ndisable_colors=true", true); + let data = inspect(&fixture, None).await; + assert_eq!(data["comparison"]["status"], "process_deferred"); + assert_eq!( + data["comparison"]["deferred_process_fields"], + json!(["general.disable_colors"]) + ); + let scoped = inspect(&fixture, Some("sections=network")).await; + assert_eq!(scoped["comparison"]["status"], "published_match"); +} + +#[tokio::test] +async fn known_source_default_inheritance_and_runtime_derived_provenance() { + let text = "[general]\nlog_level='debug'\n[[upstreams]]\ntype='direct'\n[censorship]\ntls_emulation=false"; + let (fixture, _sender) = fixture(text, true).await; + let data = inspect(&fixture, None).await; + assert_eq!( + field(&data, "desired", "general.log_level")["origin"], + "source" + ); + assert_eq!( + field(&data, "desired", "general.disable_colors")["origin"], + "type_default" + ); + assert_eq!( + field(&data, "desired", "upstreams[0].prefer")["origin"], + "inherited" + ); + assert_eq!( + field(&data, "desired", "upstreams[0].prefer")["inherited_from"], + "network.prefer" + ); + assert_eq!( + field(&data, "desired", "censorship.fake_cert_len")["origin"], + "runtime_derived" + ); + assert!( + field(&data, "desired", "censorship.fake_cert_len") + .get("value") + .is_none() + ); + assert_eq!(data["comparison"]["status"], "published_match"); + assert_eq!( + data["comparison"]["runtime_derived_paths"], + json!(["censorship.fake_cert_len"]) + ); + assert_eq!(data["desired"]["source_revision"], data["desired_revision"]); + assert_eq!(data["desired"]["source_count"], 1); +} + +#[tokio::test] +async fn reload_metadata_distinguishes_acceptance_failure_and_rollback_without_raw_errors() { + let (mut fixture, _sender) = fixture("", true).await; + let (control, mut commands) = ReloadControl::channel(1); + Arc::get_mut(&mut fixture.shared).unwrap().reload_control = control.clone(); + let revision = inspect(&fixture, None).await["desired_revision"] + .as_str() + .unwrap() + .to_owned(); + for terminal in [ReloadPhase::Failed, ReloadPhase::RolledBack] { + let accepted = control + .submit( + fixture.shared.active_runtime.load().config(), + revision.clone(), + ReloadRequest::default(), + ) + .await + .unwrap(); + let _ = commands.recv().await.unwrap(); + let data = inspect(&fixture, None).await; + assert_eq!(data["reload"]["phase"], "accepted"); + assert_eq!(data["reload_matches_desired"], true); + assert!(data["reload"]["activated_generation"].is_null()); + control.record_source_generation(accepted.reload_id, 1); + control + .mark_phase(accepted.reload_id, ReloadPhase::Preparing) + .await; + control + .add_warning(accepted.reload_id, "synthetic-private-warning") + .await; + match terminal { + ReloadPhase::Failed => { + control + .fail(accepted.reload_id, "synthetic-private-error") + .await + } + ReloadPhase::RolledBack => { + control + .rolled_back(accepted.reload_id, "synthetic-private-error") + .await + } + _ => unreachable!(), + } + let data = inspect(&fixture, None).await; + assert_eq!( + data["reload"]["phase"], + serde_json::to_value(terminal).unwrap() + ); + assert_eq!(data["reload"]["source_generation"], 1); + assert_eq!(data["reload"]["error_present"], true); + assert_eq!(data["reload"]["warning_count"], 1); + assert!(data["reload"]["retiring_generation"].is_null()); + assert!(!data.to_string().contains("synthetic-private")); + } +} diff --git a/src/api/config_store.rs b/src/api/config_store.rs index 1ac27fc..c66c212 100644 --- a/src/api/config_store.rs +++ b/src/api/config_store.rs @@ -108,23 +108,11 @@ pub(super) fn compute_revision(content: &str) -> String { /// Hashes source documents only; prepared DNS answers never change a revision. pub(super) fn compute_snapshot_revision(source_contents: &BTreeMap) -> String { - compute_source_revision(&ConfigSourceGraph { - source_contents: source_contents.clone(), - rendered: String::new(), - }) + crate::config::observation::source_revision(source_contents) } pub(super) fn compute_source_revision(graph: &ConfigSourceGraph) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"telemt-config-manifest-v1\0"); - for (path, content) in &graph.source_contents { - let path = path.as_os_str().as_encoded_bytes(); - hasher.update((path.len() as u64).to_le_bytes()); - hasher.update(path); - hasher.update((content.len() as u64).to_le_bytes()); - hasher.update(content.as_bytes()); - } - hex::encode(hasher.finalize()) + compute_snapshot_revision(&graph.source_contents) } /// Reads a normalized source snapshot without resolving decoys or loading static assets. diff --git a/src/api/discovery.rs b/src/api/discovery.rs index 640be67..b0828f6 100644 --- a/src/api/discovery.rs +++ b/src/api/discovery.rs @@ -25,6 +25,8 @@ pub(super) fn config_field_paths() -> Result, ApiFailure> { schema::field_paths() } +pub(super) use schema::projection_rules; + pub(super) const SCHEMA_VERSION: u32 = 1; #[derive(Debug, Deserialize, Serialize)] diff --git a/src/api/discovery/http_tests.rs b/src/api/discovery/http_tests.rs index 464cade..57ae903 100644 --- a/src/api/discovery/http_tests.rs +++ b/src/api/discovery/http_tests.rs @@ -50,6 +50,7 @@ impl Fixture { detected_ips_rx: watch::channel((None, None)).1, mutation_lock: Arc::new(Mutex::new(())), config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)), + config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)), minimal_cache: Arc::new(Mutex::new(None)), runtime_edge_connections_cache: Arc::new(Mutex::new(None)), runtime_edge_recompute_lock: Arc::new(Mutex::new(())), diff --git a/src/api/discovery/operations.json b/src/api/discovery/operations.json index 5eec055..8dfdd7a 100644 --- a/src/api/discovery/operations.json +++ b/src/api/discovery/operations.json @@ -37,6 +37,7 @@ {"id": "users_active_ips","method": "GET","path": "/v1/stats/users/active-ips","gate": "always","success": [200]}, {"id": "users_quota","method": "GET","path": "/v1/stats/users/quota","gate": "always","success": [200]}, {"id": "config_get","method": "GET","path": "/v1/config","gate": "always","success": [200],"credential_bearing": true}, + {"id": "config_state","method": "GET","path": "/v1/config/state","gate": "always","success": [200]}, {"id": "config_validate","method": "POST","path": "/v1/config/validate","gate": "always","success": [200],"body": "PatchConfigRequest","if_match": true,"if_runtime_match": true,"read_only_operation": true}, {"id": "config_patch","if_runtime_match": true,"method": "PATCH","path": "/v1/config","gate": "always","success": [200,202],"body": "PatchConfigRequest","if_match": true}, {"id": "reload","method": "POST","path": "/v1/system/reload","gate": "always","success": [202],"body": "ReloadRequest","optional_body": true,"if_match": true}, diff --git a/src/api/discovery/schema.rs b/src/api/discovery/schema.rs index baca7a2..f65c1e3 100644 --- a/src/api/discovery/schema.rs +++ b/src/api/discovery/schema.rs @@ -112,6 +112,22 @@ pub(super) fn field_paths() -> Result, ApiFailure> { .collect()) } +pub(in crate::api) fn projection_rules() +-> Result)>, ApiFailure> { + Ok(config_schema()? + .fields + .iter() + .map(|field| { + ( + field.spec.path.clone(), + field.spec.sensitivity == "public", + field.spec.value.clone(), + field.default.clone(), + ) + }) + .collect()) +} + fn field_specs() -> Result, String> { serde_json::from_str(include_str!("fields.json")) .map_err(|error| format!("invalid compiled config field catalog: {error}")) diff --git a/src/api/handler/read_routes.rs b/src/api/handler/read_routes.rs index 49f17e7..43a9fdf 100644 --- a/src/api/handler/read_routes.rs +++ b/src/api/handler/read_routes.rs @@ -10,6 +10,9 @@ pub(super) async fn handle( ) -> Result>>, ApiFailure> { let api_cfg = &cfg.server.api; match (method.as_str(), normalized_path) { + ("GET", "/v1/config/state") => { + Ok(super::super::config_state::response(shared, query).await?) + } ("GET", "/v1/capabilities") => { discovery::reject_query(query)?; let revision = current_revision(&shared.config_path).await?; diff --git a/src/api/mod.rs b/src/api/mod.rs index 8117030..346ba42 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -37,6 +37,8 @@ use crate::web::control::WebRuntimePublication; use crate::web::trace::WebTraceStore; mod config_edit; +mod config_projection; +mod config_state; pub(crate) mod config_store; mod discovery; mod events; @@ -122,6 +124,7 @@ pub(super) struct ApiShared { pub(super) detected_ips_rx: watch::Receiver<(Option, Option)>, pub(super) mutation_lock: Arc>, pub(super) config_preflight_slots: Arc, + pub(super) config_state_slots: Arc, pub(super) minimal_cache: Arc>>, pub(super) runtime_edge_connections_cache: Arc>>, pub(super) runtime_edge_recompute_lock: Arc>, @@ -159,6 +162,7 @@ impl ApiShared { detected_ips_rx: self.detected_ips_rx.clone(), mutation_lock: self.mutation_lock.clone(), config_preflight_slots: self.config_preflight_slots.clone(), + config_state_slots: self.config_state_slots.clone(), minimal_cache: self.minimal_cache.clone(), runtime_edge_connections_cache: self.runtime_edge_connections_cache.clone(), runtime_edge_recompute_lock: self.runtime_edge_recompute_lock.clone(), @@ -269,6 +273,7 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> { "/v1/health" | "/v1/capabilities" | "/v1/config/schema" + | "/v1/config/state" | "/v1/health/ready" | "/v1/system/info" | "/v1/runtime/gates" @@ -380,6 +385,7 @@ pub(crate) async fn serve( detected_ips_rx, mutation_lock: Arc::new(Mutex::new(())), config_preflight_slots: Arc::new(Semaphore::new(2)), + config_state_slots: Arc::new(Semaphore::new(2)), minimal_cache: Arc::new(Mutex::new(None)), runtime_edge_connections_cache: Arc::new(Mutex::new(None)), runtime_edge_recompute_lock: Arc::new(Mutex::new(())), diff --git a/src/api/users/period_tests.rs b/src/api/users/period_tests.rs index 2951539..2ce4619 100644 --- a/src/api/users/period_tests.rs +++ b/src/api/users/period_tests.rs @@ -58,6 +58,7 @@ impl Fixture { detected_ips_rx: watch::channel((None, None)).1, mutation_lock: Arc::new(Mutex::new(())), config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)), + config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)), minimal_cache: Arc::new(Mutex::new(None)), runtime_edge_connections_cache: Arc::new(Mutex::new(None)), runtime_edge_recompute_lock: Arc::new(Mutex::new(())), diff --git a/src/config/hot_reload/watcher.rs b/src/config/hot_reload/watcher.rs index cf2d70c..bfcdd19 100644 --- a/src/config/hot_reload/watcher.rs +++ b/src/config/hot_reload/watcher.rs @@ -144,9 +144,26 @@ fn prepare_effective_config( let mut applied_cfg = overlay_hot_fields(old_cfg, new_cfg); applied_cfg.validate_effective_web()?; applied_cfg.rebuild_runtime_web()?; + applied_cfg.provenance = if config_equal(&applied_cfg, new_cfg) { + new_cfg.provenance.clone() + } else { + crate::config::observation::ConfigProvenance::Mixed + }; Ok(applied_cfg) } +fn record_outcome( + config: &ProxyConfig, + phase: &'static str, + revision: Option, + epoch: Option, + error: Option<&'static str>, +) { + if let Some(observation) = &config.watcher_observation { + observation.record(phase, revision, epoch, error); + } +} + /// Diffs and publishes a fully prepared configuration on the watcher owner task. fn reload_config_with_resolver( prepared: (LoadedConfig, ProxyConfig), @@ -168,6 +185,7 @@ fn reload_config_with_resolver( let next_manifest = WatchManifest::from_source_files(&source_files); let old_cfg = config_tx.borrow().clone(); + let revision = new_cfg.provenance.revision().map(str::to_owned); let old_hot = HotFields::from_config(&old_cfg); let applied_hot = HotFields::from_config(&applied_cfg); let non_hot_changed = !config_equal(&applied_cfg, &new_cfg); @@ -179,11 +197,24 @@ fn reload_config_with_resolver( } if !hot_changed { - if refresh_quota_source { - config_tx + let published = !refresh_quota_source + || config_tx .send(Arc::new(applied_cfg.with_publication_epoch())) - .ok(); - } + .is_ok(); + let epoch = config_tx.borrow().runtime_config_epoch; + record_outcome( + &old_cfg, + if !published { + "publication_failed" + } else if non_hot_changed { + "retained" + } else { + "unchanged" + }, + revision, + published.then_some(epoch), + (!published).then_some("publication_closed"), + ); reload_state.mark_applied(rendered_hash); return Some(next_manifest); } @@ -196,6 +227,13 @@ fn reload_config_with_resolver( "config reload: invalid network.dns_overrides: {}; keeping old config", e ); + record_outcome( + &old_cfg, + "rejected", + revision, + None, + Some("dns_overrides_rejected"), + ); return Some(next_manifest); } @@ -207,9 +245,23 @@ fn reload_config_with_resolver( detected_ip_v4, detected_ip_v6, ); - config_tx + let published = config_tx .send(Arc::new(applied_cfg.with_publication_epoch())) - .ok(); + .is_ok(); + let epoch = config_tx.borrow().runtime_config_epoch; + record_outcome( + &old_cfg, + if !published { + "publication_failed" + } else if non_hot_changed { + "published_partial" + } else { + "published" + }, + revision, + published.then_some(epoch), + (!published).then_some("publication_closed"), + ); reload_state.mark_applied(rendered_hash); Some(next_manifest) } @@ -265,8 +317,18 @@ pub fn spawn_config_watcher( impl std::future::Future + Send + 'static, ) { let initial_level = initial.general.log_level.clone(); - let (config_tx, config_rx) = - watch::channel(Arc::new((*initial).clone().with_publication_epoch())); + let mut initial = (*initial).clone().with_publication_epoch(); + initial.watcher_observation = Some(Arc::new( + crate::config::observation::WatchObservation::default(), + )); + record_outcome( + &initial, + "initial", + initial.provenance.revision().map(str::to_owned), + Some(initial.runtime_config_epoch), + None, + ); + let (config_tx, config_rx) = watch::channel(Arc::new(initial)); let (log_tx, log_rx) = watch::channel(initial_level); let config_path = normalize_watch_path(&config_path); @@ -287,6 +349,15 @@ pub fn spawn_config_watcher( } } let initial_loaded = preparation::read_source(&config_path).await; + if initial_loaded.is_none() { + record_outcome( + &config_tx.borrow(), + "rejected", + None, + None, + Some("source_invalid_or_unreadable"), + ); + } let initial_manifest = initial_loaded .as_ref() .map(|loaded| WatchManifest::from_source_files(&loaded.source_files)) @@ -485,3 +556,6 @@ mod decoy_dns_tests; #[cfg(test)] mod quota_period_tests; + +#[cfg(test)] +mod observation_tests; diff --git a/src/config/hot_reload/watcher/observation_tests.rs b/src/config/hot_reload/watcher/observation_tests.rs new file mode 100644 index 0000000..46aa8e3 --- /dev/null +++ b/src/config/hot_reload/watcher/observation_tests.rs @@ -0,0 +1,115 @@ +use super::*; +use crate::config::observation::{ConfigProvenance, WatchObservation}; + +#[tokio::test] +async fn watcher_outcomes_follow_actual_publication_retention_and_rejection() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.toml"); + std::fs::write(&path, "").unwrap(); + let loaded = ProxyConfig::load_prepared(path.clone()).await.unwrap(); + let mut initial = loaded.config.with_publication_epoch(); + let observer = Arc::new(WatchObservation::default()); + initial.watcher_observation = Some(observer.clone()); + let (sender, mut receiver) = watch::channel(Arc::new(initial)); + let (log, _) = watch::channel(sender.borrow().general.log_level.clone()); + let mut state = ReloadState::new(Some(loaded.rendered_hash)); + for (text, phase, changed) in [ + ("[general]\nlog_level='debug'", "published", true), + ( + "[general]\nlog_level='debug'\ndisable_colors=true", + "retained", + false, + ), + ( + "[general]\nlog_level='verbose'\ndisable_colors=true", + "published_partial", + true, + ), + ( + "[general]\nlog_level='verbose'\ndisable_colors=true", + "unchanged", + false, + ), + ( + "[general]\nlog_level='private-invalid-log-level'", + "rejected", + false, + ), + ] { + let old = receiver.borrow_and_update().clone(); + let sequence = observer.snapshot().map_or(0, |value| value.sequence); + std::fs::write(&path, text).unwrap(); + preparation::reload( + &path, &sender, &log, None, None, &mut state, None, false, None, None, + ) + .await; + let outcome = observer.snapshot().unwrap(); + assert!(outcome.sequence > sequence); + assert_eq!(outcome.phase, phase); + assert_eq!(receiver.has_changed().unwrap(), changed); + if changed { + assert!(receiver.borrow().runtime_config_epoch > old.runtime_config_epoch); + assert_eq!( + outcome.published_config_epoch, + Some(receiver.borrow().runtime_config_epoch) + ); + if phase == "published_partial" { + assert!(matches!( + &receiver.borrow().provenance, + ConfigProvenance::Mixed + )); + } else { + assert_eq!( + outcome.source_revision.as_deref(), + receiver.borrow().provenance.revision() + ); + } + } else { + assert!(Arc::ptr_eq(&old, &receiver.borrow())); + } + assert!( + !serde_json::to_string(&*outcome) + .unwrap() + .contains("private-invalid-log-level") + ); + } + let rejected = observer.snapshot().unwrap(); + assert_eq!(rejected.error_code, Some("source_invalid_or_unreadable")); + assert_eq!(rejected.published_config_epoch, None); +} + +#[tokio::test] +async fn lost_receivers_never_report_successful_publication() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.toml"); + std::fs::write(&path, "").unwrap(); + let mut initial = ProxyConfig::load_prepared(path.clone()) + .await + .unwrap() + .config; + let observer = Arc::new(WatchObservation::default()); + initial.watcher_observation = Some(observer.clone()); + let (sender, receiver) = watch::channel(Arc::new(initial.with_publication_epoch())); + let (log, _) = watch::channel(sender.borrow().general.log_level.clone()); + let original = sender.borrow().clone(); + drop(receiver); + std::fs::write(&path, "[general]\nlog_level='debug'").unwrap(); + preparation::reload( + &path, + &sender, + &log, + None, + None, + &mut ReloadState::default(), + None, + false, + None, + None, + ) + .await; + assert!(Arc::ptr_eq(&sender.borrow(), &original)); + let outcome = observer.snapshot().unwrap(); + assert_eq!(outcome.phase, "publication_failed"); + assert_eq!(outcome.published_config_epoch, None); + assert_eq!(outcome.error_code, Some("publication_closed")); +} diff --git a/src/config/hot_reload/watcher/preparation.rs b/src/config/hot_reload/watcher/preparation.rs index 369a67a..2d0c678 100644 --- a/src/config/hot_reload/watcher/preparation.rs +++ b/src/config/hot_reload/watcher/preparation.rs @@ -43,7 +43,17 @@ pub(super) async fn reload( Some(gate) => Some(gate.lock().await), None => None, }; - let source = read_source(path).await?; + let Some(source) = read_source(path).await else { + record_outcome( + &config_tx.borrow(), + "rejected", + None, + None, + Some("source_invalid_or_unreadable"), + ); + return None; + }; + let revision = source.config.provenance.revision().map(str::to_owned); let manifest = WatchManifest::from_source_files(&source.source_files); // A new file edit can restore the exact pre-creation bytes. Refresh only source // evidence for fenced accounts, retaining prepared DNS and immutable WEB state. @@ -53,16 +63,43 @@ pub(super) async fn reload( { let mut refreshed = (**config_tx.borrow()).clone(); refreshed.quota_source_read_at = source.config.quota_source_read_at; - config_tx + let published = config_tx .send(Arc::new(refreshed.with_publication_epoch())) - .ok(); + .is_ok(); + let config = config_tx.borrow(); + record_outcome( + &config, + if published { + "source_refreshed" + } else { + "publication_failed" + }, + revision, + published.then_some(config.runtime_config_epoch), + (!published).then_some("publication_closed"), + ); reload_state.mark_applied(source.rendered_hash); return Some(manifest); } drop(guard); if !force && reload_state.is_applied(source.rendered_hash) { + let config = config_tx.borrow(); + record_outcome( + &config, + "unchanged", + revision, + Some(config.runtime_config_epoch), + None, + ); return Some(manifest); } + record_outcome( + &config_tx.borrow(), + "preparing", + revision.clone(), + None, + None, + ); let loaded = match source.prepare().await { Ok(loaded) => loaded, Err(error) => { @@ -70,6 +107,13 @@ pub(super) async fn reload( "config reload: preparation failed: {}; keeping old config", error ); + record_outcome( + &config_tx.borrow(), + "rejected", + revision, + None, + Some("preparation_failed"), + ); return Some(manifest); } }; @@ -96,6 +140,13 @@ pub(super) async fn reload( "config reload: effective WEB validation failed: {}; keeping old config", error ); + record_outcome( + &config_tx.borrow(), + "rejected", + revision, + None, + Some("effective_config_invalid"), + ); Some(manifest) } Err(error) => { @@ -103,6 +154,13 @@ pub(super) async fn reload( "config reload: runtime builder failed: {}; keeping old config", error ); + record_outcome( + &config_tx.borrow(), + "rejected", + revision, + None, + Some("preparation_worker_failed"), + ); Some(manifest) } } diff --git a/src/config/load.rs b/src/config/load.rs index e8d22c9..d098c61 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -160,6 +160,10 @@ pub struct ProxyConfig { /// Process-local identity assigned by the owner of a published config snapshot. #[serde(skip)] pub(crate) runtime_config_epoch: u64, + #[serde(skip)] + pub(crate) provenance: super::observation::ConfigProvenance, + #[serde(skip)] + pub(crate) watcher_observation: Option>, /// Monotonic start of source acquisition, used only for cold quota policy fencing. #[serde(skip)] @@ -265,6 +269,11 @@ impl ProxyConfig { Self::parse_source_with_overrides(path, &BTreeMap::new()) } + /// Parses an already captured source manifest without DNS or static-site reads. + pub(crate) fn parse_captured_source(graph: ConfigSourceGraph) -> Result { + pipeline::parse_source_graph(graph) + } + /// Checks decoy mode-specific keys before a source or API patch loses unknown fields. pub(crate) fn validate_decoy_source_keys(document: &toml::Value) -> Result<()> { decoy_dns::validate_mode_keys(document) diff --git a/src/config/load/decode.rs b/src/config/load/decode.rs index c23d6c6..6bb63cd 100644 --- a/src/config/load/decode.rs +++ b/src/config/load/decode.rs @@ -20,6 +20,8 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result Result) -> String { + let mut hash = Sha256::new(); + hash.update(b"telemt-config-manifest-v1\0"); + for (path, content) in sources { + let path = path.as_os_str().as_encoded_bytes(); + hash.update((path.len() as u64).to_le_bytes()); + hash.update(path); + hash.update((content.len() as u64).to_le_bytes()); + hash.update(content.as_bytes()); + } + hex::encode(hash.finalize()) +} + +#[derive(Debug, Clone, Default)] +pub(crate) enum ConfigProvenance { + #[default] + Unknown, + Source(Arc), + /// The published config combines sources or retained process policy. Do not + /// attribute all its values to the last source read by the watcher. + Mixed, +} + +impl ConfigProvenance { + pub(crate) fn revision(&self) -> Option<&str> { + match self { + Self::Source(source) => Some(&source.revision), + _ => None, + } + } +} + +#[derive(Debug)] +pub(crate) struct SourceEvidence { + pub(crate) revision: String, + pub(crate) explicit_paths: BTreeSet, + pub(crate) complete: bool, + pub(crate) source_count: usize, + pub(crate) generated_fake_cert_len: bool, +} + +impl SourceEvidence { + pub(crate) fn capture( + document: &toml::Value, + sources: &BTreeMap, + ) -> Arc { + fn visit( + value: &toml::Value, + path: &str, + paths: &mut BTreeSet, + complete: &mut bool, + ) { + if paths.len() >= 4096 || path.len() > 256 { + *complete = false; + return; + } + if !path.is_empty() { + paths.insert(path.to_owned()); + } + // Open maps need only presence provenance, never their private keys. + if path == "dc_overrides" + || path == "censorship.exclusive_mask" + || path.starts_with("access.") + { + return; + } + match value { + toml::Value::Table(table) => { + for (key, value) in table { + let key = if path.starts_with("upstreams[") + && path.ends_with(']') + && key == "force_bind" + { + "bindtodevice" + } else if path == "server" && key == "admin_api" { + "api" + } else { + key + }; + let child = if path.is_empty() { + key.to_owned() + } else { + format!("{path}.{key}") + }; + visit(value, &child, paths, complete); + } + } + toml::Value::Array(array) => { + for (index, value) in array.iter().enumerate() { + visit(value, &format!("{path}[{index}]"), paths, complete); + } + } + _ => {} + } + } + let mut explicit_paths = BTreeSet::new(); + let mut complete = true; + visit(document, "", &mut explicit_paths, &mut complete); + let censorship = document.get("censorship"); + let generated_fake_cert_len = censorship + .and_then(|v| v.get("tls_emulation")) + .and_then(toml::Value::as_bool) + == Some(false) + && censorship + .and_then(|v| v.get("fake_cert_len")) + .and_then(toml::Value::as_integer) + .is_none_or(|value| value == super::defaults::default_fake_cert_len() as i64); + Arc::new(Self { + revision: source_revision(sources), + explicit_paths, + complete, + source_count: sources.len(), + generated_fake_cert_len, + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct WatchOutcome { + pub(crate) sequence: u64, + pub(crate) observed_at_epoch_secs: u64, + pub(crate) phase: &'static str, + /// Source considered by this attempt, not a hash of applied runtime state. + pub(crate) source_revision: Option, + pub(crate) published_config_epoch: Option, + pub(crate) error_code: Option<&'static str>, +} + +#[derive(Debug, Default)] +pub(crate) struct WatchObservation { + next_sequence: AtomicU64, + latest: ArcSwapOption, +} + +impl WatchObservation { + pub(crate) fn record( + &self, + phase: &'static str, + revision: Option, + epoch: Option, + error_code: Option<&'static str>, + ) { + self.latest.store(Some(Arc::new(WatchOutcome { + sequence: self + .next_sequence + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1), + observed_at_epoch_secs: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + phase, + source_revision: revision, + published_config_epoch: epoch, + error_code, + }))); + } + + pub(crate) fn snapshot(&self) -> Option> { + self.latest.load_full() + } +} diff --git a/src/maestro/bootstrap.rs b/src/maestro/bootstrap.rs index f8931ba..5c247b4 100644 --- a/src/maestro/bootstrap.rs +++ b/src/maestro/bootstrap.rs @@ -207,6 +207,9 @@ pub(super) async fn bootstrap( validate_synlimit_privilege_drop(&config, privilege_drop_requested)?; if let Some(p) = data_path { + if config.general.data_path.as_ref() != Some(&p) { + config.provenance = crate::config::observation::ConfigProvenance::Mixed; + } config.general.data_path = Some(p); } diff --git a/src/maestro/orchestrator.rs b/src/maestro/orchestrator.rs index 7859ab3..881e9cf 100644 --- a/src/maestro/orchestrator.rs +++ b/src/maestro/orchestrator.rs @@ -51,6 +51,7 @@ pub(super) async fn run_telemt_core( if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control { warn!("Inline conntrack control is disabled when process privileges are dropped"); config.server.conntrack_control.inline_conntrack_control = false; + config.provenance = crate::config::observation::ConfigProvenance::Mixed; } let quota_store = Arc::new(QuotaStore::default()); diff --git a/src/maestro/reload.rs b/src/maestro/reload.rs index 8773f73..a65d0da 100644 --- a/src/maestro/reload.rs +++ b/src/maestro/reload.rs @@ -9,6 +9,10 @@ use tokio::sync::mpsc; use crate::config::ProxyConfig; +#[path = "reload_observation.rs"] +mod observation; +pub(crate) use observation::ReloadObservation; + const RELOAD_HISTORY_CAPACITY: usize = 32; const RELOAD_COMMAND_CAPACITY: usize = 1; const MAX_DRAIN_TIMEOUT_SECS: u64 = 3_600; @@ -151,6 +155,16 @@ pub(crate) struct ReloadStatus { pub(crate) warnings: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) error: Option, + #[serde(skip)] + source_generation: Option, + #[serde(skip)] + activated_generation: Option, + #[serde(skip)] + retiring_generation: Option, + #[serde(skip)] + retiring_config_epoch: Option, + #[serde(skip)] + retirement: Option<&'static str>, } /// Accepted operation metadata returned before asynchronous preparation starts. @@ -395,6 +409,11 @@ impl ReloadStatusStore { deferred_fields: Vec::new(), warnings: Vec::new(), error: None, + source_generation: None, + activated_generation: None, + retiring_generation: None, + retiring_config_epoch: None, + retirement: None, }; state.active_reload_id = Some(reload_id); state.statuses.push_back(status.clone()); @@ -416,6 +435,9 @@ impl ReloadStatusStore { fn mark_phase(&self, reload_id: u64, phase: ReloadPhase) { self.update(reload_id, |status| { status.state = phase; + if phase == ReloadPhase::Draining { + status.retirement = Some("draining"); + } if status.started_at_epoch_secs.is_none() && phase != ReloadPhase::Accepted { status.started_at_epoch_secs = Some(now_epoch_secs()); } diff --git a/src/maestro/reload_observation.rs b/src/maestro/reload_observation.rs new file mode 100644 index 0000000..0dfc64b --- /dev/null +++ b/src/maestro/reload_observation.rs @@ -0,0 +1,69 @@ +use serde::Serialize; + +use super::{ReloadControl, ReloadPhase}; + +/// Sanitized owner snapshot. Raw reload errors/warnings can contain source data. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct ReloadObservation { + pub(crate) reload_id: u64, + pub(crate) target_generation: u64, + pub(crate) source_revision: String, + pub(crate) phase: ReloadPhase, + pub(crate) source_generation: Option, + pub(crate) activated_generation: Option, + pub(crate) retiring_generation: Option, + pub(crate) retiring_config_epoch: Option, + pub(crate) retirement: Option<&'static str>, + pub(crate) error_present: bool, + pub(crate) warning_count: usize, +} + +impl ReloadControl { + /// Reads bounded owner metadata without waiting for a contended status lock. + pub(crate) fn observation(&self) -> Option> { + let state = self.status_store.state.try_lock()?; + Some(state.statuses.back().map(|status| ReloadObservation { + reload_id: status.reload_id, + target_generation: status.target_generation, + source_revision: status.config_revision.clone(), + phase: status.state, + source_generation: status.source_generation, + activated_generation: status.activated_generation, + retiring_generation: status.retiring_generation, + retiring_config_epoch: status.retiring_config_epoch, + retirement: status.retirement, + error_present: status.error.is_some(), + warning_count: status.warnings.len(), + })) + } + + pub(crate) fn record_source_generation(&self, id: u64, generation: u64) { + self.status_store + .update(id, |status| status.source_generation = Some(generation)); + } + + pub(crate) fn record_activation( + &self, + id: u64, + active: u64, + retiring: u64, + retiring_epoch: u64, + ) { + self.status_store.update(id, |status| { + status.activated_generation = Some(active); + status.retiring_generation = Some(retiring); + status.retiring_config_epoch = Some(retiring_epoch); + status.retirement = Some("retiring"); + }); + } + + pub(crate) fn record_retirement(&self, id: u64, timed_out: bool) { + self.status_store.update(id, |status| { + status.retirement = Some(if timed_out { + "cleanup_timeout" + } else { + "retired" + }) + }); + } +} diff --git a/src/maestro/reload_supervisor.rs b/src/maestro/reload_supervisor.rs index ffaacc4..0991ab3 100644 --- a/src/maestro/reload_supervisor.rs +++ b/src/maestro/reload_supervisor.rs @@ -163,6 +163,8 @@ impl ReloadSupervisor { .mark_phase(command.reload_id, ReloadPhase::Preparing) .await; let old_runtime = self.active_runtime.load_full(); + self.control + .record_source_generation(command.reload_id, old_runtime.id); let resolved = match resolve_reload_config(&old_runtime.config(), &command.config) { Ok(resolved) => resolved, Err(error) => { @@ -351,6 +353,12 @@ impl ReloadSupervisor { old_runtime.stop_accepting_sessions(); listener_manager.activate_runtime_generation(new_runtime.clone()) }; + self.control.record_activation( + command.reload_id, + new_runtime.id, + replaced.id, + replaced.config().runtime_config_epoch, + ); // A rejected admission epoch must not be bypassed by reapplying the prepared // snapshot. Quota publication retains its credential and source checks. let config = new_runtime.config(); @@ -422,7 +430,10 @@ impl ReloadSupervisor { } } - if stop_background_and_middle_end(&replaced).await { + let retirement_timed_out = stop_background_and_middle_end(&replaced).await; + self.control + .record_retirement(command.reload_id, retirement_timed_out); + if retirement_timed_out { let warning = format!( "generation {} Middle-End lifecycle shutdown timed out", replaced.id diff --git a/src/maestro/reload_supervisor_tests.rs b/src/maestro/reload_supervisor_tests.rs index 0e3df41..039b32f 100644 --- a/src/maestro/reload_supervisor_tests.rs +++ b/src/maestro/reload_supervisor_tests.rs @@ -158,6 +158,11 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() { assert!(fixture.old_runtime.spawn_session(async {})); let status = fixture.control.status(1).await.unwrap(); assert_eq!(status.state, ReloadPhase::RolledBack); + let observed = fixture.control.observation().unwrap().unwrap(); + assert_eq!(observed.phase, ReloadPhase::RolledBack); + assert_eq!(observed.activated_generation, None); + assert_eq!(observed.retiring_generation, None); + assert_eq!(observed.retirement, None); fixture.old_runtime.stop_sessions().await; } @@ -206,12 +211,33 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() { assert!(!activation.is_finished()); assert!(!fixture.old_runtime.spawn_session(async {})); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.control.status(1).await.unwrap().state != ReloadPhase::Draining { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let observed = fixture.control.observation().unwrap().unwrap(); + assert_eq!(observed.phase, ReloadPhase::Draining); + assert_eq!(observed.activated_generation, Some(2)); + assert_eq!(observed.retiring_generation, Some(1)); + assert_eq!( + observed.retiring_config_epoch, + Some(fixture.old_runtime.config().runtime_config_epoch) + ); + assert_eq!(observed.retirement, Some("draining")); + old_release.notify_one(); activation.await.unwrap(); assert_eq!( fixture.control.status(1).await.unwrap().state, ReloadPhase::Succeeded ); + let observed = fixture.control.observation().unwrap().unwrap(); + assert_eq!(observed.phase, ReloadPhase::Succeeded); + assert_eq!(observed.retiring_generation, Some(1)); + assert_eq!(observed.retirement, Some("retired")); fixture.new_runtime.stop_sessions().await; } @@ -257,6 +283,23 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() { fixture.new_runtime.stop_sessions().await; } +#[tokio::test] +async fn rejected_reload_observes_source_generation_without_activation() { + let mut fixture = fixture(ReloadRequest::default()).await; + Arc::make_mut(&mut fixture.command.config).web.enabled = true; + fixture.supervisor.reload(fixture.command).await; + let observed = fixture.control.observation().unwrap().unwrap(); + assert_eq!(observed.phase, ReloadPhase::Failed); + assert_eq!(observed.source_generation, Some(1)); + assert_eq!(observed.activated_generation, None); + assert_eq!(observed.retiring_generation, None); + assert!(observed.error_present); + assert!(Arc::ptr_eq( + &fixture.supervisor.active_runtime.load_full(), + &fixture.old_runtime + )); +} + #[tokio::test] async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() { let runtime = test_runtime_generation(1, ProxyConfig::default()); diff --git a/src/maestro/reload_tests.rs b/src/maestro/reload_tests.rs index 53cb11c..ea129ef 100644 --- a/src/maestro/reload_tests.rs +++ b/src/maestro/reload_tests.rs @@ -48,6 +48,11 @@ fn status_uses_documented_deferred_process_fields_key() { deferred_fields: vec!["server.listeners".to_string()], warnings: Vec::new(), error: None, + source_generation: Some(1), + activated_generation: Some(2), + retiring_generation: Some(1), + retiring_config_epoch: Some(3), + retirement: Some("retired"), }; let value = serde_json::to_value(status).unwrap(); @@ -56,6 +61,27 @@ fn status_uses_documented_deferred_process_fields_key() { serde_json::json!(["server.listeners"]) ); assert!(value.get("deferred_fields").is_none()); + for field in [ + "source_generation", + "activated_generation", + "retiring_generation", + "retiring_config_epoch", + "retirement", + ] { + assert!( + value.get(field).is_none(), + "Legacy reload response gained {field}" + ); + } +} + +#[test] +fn state_observation_never_waits_on_the_reload_status_lock() { + let (control, _receiver) = ReloadControl::channel(1); + let guard = control.status_store.state.lock(); + assert!(control.observation().is_none()); + drop(guard); + assert!(control.observation().unwrap().is_none()); } #[tokio::test] diff --git a/src/maestro/runtime_build.rs b/src/maestro/runtime_build.rs index 0ab9240..b21ea3d 100644 --- a/src/maestro/runtime_build.rs +++ b/src/maestro/runtime_build.rs @@ -334,6 +334,29 @@ pub(crate) fn resolve_reload_config( old: &ProxyConfig, desired: &ProxyConfig, ) -> Result { + let mut resolved = project_reload_config(old, desired); + resolved + .effective + .validate_effective_web() + .map_err(|error| format!("effective WEB configuration is invalid: {error}"))?; + resolved + .effective + .rebuild_runtime_user_auth() + .map_err(|error| format!("effective user runtime preparation failed: {error}"))?; + resolved + .effective + .rebuild_runtime_web() + .map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?; + resolved.runtime_changed = !configs_equal(old, &resolved.effective); + Ok(resolved) +} + +/// Projects retained process policy without DNS, static acquisition or validation. +/// State inspection uses this projection; preflight still uses the full resolver. +pub(crate) fn project_reload_config( + old: &ProxyConfig, + desired: &ProxyConfig, +) -> ResolvedReloadConfig { let mut effective = desired.clone(); let mut fields = Vec::new(); let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server); @@ -462,21 +485,15 @@ pub(crate) fn resolve_reload_config( fields.push("web.debug".to_string()); effective.web.debug = old.web.debug.clone(); } - effective - .validate_effective_web() - .map_err(|error| format!("effective WEB configuration is invalid: {error}"))?; - effective - .rebuild_runtime_user_auth() - .map_err(|error| format!("effective user runtime preparation failed: {error}"))?; - effective - .rebuild_runtime_web() - .map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?; + if !fields.is_empty() { + effective.provenance = crate::config::observation::ConfigProvenance::Mixed; + } let runtime_changed = !configs_equal(old, &effective); - Ok(ResolvedReloadConfig { + ResolvedReloadConfig { effective, deferred_process_fields: fields, runtime_changed, - }) + } } fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool { diff --git a/src/maestro/runtime_startup.rs b/src/maestro/runtime_startup.rs index d580fb0..41ca861 100644 --- a/src/maestro/runtime_startup.rs +++ b/src/maestro/runtime_startup.rs @@ -172,6 +172,9 @@ pub(super) async fn prepare_runtime( } else { let _ = use_middle_proxy; use_middle_proxy = false; + if config.general.use_middle_proxy { + config.provenance = crate::config::observation::ConfigProvenance::Mixed; + } config.general.use_middle_proxy = false; startup_tracker.set_transport_mode("direct").await; startup_tracker.set_degraded(true).await;