API: expose desired and effective state with runtime status

This commit is contained in:
Alexey
2026-11-03 22:10:01 +03:00
parent 5040195ca7
commit 996274bed3
32 changed files with 2114 additions and 121 deletions
+1
View File
@@ -27,6 +27,7 @@ use std::sync::Arc;
mod diagnostics;
mod preflight;
mod preparation;
pub(super) use preflight::{RuntimeBaseline, RuntimeFence};
pub(super) use preflight::{parse_runtime_match, validate_config};
use preparation::prepare_patch_to_path;
+1 -75
View File
@@ -1,9 +1,7 @@
use std::collections::BTreeSet;
use super::*;
use crate::api::model::FieldIssue;
pub(super) const MAX_PATHS: usize = 4096;
pub(super) use crate::api::config_projection::{MAX_PATHS, changed_paths};
pub(super) fn issue(
path: impl Into<String>,
@@ -114,78 +112,6 @@ pub(super) fn reject_unknown(patch: &Json) -> Result<(), ApiFailure> {
Err(error)
}
/// Paths only: no before/after values or dynamic keys are returned. Arrays use
/// indices, open maps use their documented placeholder to avoid leaking identities.
pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec<String> {
fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet<String>) {
if old == new {
return;
}
if out.len() > MAX_PATHS {
return;
}
let map_key = if path == "dc_overrides" {
Some("{dc}")
} else if path == "censorship.exclusive_mask" {
Some("{hostname}")
} else if path.starts_with("access.") {
Some("{entry}")
} else {
None
};
match (old, new) {
(Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => {
let a = old.and_then(Json::as_object);
let b = new.and_then(Json::as_object);
let keys: BTreeSet<_> = a
.into_iter()
.flat_map(|v| v.keys())
.chain(b.into_iter().flat_map(|v| v.keys()))
.collect();
if keys.is_empty() {
out.insert(path.to_string());
}
for key in keys {
let key_path = map_key.unwrap_or(key);
let next = if path.is_empty() {
key_path.to_string()
} else {
format!("{path}.{key_path}")
};
walk(
a.and_then(|v| v.get(key)),
b.and_then(|v| v.get(key)),
&next,
out,
);
}
}
(Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => {
let a = old
.and_then(Json::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
let b = new
.and_then(Json::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
if a.is_empty() && b.is_empty() {
out.insert(path.to_string());
}
for index in 0..a.len().max(b.len()) {
walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out);
}
}
_ => {
out.insert(path.to_string());
}
}
}
let mut paths = BTreeSet::new();
walk(Some(old), Some(new), "", &mut paths);
paths.into_iter().collect()
}
#[derive(Debug, Serialize)]
pub(in crate::api) struct NormalizationEffect {
pub(super) path: String,
+12 -9
View File
@@ -10,21 +10,24 @@ use hyper::{HeaderMap, StatusCode};
use super::*;
#[derive(Debug, Serialize)]
pub(super) struct RuntimeBaseline {
pub(super) active_generation: u64,
pub(super) config_epoch: u64,
pub(super) runtime_match: String,
pub(in crate::api) struct RuntimeBaseline {
pub(in crate::api) active_generation: u64,
pub(in crate::api) config_epoch: u64,
pub(in crate::api) runtime_match: String,
}
#[derive(Clone)]
pub(super) struct RuntimeFence {
pub(in crate::api) struct RuntimeFence {
active: Arc<arc_swap::ArcSwap<RuntimeGeneration>>,
generation: Arc<RuntimeGeneration>,
pub(super) config: Arc<ProxyConfig>,
pub(in crate::api) config: Arc<ProxyConfig>,
}
impl RuntimeFence {
pub(super) fn capture(shared: &ApiShared, expected: Option<&str>) -> Result<Self, ApiFailure> {
pub(in crate::api) fn capture(
shared: &ApiShared,
expected: Option<&str>,
) -> Result<Self, ApiFailure> {
let generation = shared.active_runtime.load_full();
let config = generation.config();
let fence = Self {
@@ -39,7 +42,7 @@ impl RuntimeFence {
Ok(fence)
}
pub(super) fn baseline(&self) -> RuntimeBaseline {
pub(in crate::api) fn baseline(&self) -> RuntimeBaseline {
static PROCESS: OnceLock<String> = OnceLock::new();
let process = PROCESS.get_or_init(|| {
let mut bytes = [0_u8; 16];
@@ -63,7 +66,7 @@ impl RuntimeFence {
&& Arc::ptr_eq(&self.active.load_full(), &self.generation)
}
pub(super) fn ensure_current(&self) -> Result<(), ApiFailure> {
pub(in crate::api) fn ensure_current(&self) -> Result<(), ApiFailure> {
if self.matches() {
Ok(())
} else {
+350
View File
@@ -0,0 +1,350 @@
//! Shared value-free differences and conservative read-only config projections.
use serde::Serialize;
use serde_json::Value as Json;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::sync::OnceLock;
use super::model::ApiFailure;
use crate::config::{ProxyConfig, observation::ConfigProvenance};
pub(super) const MAX_PATHS: usize = 4096;
pub(super) const SECTIONS: &[&str] = &[
"general",
"network",
"server",
"timeouts",
"censorship",
"upstreams",
"dc_overrides",
"web",
"access",
"logging",
"root",
];
const MAX_FIELDS: usize = 1024;
pub(super) fn selected(path: &str, sections: &BTreeSet<String>) -> bool {
let root = path.split(['.', '[']).next().unwrap_or_default();
sections.contains(if SECTIONS.contains(&root) {
root
} else {
"root"
})
}
pub(super) fn retain_sections(value: &mut Json, sections: &BTreeSet<String>) {
if let Some(object) = value.as_object_mut() {
object.retain(|key, _| selected(key, sections));
}
}
struct Rule {
public: bool,
schema: Json,
default: Option<Json>,
}
fn rules() -> Result<&'static BTreeMap<String, Rule>, ApiFailure> {
static RULES: OnceLock<Result<BTreeMap<String, Rule>, String>> = OnceLock::new();
RULES
.get_or_init(|| {
super::discovery::projection_rules()
.map(|fields| {
fields
.into_iter()
.map(|(path, public, schema, default)| {
(
path,
Rule {
public,
schema,
default,
},
)
})
.collect()
})
.map_err(|_| "Cannot load configuration projection policy".to_owned())
})
.as_ref()
.map_err(|error| ApiFailure::internal(error.clone()))
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct FieldProjection {
pub(super) path: String,
#[serde(skip_serializing_if = "Option::is_none")]
value: Option<Json>,
redacted: bool,
present: bool,
entries: Option<usize>,
differs: Option<bool>,
origin: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
inherited_from: Option<&'static str>,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct Projection {
pub(super) fields: Vec<FieldProjection>,
pub(super) truncated: bool,
provenance: &'static str,
/// Input provenance only; this is deliberately not named applied_revision.
source_revision: Option<String>,
source_count: Option<usize>,
}
pub(super) fn project(
config: &ProxyConfig,
value: &Json,
other: Option<&Json>,
sections: &BTreeSet<String>,
desired: bool,
) -> Result<Projection, ApiFailure> {
let rules = rules()?;
let (provenance, source_revision, source_count) = match &config.provenance {
ConfigProvenance::Source(source) => (
"normalized_source",
Some(source.revision.clone()),
Some(source.source_count),
),
ConfigProvenance::Mixed => ("mixed_publication", None, None),
ConfigProvenance::Unknown => ("unknown", None, None),
};
let mut output = Projection {
fields: Vec::new(),
truncated: false,
provenance,
source_revision,
source_count,
};
struct Walk<'a> {
config: &'a ProxyConfig,
rules: &'a BTreeMap<String, Rule>,
desired: bool,
comparing: bool,
}
impl Walk<'_> {
fn visit(
&self,
value: &Json,
other: Option<&Json>,
path: &str,
canonical: &str,
output: &mut Projection,
) {
if output.fields.len() >= MAX_FIELDS || path.len() > 256 {
output.truncated = true;
return;
}
let rule = self.rules.get(canonical);
let public = rule.is_some_and(|rule| rule.public);
let open_map = rule.is_some_and(|rule| {
rule.schema
.get("additionalProperties")
.is_some_and(|value| value != &Json::Bool(false))
}) || path == "dc_overrides"
|| path == "censorship.exclusive_mask";
if public
&& !open_map
&& let Some(object) = value.as_object()
{
if !object.is_empty() {
for (key, child) in object {
self.visit(
child,
other.and_then(|value| value.get(key)),
&format!("{path}.{key}"),
&format!("{canonical}.{key}"),
output,
);
}
return;
}
}
if public
&& let Some(array) = value.as_array()
&& rule.is_some_and(|rule| rule.schema["items"]["type"] == "object")
{
if !array.is_empty() {
for (index, child) in array.iter().enumerate() {
if let Some(object) = child.as_object() {
for (key, field) in object {
self.visit(
field,
other.and_then(|v| v.get(index)).and_then(|v| v.get(key)),
&format!("{path}[{index}].{key}"),
&format!("{canonical}[].{key}"),
output,
);
}
}
}
return;
}
}
// Arbitrary strings and all dynamic identities are private, including
// URL userinfo in a field whose static metadata says "public".
let safe_scalar = |value: &Json, schema: &Json| {
value.is_null()
|| value.is_boolean()
|| value.is_number()
|| schema
.get("enum")
.and_then(Json::as_array)
.is_some_and(|allowed| allowed.contains(value))
};
let runtime_derived = path == "censorship.fake_cert_len"
&& matches!(&self.config.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len);
let generated = self.desired && runtime_derived;
let visible = public
&& !open_map
&& !generated
&& rule.is_some_and(|rule| {
safe_scalar(value, &rule.schema)
|| value.as_array().is_some_and(|array| {
array.len() <= 32
&& array
.iter()
.all(|item| safe_scalar(item, &rule.schema["items"]))
})
});
let (origin, inherited_from) = if runtime_derived {
("runtime_derived", None)
} else if canonical == "upstreams[].prefer" && value.is_null() {
("inherited", Some("network.prefer"))
} else {
(
match &self.config.provenance {
ConfigProvenance::Source(source)
if source.explicit_paths.contains(path) =>
{
"source"
}
ConfigProvenance::Source(source) if !source.complete => "unknown",
ConfigProvenance::Source(_)
if rule.and_then(|rule| rule.default.as_ref()) == Some(value) =>
{
"type_default"
}
ConfigProvenance::Source(_) => "default_or_derived",
ConfigProvenance::Mixed => "mixed_unknown",
ConfigProvenance::Unknown => "unknown",
},
None,
)
};
let present = !value.is_null()
&& value.as_str().is_none_or(|v| !v.is_empty())
&& value.as_object().is_none_or(|v| !v.is_empty())
&& value.as_array().is_none_or(|v| !v.is_empty());
output.fields.push(FieldProjection {
path: path.to_owned(),
value: visible.then(|| value.clone()),
redacted: !visible,
present,
entries: value
.as_object()
.map(|v| v.len())
.or_else(|| value.as_array().map(Vec::len)),
differs: (self.comparing && !runtime_derived).then(|| other != Some(value)),
origin,
inherited_from,
});
}
}
let walk = Walk {
config,
rules,
desired,
comparing: other.is_some(),
};
if let Some(object) = value.as_object() {
for (key, child) in object {
if selected(key, sections) {
walk.visit(
child,
other.and_then(|value| value.get(key)),
key,
key,
&mut output,
);
}
}
}
Ok(output)
}
/// Paths only: no before/after values or dynamic keys are returned. Arrays use
/// indices, open maps use their documented placeholder to avoid leaking identities.
pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec<String> {
fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet<String>) {
if old == new {
return;
}
if out.len() > MAX_PATHS {
return;
}
let map_key = if path == "dc_overrides" {
Some("{dc}")
} else if path == "censorship.exclusive_mask" {
Some("{hostname}")
} else if path.starts_with("access.") {
Some("{entry}")
} else {
None
};
match (old, new) {
(Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => {
let a = old.and_then(Json::as_object);
let b = new.and_then(Json::as_object);
let keys: BTreeSet<_> = a
.into_iter()
.flat_map(|v| v.keys())
.chain(b.into_iter().flat_map(|v| v.keys()))
.collect();
if keys.is_empty() {
out.insert(path.to_string());
}
for key in keys {
let key_path = map_key.unwrap_or(key);
let next = if path.is_empty() {
key_path.to_string()
} else {
format!("{path}.{key_path}")
};
walk(
a.and_then(|v| v.get(key)),
b.and_then(|v| v.get(key)),
&next,
out,
);
}
}
(Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => {
let a = old
.and_then(Json::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
let b = new
.and_then(Json::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
if a.is_empty() && b.is_empty() {
out.insert(path.to_string());
}
for index in 0..a.len().max(b.len()) {
walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out);
}
}
_ => {
out.insert(path.to_string());
}
}
}
let mut paths = BTreeSet::new();
walk(Some(old), Some(new), "", &mut paths);
paths.into_iter().collect()
}
+372
View File
@@ -0,0 +1,372 @@
//! Bounded inspection of desired source and actual owner publications.
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use http_body_util::Full;
use hyper::{Response, StatusCode, body::Bytes};
use serde::Serialize;
use serde_json::Value as Json;
use super::config_edit::{RuntimeBaseline, RuntimeFence};
use super::model::{ApiFailure, FieldIssue};
use super::{ApiShared, config_projection as projection};
use crate::config::{
ProxyConfig,
observation::{ConfigProvenance, WatchOutcome},
};
use crate::maestro::{reload::ReloadObservation, runtime_build::project_reload_config};
const MAX_RESPONSE_BYTES: usize = 512 * 1024;
const MAX_STATE_PATHS: usize = 512;
#[derive(Debug, Serialize)]
struct Comparison {
scope: &'static str,
status: &'static str,
pending_paths: Vec<String>,
watcher_pending_paths: Vec<String>,
generation_pending_paths: Vec<String>,
process_deferred_paths: Vec<String>,
deferred_process_fields: Vec<String>,
runtime_derived_paths: Vec<String>,
truncated: bool,
}
impl Default for Comparison {
fn default() -> Self {
Self {
scope: "selected_sections",
status: "unknown",
pending_paths: Vec::new(),
watcher_pending_paths: Vec::new(),
generation_pending_paths: Vec::new(),
process_deferred_paths: Vec::new(),
deferred_process_fields: Vec::new(),
runtime_derived_paths: Vec::new(),
truncated: false,
}
}
}
#[derive(Debug, Serialize)]
pub(super) struct ConfigState {
scope: &'static str,
sections: BTreeSet<String>,
runtime: RuntimeBaseline,
runtime_sample_current: bool,
pub(super) desired_revision: Option<String>,
desired_status: &'static str,
desired_error: Option<FieldIssue>,
desired: Option<projection::Projection>,
effective: Option<projection::Projection>,
projection_error: Option<FieldIssue>,
comparison: Comparison,
watcher: Option<WatchOutcome>,
watcher_matches_desired: Option<bool>,
reload: Option<ReloadObservation>,
reload_observation: &'static str,
reload_matches_desired: Option<bool>,
subsystem_convergence: &'static str,
}
fn issue(code: &'static str, message: &'static str) -> FieldIssue {
FieldIssue {
path: "$".into(),
code,
message: message.into(),
}
}
fn sections(query: Option<&str>) -> Result<BTreeSet<String>, ApiFailure> {
let Some(query) = query.filter(|query| !query.is_empty()) else {
return Ok(projection::SECTIONS
.iter()
.map(|section| (*section).to_owned())
.collect());
};
if query.len() > 256 {
return Err(ApiFailure::bad_request(
"Config state query exceeds 256 bytes",
));
}
let mut parsed = url::form_urlencoded::parse(query.as_bytes());
let Some((key, value)) = parsed.next() else {
return Err(ApiFailure::bad_request("Expected sections query"));
};
if key != "sections" || parsed.next().is_some() {
return Err(ApiFailure::bad_request(
"Only one sections query parameter is supported",
));
}
let mut sections = BTreeSet::new();
for section in value.split(',') {
if !projection::SECTIONS.contains(&section) || !sections.insert(section.to_owned()) {
return Err(ApiFailure::bad_request(
"Unknown, empty or repeated configuration section",
));
}
}
Ok(sections)
}
fn json(config: &ProxyConfig) -> Result<Json, ApiFailure> {
serde_json::to_value(config)
.map_err(|_| ApiFailure::internal("Cannot project configuration snapshot"))
}
struct Desired {
revision: Option<String>,
status: &'static str,
error: Option<FieldIssue>,
projection: Option<projection::Projection>,
effective: Option<projection::Projection>,
comparison: Comparison,
}
fn unavailable(revision: Option<String>, status: &'static str, code: &'static str) -> Desired {
Desired {
revision,
status,
error: Some(issue(
code,
"Desired configuration is unavailable; effective publication remains readable",
)),
projection: None,
effective: None,
comparison: Comparison::default(),
}
}
fn read_desired(
path: &std::path::Path,
active: &ProxyConfig,
sections: &BTreeSet<String>,
) -> Result<Desired, ApiFailure> {
let graph = match ProxyConfig::read_source_graph(path) {
Ok(graph) => graph,
Err(_) => return Ok(unavailable(None, "unreadable", "desired_source_unreadable")),
};
let revision = super::config_store::compute_source_revision(&graph);
let mut desired = match ProxyConfig::parse_captured_source(graph) {
Ok(parsed) => parsed.config,
Err(_) => {
return Ok(unavailable(
Some(revision),
"invalid",
"desired_config_invalid",
));
}
};
let mut derived = Vec::new();
if matches!(&desired.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len)
{
desired.censorship.fake_cert_len = active.censorship.fake_cert_len;
if projection::selected("censorship.fake_cert_len", sections) {
derived.push("censorship.fake_cert_len".into());
}
}
let mut active_json = json(active)?;
let mut desired_json = json(&desired)?;
let effective =
projection::project(active, &active_json, Some(&desired_json), sections, false)?;
let desired_projection =
projection::project(&desired, &desired_json, Some(&active_json), sections, true)?;
let hot = crate::config::hot_reload::overlay_hot_fields(active, &desired);
let projected = project_reload_config(active, &desired);
let mut hot_json = json(&hot)?;
let mut projected_json = json(&projected.effective)?;
for value in [
&mut active_json,
&mut desired_json,
&mut hot_json,
&mut projected_json,
] {
projection::retain_sections(value, sections);
}
let mut comparison = Comparison {
pending_paths: projection::changed_paths(&active_json, &desired_json),
watcher_pending_paths: projection::changed_paths(&active_json, &hot_json),
generation_pending_paths: projection::changed_paths(&hot_json, &projected_json),
process_deferred_paths: projection::changed_paths(&projected_json, &desired_json),
deferred_process_fields: projected
.deferred_process_fields
.into_iter()
.filter(|path| projection::selected(path, sections))
.collect(),
runtime_derived_paths: derived,
..Comparison::default()
};
comparison.status = if comparison.pending_paths.is_empty() {
"published_match"
} else if comparison.watcher_pending_paths.is_empty()
&& comparison.generation_pending_paths.is_empty()
{
"process_deferred"
} else {
"pending_publication"
};
for paths in [
&mut comparison.pending_paths,
&mut comparison.watcher_pending_paths,
&mut comparison.generation_pending_paths,
&mut comparison.process_deferred_paths,
] {
comparison.truncated |= paths.len() > MAX_STATE_PATHS;
paths.truncate(MAX_STATE_PATHS);
}
Ok(Desired {
revision: Some(revision),
status: "valid",
error: None,
projection: Some(desired_projection),
effective: Some(effective),
comparison,
})
}
pub(super) async fn state(
shared: &ApiShared,
query: Option<&str>,
) -> Result<ConfigState, ApiFailure> {
let sections = sections(query)?;
let fence = RuntimeFence::capture(shared, None)?;
let mut data = ConfigState {
scope: "published_configuration_not_subsystem_acknowledgment",
sections: sections.clone(),
runtime: fence.baseline(),
runtime_sample_current: true,
desired_revision: None,
desired_status: "busy",
desired_error: None,
desired: None,
effective: None,
projection_error: None,
comparison: Comparison::default(),
watcher: None,
watcher_matches_desired: None,
reload: None,
reload_observation: "unknown",
reload_matches_desired: None,
subsystem_convergence: "unknown",
};
if let Ok(permit) = shared.config_state_slots.clone().try_acquire_owned() {
let permit = Arc::new(permit);
let guard = permit.clone();
let active = fence.config.clone();
let scope = sections.clone();
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
let effective = tokio::task::spawn_blocking(move || {
let _guard = guard;
projection::project(&active, &json(&active)?, None, &scope, false)
});
match tokio::time::timeout_at(deadline, effective).await {
Ok(Ok(Ok(effective))) => data.effective = Some(effective),
_ => {
data.projection_error = Some(issue(
"effective_projection_unavailable",
"Runtime identity is available but its projection could not complete",
))
}
}
if data.effective.is_some() {
let active = fence.config.clone();
let path = shared.config_path.clone();
let desired = tokio::task::spawn_blocking(move || {
let _guard = permit;
read_desired(&path, &active, &sections)
});
let desired = match tokio::time::timeout_at(deadline, desired).await {
Ok(Ok(Ok(desired))) => desired,
Ok(_) => unavailable(None, "unavailable", "desired_projection_failed"),
Err(_) => unavailable(None, "timeout", "desired_read_timeout"),
};
data.desired_revision = desired.revision;
data.desired_status = desired.status;
data.desired_error = desired.error;
data.desired = desired.projection;
if desired.effective.is_some() {
data.effective = desired.effective;
}
data.comparison = desired.comparison;
} else {
data.desired_status = "unavailable";
data.desired_error = Some(issue(
"state_projection_unavailable",
"Configuration projection could not complete within the read budget",
));
}
} else {
data.desired_error = Some(issue(
"config_state_busy",
"Both configuration inspection slots are occupied",
));
}
data.runtime_sample_current = fence.ensure_current().is_ok();
data.watcher = fence
.config
.watcher_observation
.as_ref()
.and_then(|observer| observer.snapshot())
.map(|outcome| (*outcome).clone());
data.watcher_matches_desired = data.desired_revision.as_ref().and_then(|revision| {
data.watcher
.as_ref()?
.source_revision
.as_ref()
.map(|source| source == revision)
});
match shared.reload_control.observation() {
Some(reload) => {
data.reload = reload;
data.reload_observation = "sampled";
}
None => data.reload_observation = "busy",
}
data.reload_matches_desired = data.desired_revision.as_ref().and_then(|revision| {
data.reload
.as_ref()
.map(|reload| &reload.source_revision == revision)
});
Ok(data)
}
pub(super) async fn response(
shared: &ApiShared,
query: Option<&str>,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let mut data = state(shared, query).await?;
loop {
let body = serde_json::to_vec(
&serde_json::json!({"ok": true, "revision": data.desired_revision, "data": data}),
)
.map_err(|_| ApiFailure::internal("Cannot encode configuration state"))?;
if body.len() <= MAX_RESPONSE_BYTES {
return Response::builder()
.status(StatusCode::OK)
.header("content-type", "application/json; charset=utf-8")
.body(Full::new(Bytes::from(body)))
.map_err(|_| ApiFailure::internal("Cannot build configuration state response"));
}
// Each list was already capped. Shrinking the two projections guarantees
// that oversized deployment shapes cannot exceed the response envelope.
let mut shrunk = false;
for projection in [&mut data.desired, &mut data.effective]
.into_iter()
.flatten()
{
shrunk |= !projection.fields.is_empty();
projection.fields.truncate(projection.fields.len() / 2);
projection.truncated = true;
}
if !shrunk {
return Err(ApiFailure::internal(
"Configuration state metadata exceeded the response bound",
));
}
}
}
#[cfg(test)]
mod tests;
+198
View File
@@ -0,0 +1,198 @@
use super::*;
use http_body_util::BodyExt;
#[tokio::test]
async fn busy_state_preserves_runtime_identity_and_does_not_queue_more_work() {
let (fixture, _sender) = fixture("", true).await;
let permits = fixture
.shared
.config_state_slots
.clone()
.acquire_many_owned(2)
.await
.unwrap();
let data = inspect(&fixture, None).await;
assert_eq!(data["desired_status"], "busy");
assert_eq!(data["desired_error"]["code"], "config_state_busy");
assert_eq!(data["runtime"]["active_generation"], 1);
assert!(data["effective"].is_null());
assert_eq!(data["comparison"]["status"], "unknown");
drop(permits);
assert_eq!(inspect(&fixture, None).await["desired_status"], "valid");
}
#[tokio::test]
async fn large_deployment_bounds_fields_paths_bytes_and_dynamic_identities() {
let (fixture, _sender) = fixture("", true).await;
let mut text =
String::from("[access.users]\nprivate_identity='000102030405060708090a0b0c0d0e0f'\n");
for _ in 0..1200 {
text.push_str("[[upstreams]]\ntype='socks5'\naddress='127.0.0.1:1080'\nusername='private_socks_user'\npassword='private_socks_password'\n");
}
source(&fixture, &text, true);
let response = response(&fixture.shared, None).await.unwrap();
let bytes = response.into_body().collect().await.unwrap().to_bytes();
assert!(bytes.len() <= MAX_RESPONSE_BYTES);
let data: Json = serde_json::from_slice(&bytes).unwrap();
assert_eq!(data["data"]["desired_status"], "valid");
assert_eq!(data["data"]["desired"]["truncated"], true);
assert!(data["data"]["desired"]["fields"].as_array().unwrap().len() <= 1024);
assert_eq!(data["data"]["comparison"]["truncated"], true);
assert!(
data["data"]["comparison"]["pending_paths"]
.as_array()
.unwrap()
.len()
<= MAX_STATE_PATHS
);
let encoded = String::from_utf8(bytes.to_vec()).unwrap();
for private in [
"private_identity",
"private_socks_user",
"private_socks_password",
"000102030405060708090a0b0c0d0e0f",
"Bearer discovery-test",
] {
assert!(!encoded.contains(private));
}
}
fn abandoned_state_retains_budget(timeout: bool) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.max_blocking_threads(1)
.build()
.unwrap();
runtime.block_on(async {
let (fixture, _sender) = fixture("", true).await;
let original = std::fs::read(&fixture.shared.config_path).unwrap();
let active = fixture.shared.active_runtime.load_full();
let (release, wait) = std::sync::mpsc::channel();
let (started, ready) = tokio::sync::oneshot::channel();
let blocker = tokio::task::spawn_blocking(move || {
started.send(()).unwrap();
wait.recv().unwrap();
});
ready.await.unwrap();
tokio::time::pause();
let shared = fixture.shared.clone();
let task = tokio::spawn(async move { state(&shared, None).await });
while fixture.shared.config_state_slots.available_permits() == 2 {
tokio::task::yield_now().await;
}
if timeout {
fixture
.shared
.active_runtime
.store(test_runtime_generation(2, active.config().as_ref().clone()));
tokio::time::advance(Duration::from_secs(6)).await;
let data = serde_json::to_value(task.await.unwrap().unwrap()).unwrap();
assert_eq!(data["runtime"]["active_generation"], 1);
assert_eq!(data["runtime_sample_current"], false);
assert_eq!(data["desired_status"], "unavailable");
assert_eq!(
data["projection_error"]["code"],
"effective_projection_unavailable"
);
} else {
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
}
assert_eq!(fixture.shared.config_state_slots.available_permits(), 1);
tokio::time::resume();
release.send(()).unwrap();
blocker.await.unwrap();
let _permits = tokio::time::timeout(
Duration::from_secs(5),
fixture
.shared
.config_state_slots
.clone()
.acquire_many_owned(2),
)
.await
.unwrap()
.unwrap();
assert_eq!(
std::fs::read(&fixture.shared.config_path).unwrap(),
original
);
assert!(fixture.shared.reload_control.status(1).await.is_none());
});
}
#[test]
fn cancellation_retains_read_slot_until_detached_worker_finishes() {
abandoned_state_retains_budget(false);
}
#[test]
fn timeout_retains_read_slot_and_reports_a_stale_runtime_sample() {
abandoned_state_retains_budget(true);
}
#[test]
fn desired_worker_timeout_preserves_effective_and_retains_its_read_slot() {
use std::future::Future;
use std::task::Poll;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.max_blocking_threads(1)
.build()
.unwrap();
runtime.block_on(async {
let (fixture, _sender) = fixture("", true).await;
let (release_initial, wait_initial) = std::sync::mpsc::channel();
let (initial_started, initial_ready) = tokio::sync::oneshot::channel();
let initial_blocker = tokio::task::spawn_blocking(move || {
initial_started.send(()).unwrap();
wait_initial.recv().unwrap();
});
initial_ready.await.unwrap();
let mut read = Box::pin(state(&fixture.shared, None));
// Queue effective projection, then occupy the only blocking thread once
// it completes. The desired-source worker will have to wait behind it.
std::future::poll_fn(|context| {
assert!(read.as_mut().poll(context).is_pending());
Poll::Ready(())
})
.await;
let (release, wait) = std::sync::mpsc::channel();
let (started, ready) = tokio::sync::oneshot::channel();
let blocker = tokio::task::spawn_blocking(move || {
started.send(()).unwrap();
wait.recv().unwrap();
});
release_initial.send(()).unwrap();
initial_blocker.await.unwrap();
ready.await.unwrap();
tokio::time::pause();
std::future::poll_fn(|context| {
assert!(read.as_mut().poll(context).is_pending());
Poll::Ready(())
})
.await;
tokio::time::advance(Duration::from_secs(6)).await;
let data = serde_json::to_value(read.await.unwrap()).unwrap();
assert_eq!(data["desired_status"], "timeout");
assert_eq!(data["desired_error"]["code"], "desired_read_timeout");
assert!(!data["effective"].is_null());
assert_eq!(data["comparison"]["status"], "unknown");
assert_eq!(fixture.shared.config_state_slots.available_permits(), 1);
tokio::time::resume();
release.send(()).unwrap();
blocker.await.unwrap();
let _permits = tokio::time::timeout(
Duration::from_secs(5),
fixture
.shared
.config_state_slots
.clone()
.acquire_many_owned(2),
)
.await
.unwrap()
.unwrap();
});
}
+143
View File
@@ -0,0 +1,143 @@
use super::*;
const WEB: &str = r#"
[access.users]
alice = "000102030405060708090a0b0c0d0e0f"
[[server.listeners]]
ip = "127.0.0.1"
port = 18080
transport = "web"
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
[web]
enabled = true
[[web.vhosts]]
host = "proxy.example.com"
public_addr = "203.0.113.10:443"
[web.vhosts.decoy]
mode = "http_upstream"
upstream = "http://127.0.0.1:18081"
[[web.vhosts.profiles]]
user = "alice"
secret_mode = "dd"
"#;
#[tokio::test]
async fn inspection_does_not_acquire_decoy_dns_or_static_files() {
let (fixture, _sender) = fixture(WEB, true).await;
source(
&fixture,
&WEB.replace(
"http://127.0.0.1:18081",
"http://unresolvable.invalid:18081",
)
.replace(
"[[web.vhosts.profiles]]",
"resolve = 'startup'\n[[web.vhosts.profiles]]",
),
true,
);
let data = inspect(&fixture, Some("sections=web")).await;
assert_eq!(data["desired_status"], "valid");
assert_eq!(
field(&data, "desired", "web.vhosts[0].decoy.upstream")["redacted"],
true
);
assert_eq!(data["comparison"]["status"], "pending_publication");
let directory = tempfile::tempdir().unwrap();
std::fs::write(
directory.path().join("index.html"),
"private synthetic asset",
)
.unwrap();
let static_source = WEB.replace(
"mode = \"http_upstream\"\nupstream = \"http://127.0.0.1:18081\"",
&format!(
"mode = 'static_directory'\ndirectory = '{}'\nindex = 'index.html'",
directory.path().display()
),
);
let (fixture, _sender) = self::fixture(&static_source, true).await;
directory.close().unwrap();
assert!(
ProxyConfig::load_prepared(fixture.shared.config_path.clone())
.await
.is_err()
);
let data = inspect(&fixture, None).await;
assert_eq!(data["desired_status"], "valid");
assert_eq!(data["comparison"]["status"], "published_match");
assert_eq!(data["subsystem_convergence"], "unknown");
assert!(!data.to_string().contains("private synthetic asset"));
}
#[tokio::test]
async fn conditional_web_capacity_and_includes_preserve_source_evidence() {
let (fixture, _sender) = fixture("[web.limits]\ndebug_bytes_global=4096", true).await;
source(
&fixture,
"[web.limits]\ndebug_bytes_global=8192\n[web.debug]\nbody_prefix_bytes=6144",
true,
);
let data = inspect(&fixture, Some("sections=web")).await;
assert_eq!(data["comparison"]["status"], "process_deferred");
assert!(
data["comparison"]["process_deferred_paths"]
.as_array()
.unwrap()
.contains(&json!("web.debug.body_prefix_bytes"))
);
assert!(
data["comparison"]["deferred_process_fields"]
.as_array()
.unwrap()
.contains(&json!("web.debug"))
);
source(&fixture, "include = \"extra.toml\"", true);
std::fs::write(
fixture
.shared
.config_path
.parent()
.unwrap()
.join("extra.toml"),
"[general]\nlog_level='debug'",
)
.unwrap();
let data = inspect(&fixture, Some("sections=general")).await;
assert_eq!(data["desired"]["source_count"], 2);
assert_eq!(
field(&data, "desired", "general.log_level")["origin"],
"source"
);
assert!(!data.to_string().contains("extra.toml"));
}
#[test]
fn projection_fails_closed_for_arbitrary_strings_unknown_fields_and_private_maps() {
let config = ProxyConfig::default();
let raw = json!({"general":{"proxy_secret_url":"https://private-user:private-pass@example.invalid/key", "unknown_field":"private-value"},
"access":{"user_rate_limits":{"private-name":{"up_bps":12345}}},
"dc_overrides":{"private-dc":{"address":"private-value"}}});
let scope = sections(None).unwrap();
let projection = projection::project(&config, &raw, Some(&json!({})), &scope, false).unwrap();
let result = serde_json::to_value(projection).unwrap();
for value in [
"private-user",
"private-pass",
"example.invalid",
"private-value",
"private-name",
"private-dc",
"12345",
] {
assert!(!result.to_string().contains(value));
}
assert!(
result["fields"]
.as_array()
.unwrap()
.iter()
.all(|field| field["redacted"] == true && field.get("value").is_none())
);
}
+352
View File
@@ -0,0 +1,352 @@
use super::*;
use crate::api::discovery::http_tests::Fixture;
use crate::config::observation::WatchObservation;
use crate::maestro::generation::test_runtime_generation;
use crate::maestro::reload::{ReloadControl, ReloadPhase, ReloadRequest};
use serde_json::json;
use tokio::sync::watch;
#[path = "bounds_tests.rs"]
mod bounds;
#[path = "preparation_tests.rs"]
mod preparation;
fn source(fixture: &Fixture, text: &str, read_only: bool) {
std::fs::write(&fixture.shared.config_path, format!("{text}\n[server.api]\nenabled = true\nread_only = {read_only}\nauth_header = \"Bearer discovery-test\"\n")).unwrap();
}
async fn fixture(text: &str, read_only: bool) -> (Fixture, watch::Sender<Arc<ProxyConfig>>) {
let fixture = Fixture::new(false);
source(&fixture, text, read_only);
let mut config = ProxyConfig::load_prepared(fixture.shared.config_path.clone())
.await
.unwrap()
.config;
config.watcher_observation = Some(Arc::new(WatchObservation::default()));
let mut runtime = test_runtime_generation(1, config);
let (sender, receiver) = watch::channel(runtime.config());
Arc::get_mut(&mut runtime).unwrap().config_rx = receiver;
fixture.shared.active_runtime.store(runtime);
(fixture, sender)
}
async fn inspect(fixture: &Fixture, query: Option<&str>) -> Json {
serde_json::to_value(state(&fixture.shared, query).await.unwrap()).unwrap()
}
fn field<'a>(data: &'a Json, projection: &str, path: &str) -> &'a Json {
data[projection]["fields"]
.as_array()
.unwrap()
.iter()
.find(|field| field["path"] == path)
.unwrap_or_else(|| panic!("missing {projection} field {path}"))
}
#[tokio::test]
async fn state_http_auth_scope_queries_read_only_and_no_mutation() {
let (fixture, _sender) = fixture("", true).await;
let bytes = std::fs::read(&fixture.shared.config_path).unwrap();
let active = fixture.shared.active_runtime.load_full();
for (method, path, authorized, expected) in [
("GET", "/v1/config/state", false, 401),
("POST", "/v1/config/state", true, 405),
(
"GET",
"/v1/config/state?sections=general,general",
true,
400,
),
("GET", "/v1/config/state?sections=", true, 400),
("GET", "/v1/config/state?sections=unknown", true, 400),
(
"GET",
"/v1/config/state?sections=general&sections=web",
true,
400,
),
("GET", "/v1/config/state?debug=true", true, 400),
] {
let (status, allow, _) = fixture.request(method, path, authorized).await;
assert_eq!(status, expected, "{method} {path}");
if status == 405 {
assert_eq!(allow, "GET");
}
}
assert!(sections(Some(&format!("sections={}", "a".repeat(256)))).is_err());
let (status, _, response) = fixture
.request("GET", "/v1/config/state?sections=general,access", true)
.await;
assert_eq!(status, 200, "{response}");
let data = &response["data"];
assert_eq!(response["revision"], data["desired_revision"]);
assert_eq!(data["sections"], json!(["access", "general"]));
assert_eq!(data["comparison"]["status"], "published_match");
assert_eq!(data["runtime"]["active_generation"], 1);
assert_eq!(data["runtime_sample_current"], true);
assert_eq!(data["subsystem_convergence"], "unknown");
assert!(
data["effective"]["fields"]
.as_array()
.unwrap()
.iter()
.all(|field| {
let path = field["path"].as_str().unwrap();
path.starts_with("general.") || path.starts_with("access.")
})
);
assert_eq!(std::fs::read(&fixture.shared.config_path).unwrap(), bytes);
assert!(Arc::ptr_eq(
&active,
&fixture.shared.active_runtime.load_full()
));
assert!(fixture.shared.reload_control.status(1).await.is_none());
assert!(fixture.shared.runtime_events.snapshot(64).events.is_empty());
let denied = Fixture::new(true);
assert_eq!(denied.request("GET", "/v1/config/state", true).await.0, 403);
let caps = fixture.request("GET", "/v1/capabilities", true).await.2;
let operation = caps["data"]["operations"]
.as_array()
.unwrap()
.iter()
.find(|op| op["id"] == "config_state")
.unwrap();
assert_eq!(operation["available"], true);
assert_eq!(operation["mutable"], false);
}
#[tokio::test]
async fn invalid_or_unreadable_desired_preserves_effective_and_hides_errors() {
let (fixture, _sender) = fixture("[general]\nlog_level='debug'", true).await;
let epoch = fixture
.shared
.active_runtime
.load()
.config()
.runtime_config_epoch;
for invalid in [
"[general]\nme_writer_pick_mode='synthetic-private-value'",
"[broken synthetic-private-value",
] {
std::fs::write(&fixture.shared.config_path, invalid).unwrap();
let (status, _, response) = fixture.request("GET", "/v1/config/state", true).await;
assert_eq!(status, 200);
let data = &response["data"];
assert_eq!(data["desired_status"], "invalid");
assert!(response["revision"].is_string());
assert_eq!(data["desired_error"]["code"], "desired_config_invalid");
assert_eq!(data["runtime"]["config_epoch"], epoch);
assert_eq!(
field(data, "effective", "general.log_level")["value"],
"debug"
);
assert!(data["desired"].is_null());
assert_eq!(data["comparison"]["status"], "unknown");
assert!(!response.to_string().contains("synthetic-private-value"));
}
std::fs::remove_file(&fixture.shared.config_path).unwrap();
let response = fixture.request("GET", "/v1/config/state", true).await.2;
assert!(response["revision"].is_null());
assert_eq!(response["data"]["desired_status"], "unreadable");
assert!(!response["data"]["effective"].is_null());
assert!(
!response
.to_string()
.contains(fixture.shared.config_path.to_str().unwrap())
);
}
#[tokio::test]
async fn secret_rotation_stays_pending_until_publication_and_never_claims_convergence() {
const OLD: &str = "000102030405060708090a0b0c0d0e0f";
const NEW: &str = "f0e0d0c0b0a090807060504030201000";
let (fixture, sender) =
fixture(&format!("[access.users]\nprivate_account='{OLD}'"), false).await;
let before = inspect(&fixture, None).await;
let (status, _, response) = fixture
.request_body(
"POST",
"/v1/users/private_account/rotate-secret",
true,
Some(&json!({"secret": NEW})),
&[],
)
.await;
assert_eq!(status, 200, "{response}");
let pending = inspect(&fixture, None).await;
assert_ne!(pending["desired_revision"], before["desired_revision"]);
assert_eq!(pending["runtime"], before["runtime"]);
assert_eq!(pending["comparison"]["status"], "pending_publication");
assert!(
pending["comparison"]["watcher_pending_paths"]
.as_array()
.unwrap()
.contains(&json!("access.users.{entry}"))
);
let users = field(&pending, "effective", "access.users");
assert_eq!(users["differs"], true);
assert_eq!(users["present"], true);
assert_eq!(users["entries"], 1);
assert_eq!(users["redacted"], true);
assert!(users.get("value").is_none());
for private in [OLD, NEW, "private_account", "Bearer discovery-test"] {
assert!(!pending.to_string().contains(private));
}
let mut published = ProxyConfig::load_prepared(fixture.shared.config_path.clone())
.await
.unwrap()
.config
.with_publication_epoch();
published.watcher_observation = sender.borrow().watcher_observation.clone();
published.watcher_observation.as_ref().unwrap().record(
"published",
published.provenance.revision().map(str::to_owned),
Some(published.runtime_config_epoch),
None,
);
sender.send(Arc::new(published)).unwrap();
let after = inspect(&fixture, None).await;
assert_eq!(after["comparison"]["status"], "published_match");
assert_ne!(
after["runtime"]["config_epoch"],
before["runtime"]["config_epoch"]
);
assert_eq!(after["watcher_matches_desired"], true);
assert_eq!(after["subsystem_convergence"], "unknown");
assert_eq!(field(&after, "effective", "access.users")["differs"], false);
}
#[tokio::test]
async fn hot_generation_and_process_changes_have_distinct_paths() {
let (fixture, _sender) = fixture("", true).await;
source(
&fixture,
"[general]\nlog_level='debug'\ndisable_colors=true\n[network]\nipv6=true",
true,
);
let data = inspect(&fixture, None).await;
let comparison = &data["comparison"];
assert!(
comparison["watcher_pending_paths"]
.as_array()
.unwrap()
.contains(&json!("general.log_level"))
);
assert!(
comparison["generation_pending_paths"]
.as_array()
.unwrap()
.contains(&json!("network.ipv6"))
);
assert!(
comparison["process_deferred_paths"]
.as_array()
.unwrap()
.contains(&json!("general.disable_colors"))
);
source(&fixture, "[general]\ndisable_colors=true", true);
let data = inspect(&fixture, None).await;
assert_eq!(data["comparison"]["status"], "process_deferred");
assert_eq!(
data["comparison"]["deferred_process_fields"],
json!(["general.disable_colors"])
);
let scoped = inspect(&fixture, Some("sections=network")).await;
assert_eq!(scoped["comparison"]["status"], "published_match");
}
#[tokio::test]
async fn known_source_default_inheritance_and_runtime_derived_provenance() {
let text = "[general]\nlog_level='debug'\n[[upstreams]]\ntype='direct'\n[censorship]\ntls_emulation=false";
let (fixture, _sender) = fixture(text, true).await;
let data = inspect(&fixture, None).await;
assert_eq!(
field(&data, "desired", "general.log_level")["origin"],
"source"
);
assert_eq!(
field(&data, "desired", "general.disable_colors")["origin"],
"type_default"
);
assert_eq!(
field(&data, "desired", "upstreams[0].prefer")["origin"],
"inherited"
);
assert_eq!(
field(&data, "desired", "upstreams[0].prefer")["inherited_from"],
"network.prefer"
);
assert_eq!(
field(&data, "desired", "censorship.fake_cert_len")["origin"],
"runtime_derived"
);
assert!(
field(&data, "desired", "censorship.fake_cert_len")
.get("value")
.is_none()
);
assert_eq!(data["comparison"]["status"], "published_match");
assert_eq!(
data["comparison"]["runtime_derived_paths"],
json!(["censorship.fake_cert_len"])
);
assert_eq!(data["desired"]["source_revision"], data["desired_revision"]);
assert_eq!(data["desired"]["source_count"], 1);
}
#[tokio::test]
async fn reload_metadata_distinguishes_acceptance_failure_and_rollback_without_raw_errors() {
let (mut fixture, _sender) = fixture("", true).await;
let (control, mut commands) = ReloadControl::channel(1);
Arc::get_mut(&mut fixture.shared).unwrap().reload_control = control.clone();
let revision = inspect(&fixture, None).await["desired_revision"]
.as_str()
.unwrap()
.to_owned();
for terminal in [ReloadPhase::Failed, ReloadPhase::RolledBack] {
let accepted = control
.submit(
fixture.shared.active_runtime.load().config(),
revision.clone(),
ReloadRequest::default(),
)
.await
.unwrap();
let _ = commands.recv().await.unwrap();
let data = inspect(&fixture, None).await;
assert_eq!(data["reload"]["phase"], "accepted");
assert_eq!(data["reload_matches_desired"], true);
assert!(data["reload"]["activated_generation"].is_null());
control.record_source_generation(accepted.reload_id, 1);
control
.mark_phase(accepted.reload_id, ReloadPhase::Preparing)
.await;
control
.add_warning(accepted.reload_id, "synthetic-private-warning")
.await;
match terminal {
ReloadPhase::Failed => {
control
.fail(accepted.reload_id, "synthetic-private-error")
.await
}
ReloadPhase::RolledBack => {
control
.rolled_back(accepted.reload_id, "synthetic-private-error")
.await
}
_ => unreachable!(),
}
let data = inspect(&fixture, None).await;
assert_eq!(
data["reload"]["phase"],
serde_json::to_value(terminal).unwrap()
);
assert_eq!(data["reload"]["source_generation"], 1);
assert_eq!(data["reload"]["error_present"], true);
assert_eq!(data["reload"]["warning_count"], 1);
assert!(data["reload"]["retiring_generation"].is_null());
assert!(!data.to_string().contains("synthetic-private"));
}
}
+2 -14
View File
@@ -108,23 +108,11 @@ pub(super) fn compute_revision(content: &str) -> String {
/// Hashes source documents only; prepared DNS answers never change a revision.
pub(super) fn compute_snapshot_revision(source_contents: &BTreeMap<PathBuf, String>) -> String {
compute_source_revision(&ConfigSourceGraph {
source_contents: source_contents.clone(),
rendered: String::new(),
})
crate::config::observation::source_revision(source_contents)
}
pub(super) fn compute_source_revision(graph: &ConfigSourceGraph) -> String {
let mut hasher = Sha256::new();
hasher.update(b"telemt-config-manifest-v1\0");
for (path, content) in &graph.source_contents {
let path = path.as_os_str().as_encoded_bytes();
hasher.update((path.len() as u64).to_le_bytes());
hasher.update(path);
hasher.update((content.len() as u64).to_le_bytes());
hasher.update(content.as_bytes());
}
hex::encode(hasher.finalize())
compute_snapshot_revision(&graph.source_contents)
}
/// Reads a normalized source snapshot without resolving decoys or loading static assets.
+2
View File
@@ -25,6 +25,8 @@ pub(super) fn config_field_paths() -> Result<Vec<&'static str>, ApiFailure> {
schema::field_paths()
}
pub(super) use schema::projection_rules;
pub(super) const SCHEMA_VERSION: u32 = 1;
#[derive(Debug, Deserialize, Serialize)]
+1
View File
@@ -50,6 +50,7 @@ impl Fixture {
detected_ips_rx: watch::channel((None, None)).1,
mutation_lock: Arc::new(Mutex::new(())),
config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)),
config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)),
minimal_cache: Arc::new(Mutex::new(None)),
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
+1
View File
@@ -37,6 +37,7 @@
{"id": "users_active_ips","method": "GET","path": "/v1/stats/users/active-ips","gate": "always","success": [200]},
{"id": "users_quota","method": "GET","path": "/v1/stats/users/quota","gate": "always","success": [200]},
{"id": "config_get","method": "GET","path": "/v1/config","gate": "always","success": [200],"credential_bearing": true},
{"id": "config_state","method": "GET","path": "/v1/config/state","gate": "always","success": [200]},
{"id": "config_validate","method": "POST","path": "/v1/config/validate","gate": "always","success": [200],"body": "PatchConfigRequest","if_match": true,"if_runtime_match": true,"read_only_operation": true},
{"id": "config_patch","if_runtime_match": true,"method": "PATCH","path": "/v1/config","gate": "always","success": [200,202],"body": "PatchConfigRequest","if_match": true},
{"id": "reload","method": "POST","path": "/v1/system/reload","gate": "always","success": [202],"body": "ReloadRequest","optional_body": true,"if_match": true},
+16
View File
@@ -112,6 +112,22 @@ pub(super) fn field_paths() -> Result<Vec<&'static str>, ApiFailure> {
.collect())
}
pub(in crate::api) fn projection_rules()
-> Result<Vec<(String, bool, Value, Option<Value>)>, ApiFailure> {
Ok(config_schema()?
.fields
.iter()
.map(|field| {
(
field.spec.path.clone(),
field.spec.sensitivity == "public",
field.spec.value.clone(),
field.default.clone(),
)
})
.collect())
}
fn field_specs() -> Result<Vec<FieldSpec>, String> {
serde_json::from_str(include_str!("fields.json"))
.map_err(|error| format!("invalid compiled config field catalog: {error}"))
+3
View File
@@ -10,6 +10,9 @@ pub(super) async fn handle(
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
let api_cfg = &cfg.server.api;
match (method.as_str(), normalized_path) {
("GET", "/v1/config/state") => {
Ok(super::super::config_state::response(shared, query).await?)
}
("GET", "/v1/capabilities") => {
discovery::reject_query(query)?;
let revision = current_revision(&shared.config_path).await?;
+6
View File
@@ -37,6 +37,8 @@ use crate::web::control::WebRuntimePublication;
use crate::web::trace::WebTraceStore;
mod config_edit;
mod config_projection;
mod config_state;
pub(crate) mod config_store;
mod discovery;
mod events;
@@ -122,6 +124,7 @@ pub(super) struct ApiShared {
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
pub(super) mutation_lock: Arc<Mutex<()>>,
pub(super) config_preflight_slots: Arc<Semaphore>,
pub(super) config_state_slots: Arc<Semaphore>,
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
pub(super) runtime_edge_connections_cache: Arc<Mutex<Option<EdgeConnectionsCacheEntry>>>,
pub(super) runtime_edge_recompute_lock: Arc<Mutex<()>>,
@@ -159,6 +162,7 @@ impl ApiShared {
detected_ips_rx: self.detected_ips_rx.clone(),
mutation_lock: self.mutation_lock.clone(),
config_preflight_slots: self.config_preflight_slots.clone(),
config_state_slots: self.config_state_slots.clone(),
minimal_cache: self.minimal_cache.clone(),
runtime_edge_connections_cache: self.runtime_edge_connections_cache.clone(),
runtime_edge_recompute_lock: self.runtime_edge_recompute_lock.clone(),
@@ -269,6 +273,7 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
"/v1/health"
| "/v1/capabilities"
| "/v1/config/schema"
| "/v1/config/state"
| "/v1/health/ready"
| "/v1/system/info"
| "/v1/runtime/gates"
@@ -380,6 +385,7 @@ pub(crate) async fn serve(
detected_ips_rx,
mutation_lock: Arc::new(Mutex::new(())),
config_preflight_slots: Arc::new(Semaphore::new(2)),
config_state_slots: Arc::new(Semaphore::new(2)),
minimal_cache: Arc::new(Mutex::new(None)),
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
+1
View File
@@ -58,6 +58,7 @@ impl Fixture {
detected_ips_rx: watch::channel((None, None)).1,
mutation_lock: Arc::new(Mutex::new(())),
config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)),
config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)),
minimal_cache: Arc::new(Mutex::new(None)),
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
+82 -8
View File
@@ -144,9 +144,26 @@ fn prepare_effective_config(
let mut applied_cfg = overlay_hot_fields(old_cfg, new_cfg);
applied_cfg.validate_effective_web()?;
applied_cfg.rebuild_runtime_web()?;
applied_cfg.provenance = if config_equal(&applied_cfg, new_cfg) {
new_cfg.provenance.clone()
} else {
crate::config::observation::ConfigProvenance::Mixed
};
Ok(applied_cfg)
}
fn record_outcome(
config: &ProxyConfig,
phase: &'static str,
revision: Option<String>,
epoch: Option<u64>,
error: Option<&'static str>,
) {
if let Some(observation) = &config.watcher_observation {
observation.record(phase, revision, epoch, error);
}
}
/// Diffs and publishes a fully prepared configuration on the watcher owner task.
fn reload_config_with_resolver(
prepared: (LoadedConfig, ProxyConfig),
@@ -168,6 +185,7 @@ fn reload_config_with_resolver(
let next_manifest = WatchManifest::from_source_files(&source_files);
let old_cfg = config_tx.borrow().clone();
let revision = new_cfg.provenance.revision().map(str::to_owned);
let old_hot = HotFields::from_config(&old_cfg);
let applied_hot = HotFields::from_config(&applied_cfg);
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
@@ -179,11 +197,24 @@ fn reload_config_with_resolver(
}
if !hot_changed {
if refresh_quota_source {
config_tx
let published = !refresh_quota_source
|| config_tx
.send(Arc::new(applied_cfg.with_publication_epoch()))
.ok();
}
.is_ok();
let epoch = config_tx.borrow().runtime_config_epoch;
record_outcome(
&old_cfg,
if !published {
"publication_failed"
} else if non_hot_changed {
"retained"
} else {
"unchanged"
},
revision,
published.then_some(epoch),
(!published).then_some("publication_closed"),
);
reload_state.mark_applied(rendered_hash);
return Some(next_manifest);
}
@@ -196,6 +227,13 @@ fn reload_config_with_resolver(
"config reload: invalid network.dns_overrides: {}; keeping old config",
e
);
record_outcome(
&old_cfg,
"rejected",
revision,
None,
Some("dns_overrides_rejected"),
);
return Some(next_manifest);
}
@@ -207,9 +245,23 @@ fn reload_config_with_resolver(
detected_ip_v4,
detected_ip_v6,
);
config_tx
let published = config_tx
.send(Arc::new(applied_cfg.with_publication_epoch()))
.ok();
.is_ok();
let epoch = config_tx.borrow().runtime_config_epoch;
record_outcome(
&old_cfg,
if !published {
"publication_failed"
} else if non_hot_changed {
"published_partial"
} else {
"published"
},
revision,
published.then_some(epoch),
(!published).then_some("publication_closed"),
);
reload_state.mark_applied(rendered_hash);
Some(next_manifest)
}
@@ -265,8 +317,18 @@ pub fn spawn_config_watcher(
impl std::future::Future<Output = ()> + Send + 'static,
) {
let initial_level = initial.general.log_level.clone();
let (config_tx, config_rx) =
watch::channel(Arc::new((*initial).clone().with_publication_epoch()));
let mut initial = (*initial).clone().with_publication_epoch();
initial.watcher_observation = Some(Arc::new(
crate::config::observation::WatchObservation::default(),
));
record_outcome(
&initial,
"initial",
initial.provenance.revision().map(str::to_owned),
Some(initial.runtime_config_epoch),
None,
);
let (config_tx, config_rx) = watch::channel(Arc::new(initial));
let (log_tx, log_rx) = watch::channel(initial_level);
let config_path = normalize_watch_path(&config_path);
@@ -287,6 +349,15 @@ pub fn spawn_config_watcher(
}
}
let initial_loaded = preparation::read_source(&config_path).await;
if initial_loaded.is_none() {
record_outcome(
&config_tx.borrow(),
"rejected",
None,
None,
Some("source_invalid_or_unreadable"),
);
}
let initial_manifest = initial_loaded
.as_ref()
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
@@ -485,3 +556,6 @@ mod decoy_dns_tests;
#[cfg(test)]
mod quota_period_tests;
#[cfg(test)]
mod observation_tests;
@@ -0,0 +1,115 @@
use super::*;
use crate::config::observation::{ConfigProvenance, WatchObservation};
#[tokio::test]
async fn watcher_outcomes_follow_actual_publication_retention_and_rejection() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("config.toml");
std::fs::write(&path, "").unwrap();
let loaded = ProxyConfig::load_prepared(path.clone()).await.unwrap();
let mut initial = loaded.config.with_publication_epoch();
let observer = Arc::new(WatchObservation::default());
initial.watcher_observation = Some(observer.clone());
let (sender, mut receiver) = watch::channel(Arc::new(initial));
let (log, _) = watch::channel(sender.borrow().general.log_level.clone());
let mut state = ReloadState::new(Some(loaded.rendered_hash));
for (text, phase, changed) in [
("[general]\nlog_level='debug'", "published", true),
(
"[general]\nlog_level='debug'\ndisable_colors=true",
"retained",
false,
),
(
"[general]\nlog_level='verbose'\ndisable_colors=true",
"published_partial",
true,
),
(
"[general]\nlog_level='verbose'\ndisable_colors=true",
"unchanged",
false,
),
(
"[general]\nlog_level='private-invalid-log-level'",
"rejected",
false,
),
] {
let old = receiver.borrow_and_update().clone();
let sequence = observer.snapshot().map_or(0, |value| value.sequence);
std::fs::write(&path, text).unwrap();
preparation::reload(
&path, &sender, &log, None, None, &mut state, None, false, None, None,
)
.await;
let outcome = observer.snapshot().unwrap();
assert!(outcome.sequence > sequence);
assert_eq!(outcome.phase, phase);
assert_eq!(receiver.has_changed().unwrap(), changed);
if changed {
assert!(receiver.borrow().runtime_config_epoch > old.runtime_config_epoch);
assert_eq!(
outcome.published_config_epoch,
Some(receiver.borrow().runtime_config_epoch)
);
if phase == "published_partial" {
assert!(matches!(
&receiver.borrow().provenance,
ConfigProvenance::Mixed
));
} else {
assert_eq!(
outcome.source_revision.as_deref(),
receiver.borrow().provenance.revision()
);
}
} else {
assert!(Arc::ptr_eq(&old, &receiver.borrow()));
}
assert!(
!serde_json::to_string(&*outcome)
.unwrap()
.contains("private-invalid-log-level")
);
}
let rejected = observer.snapshot().unwrap();
assert_eq!(rejected.error_code, Some("source_invalid_or_unreadable"));
assert_eq!(rejected.published_config_epoch, None);
}
#[tokio::test]
async fn lost_receivers_never_report_successful_publication() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("config.toml");
std::fs::write(&path, "").unwrap();
let mut initial = ProxyConfig::load_prepared(path.clone())
.await
.unwrap()
.config;
let observer = Arc::new(WatchObservation::default());
initial.watcher_observation = Some(observer.clone());
let (sender, receiver) = watch::channel(Arc::new(initial.with_publication_epoch()));
let (log, _) = watch::channel(sender.borrow().general.log_level.clone());
let original = sender.borrow().clone();
drop(receiver);
std::fs::write(&path, "[general]\nlog_level='debug'").unwrap();
preparation::reload(
&path,
&sender,
&log,
None,
None,
&mut ReloadState::default(),
None,
false,
None,
None,
)
.await;
assert!(Arc::ptr_eq(&sender.borrow(), &original));
let outcome = observer.snapshot().unwrap();
assert_eq!(outcome.phase, "publication_failed");
assert_eq!(outcome.published_config_epoch, None);
assert_eq!(outcome.error_code, Some("publication_closed"));
}
+61 -3
View File
@@ -43,7 +43,17 @@ pub(super) async fn reload(
Some(gate) => Some(gate.lock().await),
None => None,
};
let source = read_source(path).await?;
let Some(source) = read_source(path).await else {
record_outcome(
&config_tx.borrow(),
"rejected",
None,
None,
Some("source_invalid_or_unreadable"),
);
return None;
};
let revision = source.config.provenance.revision().map(str::to_owned);
let manifest = WatchManifest::from_source_files(&source.source_files);
// A new file edit can restore the exact pre-creation bytes. Refresh only source
// evidence for fenced accounts, retaining prepared DNS and immutable WEB state.
@@ -53,16 +63,43 @@ pub(super) async fn reload(
{
let mut refreshed = (**config_tx.borrow()).clone();
refreshed.quota_source_read_at = source.config.quota_source_read_at;
config_tx
let published = config_tx
.send(Arc::new(refreshed.with_publication_epoch()))
.ok();
.is_ok();
let config = config_tx.borrow();
record_outcome(
&config,
if published {
"source_refreshed"
} else {
"publication_failed"
},
revision,
published.then_some(config.runtime_config_epoch),
(!published).then_some("publication_closed"),
);
reload_state.mark_applied(source.rendered_hash);
return Some(manifest);
}
drop(guard);
if !force && reload_state.is_applied(source.rendered_hash) {
let config = config_tx.borrow();
record_outcome(
&config,
"unchanged",
revision,
Some(config.runtime_config_epoch),
None,
);
return Some(manifest);
}
record_outcome(
&config_tx.borrow(),
"preparing",
revision.clone(),
None,
None,
);
let loaded = match source.prepare().await {
Ok(loaded) => loaded,
Err(error) => {
@@ -70,6 +107,13 @@ pub(super) async fn reload(
"config reload: preparation failed: {}; keeping old config",
error
);
record_outcome(
&config_tx.borrow(),
"rejected",
revision,
None,
Some("preparation_failed"),
);
return Some(manifest);
}
};
@@ -96,6 +140,13 @@ pub(super) async fn reload(
"config reload: effective WEB validation failed: {}; keeping old config",
error
);
record_outcome(
&config_tx.borrow(),
"rejected",
revision,
None,
Some("effective_config_invalid"),
);
Some(manifest)
}
Err(error) => {
@@ -103,6 +154,13 @@ pub(super) async fn reload(
"config reload: runtime builder failed: {}; keeping old config",
error
);
record_outcome(
&config_tx.borrow(),
"rejected",
revision,
None,
Some("preparation_worker_failed"),
);
Some(manifest)
}
}
+9
View File
@@ -160,6 +160,10 @@ pub struct ProxyConfig {
/// Process-local identity assigned by the owner of a published config snapshot.
#[serde(skip)]
pub(crate) runtime_config_epoch: u64,
#[serde(skip)]
pub(crate) provenance: super::observation::ConfigProvenance,
#[serde(skip)]
pub(crate) watcher_observation: Option<Arc<super::observation::WatchObservation>>,
/// Monotonic start of source acquisition, used only for cold quota policy fencing.
#[serde(skip)]
@@ -265,6 +269,11 @@ impl ProxyConfig {
Self::parse_source_with_overrides(path, &BTreeMap::new())
}
/// Parses an already captured source manifest without DNS or static-site reads.
pub(crate) fn parse_captured_source(graph: ConfigSourceGraph) -> Result<ParsedConfigSource> {
pipeline::parse_source_graph(graph)
}
/// Checks decoy mode-specific keys before a source or API patch loses unknown fields.
pub(crate) fn validate_decoy_source_keys(document: &toml::Value) -> Result<()> {
decoy_dns::validate_mode_keys(document)
+3
View File
@@ -20,6 +20,8 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSou
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
handle_unknown_config_keys(&parsed_toml)?;
ProxyConfig::validate_decoy_source_keys(&parsed_toml)?;
let source_evidence =
crate::config::observation::SourceEvidence::capture(&parsed_toml, &source_contents);
let general_table = parsed_toml
.get("general")
.and_then(|value| value.as_table());
@@ -65,6 +67,7 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSou
let mut config: ProxyConfig = parsed_toml
.try_into()
.map_err(|e| ProxyError::Config(e.to_string()))?;
config.provenance = crate::config::observation::ConfigProvenance::Source(source_evidence);
config
.server
.conntrack_control
+1
View File
@@ -3,6 +3,7 @@
pub(crate) mod defaults;
pub mod hot_reload;
mod load;
pub(crate) mod observation;
mod types;
pub use load::ProxyConfig;
+175
View File
@@ -0,0 +1,175 @@
//! Bounded source evidence and owner-authored configuration observations.
//! These records describe publication, never asynchronous consumer convergence.
use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use arc_swap::ArcSwapOption;
use serde::Serialize;
use sha2::{Digest, Sha256};
pub(crate) fn source_revision(sources: &BTreeMap<PathBuf, String>) -> String {
let mut hash = Sha256::new();
hash.update(b"telemt-config-manifest-v1\0");
for (path, content) in sources {
let path = path.as_os_str().as_encoded_bytes();
hash.update((path.len() as u64).to_le_bytes());
hash.update(path);
hash.update((content.len() as u64).to_le_bytes());
hash.update(content.as_bytes());
}
hex::encode(hash.finalize())
}
#[derive(Debug, Clone, Default)]
pub(crate) enum ConfigProvenance {
#[default]
Unknown,
Source(Arc<SourceEvidence>),
/// The published config combines sources or retained process policy. Do not
/// attribute all its values to the last source read by the watcher.
Mixed,
}
impl ConfigProvenance {
pub(crate) fn revision(&self) -> Option<&str> {
match self {
Self::Source(source) => Some(&source.revision),
_ => None,
}
}
}
#[derive(Debug)]
pub(crate) struct SourceEvidence {
pub(crate) revision: String,
pub(crate) explicit_paths: BTreeSet<String>,
pub(crate) complete: bool,
pub(crate) source_count: usize,
pub(crate) generated_fake_cert_len: bool,
}
impl SourceEvidence {
pub(crate) fn capture(
document: &toml::Value,
sources: &BTreeMap<PathBuf, String>,
) -> Arc<Self> {
fn visit(
value: &toml::Value,
path: &str,
paths: &mut BTreeSet<String>,
complete: &mut bool,
) {
if paths.len() >= 4096 || path.len() > 256 {
*complete = false;
return;
}
if !path.is_empty() {
paths.insert(path.to_owned());
}
// Open maps need only presence provenance, never their private keys.
if path == "dc_overrides"
|| path == "censorship.exclusive_mask"
|| path.starts_with("access.")
{
return;
}
match value {
toml::Value::Table(table) => {
for (key, value) in table {
let key = if path.starts_with("upstreams[")
&& path.ends_with(']')
&& key == "force_bind"
{
"bindtodevice"
} else if path == "server" && key == "admin_api" {
"api"
} else {
key
};
let child = if path.is_empty() {
key.to_owned()
} else {
format!("{path}.{key}")
};
visit(value, &child, paths, complete);
}
}
toml::Value::Array(array) => {
for (index, value) in array.iter().enumerate() {
visit(value, &format!("{path}[{index}]"), paths, complete);
}
}
_ => {}
}
}
let mut explicit_paths = BTreeSet::new();
let mut complete = true;
visit(document, "", &mut explicit_paths, &mut complete);
let censorship = document.get("censorship");
let generated_fake_cert_len = censorship
.and_then(|v| v.get("tls_emulation"))
.and_then(toml::Value::as_bool)
== Some(false)
&& censorship
.and_then(|v| v.get("fake_cert_len"))
.and_then(toml::Value::as_integer)
.is_none_or(|value| value == super::defaults::default_fake_cert_len() as i64);
Arc::new(Self {
revision: source_revision(sources),
explicit_paths,
complete,
source_count: sources.len(),
generated_fake_cert_len,
})
}
}
#[derive(Debug, Clone, Serialize)]
pub(crate) struct WatchOutcome {
pub(crate) sequence: u64,
pub(crate) observed_at_epoch_secs: u64,
pub(crate) phase: &'static str,
/// Source considered by this attempt, not a hash of applied runtime state.
pub(crate) source_revision: Option<String>,
pub(crate) published_config_epoch: Option<u64>,
pub(crate) error_code: Option<&'static str>,
}
#[derive(Debug, Default)]
pub(crate) struct WatchObservation {
next_sequence: AtomicU64,
latest: ArcSwapOption<WatchOutcome>,
}
impl WatchObservation {
pub(crate) fn record(
&self,
phase: &'static str,
revision: Option<String>,
epoch: Option<u64>,
error_code: Option<&'static str>,
) {
self.latest.store(Some(Arc::new(WatchOutcome {
sequence: self
.next_sequence
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1),
observed_at_epoch_secs: SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
phase,
source_revision: revision,
published_config_epoch: epoch,
error_code,
})));
}
pub(crate) fn snapshot(&self) -> Option<Arc<WatchOutcome>> {
self.latest.load_full()
}
}
+3
View File
@@ -207,6 +207,9 @@ pub(super) async fn bootstrap(
validate_synlimit_privilege_drop(&config, privilege_drop_requested)?;
if let Some(p) = data_path {
if config.general.data_path.as_ref() != Some(&p) {
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
}
config.general.data_path = Some(p);
}
+1
View File
@@ -51,6 +51,7 @@ pub(super) async fn run_telemt_core(
if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control {
warn!("Inline conntrack control is disabled when process privileges are dropped");
config.server.conntrack_control.inline_conntrack_control = false;
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
}
let quota_store = Arc::new(QuotaStore::default());
+22
View File
@@ -9,6 +9,10 @@ use tokio::sync::mpsc;
use crate::config::ProxyConfig;
#[path = "reload_observation.rs"]
mod observation;
pub(crate) use observation::ReloadObservation;
const RELOAD_HISTORY_CAPACITY: usize = 32;
const RELOAD_COMMAND_CAPACITY: usize = 1;
const MAX_DRAIN_TIMEOUT_SECS: u64 = 3_600;
@@ -151,6 +155,16 @@ pub(crate) struct ReloadStatus {
pub(crate) warnings: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) error: Option<String>,
#[serde(skip)]
source_generation: Option<u64>,
#[serde(skip)]
activated_generation: Option<u64>,
#[serde(skip)]
retiring_generation: Option<u64>,
#[serde(skip)]
retiring_config_epoch: Option<u64>,
#[serde(skip)]
retirement: Option<&'static str>,
}
/// Accepted operation metadata returned before asynchronous preparation starts.
@@ -395,6 +409,11 @@ impl ReloadStatusStore {
deferred_fields: Vec::new(),
warnings: Vec::new(),
error: None,
source_generation: None,
activated_generation: None,
retiring_generation: None,
retiring_config_epoch: None,
retirement: None,
};
state.active_reload_id = Some(reload_id);
state.statuses.push_back(status.clone());
@@ -416,6 +435,9 @@ impl ReloadStatusStore {
fn mark_phase(&self, reload_id: u64, phase: ReloadPhase) {
self.update(reload_id, |status| {
status.state = phase;
if phase == ReloadPhase::Draining {
status.retirement = Some("draining");
}
if status.started_at_epoch_secs.is_none() && phase != ReloadPhase::Accepted {
status.started_at_epoch_secs = Some(now_epoch_secs());
}
+69
View File
@@ -0,0 +1,69 @@
use serde::Serialize;
use super::{ReloadControl, ReloadPhase};
/// Sanitized owner snapshot. Raw reload errors/warnings can contain source data.
#[derive(Debug, Clone, Serialize)]
pub(crate) struct ReloadObservation {
pub(crate) reload_id: u64,
pub(crate) target_generation: u64,
pub(crate) source_revision: String,
pub(crate) phase: ReloadPhase,
pub(crate) source_generation: Option<u64>,
pub(crate) activated_generation: Option<u64>,
pub(crate) retiring_generation: Option<u64>,
pub(crate) retiring_config_epoch: Option<u64>,
pub(crate) retirement: Option<&'static str>,
pub(crate) error_present: bool,
pub(crate) warning_count: usize,
}
impl ReloadControl {
/// Reads bounded owner metadata without waiting for a contended status lock.
pub(crate) fn observation(&self) -> Option<Option<ReloadObservation>> {
let state = self.status_store.state.try_lock()?;
Some(state.statuses.back().map(|status| ReloadObservation {
reload_id: status.reload_id,
target_generation: status.target_generation,
source_revision: status.config_revision.clone(),
phase: status.state,
source_generation: status.source_generation,
activated_generation: status.activated_generation,
retiring_generation: status.retiring_generation,
retiring_config_epoch: status.retiring_config_epoch,
retirement: status.retirement,
error_present: status.error.is_some(),
warning_count: status.warnings.len(),
}))
}
pub(crate) fn record_source_generation(&self, id: u64, generation: u64) {
self.status_store
.update(id, |status| status.source_generation = Some(generation));
}
pub(crate) fn record_activation(
&self,
id: u64,
active: u64,
retiring: u64,
retiring_epoch: u64,
) {
self.status_store.update(id, |status| {
status.activated_generation = Some(active);
status.retiring_generation = Some(retiring);
status.retiring_config_epoch = Some(retiring_epoch);
status.retirement = Some("retiring");
});
}
pub(crate) fn record_retirement(&self, id: u64, timed_out: bool) {
self.status_store.update(id, |status| {
status.retirement = Some(if timed_out {
"cleanup_timeout"
} else {
"retired"
})
});
}
}
+12 -1
View File
@@ -163,6 +163,8 @@ impl ReloadSupervisor {
.mark_phase(command.reload_id, ReloadPhase::Preparing)
.await;
let old_runtime = self.active_runtime.load_full();
self.control
.record_source_generation(command.reload_id, old_runtime.id);
let resolved = match resolve_reload_config(&old_runtime.config(), &command.config) {
Ok(resolved) => resolved,
Err(error) => {
@@ -351,6 +353,12 @@ impl ReloadSupervisor {
old_runtime.stop_accepting_sessions();
listener_manager.activate_runtime_generation(new_runtime.clone())
};
self.control.record_activation(
command.reload_id,
new_runtime.id,
replaced.id,
replaced.config().runtime_config_epoch,
);
// A rejected admission epoch must not be bypassed by reapplying the prepared
// snapshot. Quota publication retains its credential and source checks.
let config = new_runtime.config();
@@ -422,7 +430,10 @@ impl ReloadSupervisor {
}
}
if stop_background_and_middle_end(&replaced).await {
let retirement_timed_out = stop_background_and_middle_end(&replaced).await;
self.control
.record_retirement(command.reload_id, retirement_timed_out);
if retirement_timed_out {
let warning = format!(
"generation {} Middle-End lifecycle shutdown timed out",
replaced.id
+43
View File
@@ -158,6 +158,11 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
assert!(fixture.old_runtime.spawn_session(async {}));
let status = fixture.control.status(1).await.unwrap();
assert_eq!(status.state, ReloadPhase::RolledBack);
let observed = fixture.control.observation().unwrap().unwrap();
assert_eq!(observed.phase, ReloadPhase::RolledBack);
assert_eq!(observed.activated_generation, None);
assert_eq!(observed.retiring_generation, None);
assert_eq!(observed.retirement, None);
fixture.old_runtime.stop_sessions().await;
}
@@ -206,12 +211,33 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
assert!(!activation.is_finished());
assert!(!fixture.old_runtime.spawn_session(async {}));
tokio::time::timeout(Duration::from_secs(1), async {
while fixture.control.status(1).await.unwrap().state != ReloadPhase::Draining {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let observed = fixture.control.observation().unwrap().unwrap();
assert_eq!(observed.phase, ReloadPhase::Draining);
assert_eq!(observed.activated_generation, Some(2));
assert_eq!(observed.retiring_generation, Some(1));
assert_eq!(
observed.retiring_config_epoch,
Some(fixture.old_runtime.config().runtime_config_epoch)
);
assert_eq!(observed.retirement, Some("draining"));
old_release.notify_one();
activation.await.unwrap();
assert_eq!(
fixture.control.status(1).await.unwrap().state,
ReloadPhase::Succeeded
);
let observed = fixture.control.observation().unwrap().unwrap();
assert_eq!(observed.phase, ReloadPhase::Succeeded);
assert_eq!(observed.retiring_generation, Some(1));
assert_eq!(observed.retirement, Some("retired"));
fixture.new_runtime.stop_sessions().await;
}
@@ -257,6 +283,23 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
fixture.new_runtime.stop_sessions().await;
}
#[tokio::test]
async fn rejected_reload_observes_source_generation_without_activation() {
let mut fixture = fixture(ReloadRequest::default()).await;
Arc::make_mut(&mut fixture.command.config).web.enabled = true;
fixture.supervisor.reload(fixture.command).await;
let observed = fixture.control.observation().unwrap().unwrap();
assert_eq!(observed.phase, ReloadPhase::Failed);
assert_eq!(observed.source_generation, Some(1));
assert_eq!(observed.activated_generation, None);
assert_eq!(observed.retiring_generation, None);
assert!(observed.error_present);
assert!(Arc::ptr_eq(
&fixture.supervisor.active_runtime.load_full(),
&fixture.old_runtime
));
}
#[tokio::test]
async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
let runtime = test_runtime_generation(1, ProxyConfig::default());
+26
View File
@@ -48,6 +48,11 @@ fn status_uses_documented_deferred_process_fields_key() {
deferred_fields: vec!["server.listeners".to_string()],
warnings: Vec::new(),
error: None,
source_generation: Some(1),
activated_generation: Some(2),
retiring_generation: Some(1),
retiring_config_epoch: Some(3),
retirement: Some("retired"),
};
let value = serde_json::to_value(status).unwrap();
@@ -56,6 +61,27 @@ fn status_uses_documented_deferred_process_fields_key() {
serde_json::json!(["server.listeners"])
);
assert!(value.get("deferred_fields").is_none());
for field in [
"source_generation",
"activated_generation",
"retiring_generation",
"retiring_config_epoch",
"retirement",
] {
assert!(
value.get(field).is_none(),
"Legacy reload response gained {field}"
);
}
}
#[test]
fn state_observation_never_waits_on_the_reload_status_lock() {
let (control, _receiver) = ReloadControl::channel(1);
let guard = control.status_store.state.lock();
assert!(control.observation().is_none());
drop(guard);
assert!(control.observation().unwrap().is_none());
}
#[tokio::test]
+28 -11
View File
@@ -334,6 +334,29 @@ pub(crate) fn resolve_reload_config(
old: &ProxyConfig,
desired: &ProxyConfig,
) -> Result<ResolvedReloadConfig, String> {
let mut resolved = project_reload_config(old, desired);
resolved
.effective
.validate_effective_web()
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
resolved
.effective
.rebuild_runtime_user_auth()
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
resolved
.effective
.rebuild_runtime_web()
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
resolved.runtime_changed = !configs_equal(old, &resolved.effective);
Ok(resolved)
}
/// Projects retained process policy without DNS, static acquisition or validation.
/// State inspection uses this projection; preflight still uses the full resolver.
pub(crate) fn project_reload_config(
old: &ProxyConfig,
desired: &ProxyConfig,
) -> ResolvedReloadConfig {
let mut effective = desired.clone();
let mut fields = Vec::new();
let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server);
@@ -462,21 +485,15 @@ pub(crate) fn resolve_reload_config(
fields.push("web.debug".to_string());
effective.web.debug = old.web.debug.clone();
}
effective
.validate_effective_web()
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
effective
.rebuild_runtime_user_auth()
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
effective
.rebuild_runtime_web()
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
if !fields.is_empty() {
effective.provenance = crate::config::observation::ConfigProvenance::Mixed;
}
let runtime_changed = !configs_equal(old, &effective);
Ok(ResolvedReloadConfig {
ResolvedReloadConfig {
effective,
deferred_process_fields: fields,
runtime_changed,
})
}
}
fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool {
+3
View File
@@ -172,6 +172,9 @@ pub(super) async fn prepare_runtime(
} else {
let _ = use_middle_proxy;
use_middle_proxy = false;
if config.general.use_middle_proxy {
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
}
config.general.use_middle_proxy = false;
startup_tracker.set_transport_mode("direct").await;
startup_tracker.set_degraded(true).await;