mirror of
https://github.com/telemt/telemt.git
synced 2026-11-22 15:26:37 +03:00
API: expose desired and effective state with runtime status
This commit is contained in:
@@ -27,6 +27,7 @@ use std::sync::Arc;
|
||||
mod diagnostics;
|
||||
mod preflight;
|
||||
mod preparation;
|
||||
pub(super) use preflight::{RuntimeBaseline, RuntimeFence};
|
||||
pub(super) use preflight::{parse_runtime_match, validate_config};
|
||||
use preparation::prepare_patch_to_path;
|
||||
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
use crate::api::model::FieldIssue;
|
||||
|
||||
pub(super) const MAX_PATHS: usize = 4096;
|
||||
pub(super) use crate::api::config_projection::{MAX_PATHS, changed_paths};
|
||||
|
||||
pub(super) fn issue(
|
||||
path: impl Into<String>,
|
||||
@@ -114,78 +112,6 @@ pub(super) fn reject_unknown(patch: &Json) -> Result<(), ApiFailure> {
|
||||
Err(error)
|
||||
}
|
||||
|
||||
/// Paths only: no before/after values or dynamic keys are returned. Arrays use
|
||||
/// indices, open maps use their documented placeholder to avoid leaking identities.
|
||||
pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec<String> {
|
||||
fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet<String>) {
|
||||
if old == new {
|
||||
return;
|
||||
}
|
||||
if out.len() > MAX_PATHS {
|
||||
return;
|
||||
}
|
||||
let map_key = if path == "dc_overrides" {
|
||||
Some("{dc}")
|
||||
} else if path == "censorship.exclusive_mask" {
|
||||
Some("{hostname}")
|
||||
} else if path.starts_with("access.") {
|
||||
Some("{entry}")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match (old, new) {
|
||||
(Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => {
|
||||
let a = old.and_then(Json::as_object);
|
||||
let b = new.and_then(Json::as_object);
|
||||
let keys: BTreeSet<_> = a
|
||||
.into_iter()
|
||||
.flat_map(|v| v.keys())
|
||||
.chain(b.into_iter().flat_map(|v| v.keys()))
|
||||
.collect();
|
||||
if keys.is_empty() {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
for key in keys {
|
||||
let key_path = map_key.unwrap_or(key);
|
||||
let next = if path.is_empty() {
|
||||
key_path.to_string()
|
||||
} else {
|
||||
format!("{path}.{key_path}")
|
||||
};
|
||||
walk(
|
||||
a.and_then(|v| v.get(key)),
|
||||
b.and_then(|v| v.get(key)),
|
||||
&next,
|
||||
out,
|
||||
);
|
||||
}
|
||||
}
|
||||
(Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => {
|
||||
let a = old
|
||||
.and_then(Json::as_array)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or_default();
|
||||
let b = new
|
||||
.and_then(Json::as_array)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or_default();
|
||||
if a.is_empty() && b.is_empty() {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
for index in 0..a.len().max(b.len()) {
|
||||
walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
walk(Some(old), Some(new), "", &mut paths);
|
||||
paths.into_iter().collect()
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(in crate::api) struct NormalizationEffect {
|
||||
pub(super) path: String,
|
||||
|
||||
@@ -10,21 +10,24 @@ use hyper::{HeaderMap, StatusCode};
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct RuntimeBaseline {
|
||||
pub(super) active_generation: u64,
|
||||
pub(super) config_epoch: u64,
|
||||
pub(super) runtime_match: String,
|
||||
pub(in crate::api) struct RuntimeBaseline {
|
||||
pub(in crate::api) active_generation: u64,
|
||||
pub(in crate::api) config_epoch: u64,
|
||||
pub(in crate::api) runtime_match: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RuntimeFence {
|
||||
pub(in crate::api) struct RuntimeFence {
|
||||
active: Arc<arc_swap::ArcSwap<RuntimeGeneration>>,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
pub(super) config: Arc<ProxyConfig>,
|
||||
pub(in crate::api) config: Arc<ProxyConfig>,
|
||||
}
|
||||
|
||||
impl RuntimeFence {
|
||||
pub(super) fn capture(shared: &ApiShared, expected: Option<&str>) -> Result<Self, ApiFailure> {
|
||||
pub(in crate::api) fn capture(
|
||||
shared: &ApiShared,
|
||||
expected: Option<&str>,
|
||||
) -> Result<Self, ApiFailure> {
|
||||
let generation = shared.active_runtime.load_full();
|
||||
let config = generation.config();
|
||||
let fence = Self {
|
||||
@@ -39,7 +42,7 @@ impl RuntimeFence {
|
||||
Ok(fence)
|
||||
}
|
||||
|
||||
pub(super) fn baseline(&self) -> RuntimeBaseline {
|
||||
pub(in crate::api) fn baseline(&self) -> RuntimeBaseline {
|
||||
static PROCESS: OnceLock<String> = OnceLock::new();
|
||||
let process = PROCESS.get_or_init(|| {
|
||||
let mut bytes = [0_u8; 16];
|
||||
@@ -63,7 +66,7 @@ impl RuntimeFence {
|
||||
&& Arc::ptr_eq(&self.active.load_full(), &self.generation)
|
||||
}
|
||||
|
||||
pub(super) fn ensure_current(&self) -> Result<(), ApiFailure> {
|
||||
pub(in crate::api) fn ensure_current(&self) -> Result<(), ApiFailure> {
|
||||
if self.matches() {
|
||||
Ok(())
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
//! Shared value-free differences and conservative read-only config projections.
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as Json;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use super::model::ApiFailure;
|
||||
use crate::config::{ProxyConfig, observation::ConfigProvenance};
|
||||
|
||||
pub(super) const MAX_PATHS: usize = 4096;
|
||||
|
||||
pub(super) const SECTIONS: &[&str] = &[
|
||||
"general",
|
||||
"network",
|
||||
"server",
|
||||
"timeouts",
|
||||
"censorship",
|
||||
"upstreams",
|
||||
"dc_overrides",
|
||||
"web",
|
||||
"access",
|
||||
"logging",
|
||||
"root",
|
||||
];
|
||||
const MAX_FIELDS: usize = 1024;
|
||||
|
||||
pub(super) fn selected(path: &str, sections: &BTreeSet<String>) -> bool {
|
||||
let root = path.split(['.', '[']).next().unwrap_or_default();
|
||||
sections.contains(if SECTIONS.contains(&root) {
|
||||
root
|
||||
} else {
|
||||
"root"
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn retain_sections(value: &mut Json, sections: &BTreeSet<String>) {
|
||||
if let Some(object) = value.as_object_mut() {
|
||||
object.retain(|key, _| selected(key, sections));
|
||||
}
|
||||
}
|
||||
|
||||
struct Rule {
|
||||
public: bool,
|
||||
schema: Json,
|
||||
default: Option<Json>,
|
||||
}
|
||||
fn rules() -> Result<&'static BTreeMap<String, Rule>, ApiFailure> {
|
||||
static RULES: OnceLock<Result<BTreeMap<String, Rule>, String>> = OnceLock::new();
|
||||
RULES
|
||||
.get_or_init(|| {
|
||||
super::discovery::projection_rules()
|
||||
.map(|fields| {
|
||||
fields
|
||||
.into_iter()
|
||||
.map(|(path, public, schema, default)| {
|
||||
(
|
||||
path,
|
||||
Rule {
|
||||
public,
|
||||
schema,
|
||||
default,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.map_err(|_| "Cannot load configuration projection policy".to_owned())
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|error| ApiFailure::internal(error.clone()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct FieldProjection {
|
||||
pub(super) path: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
value: Option<Json>,
|
||||
redacted: bool,
|
||||
present: bool,
|
||||
entries: Option<usize>,
|
||||
differs: Option<bool>,
|
||||
origin: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
inherited_from: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct Projection {
|
||||
pub(super) fields: Vec<FieldProjection>,
|
||||
pub(super) truncated: bool,
|
||||
provenance: &'static str,
|
||||
/// Input provenance only; this is deliberately not named applied_revision.
|
||||
source_revision: Option<String>,
|
||||
source_count: Option<usize>,
|
||||
}
|
||||
|
||||
pub(super) fn project(
|
||||
config: &ProxyConfig,
|
||||
value: &Json,
|
||||
other: Option<&Json>,
|
||||
sections: &BTreeSet<String>,
|
||||
desired: bool,
|
||||
) -> Result<Projection, ApiFailure> {
|
||||
let rules = rules()?;
|
||||
let (provenance, source_revision, source_count) = match &config.provenance {
|
||||
ConfigProvenance::Source(source) => (
|
||||
"normalized_source",
|
||||
Some(source.revision.clone()),
|
||||
Some(source.source_count),
|
||||
),
|
||||
ConfigProvenance::Mixed => ("mixed_publication", None, None),
|
||||
ConfigProvenance::Unknown => ("unknown", None, None),
|
||||
};
|
||||
let mut output = Projection {
|
||||
fields: Vec::new(),
|
||||
truncated: false,
|
||||
provenance,
|
||||
source_revision,
|
||||
source_count,
|
||||
};
|
||||
struct Walk<'a> {
|
||||
config: &'a ProxyConfig,
|
||||
rules: &'a BTreeMap<String, Rule>,
|
||||
desired: bool,
|
||||
comparing: bool,
|
||||
}
|
||||
impl Walk<'_> {
|
||||
fn visit(
|
||||
&self,
|
||||
value: &Json,
|
||||
other: Option<&Json>,
|
||||
path: &str,
|
||||
canonical: &str,
|
||||
output: &mut Projection,
|
||||
) {
|
||||
if output.fields.len() >= MAX_FIELDS || path.len() > 256 {
|
||||
output.truncated = true;
|
||||
return;
|
||||
}
|
||||
let rule = self.rules.get(canonical);
|
||||
let public = rule.is_some_and(|rule| rule.public);
|
||||
let open_map = rule.is_some_and(|rule| {
|
||||
rule.schema
|
||||
.get("additionalProperties")
|
||||
.is_some_and(|value| value != &Json::Bool(false))
|
||||
}) || path == "dc_overrides"
|
||||
|| path == "censorship.exclusive_mask";
|
||||
if public
|
||||
&& !open_map
|
||||
&& let Some(object) = value.as_object()
|
||||
{
|
||||
if !object.is_empty() {
|
||||
for (key, child) in object {
|
||||
self.visit(
|
||||
child,
|
||||
other.and_then(|value| value.get(key)),
|
||||
&format!("{path}.{key}"),
|
||||
&format!("{canonical}.{key}"),
|
||||
output,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
if public
|
||||
&& let Some(array) = value.as_array()
|
||||
&& rule.is_some_and(|rule| rule.schema["items"]["type"] == "object")
|
||||
{
|
||||
if !array.is_empty() {
|
||||
for (index, child) in array.iter().enumerate() {
|
||||
if let Some(object) = child.as_object() {
|
||||
for (key, field) in object {
|
||||
self.visit(
|
||||
field,
|
||||
other.and_then(|v| v.get(index)).and_then(|v| v.get(key)),
|
||||
&format!("{path}[{index}].{key}"),
|
||||
&format!("{canonical}[].{key}"),
|
||||
output,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Arbitrary strings and all dynamic identities are private, including
|
||||
// URL userinfo in a field whose static metadata says "public".
|
||||
let safe_scalar = |value: &Json, schema: &Json| {
|
||||
value.is_null()
|
||||
|| value.is_boolean()
|
||||
|| value.is_number()
|
||||
|| schema
|
||||
.get("enum")
|
||||
.and_then(Json::as_array)
|
||||
.is_some_and(|allowed| allowed.contains(value))
|
||||
};
|
||||
let runtime_derived = path == "censorship.fake_cert_len"
|
||||
&& matches!(&self.config.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len);
|
||||
let generated = self.desired && runtime_derived;
|
||||
let visible = public
|
||||
&& !open_map
|
||||
&& !generated
|
||||
&& rule.is_some_and(|rule| {
|
||||
safe_scalar(value, &rule.schema)
|
||||
|| value.as_array().is_some_and(|array| {
|
||||
array.len() <= 32
|
||||
&& array
|
||||
.iter()
|
||||
.all(|item| safe_scalar(item, &rule.schema["items"]))
|
||||
})
|
||||
});
|
||||
let (origin, inherited_from) = if runtime_derived {
|
||||
("runtime_derived", None)
|
||||
} else if canonical == "upstreams[].prefer" && value.is_null() {
|
||||
("inherited", Some("network.prefer"))
|
||||
} else {
|
||||
(
|
||||
match &self.config.provenance {
|
||||
ConfigProvenance::Source(source)
|
||||
if source.explicit_paths.contains(path) =>
|
||||
{
|
||||
"source"
|
||||
}
|
||||
ConfigProvenance::Source(source) if !source.complete => "unknown",
|
||||
ConfigProvenance::Source(_)
|
||||
if rule.and_then(|rule| rule.default.as_ref()) == Some(value) =>
|
||||
{
|
||||
"type_default"
|
||||
}
|
||||
ConfigProvenance::Source(_) => "default_or_derived",
|
||||
ConfigProvenance::Mixed => "mixed_unknown",
|
||||
ConfigProvenance::Unknown => "unknown",
|
||||
},
|
||||
None,
|
||||
)
|
||||
};
|
||||
let present = !value.is_null()
|
||||
&& value.as_str().is_none_or(|v| !v.is_empty())
|
||||
&& value.as_object().is_none_or(|v| !v.is_empty())
|
||||
&& value.as_array().is_none_or(|v| !v.is_empty());
|
||||
output.fields.push(FieldProjection {
|
||||
path: path.to_owned(),
|
||||
value: visible.then(|| value.clone()),
|
||||
redacted: !visible,
|
||||
present,
|
||||
entries: value
|
||||
.as_object()
|
||||
.map(|v| v.len())
|
||||
.or_else(|| value.as_array().map(Vec::len)),
|
||||
differs: (self.comparing && !runtime_derived).then(|| other != Some(value)),
|
||||
origin,
|
||||
inherited_from,
|
||||
});
|
||||
}
|
||||
}
|
||||
let walk = Walk {
|
||||
config,
|
||||
rules,
|
||||
desired,
|
||||
comparing: other.is_some(),
|
||||
};
|
||||
if let Some(object) = value.as_object() {
|
||||
for (key, child) in object {
|
||||
if selected(key, sections) {
|
||||
walk.visit(
|
||||
child,
|
||||
other.and_then(|value| value.get(key)),
|
||||
key,
|
||||
key,
|
||||
&mut output,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Paths only: no before/after values or dynamic keys are returned. Arrays use
|
||||
/// indices, open maps use their documented placeholder to avoid leaking identities.
|
||||
pub(super) fn changed_paths(old: &Json, new: &Json) -> Vec<String> {
|
||||
fn walk(old: Option<&Json>, new: Option<&Json>, path: &str, out: &mut BTreeSet<String>) {
|
||||
if old == new {
|
||||
return;
|
||||
}
|
||||
if out.len() > MAX_PATHS {
|
||||
return;
|
||||
}
|
||||
let map_key = if path == "dc_overrides" {
|
||||
Some("{dc}")
|
||||
} else if path == "censorship.exclusive_mask" {
|
||||
Some("{hostname}")
|
||||
} else if path.starts_with("access.") {
|
||||
Some("{entry}")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match (old, new) {
|
||||
(Some(Json::Object(_)), _) | (_, Some(Json::Object(_))) => {
|
||||
let a = old.and_then(Json::as_object);
|
||||
let b = new.and_then(Json::as_object);
|
||||
let keys: BTreeSet<_> = a
|
||||
.into_iter()
|
||||
.flat_map(|v| v.keys())
|
||||
.chain(b.into_iter().flat_map(|v| v.keys()))
|
||||
.collect();
|
||||
if keys.is_empty() {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
for key in keys {
|
||||
let key_path = map_key.unwrap_or(key);
|
||||
let next = if path.is_empty() {
|
||||
key_path.to_string()
|
||||
} else {
|
||||
format!("{path}.{key_path}")
|
||||
};
|
||||
walk(
|
||||
a.and_then(|v| v.get(key)),
|
||||
b.and_then(|v| v.get(key)),
|
||||
&next,
|
||||
out,
|
||||
);
|
||||
}
|
||||
}
|
||||
(Some(Json::Array(_)), _) | (_, Some(Json::Array(_))) => {
|
||||
let a = old
|
||||
.and_then(Json::as_array)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or_default();
|
||||
let b = new
|
||||
.and_then(Json::as_array)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or_default();
|
||||
if a.is_empty() && b.is_empty() {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
for index in 0..a.len().max(b.len()) {
|
||||
walk(a.get(index), b.get(index), &format!("{path}[{index}]"), out);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
out.insert(path.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
walk(Some(old), Some(new), "", &mut paths);
|
||||
paths.into_iter().collect()
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
//! Bounded inspection of desired source and actual owner publications.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use http_body_util::Full;
|
||||
use hyper::{Response, StatusCode, body::Bytes};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as Json;
|
||||
|
||||
use super::config_edit::{RuntimeBaseline, RuntimeFence};
|
||||
use super::model::{ApiFailure, FieldIssue};
|
||||
use super::{ApiShared, config_projection as projection};
|
||||
use crate::config::{
|
||||
ProxyConfig,
|
||||
observation::{ConfigProvenance, WatchOutcome},
|
||||
};
|
||||
use crate::maestro::{reload::ReloadObservation, runtime_build::project_reload_config};
|
||||
|
||||
const MAX_RESPONSE_BYTES: usize = 512 * 1024;
|
||||
const MAX_STATE_PATHS: usize = 512;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Comparison {
|
||||
scope: &'static str,
|
||||
status: &'static str,
|
||||
pending_paths: Vec<String>,
|
||||
watcher_pending_paths: Vec<String>,
|
||||
generation_pending_paths: Vec<String>,
|
||||
process_deferred_paths: Vec<String>,
|
||||
deferred_process_fields: Vec<String>,
|
||||
runtime_derived_paths: Vec<String>,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
impl Default for Comparison {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
scope: "selected_sections",
|
||||
status: "unknown",
|
||||
pending_paths: Vec::new(),
|
||||
watcher_pending_paths: Vec::new(),
|
||||
generation_pending_paths: Vec::new(),
|
||||
process_deferred_paths: Vec::new(),
|
||||
deferred_process_fields: Vec::new(),
|
||||
runtime_derived_paths: Vec::new(),
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct ConfigState {
|
||||
scope: &'static str,
|
||||
sections: BTreeSet<String>,
|
||||
runtime: RuntimeBaseline,
|
||||
runtime_sample_current: bool,
|
||||
pub(super) desired_revision: Option<String>,
|
||||
desired_status: &'static str,
|
||||
desired_error: Option<FieldIssue>,
|
||||
desired: Option<projection::Projection>,
|
||||
effective: Option<projection::Projection>,
|
||||
projection_error: Option<FieldIssue>,
|
||||
comparison: Comparison,
|
||||
watcher: Option<WatchOutcome>,
|
||||
watcher_matches_desired: Option<bool>,
|
||||
reload: Option<ReloadObservation>,
|
||||
reload_observation: &'static str,
|
||||
reload_matches_desired: Option<bool>,
|
||||
subsystem_convergence: &'static str,
|
||||
}
|
||||
|
||||
fn issue(code: &'static str, message: &'static str) -> FieldIssue {
|
||||
FieldIssue {
|
||||
path: "$".into(),
|
||||
code,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sections(query: Option<&str>) -> Result<BTreeSet<String>, ApiFailure> {
|
||||
let Some(query) = query.filter(|query| !query.is_empty()) else {
|
||||
return Ok(projection::SECTIONS
|
||||
.iter()
|
||||
.map(|section| (*section).to_owned())
|
||||
.collect());
|
||||
};
|
||||
if query.len() > 256 {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"Config state query exceeds 256 bytes",
|
||||
));
|
||||
}
|
||||
let mut parsed = url::form_urlencoded::parse(query.as_bytes());
|
||||
let Some((key, value)) = parsed.next() else {
|
||||
return Err(ApiFailure::bad_request("Expected sections query"));
|
||||
};
|
||||
if key != "sections" || parsed.next().is_some() {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"Only one sections query parameter is supported",
|
||||
));
|
||||
}
|
||||
let mut sections = BTreeSet::new();
|
||||
for section in value.split(',') {
|
||||
if !projection::SECTIONS.contains(§ion) || !sections.insert(section.to_owned()) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"Unknown, empty or repeated configuration section",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(sections)
|
||||
}
|
||||
|
||||
fn json(config: &ProxyConfig) -> Result<Json, ApiFailure> {
|
||||
serde_json::to_value(config)
|
||||
.map_err(|_| ApiFailure::internal("Cannot project configuration snapshot"))
|
||||
}
|
||||
|
||||
struct Desired {
|
||||
revision: Option<String>,
|
||||
status: &'static str,
|
||||
error: Option<FieldIssue>,
|
||||
projection: Option<projection::Projection>,
|
||||
effective: Option<projection::Projection>,
|
||||
comparison: Comparison,
|
||||
}
|
||||
|
||||
fn unavailable(revision: Option<String>, status: &'static str, code: &'static str) -> Desired {
|
||||
Desired {
|
||||
revision,
|
||||
status,
|
||||
error: Some(issue(
|
||||
code,
|
||||
"Desired configuration is unavailable; effective publication remains readable",
|
||||
)),
|
||||
projection: None,
|
||||
effective: None,
|
||||
comparison: Comparison::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_desired(
|
||||
path: &std::path::Path,
|
||||
active: &ProxyConfig,
|
||||
sections: &BTreeSet<String>,
|
||||
) -> Result<Desired, ApiFailure> {
|
||||
let graph = match ProxyConfig::read_source_graph(path) {
|
||||
Ok(graph) => graph,
|
||||
Err(_) => return Ok(unavailable(None, "unreadable", "desired_source_unreadable")),
|
||||
};
|
||||
let revision = super::config_store::compute_source_revision(&graph);
|
||||
let mut desired = match ProxyConfig::parse_captured_source(graph) {
|
||||
Ok(parsed) => parsed.config,
|
||||
Err(_) => {
|
||||
return Ok(unavailable(
|
||||
Some(revision),
|
||||
"invalid",
|
||||
"desired_config_invalid",
|
||||
));
|
||||
}
|
||||
};
|
||||
let mut derived = Vec::new();
|
||||
if matches!(&desired.provenance, ConfigProvenance::Source(source) if source.generated_fake_cert_len)
|
||||
{
|
||||
desired.censorship.fake_cert_len = active.censorship.fake_cert_len;
|
||||
if projection::selected("censorship.fake_cert_len", sections) {
|
||||
derived.push("censorship.fake_cert_len".into());
|
||||
}
|
||||
}
|
||||
let mut active_json = json(active)?;
|
||||
let mut desired_json = json(&desired)?;
|
||||
let effective =
|
||||
projection::project(active, &active_json, Some(&desired_json), sections, false)?;
|
||||
let desired_projection =
|
||||
projection::project(&desired, &desired_json, Some(&active_json), sections, true)?;
|
||||
let hot = crate::config::hot_reload::overlay_hot_fields(active, &desired);
|
||||
let projected = project_reload_config(active, &desired);
|
||||
let mut hot_json = json(&hot)?;
|
||||
let mut projected_json = json(&projected.effective)?;
|
||||
for value in [
|
||||
&mut active_json,
|
||||
&mut desired_json,
|
||||
&mut hot_json,
|
||||
&mut projected_json,
|
||||
] {
|
||||
projection::retain_sections(value, sections);
|
||||
}
|
||||
let mut comparison = Comparison {
|
||||
pending_paths: projection::changed_paths(&active_json, &desired_json),
|
||||
watcher_pending_paths: projection::changed_paths(&active_json, &hot_json),
|
||||
generation_pending_paths: projection::changed_paths(&hot_json, &projected_json),
|
||||
process_deferred_paths: projection::changed_paths(&projected_json, &desired_json),
|
||||
deferred_process_fields: projected
|
||||
.deferred_process_fields
|
||||
.into_iter()
|
||||
.filter(|path| projection::selected(path, sections))
|
||||
.collect(),
|
||||
runtime_derived_paths: derived,
|
||||
..Comparison::default()
|
||||
};
|
||||
comparison.status = if comparison.pending_paths.is_empty() {
|
||||
"published_match"
|
||||
} else if comparison.watcher_pending_paths.is_empty()
|
||||
&& comparison.generation_pending_paths.is_empty()
|
||||
{
|
||||
"process_deferred"
|
||||
} else {
|
||||
"pending_publication"
|
||||
};
|
||||
for paths in [
|
||||
&mut comparison.pending_paths,
|
||||
&mut comparison.watcher_pending_paths,
|
||||
&mut comparison.generation_pending_paths,
|
||||
&mut comparison.process_deferred_paths,
|
||||
] {
|
||||
comparison.truncated |= paths.len() > MAX_STATE_PATHS;
|
||||
paths.truncate(MAX_STATE_PATHS);
|
||||
}
|
||||
Ok(Desired {
|
||||
revision: Some(revision),
|
||||
status: "valid",
|
||||
error: None,
|
||||
projection: Some(desired_projection),
|
||||
effective: Some(effective),
|
||||
comparison,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn state(
|
||||
shared: &ApiShared,
|
||||
query: Option<&str>,
|
||||
) -> Result<ConfigState, ApiFailure> {
|
||||
let sections = sections(query)?;
|
||||
let fence = RuntimeFence::capture(shared, None)?;
|
||||
let mut data = ConfigState {
|
||||
scope: "published_configuration_not_subsystem_acknowledgment",
|
||||
sections: sections.clone(),
|
||||
runtime: fence.baseline(),
|
||||
runtime_sample_current: true,
|
||||
desired_revision: None,
|
||||
desired_status: "busy",
|
||||
desired_error: None,
|
||||
desired: None,
|
||||
effective: None,
|
||||
projection_error: None,
|
||||
comparison: Comparison::default(),
|
||||
watcher: None,
|
||||
watcher_matches_desired: None,
|
||||
reload: None,
|
||||
reload_observation: "unknown",
|
||||
reload_matches_desired: None,
|
||||
subsystem_convergence: "unknown",
|
||||
};
|
||||
if let Ok(permit) = shared.config_state_slots.clone().try_acquire_owned() {
|
||||
let permit = Arc::new(permit);
|
||||
let guard = permit.clone();
|
||||
let active = fence.config.clone();
|
||||
let scope = sections.clone();
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
let effective = tokio::task::spawn_blocking(move || {
|
||||
let _guard = guard;
|
||||
projection::project(&active, &json(&active)?, None, &scope, false)
|
||||
});
|
||||
match tokio::time::timeout_at(deadline, effective).await {
|
||||
Ok(Ok(Ok(effective))) => data.effective = Some(effective),
|
||||
_ => {
|
||||
data.projection_error = Some(issue(
|
||||
"effective_projection_unavailable",
|
||||
"Runtime identity is available but its projection could not complete",
|
||||
))
|
||||
}
|
||||
}
|
||||
if data.effective.is_some() {
|
||||
let active = fence.config.clone();
|
||||
let path = shared.config_path.clone();
|
||||
let desired = tokio::task::spawn_blocking(move || {
|
||||
let _guard = permit;
|
||||
read_desired(&path, &active, §ions)
|
||||
});
|
||||
let desired = match tokio::time::timeout_at(deadline, desired).await {
|
||||
Ok(Ok(Ok(desired))) => desired,
|
||||
Ok(_) => unavailable(None, "unavailable", "desired_projection_failed"),
|
||||
Err(_) => unavailable(None, "timeout", "desired_read_timeout"),
|
||||
};
|
||||
data.desired_revision = desired.revision;
|
||||
data.desired_status = desired.status;
|
||||
data.desired_error = desired.error;
|
||||
data.desired = desired.projection;
|
||||
if desired.effective.is_some() {
|
||||
data.effective = desired.effective;
|
||||
}
|
||||
data.comparison = desired.comparison;
|
||||
} else {
|
||||
data.desired_status = "unavailable";
|
||||
data.desired_error = Some(issue(
|
||||
"state_projection_unavailable",
|
||||
"Configuration projection could not complete within the read budget",
|
||||
));
|
||||
}
|
||||
} else {
|
||||
data.desired_error = Some(issue(
|
||||
"config_state_busy",
|
||||
"Both configuration inspection slots are occupied",
|
||||
));
|
||||
}
|
||||
data.runtime_sample_current = fence.ensure_current().is_ok();
|
||||
data.watcher = fence
|
||||
.config
|
||||
.watcher_observation
|
||||
.as_ref()
|
||||
.and_then(|observer| observer.snapshot())
|
||||
.map(|outcome| (*outcome).clone());
|
||||
data.watcher_matches_desired = data.desired_revision.as_ref().and_then(|revision| {
|
||||
data.watcher
|
||||
.as_ref()?
|
||||
.source_revision
|
||||
.as_ref()
|
||||
.map(|source| source == revision)
|
||||
});
|
||||
match shared.reload_control.observation() {
|
||||
Some(reload) => {
|
||||
data.reload = reload;
|
||||
data.reload_observation = "sampled";
|
||||
}
|
||||
None => data.reload_observation = "busy",
|
||||
}
|
||||
data.reload_matches_desired = data.desired_revision.as_ref().and_then(|revision| {
|
||||
data.reload
|
||||
.as_ref()
|
||||
.map(|reload| &reload.source_revision == revision)
|
||||
});
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(super) async fn response(
|
||||
shared: &ApiShared,
|
||||
query: Option<&str>,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let mut data = state(shared, query).await?;
|
||||
loop {
|
||||
let body = serde_json::to_vec(
|
||||
&serde_json::json!({"ok": true, "revision": data.desired_revision, "data": data}),
|
||||
)
|
||||
.map_err(|_| ApiFailure::internal("Cannot encode configuration state"))?;
|
||||
if body.len() <= MAX_RESPONSE_BYTES {
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "application/json; charset=utf-8")
|
||||
.body(Full::new(Bytes::from(body)))
|
||||
.map_err(|_| ApiFailure::internal("Cannot build configuration state response"));
|
||||
}
|
||||
// Each list was already capped. Shrinking the two projections guarantees
|
||||
// that oversized deployment shapes cannot exceed the response envelope.
|
||||
let mut shrunk = false;
|
||||
for projection in [&mut data.desired, &mut data.effective]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
shrunk |= !projection.fields.is_empty();
|
||||
projection.fields.truncate(projection.fields.len() / 2);
|
||||
projection.truncated = true;
|
||||
}
|
||||
if !shrunk {
|
||||
return Err(ApiFailure::internal(
|
||||
"Configuration state metadata exceeded the response bound",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,198 @@
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn busy_state_preserves_runtime_identity_and_does_not_queue_more_work() {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
let permits = fixture
|
||||
.shared
|
||||
.config_state_slots
|
||||
.clone()
|
||||
.acquire_many_owned(2)
|
||||
.await
|
||||
.unwrap();
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(data["desired_status"], "busy");
|
||||
assert_eq!(data["desired_error"]["code"], "config_state_busy");
|
||||
assert_eq!(data["runtime"]["active_generation"], 1);
|
||||
assert!(data["effective"].is_null());
|
||||
assert_eq!(data["comparison"]["status"], "unknown");
|
||||
drop(permits);
|
||||
assert_eq!(inspect(&fixture, None).await["desired_status"], "valid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_deployment_bounds_fields_paths_bytes_and_dynamic_identities() {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
let mut text =
|
||||
String::from("[access.users]\nprivate_identity='000102030405060708090a0b0c0d0e0f'\n");
|
||||
for _ in 0..1200 {
|
||||
text.push_str("[[upstreams]]\ntype='socks5'\naddress='127.0.0.1:1080'\nusername='private_socks_user'\npassword='private_socks_password'\n");
|
||||
}
|
||||
source(&fixture, &text, true);
|
||||
let response = response(&fixture.shared, None).await.unwrap();
|
||||
let bytes = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert!(bytes.len() <= MAX_RESPONSE_BYTES);
|
||||
let data: Json = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(data["data"]["desired_status"], "valid");
|
||||
assert_eq!(data["data"]["desired"]["truncated"], true);
|
||||
assert!(data["data"]["desired"]["fields"].as_array().unwrap().len() <= 1024);
|
||||
assert_eq!(data["data"]["comparison"]["truncated"], true);
|
||||
assert!(
|
||||
data["data"]["comparison"]["pending_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len()
|
||||
<= MAX_STATE_PATHS
|
||||
);
|
||||
let encoded = String::from_utf8(bytes.to_vec()).unwrap();
|
||||
for private in [
|
||||
"private_identity",
|
||||
"private_socks_user",
|
||||
"private_socks_password",
|
||||
"000102030405060708090a0b0c0d0e0f",
|
||||
"Bearer discovery-test",
|
||||
] {
|
||||
assert!(!encoded.contains(private));
|
||||
}
|
||||
}
|
||||
|
||||
fn abandoned_state_retains_budget(timeout: bool) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.unwrap();
|
||||
runtime.block_on(async {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
let original = std::fs::read(&fixture.shared.config_path).unwrap();
|
||||
let active = fixture.shared.active_runtime.load_full();
|
||||
let (release, wait) = std::sync::mpsc::channel();
|
||||
let (started, ready) = tokio::sync::oneshot::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
started.send(()).unwrap();
|
||||
wait.recv().unwrap();
|
||||
});
|
||||
ready.await.unwrap();
|
||||
tokio::time::pause();
|
||||
let shared = fixture.shared.clone();
|
||||
let task = tokio::spawn(async move { state(&shared, None).await });
|
||||
while fixture.shared.config_state_slots.available_permits() == 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
if timeout {
|
||||
fixture
|
||||
.shared
|
||||
.active_runtime
|
||||
.store(test_runtime_generation(2, active.config().as_ref().clone()));
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
let data = serde_json::to_value(task.await.unwrap().unwrap()).unwrap();
|
||||
assert_eq!(data["runtime"]["active_generation"], 1);
|
||||
assert_eq!(data["runtime_sample_current"], false);
|
||||
assert_eq!(data["desired_status"], "unavailable");
|
||||
assert_eq!(
|
||||
data["projection_error"]["code"],
|
||||
"effective_projection_unavailable"
|
||||
);
|
||||
} else {
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
}
|
||||
assert_eq!(fixture.shared.config_state_slots.available_permits(), 1);
|
||||
tokio::time::resume();
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap();
|
||||
let _permits = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
fixture
|
||||
.shared
|
||||
.config_state_slots
|
||||
.clone()
|
||||
.acquire_many_owned(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.shared.config_path).unwrap(),
|
||||
original
|
||||
);
|
||||
assert!(fixture.shared.reload_control.status(1).await.is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_retains_read_slot_until_detached_worker_finishes() {
|
||||
abandoned_state_retains_budget(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeout_retains_read_slot_and_reports_a_stale_runtime_sample() {
|
||||
abandoned_state_retains_budget(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desired_worker_timeout_preserves_effective_and_retains_its_read_slot() {
|
||||
use std::future::Future;
|
||||
use std::task::Poll;
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.unwrap();
|
||||
runtime.block_on(async {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
let (release_initial, wait_initial) = std::sync::mpsc::channel();
|
||||
let (initial_started, initial_ready) = tokio::sync::oneshot::channel();
|
||||
let initial_blocker = tokio::task::spawn_blocking(move || {
|
||||
initial_started.send(()).unwrap();
|
||||
wait_initial.recv().unwrap();
|
||||
});
|
||||
initial_ready.await.unwrap();
|
||||
let mut read = Box::pin(state(&fixture.shared, None));
|
||||
// Queue effective projection, then occupy the only blocking thread once
|
||||
// it completes. The desired-source worker will have to wait behind it.
|
||||
std::future::poll_fn(|context| {
|
||||
assert!(read.as_mut().poll(context).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
let (release, wait) = std::sync::mpsc::channel();
|
||||
let (started, ready) = tokio::sync::oneshot::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
started.send(()).unwrap();
|
||||
wait.recv().unwrap();
|
||||
});
|
||||
release_initial.send(()).unwrap();
|
||||
initial_blocker.await.unwrap();
|
||||
ready.await.unwrap();
|
||||
tokio::time::pause();
|
||||
std::future::poll_fn(|context| {
|
||||
assert!(read.as_mut().poll(context).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
let data = serde_json::to_value(read.await.unwrap()).unwrap();
|
||||
assert_eq!(data["desired_status"], "timeout");
|
||||
assert_eq!(data["desired_error"]["code"], "desired_read_timeout");
|
||||
assert!(!data["effective"].is_null());
|
||||
assert_eq!(data["comparison"]["status"], "unknown");
|
||||
assert_eq!(fixture.shared.config_state_slots.available_permits(), 1);
|
||||
tokio::time::resume();
|
||||
release.send(()).unwrap();
|
||||
blocker.await.unwrap();
|
||||
let _permits = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
fixture
|
||||
.shared
|
||||
.config_state_slots
|
||||
.clone()
|
||||
.acquire_many_owned(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use super::*;
|
||||
|
||||
const WEB: &str = r#"
|
||||
[access.users]
|
||||
alice = "000102030405060708090a0b0c0d0e0f"
|
||||
[[server.listeners]]
|
||||
ip = "127.0.0.1"
|
||||
port = 18080
|
||||
transport = "web"
|
||||
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||
[web]
|
||||
enabled = true
|
||||
[[web.vhosts]]
|
||||
host = "proxy.example.com"
|
||||
public_addr = "203.0.113.10:443"
|
||||
[web.vhosts.decoy]
|
||||
mode = "http_upstream"
|
||||
upstream = "http://127.0.0.1:18081"
|
||||
[[web.vhosts.profiles]]
|
||||
user = "alice"
|
||||
secret_mode = "dd"
|
||||
"#;
|
||||
|
||||
#[tokio::test]
|
||||
async fn inspection_does_not_acquire_decoy_dns_or_static_files() {
|
||||
let (fixture, _sender) = fixture(WEB, true).await;
|
||||
source(
|
||||
&fixture,
|
||||
&WEB.replace(
|
||||
"http://127.0.0.1:18081",
|
||||
"http://unresolvable.invalid:18081",
|
||||
)
|
||||
.replace(
|
||||
"[[web.vhosts.profiles]]",
|
||||
"resolve = 'startup'\n[[web.vhosts.profiles]]",
|
||||
),
|
||||
true,
|
||||
);
|
||||
let data = inspect(&fixture, Some("sections=web")).await;
|
||||
assert_eq!(data["desired_status"], "valid");
|
||||
assert_eq!(
|
||||
field(&data, "desired", "web.vhosts[0].decoy.upstream")["redacted"],
|
||||
true
|
||||
);
|
||||
assert_eq!(data["comparison"]["status"], "pending_publication");
|
||||
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
std::fs::write(
|
||||
directory.path().join("index.html"),
|
||||
"private synthetic asset",
|
||||
)
|
||||
.unwrap();
|
||||
let static_source = WEB.replace(
|
||||
"mode = \"http_upstream\"\nupstream = \"http://127.0.0.1:18081\"",
|
||||
&format!(
|
||||
"mode = 'static_directory'\ndirectory = '{}'\nindex = 'index.html'",
|
||||
directory.path().display()
|
||||
),
|
||||
);
|
||||
let (fixture, _sender) = self::fixture(&static_source, true).await;
|
||||
directory.close().unwrap();
|
||||
assert!(
|
||||
ProxyConfig::load_prepared(fixture.shared.config_path.clone())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(data["desired_status"], "valid");
|
||||
assert_eq!(data["comparison"]["status"], "published_match");
|
||||
assert_eq!(data["subsystem_convergence"], "unknown");
|
||||
assert!(!data.to_string().contains("private synthetic asset"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_web_capacity_and_includes_preserve_source_evidence() {
|
||||
let (fixture, _sender) = fixture("[web.limits]\ndebug_bytes_global=4096", true).await;
|
||||
source(
|
||||
&fixture,
|
||||
"[web.limits]\ndebug_bytes_global=8192\n[web.debug]\nbody_prefix_bytes=6144",
|
||||
true,
|
||||
);
|
||||
let data = inspect(&fixture, Some("sections=web")).await;
|
||||
assert_eq!(data["comparison"]["status"], "process_deferred");
|
||||
assert!(
|
||||
data["comparison"]["process_deferred_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("web.debug.body_prefix_bytes"))
|
||||
);
|
||||
assert!(
|
||||
data["comparison"]["deferred_process_fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("web.debug"))
|
||||
);
|
||||
source(&fixture, "include = \"extra.toml\"", true);
|
||||
std::fs::write(
|
||||
fixture
|
||||
.shared
|
||||
.config_path
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("extra.toml"),
|
||||
"[general]\nlog_level='debug'",
|
||||
)
|
||||
.unwrap();
|
||||
let data = inspect(&fixture, Some("sections=general")).await;
|
||||
assert_eq!(data["desired"]["source_count"], 2);
|
||||
assert_eq!(
|
||||
field(&data, "desired", "general.log_level")["origin"],
|
||||
"source"
|
||||
);
|
||||
assert!(!data.to_string().contains("extra.toml"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_fails_closed_for_arbitrary_strings_unknown_fields_and_private_maps() {
|
||||
let config = ProxyConfig::default();
|
||||
let raw = json!({"general":{"proxy_secret_url":"https://private-user:private-pass@example.invalid/key", "unknown_field":"private-value"},
|
||||
"access":{"user_rate_limits":{"private-name":{"up_bps":12345}}},
|
||||
"dc_overrides":{"private-dc":{"address":"private-value"}}});
|
||||
let scope = sections(None).unwrap();
|
||||
let projection = projection::project(&config, &raw, Some(&json!({})), &scope, false).unwrap();
|
||||
let result = serde_json::to_value(projection).unwrap();
|
||||
for value in [
|
||||
"private-user",
|
||||
"private-pass",
|
||||
"example.invalid",
|
||||
"private-value",
|
||||
"private-name",
|
||||
"private-dc",
|
||||
"12345",
|
||||
] {
|
||||
assert!(!result.to_string().contains(value));
|
||||
}
|
||||
assert!(
|
||||
result["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|field| field["redacted"] == true && field.get("value").is_none())
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
use super::*;
|
||||
use crate::api::discovery::http_tests::Fixture;
|
||||
use crate::config::observation::WatchObservation;
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::maestro::reload::{ReloadControl, ReloadPhase, ReloadRequest};
|
||||
use serde_json::json;
|
||||
use tokio::sync::watch;
|
||||
|
||||
#[path = "bounds_tests.rs"]
|
||||
mod bounds;
|
||||
|
||||
#[path = "preparation_tests.rs"]
|
||||
mod preparation;
|
||||
|
||||
fn source(fixture: &Fixture, text: &str, read_only: bool) {
|
||||
std::fs::write(&fixture.shared.config_path, format!("{text}\n[server.api]\nenabled = true\nread_only = {read_only}\nauth_header = \"Bearer discovery-test\"\n")).unwrap();
|
||||
}
|
||||
|
||||
async fn fixture(text: &str, read_only: bool) -> (Fixture, watch::Sender<Arc<ProxyConfig>>) {
|
||||
let fixture = Fixture::new(false);
|
||||
source(&fixture, text, read_only);
|
||||
let mut config = ProxyConfig::load_prepared(fixture.shared.config_path.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.config;
|
||||
config.watcher_observation = Some(Arc::new(WatchObservation::default()));
|
||||
let mut runtime = test_runtime_generation(1, config);
|
||||
let (sender, receiver) = watch::channel(runtime.config());
|
||||
Arc::get_mut(&mut runtime).unwrap().config_rx = receiver;
|
||||
fixture.shared.active_runtime.store(runtime);
|
||||
(fixture, sender)
|
||||
}
|
||||
|
||||
async fn inspect(fixture: &Fixture, query: Option<&str>) -> Json {
|
||||
serde_json::to_value(state(&fixture.shared, query).await.unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn field<'a>(data: &'a Json, projection: &str, path: &str) -> &'a Json {
|
||||
data[projection]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|field| field["path"] == path)
|
||||
.unwrap_or_else(|| panic!("missing {projection} field {path}"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn state_http_auth_scope_queries_read_only_and_no_mutation() {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
let bytes = std::fs::read(&fixture.shared.config_path).unwrap();
|
||||
let active = fixture.shared.active_runtime.load_full();
|
||||
for (method, path, authorized, expected) in [
|
||||
("GET", "/v1/config/state", false, 401),
|
||||
("POST", "/v1/config/state", true, 405),
|
||||
(
|
||||
"GET",
|
||||
"/v1/config/state?sections=general,general",
|
||||
true,
|
||||
400,
|
||||
),
|
||||
("GET", "/v1/config/state?sections=", true, 400),
|
||||
("GET", "/v1/config/state?sections=unknown", true, 400),
|
||||
(
|
||||
"GET",
|
||||
"/v1/config/state?sections=general§ions=web",
|
||||
true,
|
||||
400,
|
||||
),
|
||||
("GET", "/v1/config/state?debug=true", true, 400),
|
||||
] {
|
||||
let (status, allow, _) = fixture.request(method, path, authorized).await;
|
||||
assert_eq!(status, expected, "{method} {path}");
|
||||
if status == 405 {
|
||||
assert_eq!(allow, "GET");
|
||||
}
|
||||
}
|
||||
assert!(sections(Some(&format!("sections={}", "a".repeat(256)))).is_err());
|
||||
let (status, _, response) = fixture
|
||||
.request("GET", "/v1/config/state?sections=general,access", true)
|
||||
.await;
|
||||
assert_eq!(status, 200, "{response}");
|
||||
let data = &response["data"];
|
||||
assert_eq!(response["revision"], data["desired_revision"]);
|
||||
assert_eq!(data["sections"], json!(["access", "general"]));
|
||||
assert_eq!(data["comparison"]["status"], "published_match");
|
||||
assert_eq!(data["runtime"]["active_generation"], 1);
|
||||
assert_eq!(data["runtime_sample_current"], true);
|
||||
assert_eq!(data["subsystem_convergence"], "unknown");
|
||||
assert!(
|
||||
data["effective"]["fields"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|field| {
|
||||
let path = field["path"].as_str().unwrap();
|
||||
path.starts_with("general.") || path.starts_with("access.")
|
||||
})
|
||||
);
|
||||
assert_eq!(std::fs::read(&fixture.shared.config_path).unwrap(), bytes);
|
||||
assert!(Arc::ptr_eq(
|
||||
&active,
|
||||
&fixture.shared.active_runtime.load_full()
|
||||
));
|
||||
assert!(fixture.shared.reload_control.status(1).await.is_none());
|
||||
assert!(fixture.shared.runtime_events.snapshot(64).events.is_empty());
|
||||
let denied = Fixture::new(true);
|
||||
assert_eq!(denied.request("GET", "/v1/config/state", true).await.0, 403);
|
||||
let caps = fixture.request("GET", "/v1/capabilities", true).await.2;
|
||||
let operation = caps["data"]["operations"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|op| op["id"] == "config_state")
|
||||
.unwrap();
|
||||
assert_eq!(operation["available"], true);
|
||||
assert_eq!(operation["mutable"], false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_or_unreadable_desired_preserves_effective_and_hides_errors() {
|
||||
let (fixture, _sender) = fixture("[general]\nlog_level='debug'", true).await;
|
||||
let epoch = fixture
|
||||
.shared
|
||||
.active_runtime
|
||||
.load()
|
||||
.config()
|
||||
.runtime_config_epoch;
|
||||
for invalid in [
|
||||
"[general]\nme_writer_pick_mode='synthetic-private-value'",
|
||||
"[broken synthetic-private-value",
|
||||
] {
|
||||
std::fs::write(&fixture.shared.config_path, invalid).unwrap();
|
||||
let (status, _, response) = fixture.request("GET", "/v1/config/state", true).await;
|
||||
assert_eq!(status, 200);
|
||||
let data = &response["data"];
|
||||
assert_eq!(data["desired_status"], "invalid");
|
||||
assert!(response["revision"].is_string());
|
||||
assert_eq!(data["desired_error"]["code"], "desired_config_invalid");
|
||||
assert_eq!(data["runtime"]["config_epoch"], epoch);
|
||||
assert_eq!(
|
||||
field(data, "effective", "general.log_level")["value"],
|
||||
"debug"
|
||||
);
|
||||
assert!(data["desired"].is_null());
|
||||
assert_eq!(data["comparison"]["status"], "unknown");
|
||||
assert!(!response.to_string().contains("synthetic-private-value"));
|
||||
}
|
||||
std::fs::remove_file(&fixture.shared.config_path).unwrap();
|
||||
let response = fixture.request("GET", "/v1/config/state", true).await.2;
|
||||
assert!(response["revision"].is_null());
|
||||
assert_eq!(response["data"]["desired_status"], "unreadable");
|
||||
assert!(!response["data"]["effective"].is_null());
|
||||
assert!(
|
||||
!response
|
||||
.to_string()
|
||||
.contains(fixture.shared.config_path.to_str().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn secret_rotation_stays_pending_until_publication_and_never_claims_convergence() {
|
||||
const OLD: &str = "000102030405060708090a0b0c0d0e0f";
|
||||
const NEW: &str = "f0e0d0c0b0a090807060504030201000";
|
||||
let (fixture, sender) =
|
||||
fixture(&format!("[access.users]\nprivate_account='{OLD}'"), false).await;
|
||||
let before = inspect(&fixture, None).await;
|
||||
let (status, _, response) = fixture
|
||||
.request_body(
|
||||
"POST",
|
||||
"/v1/users/private_account/rotate-secret",
|
||||
true,
|
||||
Some(&json!({"secret": NEW})),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 200, "{response}");
|
||||
let pending = inspect(&fixture, None).await;
|
||||
assert_ne!(pending["desired_revision"], before["desired_revision"]);
|
||||
assert_eq!(pending["runtime"], before["runtime"]);
|
||||
assert_eq!(pending["comparison"]["status"], "pending_publication");
|
||||
assert!(
|
||||
pending["comparison"]["watcher_pending_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("access.users.{entry}"))
|
||||
);
|
||||
let users = field(&pending, "effective", "access.users");
|
||||
assert_eq!(users["differs"], true);
|
||||
assert_eq!(users["present"], true);
|
||||
assert_eq!(users["entries"], 1);
|
||||
assert_eq!(users["redacted"], true);
|
||||
assert!(users.get("value").is_none());
|
||||
for private in [OLD, NEW, "private_account", "Bearer discovery-test"] {
|
||||
assert!(!pending.to_string().contains(private));
|
||||
}
|
||||
let mut published = ProxyConfig::load_prepared(fixture.shared.config_path.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.config
|
||||
.with_publication_epoch();
|
||||
published.watcher_observation = sender.borrow().watcher_observation.clone();
|
||||
published.watcher_observation.as_ref().unwrap().record(
|
||||
"published",
|
||||
published.provenance.revision().map(str::to_owned),
|
||||
Some(published.runtime_config_epoch),
|
||||
None,
|
||||
);
|
||||
sender.send(Arc::new(published)).unwrap();
|
||||
let after = inspect(&fixture, None).await;
|
||||
assert_eq!(after["comparison"]["status"], "published_match");
|
||||
assert_ne!(
|
||||
after["runtime"]["config_epoch"],
|
||||
before["runtime"]["config_epoch"]
|
||||
);
|
||||
assert_eq!(after["watcher_matches_desired"], true);
|
||||
assert_eq!(after["subsystem_convergence"], "unknown");
|
||||
assert_eq!(field(&after, "effective", "access.users")["differs"], false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hot_generation_and_process_changes_have_distinct_paths() {
|
||||
let (fixture, _sender) = fixture("", true).await;
|
||||
source(
|
||||
&fixture,
|
||||
"[general]\nlog_level='debug'\ndisable_colors=true\n[network]\nipv6=true",
|
||||
true,
|
||||
);
|
||||
let data = inspect(&fixture, None).await;
|
||||
let comparison = &data["comparison"];
|
||||
assert!(
|
||||
comparison["watcher_pending_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("general.log_level"))
|
||||
);
|
||||
assert!(
|
||||
comparison["generation_pending_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("network.ipv6"))
|
||||
);
|
||||
assert!(
|
||||
comparison["process_deferred_paths"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("general.disable_colors"))
|
||||
);
|
||||
source(&fixture, "[general]\ndisable_colors=true", true);
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(data["comparison"]["status"], "process_deferred");
|
||||
assert_eq!(
|
||||
data["comparison"]["deferred_process_fields"],
|
||||
json!(["general.disable_colors"])
|
||||
);
|
||||
let scoped = inspect(&fixture, Some("sections=network")).await;
|
||||
assert_eq!(scoped["comparison"]["status"], "published_match");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn known_source_default_inheritance_and_runtime_derived_provenance() {
|
||||
let text = "[general]\nlog_level='debug'\n[[upstreams]]\ntype='direct'\n[censorship]\ntls_emulation=false";
|
||||
let (fixture, _sender) = fixture(text, true).await;
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(
|
||||
field(&data, "desired", "general.log_level")["origin"],
|
||||
"source"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&data, "desired", "general.disable_colors")["origin"],
|
||||
"type_default"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&data, "desired", "upstreams[0].prefer")["origin"],
|
||||
"inherited"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&data, "desired", "upstreams[0].prefer")["inherited_from"],
|
||||
"network.prefer"
|
||||
);
|
||||
assert_eq!(
|
||||
field(&data, "desired", "censorship.fake_cert_len")["origin"],
|
||||
"runtime_derived"
|
||||
);
|
||||
assert!(
|
||||
field(&data, "desired", "censorship.fake_cert_len")
|
||||
.get("value")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(data["comparison"]["status"], "published_match");
|
||||
assert_eq!(
|
||||
data["comparison"]["runtime_derived_paths"],
|
||||
json!(["censorship.fake_cert_len"])
|
||||
);
|
||||
assert_eq!(data["desired"]["source_revision"], data["desired_revision"]);
|
||||
assert_eq!(data["desired"]["source_count"], 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_metadata_distinguishes_acceptance_failure_and_rollback_without_raw_errors() {
|
||||
let (mut fixture, _sender) = fixture("", true).await;
|
||||
let (control, mut commands) = ReloadControl::channel(1);
|
||||
Arc::get_mut(&mut fixture.shared).unwrap().reload_control = control.clone();
|
||||
let revision = inspect(&fixture, None).await["desired_revision"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
for terminal in [ReloadPhase::Failed, ReloadPhase::RolledBack] {
|
||||
let accepted = control
|
||||
.submit(
|
||||
fixture.shared.active_runtime.load().config(),
|
||||
revision.clone(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _ = commands.recv().await.unwrap();
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(data["reload"]["phase"], "accepted");
|
||||
assert_eq!(data["reload_matches_desired"], true);
|
||||
assert!(data["reload"]["activated_generation"].is_null());
|
||||
control.record_source_generation(accepted.reload_id, 1);
|
||||
control
|
||||
.mark_phase(accepted.reload_id, ReloadPhase::Preparing)
|
||||
.await;
|
||||
control
|
||||
.add_warning(accepted.reload_id, "synthetic-private-warning")
|
||||
.await;
|
||||
match terminal {
|
||||
ReloadPhase::Failed => {
|
||||
control
|
||||
.fail(accepted.reload_id, "synthetic-private-error")
|
||||
.await
|
||||
}
|
||||
ReloadPhase::RolledBack => {
|
||||
control
|
||||
.rolled_back(accepted.reload_id, "synthetic-private-error")
|
||||
.await
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let data = inspect(&fixture, None).await;
|
||||
assert_eq!(
|
||||
data["reload"]["phase"],
|
||||
serde_json::to_value(terminal).unwrap()
|
||||
);
|
||||
assert_eq!(data["reload"]["source_generation"], 1);
|
||||
assert_eq!(data["reload"]["error_present"], true);
|
||||
assert_eq!(data["reload"]["warning_count"], 1);
|
||||
assert!(data["reload"]["retiring_generation"].is_null());
|
||||
assert!(!data.to_string().contains("synthetic-private"));
|
||||
}
|
||||
}
|
||||
+2
-14
@@ -108,23 +108,11 @@ pub(super) fn compute_revision(content: &str) -> String {
|
||||
|
||||
/// Hashes source documents only; prepared DNS answers never change a revision.
|
||||
pub(super) fn compute_snapshot_revision(source_contents: &BTreeMap<PathBuf, String>) -> String {
|
||||
compute_source_revision(&ConfigSourceGraph {
|
||||
source_contents: source_contents.clone(),
|
||||
rendered: String::new(),
|
||||
})
|
||||
crate::config::observation::source_revision(source_contents)
|
||||
}
|
||||
|
||||
pub(super) fn compute_source_revision(graph: &ConfigSourceGraph) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"telemt-config-manifest-v1\0");
|
||||
for (path, content) in &graph.source_contents {
|
||||
let path = path.as_os_str().as_encoded_bytes();
|
||||
hasher.update((path.len() as u64).to_le_bytes());
|
||||
hasher.update(path);
|
||||
hasher.update((content.len() as u64).to_le_bytes());
|
||||
hasher.update(content.as_bytes());
|
||||
}
|
||||
hex::encode(hasher.finalize())
|
||||
compute_snapshot_revision(&graph.source_contents)
|
||||
}
|
||||
|
||||
/// Reads a normalized source snapshot without resolving decoys or loading static assets.
|
||||
|
||||
@@ -25,6 +25,8 @@ pub(super) fn config_field_paths() -> Result<Vec<&'static str>, ApiFailure> {
|
||||
schema::field_paths()
|
||||
}
|
||||
|
||||
pub(super) use schema::projection_rules;
|
||||
|
||||
pub(super) const SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
|
||||
@@ -50,6 +50,7 @@ impl Fixture {
|
||||
detected_ips_rx: watch::channel((None, None)).1,
|
||||
mutation_lock: Arc::new(Mutex::new(())),
|
||||
config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)),
|
||||
config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)),
|
||||
minimal_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
{"id": "users_active_ips","method": "GET","path": "/v1/stats/users/active-ips","gate": "always","success": [200]},
|
||||
{"id": "users_quota","method": "GET","path": "/v1/stats/users/quota","gate": "always","success": [200]},
|
||||
{"id": "config_get","method": "GET","path": "/v1/config","gate": "always","success": [200],"credential_bearing": true},
|
||||
{"id": "config_state","method": "GET","path": "/v1/config/state","gate": "always","success": [200]},
|
||||
{"id": "config_validate","method": "POST","path": "/v1/config/validate","gate": "always","success": [200],"body": "PatchConfigRequest","if_match": true,"if_runtime_match": true,"read_only_operation": true},
|
||||
{"id": "config_patch","if_runtime_match": true,"method": "PATCH","path": "/v1/config","gate": "always","success": [200,202],"body": "PatchConfigRequest","if_match": true},
|
||||
{"id": "reload","method": "POST","path": "/v1/system/reload","gate": "always","success": [202],"body": "ReloadRequest","optional_body": true,"if_match": true},
|
||||
|
||||
@@ -112,6 +112,22 @@ pub(super) fn field_paths() -> Result<Vec<&'static str>, ApiFailure> {
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(in crate::api) fn projection_rules()
|
||||
-> Result<Vec<(String, bool, Value, Option<Value>)>, ApiFailure> {
|
||||
Ok(config_schema()?
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
(
|
||||
field.spec.path.clone(),
|
||||
field.spec.sensitivity == "public",
|
||||
field.spec.value.clone(),
|
||||
field.default.clone(),
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn field_specs() -> Result<Vec<FieldSpec>, String> {
|
||||
serde_json::from_str(include_str!("fields.json"))
|
||||
.map_err(|error| format!("invalid compiled config field catalog: {error}"))
|
||||
|
||||
@@ -10,6 +10,9 @@ pub(super) async fn handle(
|
||||
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/v1/config/state") => {
|
||||
Ok(super::super::config_state::response(shared, query).await?)
|
||||
}
|
||||
("GET", "/v1/capabilities") => {
|
||||
discovery::reject_query(query)?;
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
|
||||
@@ -37,6 +37,8 @@ use crate::web::control::WebRuntimePublication;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
mod config_edit;
|
||||
mod config_projection;
|
||||
mod config_state;
|
||||
pub(crate) mod config_store;
|
||||
mod discovery;
|
||||
mod events;
|
||||
@@ -122,6 +124,7 @@ pub(super) struct ApiShared {
|
||||
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||
pub(super) mutation_lock: Arc<Mutex<()>>,
|
||||
pub(super) config_preflight_slots: Arc<Semaphore>,
|
||||
pub(super) config_state_slots: Arc<Semaphore>,
|
||||
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
||||
pub(super) runtime_edge_connections_cache: Arc<Mutex<Option<EdgeConnectionsCacheEntry>>>,
|
||||
pub(super) runtime_edge_recompute_lock: Arc<Mutex<()>>,
|
||||
@@ -159,6 +162,7 @@ impl ApiShared {
|
||||
detected_ips_rx: self.detected_ips_rx.clone(),
|
||||
mutation_lock: self.mutation_lock.clone(),
|
||||
config_preflight_slots: self.config_preflight_slots.clone(),
|
||||
config_state_slots: self.config_state_slots.clone(),
|
||||
minimal_cache: self.minimal_cache.clone(),
|
||||
runtime_edge_connections_cache: self.runtime_edge_connections_cache.clone(),
|
||||
runtime_edge_recompute_lock: self.runtime_edge_recompute_lock.clone(),
|
||||
@@ -269,6 +273,7 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
"/v1/health"
|
||||
| "/v1/capabilities"
|
||||
| "/v1/config/schema"
|
||||
| "/v1/config/state"
|
||||
| "/v1/health/ready"
|
||||
| "/v1/system/info"
|
||||
| "/v1/runtime/gates"
|
||||
@@ -380,6 +385,7 @@ pub(crate) async fn serve(
|
||||
detected_ips_rx,
|
||||
mutation_lock: Arc::new(Mutex::new(())),
|
||||
config_preflight_slots: Arc::new(Semaphore::new(2)),
|
||||
config_state_slots: Arc::new(Semaphore::new(2)),
|
||||
minimal_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
|
||||
|
||||
@@ -58,6 +58,7 @@ impl Fixture {
|
||||
detected_ips_rx: watch::channel((None, None)).1,
|
||||
mutation_lock: Arc::new(Mutex::new(())),
|
||||
config_preflight_slots: Arc::new(tokio::sync::Semaphore::new(2)),
|
||||
config_state_slots: Arc::new(tokio::sync::Semaphore::new(2)),
|
||||
minimal_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
|
||||
|
||||
@@ -144,9 +144,26 @@ fn prepare_effective_config(
|
||||
let mut applied_cfg = overlay_hot_fields(old_cfg, new_cfg);
|
||||
applied_cfg.validate_effective_web()?;
|
||||
applied_cfg.rebuild_runtime_web()?;
|
||||
applied_cfg.provenance = if config_equal(&applied_cfg, new_cfg) {
|
||||
new_cfg.provenance.clone()
|
||||
} else {
|
||||
crate::config::observation::ConfigProvenance::Mixed
|
||||
};
|
||||
Ok(applied_cfg)
|
||||
}
|
||||
|
||||
fn record_outcome(
|
||||
config: &ProxyConfig,
|
||||
phase: &'static str,
|
||||
revision: Option<String>,
|
||||
epoch: Option<u64>,
|
||||
error: Option<&'static str>,
|
||||
) {
|
||||
if let Some(observation) = &config.watcher_observation {
|
||||
observation.record(phase, revision, epoch, error);
|
||||
}
|
||||
}
|
||||
|
||||
/// Diffs and publishes a fully prepared configuration on the watcher owner task.
|
||||
fn reload_config_with_resolver(
|
||||
prepared: (LoadedConfig, ProxyConfig),
|
||||
@@ -168,6 +185,7 @@ fn reload_config_with_resolver(
|
||||
let next_manifest = WatchManifest::from_source_files(&source_files);
|
||||
|
||||
let old_cfg = config_tx.borrow().clone();
|
||||
let revision = new_cfg.provenance.revision().map(str::to_owned);
|
||||
let old_hot = HotFields::from_config(&old_cfg);
|
||||
let applied_hot = HotFields::from_config(&applied_cfg);
|
||||
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
||||
@@ -179,11 +197,24 @@ fn reload_config_with_resolver(
|
||||
}
|
||||
|
||||
if !hot_changed {
|
||||
if refresh_quota_source {
|
||||
config_tx
|
||||
let published = !refresh_quota_source
|
||||
|| config_tx
|
||||
.send(Arc::new(applied_cfg.with_publication_epoch()))
|
||||
.ok();
|
||||
}
|
||||
.is_ok();
|
||||
let epoch = config_tx.borrow().runtime_config_epoch;
|
||||
record_outcome(
|
||||
&old_cfg,
|
||||
if !published {
|
||||
"publication_failed"
|
||||
} else if non_hot_changed {
|
||||
"retained"
|
||||
} else {
|
||||
"unchanged"
|
||||
},
|
||||
revision,
|
||||
published.then_some(epoch),
|
||||
(!published).then_some("publication_closed"),
|
||||
);
|
||||
reload_state.mark_applied(rendered_hash);
|
||||
return Some(next_manifest);
|
||||
}
|
||||
@@ -196,6 +227,13 @@ fn reload_config_with_resolver(
|
||||
"config reload: invalid network.dns_overrides: {}; keeping old config",
|
||||
e
|
||||
);
|
||||
record_outcome(
|
||||
&old_cfg,
|
||||
"rejected",
|
||||
revision,
|
||||
None,
|
||||
Some("dns_overrides_rejected"),
|
||||
);
|
||||
return Some(next_manifest);
|
||||
}
|
||||
|
||||
@@ -207,9 +245,23 @@ fn reload_config_with_resolver(
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
);
|
||||
config_tx
|
||||
let published = config_tx
|
||||
.send(Arc::new(applied_cfg.with_publication_epoch()))
|
||||
.ok();
|
||||
.is_ok();
|
||||
let epoch = config_tx.borrow().runtime_config_epoch;
|
||||
record_outcome(
|
||||
&old_cfg,
|
||||
if !published {
|
||||
"publication_failed"
|
||||
} else if non_hot_changed {
|
||||
"published_partial"
|
||||
} else {
|
||||
"published"
|
||||
},
|
||||
revision,
|
||||
published.then_some(epoch),
|
||||
(!published).then_some("publication_closed"),
|
||||
);
|
||||
reload_state.mark_applied(rendered_hash);
|
||||
Some(next_manifest)
|
||||
}
|
||||
@@ -265,8 +317,18 @@ pub fn spawn_config_watcher(
|
||||
impl std::future::Future<Output = ()> + Send + 'static,
|
||||
) {
|
||||
let initial_level = initial.general.log_level.clone();
|
||||
let (config_tx, config_rx) =
|
||||
watch::channel(Arc::new((*initial).clone().with_publication_epoch()));
|
||||
let mut initial = (*initial).clone().with_publication_epoch();
|
||||
initial.watcher_observation = Some(Arc::new(
|
||||
crate::config::observation::WatchObservation::default(),
|
||||
));
|
||||
record_outcome(
|
||||
&initial,
|
||||
"initial",
|
||||
initial.provenance.revision().map(str::to_owned),
|
||||
Some(initial.runtime_config_epoch),
|
||||
None,
|
||||
);
|
||||
let (config_tx, config_rx) = watch::channel(Arc::new(initial));
|
||||
let (log_tx, log_rx) = watch::channel(initial_level);
|
||||
|
||||
let config_path = normalize_watch_path(&config_path);
|
||||
@@ -287,6 +349,15 @@ pub fn spawn_config_watcher(
|
||||
}
|
||||
}
|
||||
let initial_loaded = preparation::read_source(&config_path).await;
|
||||
if initial_loaded.is_none() {
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"rejected",
|
||||
None,
|
||||
None,
|
||||
Some("source_invalid_or_unreadable"),
|
||||
);
|
||||
}
|
||||
let initial_manifest = initial_loaded
|
||||
.as_ref()
|
||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||
@@ -485,3 +556,6 @@ mod decoy_dns_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod quota_period_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod observation_tests;
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
use super::*;
|
||||
use crate::config::observation::{ConfigProvenance, WatchObservation};
|
||||
|
||||
#[tokio::test]
|
||||
async fn watcher_outcomes_follow_actual_publication_retention_and_rejection() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.toml");
|
||||
std::fs::write(&path, "").unwrap();
|
||||
let loaded = ProxyConfig::load_prepared(path.clone()).await.unwrap();
|
||||
let mut initial = loaded.config.with_publication_epoch();
|
||||
let observer = Arc::new(WatchObservation::default());
|
||||
initial.watcher_observation = Some(observer.clone());
|
||||
let (sender, mut receiver) = watch::channel(Arc::new(initial));
|
||||
let (log, _) = watch::channel(sender.borrow().general.log_level.clone());
|
||||
let mut state = ReloadState::new(Some(loaded.rendered_hash));
|
||||
for (text, phase, changed) in [
|
||||
("[general]\nlog_level='debug'", "published", true),
|
||||
(
|
||||
"[general]\nlog_level='debug'\ndisable_colors=true",
|
||||
"retained",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"[general]\nlog_level='verbose'\ndisable_colors=true",
|
||||
"published_partial",
|
||||
true,
|
||||
),
|
||||
(
|
||||
"[general]\nlog_level='verbose'\ndisable_colors=true",
|
||||
"unchanged",
|
||||
false,
|
||||
),
|
||||
(
|
||||
"[general]\nlog_level='private-invalid-log-level'",
|
||||
"rejected",
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let old = receiver.borrow_and_update().clone();
|
||||
let sequence = observer.snapshot().map_or(0, |value| value.sequence);
|
||||
std::fs::write(&path, text).unwrap();
|
||||
preparation::reload(
|
||||
&path, &sender, &log, None, None, &mut state, None, false, None, None,
|
||||
)
|
||||
.await;
|
||||
let outcome = observer.snapshot().unwrap();
|
||||
assert!(outcome.sequence > sequence);
|
||||
assert_eq!(outcome.phase, phase);
|
||||
assert_eq!(receiver.has_changed().unwrap(), changed);
|
||||
if changed {
|
||||
assert!(receiver.borrow().runtime_config_epoch > old.runtime_config_epoch);
|
||||
assert_eq!(
|
||||
outcome.published_config_epoch,
|
||||
Some(receiver.borrow().runtime_config_epoch)
|
||||
);
|
||||
if phase == "published_partial" {
|
||||
assert!(matches!(
|
||||
&receiver.borrow().provenance,
|
||||
ConfigProvenance::Mixed
|
||||
));
|
||||
} else {
|
||||
assert_eq!(
|
||||
outcome.source_revision.as_deref(),
|
||||
receiver.borrow().provenance.revision()
|
||||
);
|
||||
}
|
||||
} else {
|
||||
assert!(Arc::ptr_eq(&old, &receiver.borrow()));
|
||||
}
|
||||
assert!(
|
||||
!serde_json::to_string(&*outcome)
|
||||
.unwrap()
|
||||
.contains("private-invalid-log-level")
|
||||
);
|
||||
}
|
||||
let rejected = observer.snapshot().unwrap();
|
||||
assert_eq!(rejected.error_code, Some("source_invalid_or_unreadable"));
|
||||
assert_eq!(rejected.published_config_epoch, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_receivers_never_report_successful_publication() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.toml");
|
||||
std::fs::write(&path, "").unwrap();
|
||||
let mut initial = ProxyConfig::load_prepared(path.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
.config;
|
||||
let observer = Arc::new(WatchObservation::default());
|
||||
initial.watcher_observation = Some(observer.clone());
|
||||
let (sender, receiver) = watch::channel(Arc::new(initial.with_publication_epoch()));
|
||||
let (log, _) = watch::channel(sender.borrow().general.log_level.clone());
|
||||
let original = sender.borrow().clone();
|
||||
drop(receiver);
|
||||
std::fs::write(&path, "[general]\nlog_level='debug'").unwrap();
|
||||
preparation::reload(
|
||||
&path,
|
||||
&sender,
|
||||
&log,
|
||||
None,
|
||||
None,
|
||||
&mut ReloadState::default(),
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(Arc::ptr_eq(&sender.borrow(), &original));
|
||||
let outcome = observer.snapshot().unwrap();
|
||||
assert_eq!(outcome.phase, "publication_failed");
|
||||
assert_eq!(outcome.published_config_epoch, None);
|
||||
assert_eq!(outcome.error_code, Some("publication_closed"));
|
||||
}
|
||||
@@ -43,7 +43,17 @@ pub(super) async fn reload(
|
||||
Some(gate) => Some(gate.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let source = read_source(path).await?;
|
||||
let Some(source) = read_source(path).await else {
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"rejected",
|
||||
None,
|
||||
None,
|
||||
Some("source_invalid_or_unreadable"),
|
||||
);
|
||||
return None;
|
||||
};
|
||||
let revision = source.config.provenance.revision().map(str::to_owned);
|
||||
let manifest = WatchManifest::from_source_files(&source.source_files);
|
||||
// A new file edit can restore the exact pre-creation bytes. Refresh only source
|
||||
// evidence for fenced accounts, retaining prepared DNS and immutable WEB state.
|
||||
@@ -53,16 +63,43 @@ pub(super) async fn reload(
|
||||
{
|
||||
let mut refreshed = (**config_tx.borrow()).clone();
|
||||
refreshed.quota_source_read_at = source.config.quota_source_read_at;
|
||||
config_tx
|
||||
let published = config_tx
|
||||
.send(Arc::new(refreshed.with_publication_epoch()))
|
||||
.ok();
|
||||
.is_ok();
|
||||
let config = config_tx.borrow();
|
||||
record_outcome(
|
||||
&config,
|
||||
if published {
|
||||
"source_refreshed"
|
||||
} else {
|
||||
"publication_failed"
|
||||
},
|
||||
revision,
|
||||
published.then_some(config.runtime_config_epoch),
|
||||
(!published).then_some("publication_closed"),
|
||||
);
|
||||
reload_state.mark_applied(source.rendered_hash);
|
||||
return Some(manifest);
|
||||
}
|
||||
drop(guard);
|
||||
if !force && reload_state.is_applied(source.rendered_hash) {
|
||||
let config = config_tx.borrow();
|
||||
record_outcome(
|
||||
&config,
|
||||
"unchanged",
|
||||
revision,
|
||||
Some(config.runtime_config_epoch),
|
||||
None,
|
||||
);
|
||||
return Some(manifest);
|
||||
}
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"preparing",
|
||||
revision.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let loaded = match source.prepare().await {
|
||||
Ok(loaded) => loaded,
|
||||
Err(error) => {
|
||||
@@ -70,6 +107,13 @@ pub(super) async fn reload(
|
||||
"config reload: preparation failed: {}; keeping old config",
|
||||
error
|
||||
);
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"rejected",
|
||||
revision,
|
||||
None,
|
||||
Some("preparation_failed"),
|
||||
);
|
||||
return Some(manifest);
|
||||
}
|
||||
};
|
||||
@@ -96,6 +140,13 @@ pub(super) async fn reload(
|
||||
"config reload: effective WEB validation failed: {}; keeping old config",
|
||||
error
|
||||
);
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"rejected",
|
||||
revision,
|
||||
None,
|
||||
Some("effective_config_invalid"),
|
||||
);
|
||||
Some(manifest)
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -103,6 +154,13 @@ pub(super) async fn reload(
|
||||
"config reload: runtime builder failed: {}; keeping old config",
|
||||
error
|
||||
);
|
||||
record_outcome(
|
||||
&config_tx.borrow(),
|
||||
"rejected",
|
||||
revision,
|
||||
None,
|
||||
Some("preparation_worker_failed"),
|
||||
);
|
||||
Some(manifest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,6 +160,10 @@ pub struct ProxyConfig {
|
||||
/// Process-local identity assigned by the owner of a published config snapshot.
|
||||
#[serde(skip)]
|
||||
pub(crate) runtime_config_epoch: u64,
|
||||
#[serde(skip)]
|
||||
pub(crate) provenance: super::observation::ConfigProvenance,
|
||||
#[serde(skip)]
|
||||
pub(crate) watcher_observation: Option<Arc<super::observation::WatchObservation>>,
|
||||
|
||||
/// Monotonic start of source acquisition, used only for cold quota policy fencing.
|
||||
#[serde(skip)]
|
||||
@@ -265,6 +269,11 @@ impl ProxyConfig {
|
||||
Self::parse_source_with_overrides(path, &BTreeMap::new())
|
||||
}
|
||||
|
||||
/// Parses an already captured source manifest without DNS or static-site reads.
|
||||
pub(crate) fn parse_captured_source(graph: ConfigSourceGraph) -> Result<ParsedConfigSource> {
|
||||
pipeline::parse_source_graph(graph)
|
||||
}
|
||||
|
||||
/// Checks decoy mode-specific keys before a source or API patch loses unknown fields.
|
||||
pub(crate) fn validate_decoy_source_keys(document: &toml::Value) -> Result<()> {
|
||||
decoy_dns::validate_mode_keys(document)
|
||||
|
||||
@@ -20,6 +20,8 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSou
|
||||
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||
handle_unknown_config_keys(&parsed_toml)?;
|
||||
ProxyConfig::validate_decoy_source_keys(&parsed_toml)?;
|
||||
let source_evidence =
|
||||
crate::config::observation::SourceEvidence::capture(&parsed_toml, &source_contents);
|
||||
let general_table = parsed_toml
|
||||
.get("general")
|
||||
.and_then(|value| value.as_table());
|
||||
@@ -65,6 +67,7 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSou
|
||||
let mut config: ProxyConfig = parsed_toml
|
||||
.try_into()
|
||||
.map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||
config.provenance = crate::config::observation::ConfigProvenance::Source(source_evidence);
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
pub(crate) mod defaults;
|
||||
pub mod hot_reload;
|
||||
mod load;
|
||||
pub(crate) mod observation;
|
||||
mod types;
|
||||
|
||||
pub use load::ProxyConfig;
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Bounded source evidence and owner-authored configuration observations.
|
||||
//! These records describe publication, never asynchronous consumer convergence.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(crate) fn source_revision(sources: &BTreeMap<PathBuf, String>) -> String {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"telemt-config-manifest-v1\0");
|
||||
for (path, content) in sources {
|
||||
let path = path.as_os_str().as_encoded_bytes();
|
||||
hash.update((path.len() as u64).to_le_bytes());
|
||||
hash.update(path);
|
||||
hash.update((content.len() as u64).to_le_bytes());
|
||||
hash.update(content.as_bytes());
|
||||
}
|
||||
hex::encode(hash.finalize())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) enum ConfigProvenance {
|
||||
#[default]
|
||||
Unknown,
|
||||
Source(Arc<SourceEvidence>),
|
||||
/// The published config combines sources or retained process policy. Do not
|
||||
/// attribute all its values to the last source read by the watcher.
|
||||
Mixed,
|
||||
}
|
||||
|
||||
impl ConfigProvenance {
|
||||
pub(crate) fn revision(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Source(source) => Some(&source.revision),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SourceEvidence {
|
||||
pub(crate) revision: String,
|
||||
pub(crate) explicit_paths: BTreeSet<String>,
|
||||
pub(crate) complete: bool,
|
||||
pub(crate) source_count: usize,
|
||||
pub(crate) generated_fake_cert_len: bool,
|
||||
}
|
||||
|
||||
impl SourceEvidence {
|
||||
pub(crate) fn capture(
|
||||
document: &toml::Value,
|
||||
sources: &BTreeMap<PathBuf, String>,
|
||||
) -> Arc<Self> {
|
||||
fn visit(
|
||||
value: &toml::Value,
|
||||
path: &str,
|
||||
paths: &mut BTreeSet<String>,
|
||||
complete: &mut bool,
|
||||
) {
|
||||
if paths.len() >= 4096 || path.len() > 256 {
|
||||
*complete = false;
|
||||
return;
|
||||
}
|
||||
if !path.is_empty() {
|
||||
paths.insert(path.to_owned());
|
||||
}
|
||||
// Open maps need only presence provenance, never their private keys.
|
||||
if path == "dc_overrides"
|
||||
|| path == "censorship.exclusive_mask"
|
||||
|| path.starts_with("access.")
|
||||
{
|
||||
return;
|
||||
}
|
||||
match value {
|
||||
toml::Value::Table(table) => {
|
||||
for (key, value) in table {
|
||||
let key = if path.starts_with("upstreams[")
|
||||
&& path.ends_with(']')
|
||||
&& key == "force_bind"
|
||||
{
|
||||
"bindtodevice"
|
||||
} else if path == "server" && key == "admin_api" {
|
||||
"api"
|
||||
} else {
|
||||
key
|
||||
};
|
||||
let child = if path.is_empty() {
|
||||
key.to_owned()
|
||||
} else {
|
||||
format!("{path}.{key}")
|
||||
};
|
||||
visit(value, &child, paths, complete);
|
||||
}
|
||||
}
|
||||
toml::Value::Array(array) => {
|
||||
for (index, value) in array.iter().enumerate() {
|
||||
visit(value, &format!("{path}[{index}]"), paths, complete);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let mut explicit_paths = BTreeSet::new();
|
||||
let mut complete = true;
|
||||
visit(document, "", &mut explicit_paths, &mut complete);
|
||||
let censorship = document.get("censorship");
|
||||
let generated_fake_cert_len = censorship
|
||||
.and_then(|v| v.get("tls_emulation"))
|
||||
.and_then(toml::Value::as_bool)
|
||||
== Some(false)
|
||||
&& censorship
|
||||
.and_then(|v| v.get("fake_cert_len"))
|
||||
.and_then(toml::Value::as_integer)
|
||||
.is_none_or(|value| value == super::defaults::default_fake_cert_len() as i64);
|
||||
Arc::new(Self {
|
||||
revision: source_revision(sources),
|
||||
explicit_paths,
|
||||
complete,
|
||||
source_count: sources.len(),
|
||||
generated_fake_cert_len,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct WatchOutcome {
|
||||
pub(crate) sequence: u64,
|
||||
pub(crate) observed_at_epoch_secs: u64,
|
||||
pub(crate) phase: &'static str,
|
||||
/// Source considered by this attempt, not a hash of applied runtime state.
|
||||
pub(crate) source_revision: Option<String>,
|
||||
pub(crate) published_config_epoch: Option<u64>,
|
||||
pub(crate) error_code: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct WatchObservation {
|
||||
next_sequence: AtomicU64,
|
||||
latest: ArcSwapOption<WatchOutcome>,
|
||||
}
|
||||
|
||||
impl WatchObservation {
|
||||
pub(crate) fn record(
|
||||
&self,
|
||||
phase: &'static str,
|
||||
revision: Option<String>,
|
||||
epoch: Option<u64>,
|
||||
error_code: Option<&'static str>,
|
||||
) {
|
||||
self.latest.store(Some(Arc::new(WatchOutcome {
|
||||
sequence: self
|
||||
.next_sequence
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
.saturating_add(1),
|
||||
observed_at_epoch_secs: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs(),
|
||||
phase,
|
||||
source_revision: revision,
|
||||
published_config_epoch: epoch,
|
||||
error_code,
|
||||
})));
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> Option<Arc<WatchOutcome>> {
|
||||
self.latest.load_full()
|
||||
}
|
||||
}
|
||||
@@ -207,6 +207,9 @@ pub(super) async fn bootstrap(
|
||||
validate_synlimit_privilege_drop(&config, privilege_drop_requested)?;
|
||||
|
||||
if let Some(p) = data_path {
|
||||
if config.general.data_path.as_ref() != Some(&p) {
|
||||
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
|
||||
}
|
||||
config.general.data_path = Some(p);
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ pub(super) async fn run_telemt_core(
|
||||
if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control {
|
||||
warn!("Inline conntrack control is disabled when process privileges are dropped");
|
||||
config.server.conntrack_control.inline_conntrack_control = false;
|
||||
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
|
||||
}
|
||||
|
||||
let quota_store = Arc::new(QuotaStore::default());
|
||||
|
||||
@@ -9,6 +9,10 @@ use tokio::sync::mpsc;
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
#[path = "reload_observation.rs"]
|
||||
mod observation;
|
||||
pub(crate) use observation::ReloadObservation;
|
||||
|
||||
const RELOAD_HISTORY_CAPACITY: usize = 32;
|
||||
const RELOAD_COMMAND_CAPACITY: usize = 1;
|
||||
const MAX_DRAIN_TIMEOUT_SECS: u64 = 3_600;
|
||||
@@ -151,6 +155,16 @@ pub(crate) struct ReloadStatus {
|
||||
pub(crate) warnings: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) error: Option<String>,
|
||||
#[serde(skip)]
|
||||
source_generation: Option<u64>,
|
||||
#[serde(skip)]
|
||||
activated_generation: Option<u64>,
|
||||
#[serde(skip)]
|
||||
retiring_generation: Option<u64>,
|
||||
#[serde(skip)]
|
||||
retiring_config_epoch: Option<u64>,
|
||||
#[serde(skip)]
|
||||
retirement: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// Accepted operation metadata returned before asynchronous preparation starts.
|
||||
@@ -395,6 +409,11 @@ impl ReloadStatusStore {
|
||||
deferred_fields: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
source_generation: None,
|
||||
activated_generation: None,
|
||||
retiring_generation: None,
|
||||
retiring_config_epoch: None,
|
||||
retirement: None,
|
||||
};
|
||||
state.active_reload_id = Some(reload_id);
|
||||
state.statuses.push_back(status.clone());
|
||||
@@ -416,6 +435,9 @@ impl ReloadStatusStore {
|
||||
fn mark_phase(&self, reload_id: u64, phase: ReloadPhase) {
|
||||
self.update(reload_id, |status| {
|
||||
status.state = phase;
|
||||
if phase == ReloadPhase::Draining {
|
||||
status.retirement = Some("draining");
|
||||
}
|
||||
if status.started_at_epoch_secs.is_none() && phase != ReloadPhase::Accepted {
|
||||
status.started_at_epoch_secs = Some(now_epoch_secs());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{ReloadControl, ReloadPhase};
|
||||
|
||||
/// Sanitized owner snapshot. Raw reload errors/warnings can contain source data.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct ReloadObservation {
|
||||
pub(crate) reload_id: u64,
|
||||
pub(crate) target_generation: u64,
|
||||
pub(crate) source_revision: String,
|
||||
pub(crate) phase: ReloadPhase,
|
||||
pub(crate) source_generation: Option<u64>,
|
||||
pub(crate) activated_generation: Option<u64>,
|
||||
pub(crate) retiring_generation: Option<u64>,
|
||||
pub(crate) retiring_config_epoch: Option<u64>,
|
||||
pub(crate) retirement: Option<&'static str>,
|
||||
pub(crate) error_present: bool,
|
||||
pub(crate) warning_count: usize,
|
||||
}
|
||||
|
||||
impl ReloadControl {
|
||||
/// Reads bounded owner metadata without waiting for a contended status lock.
|
||||
pub(crate) fn observation(&self) -> Option<Option<ReloadObservation>> {
|
||||
let state = self.status_store.state.try_lock()?;
|
||||
Some(state.statuses.back().map(|status| ReloadObservation {
|
||||
reload_id: status.reload_id,
|
||||
target_generation: status.target_generation,
|
||||
source_revision: status.config_revision.clone(),
|
||||
phase: status.state,
|
||||
source_generation: status.source_generation,
|
||||
activated_generation: status.activated_generation,
|
||||
retiring_generation: status.retiring_generation,
|
||||
retiring_config_epoch: status.retiring_config_epoch,
|
||||
retirement: status.retirement,
|
||||
error_present: status.error.is_some(),
|
||||
warning_count: status.warnings.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn record_source_generation(&self, id: u64, generation: u64) {
|
||||
self.status_store
|
||||
.update(id, |status| status.source_generation = Some(generation));
|
||||
}
|
||||
|
||||
pub(crate) fn record_activation(
|
||||
&self,
|
||||
id: u64,
|
||||
active: u64,
|
||||
retiring: u64,
|
||||
retiring_epoch: u64,
|
||||
) {
|
||||
self.status_store.update(id, |status| {
|
||||
status.activated_generation = Some(active);
|
||||
status.retiring_generation = Some(retiring);
|
||||
status.retiring_config_epoch = Some(retiring_epoch);
|
||||
status.retirement = Some("retiring");
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn record_retirement(&self, id: u64, timed_out: bool) {
|
||||
self.status_store.update(id, |status| {
|
||||
status.retirement = Some(if timed_out {
|
||||
"cleanup_timeout"
|
||||
} else {
|
||||
"retired"
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -163,6 +163,8 @@ impl ReloadSupervisor {
|
||||
.mark_phase(command.reload_id, ReloadPhase::Preparing)
|
||||
.await;
|
||||
let old_runtime = self.active_runtime.load_full();
|
||||
self.control
|
||||
.record_source_generation(command.reload_id, old_runtime.id);
|
||||
let resolved = match resolve_reload_config(&old_runtime.config(), &command.config) {
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => {
|
||||
@@ -351,6 +353,12 @@ impl ReloadSupervisor {
|
||||
old_runtime.stop_accepting_sessions();
|
||||
listener_manager.activate_runtime_generation(new_runtime.clone())
|
||||
};
|
||||
self.control.record_activation(
|
||||
command.reload_id,
|
||||
new_runtime.id,
|
||||
replaced.id,
|
||||
replaced.config().runtime_config_epoch,
|
||||
);
|
||||
// A rejected admission epoch must not be bypassed by reapplying the prepared
|
||||
// snapshot. Quota publication retains its credential and source checks.
|
||||
let config = new_runtime.config();
|
||||
@@ -422,7 +430,10 @@ impl ReloadSupervisor {
|
||||
}
|
||||
}
|
||||
|
||||
if stop_background_and_middle_end(&replaced).await {
|
||||
let retirement_timed_out = stop_background_and_middle_end(&replaced).await;
|
||||
self.control
|
||||
.record_retirement(command.reload_id, retirement_timed_out);
|
||||
if retirement_timed_out {
|
||||
let warning = format!(
|
||||
"generation {} Middle-End lifecycle shutdown timed out",
|
||||
replaced.id
|
||||
|
||||
@@ -158,6 +158,11 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
||||
let observed = fixture.control.observation().unwrap().unwrap();
|
||||
assert_eq!(observed.phase, ReloadPhase::RolledBack);
|
||||
assert_eq!(observed.activated_generation, None);
|
||||
assert_eq!(observed.retiring_generation, None);
|
||||
assert_eq!(observed.retirement, None);
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
@@ -206,12 +211,33 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
assert!(!activation.is_finished());
|
||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.control.status(1).await.unwrap().state != ReloadPhase::Draining {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let observed = fixture.control.observation().unwrap().unwrap();
|
||||
assert_eq!(observed.phase, ReloadPhase::Draining);
|
||||
assert_eq!(observed.activated_generation, Some(2));
|
||||
assert_eq!(observed.retiring_generation, Some(1));
|
||||
assert_eq!(
|
||||
observed.retiring_config_epoch,
|
||||
Some(fixture.old_runtime.config().runtime_config_epoch)
|
||||
);
|
||||
assert_eq!(observed.retirement, Some("draining"));
|
||||
|
||||
old_release.notify_one();
|
||||
activation.await.unwrap();
|
||||
assert_eq!(
|
||||
fixture.control.status(1).await.unwrap().state,
|
||||
ReloadPhase::Succeeded
|
||||
);
|
||||
let observed = fixture.control.observation().unwrap().unwrap();
|
||||
assert_eq!(observed.phase, ReloadPhase::Succeeded);
|
||||
assert_eq!(observed.retiring_generation, Some(1));
|
||||
assert_eq!(observed.retirement, Some("retired"));
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
@@ -257,6 +283,23 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_reload_observes_source_generation_without_activation() {
|
||||
let mut fixture = fixture(ReloadRequest::default()).await;
|
||||
Arc::make_mut(&mut fixture.command.config).web.enabled = true;
|
||||
fixture.supervisor.reload(fixture.command).await;
|
||||
let observed = fixture.control.observation().unwrap().unwrap();
|
||||
assert_eq!(observed.phase, ReloadPhase::Failed);
|
||||
assert_eq!(observed.source_generation, Some(1));
|
||||
assert_eq!(observed.activated_generation, None);
|
||||
assert_eq!(observed.retiring_generation, None);
|
||||
assert!(observed.error_present);
|
||||
assert!(Arc::ptr_eq(
|
||||
&fixture.supervisor.active_runtime.load_full(),
|
||||
&fixture.old_runtime
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
||||
let runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||
|
||||
@@ -48,6 +48,11 @@ fn status_uses_documented_deferred_process_fields_key() {
|
||||
deferred_fields: vec!["server.listeners".to_string()],
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
source_generation: Some(1),
|
||||
activated_generation: Some(2),
|
||||
retiring_generation: Some(1),
|
||||
retiring_config_epoch: Some(3),
|
||||
retirement: Some("retired"),
|
||||
};
|
||||
let value = serde_json::to_value(status).unwrap();
|
||||
|
||||
@@ -56,6 +61,27 @@ fn status_uses_documented_deferred_process_fields_key() {
|
||||
serde_json::json!(["server.listeners"])
|
||||
);
|
||||
assert!(value.get("deferred_fields").is_none());
|
||||
for field in [
|
||||
"source_generation",
|
||||
"activated_generation",
|
||||
"retiring_generation",
|
||||
"retiring_config_epoch",
|
||||
"retirement",
|
||||
] {
|
||||
assert!(
|
||||
value.get(field).is_none(),
|
||||
"Legacy reload response gained {field}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_observation_never_waits_on_the_reload_status_lock() {
|
||||
let (control, _receiver) = ReloadControl::channel(1);
|
||||
let guard = control.status_store.state.lock();
|
||||
assert!(control.observation().is_none());
|
||||
drop(guard);
|
||||
assert!(control.observation().unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -334,6 +334,29 @@ pub(crate) fn resolve_reload_config(
|
||||
old: &ProxyConfig,
|
||||
desired: &ProxyConfig,
|
||||
) -> Result<ResolvedReloadConfig, String> {
|
||||
let mut resolved = project_reload_config(old, desired);
|
||||
resolved
|
||||
.effective
|
||||
.validate_effective_web()
|
||||
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
|
||||
resolved
|
||||
.effective
|
||||
.rebuild_runtime_user_auth()
|
||||
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
|
||||
resolved
|
||||
.effective
|
||||
.rebuild_runtime_web()
|
||||
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
|
||||
resolved.runtime_changed = !configs_equal(old, &resolved.effective);
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
/// Projects retained process policy without DNS, static acquisition or validation.
|
||||
/// State inspection uses this projection; preflight still uses the full resolver.
|
||||
pub(crate) fn project_reload_config(
|
||||
old: &ProxyConfig,
|
||||
desired: &ProxyConfig,
|
||||
) -> ResolvedReloadConfig {
|
||||
let mut effective = desired.clone();
|
||||
let mut fields = Vec::new();
|
||||
let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server);
|
||||
@@ -462,21 +485,15 @@ pub(crate) fn resolve_reload_config(
|
||||
fields.push("web.debug".to_string());
|
||||
effective.web.debug = old.web.debug.clone();
|
||||
}
|
||||
effective
|
||||
.validate_effective_web()
|
||||
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
|
||||
effective
|
||||
.rebuild_runtime_user_auth()
|
||||
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
|
||||
effective
|
||||
.rebuild_runtime_web()
|
||||
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
|
||||
if !fields.is_empty() {
|
||||
effective.provenance = crate::config::observation::ConfigProvenance::Mixed;
|
||||
}
|
||||
let runtime_changed = !configs_equal(old, &effective);
|
||||
Ok(ResolvedReloadConfig {
|
||||
ResolvedReloadConfig {
|
||||
effective,
|
||||
deferred_process_fields: fields,
|
||||
runtime_changed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool {
|
||||
|
||||
@@ -172,6 +172,9 @@ pub(super) async fn prepare_runtime(
|
||||
} else {
|
||||
let _ = use_middle_proxy;
|
||||
use_middle_proxy = false;
|
||||
if config.general.use_middle_proxy {
|
||||
config.provenance = crate::config::observation::ConfigProvenance::Mixed;
|
||||
}
|
||||
config.general.use_middle_proxy = false;
|
||||
startup_tracker.set_transport_mode("direct").await;
|
||||
startup_tracker.set_degraded(true).await;
|
||||
|
||||
Reference in New Issue
Block a user