API: expose observability settings and fix cross-generation cache races

This commit is contained in:
Alexey
2026-11-07 08:05:22 +03:00
parent b1502b629b
commit 982c4e4a34
19 changed files with 1024 additions and 52 deletions
+49 -12
View File
@@ -8,9 +8,9 @@ use toml::Value as Toml;
use super::ApiShared;
use super::config_store::{
EDITABLE_NETWORK_FIELDS, EDITABLE_SECTIONS, EDITABLE_SERVER_FIELDS, compute_snapshot_revision,
is_editable_section, load_config_snapshot, render_server_listeners, render_top_level_section,
resolve_single_source_owner, upsert_dns_overrides, upsert_toml_table,
EDITABLE_NETWORK_FIELDS, EDITABLE_SECTIONS, EDITABLE_SERVER_API_FIELDS, EDITABLE_SERVER_FIELDS,
compute_snapshot_revision, is_editable_section, load_config_snapshot, render_server_listeners,
render_top_level_section, resolve_single_source_owner, upsert_dns_overrides, upsert_toml_table,
};
#[cfg(test)]
use super::config_store::{write_atomic, write_atomic_if_unchanged};
@@ -25,6 +25,8 @@ use serde::Serialize;
use std::path::{Path, PathBuf};
use std::sync::Arc;
#[cfg(test)]
mod api_observability_tests;
mod diagnostics;
mod preflight;
mod preparation;
@@ -250,7 +252,23 @@ fn filter_server_for_read(server: &Toml) -> Option<Toml> {
}
}
}
out.insert((*field).to_string(), value.clone());
if *field == "api" {
if let Some(api) = value.as_table() {
out.insert(
"api".to_owned(),
Toml::Table(
api.iter()
.filter(|(key, _)| {
EDITABLE_SERVER_API_FIELDS.contains(&key.as_str())
})
.map(|(key, value)| (key.clone(), value.clone()))
.collect(),
),
);
}
} else {
out.insert((*field).to_string(), value.clone());
}
}
}
if out.is_empty() {
@@ -261,20 +279,15 @@ fn filter_server_for_read(server: &Toml) -> Option<Toml> {
}
/// Reject any `[server]` patch keys outside [`EDITABLE_SERVER_FIELDS`].
fn validate_server_patch(server: &Toml) -> Result<(), ApiFailure> {
let Some(table) = server.as_table() else {
fn validate_server_patch(server: &Json) -> Result<(), ApiFailure> {
let Some(table) = server.as_object() else {
return Err(ApiFailure::new(
hyper::StatusCode::BAD_REQUEST,
"section_not_editable",
"server patch must be a JSON object",
));
};
if table.is_empty() {
return Err(ApiFailure::bad_request(
"empty server patch: provide at least one editable field \
(currently: listeners)",
));
}
// Validate original keys before null removal, including forbidden siblings.
for key in table.keys() {
if !EDITABLE_SERVER_FIELDS.contains(&key.as_str()) {
return Err(ApiFailure::new(
@@ -288,6 +301,30 @@ fn validate_server_patch(server: &Toml) -> Result<(), ApiFailure> {
));
}
}
if let Some(api) = table.get("api").filter(|api| !api.is_null()) {
let fields = api
.as_object()
.ok_or_else(|| ApiFailure::bad_request("server.api patch must be a JSON object"))?;
for key in fields.keys() {
if !EDITABLE_SERVER_API_FIELDS.contains(&key.as_str()) {
return Err(ApiFailure::new(
hyper::StatusCode::BAD_REQUEST,
"field_not_editable",
format!("server.api.{key} is not editable via the config API"),
));
}
}
}
let has_listeners = table.get("listeners").is_some_and(|value| !value.is_null());
let has_api_fields = table
.get("api")
.and_then(Json::as_object)
.is_some_and(|fields| fields.values().any(|value| !value.is_null()));
if !has_listeners && !has_api_fields {
return Err(ApiFailure::bad_request(
"empty server patch: provide listeners or an editable server.api field",
));
}
Ok(())
}
@@ -0,0 +1,508 @@
use super::*;
use crate::api::discovery::http_tests::Fixture;
use crate::config::hot_reload::spawn_config_watcher;
use crate::maestro::{generation::test_runtime_generation, reload::ReloadControl};
use serde_json::json;
use std::time::Duration;
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
const API: &str = "[server.api] # keep API perimeter\nenabled=true\nlisten='127.0.0.1:9092'\nread_only=false\nauth_header='Bearer discovery-test' # keep secret\nwhitelist=['127.0.0.1/32']\ngray_action='api'\nrequest_body_limit_bytes=4096\nruntime_edge_events_capacity=64\nminimal_runtime_enabled=false\nruntime_edge_enabled=false\n";
fn patch(fields: Json) -> Json {
json!({"server": {"api": fields}})
}
#[tokio::test]
async fn get_exposes_exactly_five_settings_and_defaults_without_api_secrets() {
for source in ["", API] {
let (path, _directory) = tests::temp_config(source);
let (managed, _) = read_managed_config(&path).await.unwrap();
let fields = managed["server"]["api"].as_table().unwrap();
let config = ProxyConfig::load(&path).unwrap();
let typed = toml::Value::try_from(&config.server.api).unwrap();
assert_eq!(fields.len(), 5);
for (key, value) in fields {
assert!(EDITABLE_SERVER_API_FIELDS.contains(&key.as_str()));
assert_eq!(*value, typed[key]);
}
assert!(!managed.to_string().contains("discovery-test"));
}
}
#[tokio::test]
async fn every_forbidden_api_sibling_and_alias_is_rejected_even_when_null() {
let (path, _directory) = tests::temp_config(API);
let original = std::fs::read(&path).unwrap();
let api = serde_json::to_value(ProxyConfig::default().server.api).unwrap();
for (key, value) in api.as_object().unwrap() {
if EDITABLE_SERVER_API_FIELDS.contains(&key.as_str()) {
continue;
}
for value in [value.clone(), Json::Null] {
let error = apply_patch_to_path(
&path,
&patch(json!({key: value, "runtime_edge_enabled": true})),
None,
)
.await
.unwrap_err();
assert_eq!(error.code, "field_not_editable", "{key}: {error:?}");
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
for body in [
patch(json!({"typo": null, "runtime_edge_enabled": true})),
json!({"server": {"admin_api": null, "api": {"runtime_edge_enabled": true}}}),
json!({"server": {"port": null, "api": {"runtime_edge_enabled": true}}}),
] {
assert_eq!(
apply_patch_to_path(&path, &body, None)
.await
.unwrap_err()
.code,
"field_not_editable"
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
#[tokio::test]
async fn observability_bounds_and_types_are_checked_before_persistence() {
let (path, _directory) = tests::temp_config(API);
for (key, good, bad) in [
(
"minimal_runtime_cache_ttl_ms",
vec![json!(0), json!(60_000)],
vec![json!(-1), json!(60_001), json!(1.5), json!(false)],
),
(
"runtime_edge_cache_ttl_ms",
vec![json!(0), json!(60_000)],
vec![json!(-1), json!(60_001), json!("0")],
),
(
"runtime_edge_top_n",
vec![json!(1), json!(1000)],
vec![json!(0), json!(1001), json!(-1), json!([])],
),
(
"minimal_runtime_enabled",
vec![json!(true), json!(false)],
vec![json!(0), json!("true")],
),
(
"runtime_edge_enabled",
vec![json!(true), json!(false)],
vec![json!(1), json!({})],
),
] {
for value in good {
apply_patch_to_path(&path, &patch(json!({key: value})), None)
.await
.unwrap();
assert_eq!(
serde_json::to_value(ProxyConfig::load(&path).unwrap().server.api).unwrap()[key],
value
);
}
let original = std::fs::read(&path).unwrap();
for value in bad {
let error = apply_patch_to_path(&path, &patch(json!({key: value})), None)
.await
.unwrap_err();
assert_eq!(
error.status,
hyper::StatusCode::BAD_REQUEST,
"{key}: {error:?}"
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
let original = std::fs::read(&path).unwrap();
for value in [
json!(null),
json!({}),
json!([]),
json!(false),
json!({"runtime_edge_top_n": null}),
] {
assert!(
apply_patch_to_path(&path, &patch(value), None)
.await
.is_err()
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
#[tokio::test]
async fn sparse_observability_patch_preserves_perimeter_listeners_and_comments() {
let source = format!("{API}\n[[server.listeners]]\nip='127.0.0.1'\nport=443\n");
let (path, _directory) = tests::temp_config(&source);
let before = ProxyConfig::load(&path).unwrap();
let mut response = apply_patch_to_path(
&path,
&patch(json!({
"minimal_runtime_enabled": true, "minimal_runtime_cache_ttl_ms": 0,
"runtime_edge_enabled": true, "runtime_edge_cache_ttl_ms": 60_000,
"runtime_edge_top_n": 1,
})),
None,
)
.await
.unwrap();
let after = ProxyConfig::load(&path).unwrap();
reconcile_runtime_effect(&mut response, &before, &after).unwrap();
assert!(response.restart_required);
assert!(response.runtime_reload_required);
assert!(!response.process_restart_required);
assert!(response.deferred_process_fields.is_empty());
assert_eq!(response.changed_paths.len(), 5);
let mut restored = after.clone();
for field in EDITABLE_SERVER_API_FIELDS {
let mut api = serde_json::to_value(&restored.server.api).unwrap();
api[*field] = serde_json::to_value(&before.server.api).unwrap()[*field].clone();
restored.server.api = serde_json::from_value(api).unwrap();
}
assert_eq!(
serde_json::to_value(restored).unwrap(),
serde_json::to_value(&before).unwrap()
);
let written = std::fs::read_to_string(&path).unwrap();
for line in source.lines().filter(|line| {
!line.starts_with("minimal_runtime_enabled=") && !line.starts_with("runtime_edge_enabled=")
}) {
assert!(written.contains(line), "{line}: {written}");
}
let response = apply_patch_to_path(
&path,
&patch(json!({"runtime_edge_top_n": 2, "runtime_edge_enabled": null})),
None,
)
.await
.unwrap();
assert_eq!(response.changed_paths, ["server.api.runtime_edge_top_n"]);
assert!(
ProxyConfig::load(&path)
.unwrap()
.server
.api
.runtime_edge_enabled
);
}
#[tokio::test]
async fn precise_api_edits_preserve_inline_dotted_quoted_and_legacy_source_shapes() {
for source in [
"server = { api = { auth_header = 'secret', runtime_edge_top_n = 10 } }\n",
"server.api.auth_header = 'secret'\nserver.api.runtime_edge_top_n = 10\n",
"['server'.'api']\nauth_header='secret'\nruntime_edge_top_n=10 # retain\n",
"[server.admin_api]\nauth_header='secret'\nruntime_edge_top_n=10 # retain\n",
] {
let (path, _directory) = tests::temp_config(source);
let before = ProxyConfig::load(&path).unwrap();
apply_patch_to_path(
&path,
&patch(json!({"runtime_edge_top_n": 3, "minimal_runtime_cache_ttl_ms": 0})),
None,
)
.await
.unwrap();
let after = ProxyConfig::load(&path).unwrap();
assert_eq!(after.server.api.auth_header, before.server.api.auth_header);
assert_eq!(after.server.api.runtime_edge_top_n, 3);
assert_eq!(after.server.api.minimal_runtime_cache_ttl_ms, 0);
let written = std::fs::read_to_string(&path).unwrap();
if source.contains("admin_api") {
assert!(written.contains("[server.admin_api]"));
assert!(!written.contains("[server.api]"));
}
if source.contains("# retain") {
assert!(written.contains("# retain"));
}
}
}
#[tokio::test]
async fn included_api_owner_is_updated_and_multiple_owners_are_rejected_atomically() {
let directory = tempfile::tempdir().unwrap();
let root = directory.path().join("config.toml");
let included = directory.path().join("api.toml");
let root_text = "include=\"api.toml\"\n[general]\nlog_level='normal'\n[server]\nport=443\n";
std::fs::write(&root, root_text).unwrap();
std::fs::write(&included, API).unwrap();
apply_patch_to_path(&root, &patch(json!({"runtime_edge_top_n": 3})), None)
.await
.unwrap();
assert_eq!(std::fs::read_to_string(&root).unwrap(), root_text);
assert_eq!(
ProxyConfig::load(&root)
.unwrap()
.server
.api
.runtime_edge_top_n,
3
);
for (root_text, included_text, body) in [
(
root_text.to_owned(),
std::fs::read_to_string(&included).unwrap(),
json!({"server":{"api":{"runtime_edge_top_n": 4}}, "general":{"log_level":"debug"}}),
),
(
"include=\"api.toml\"\nserver.api.minimal_runtime_enabled=false\n".to_owned(),
"server.api.runtime_edge_enabled=false\n".to_owned(),
patch(json!({"minimal_runtime_enabled":true,"runtime_edge_enabled":true})),
),
(
"include=\"api.toml\"\nserver.api.minimal_runtime_enabled=false\n".to_owned(),
"server.api.runtime_edge_enabled=false\n".to_owned(),
patch(json!({"runtime_edge_top_n":3})),
),
] {
std::fs::write(&root, &root_text).unwrap();
std::fs::write(&included, &included_text).unwrap();
let error = apply_patch_to_path(&root, &body, None).await.unwrap_err();
assert_eq!(error.code, "config_patch_not_atomic", "{error:?}");
assert_eq!(std::fs::read_to_string(&root).unwrap(), root_text);
assert_eq!(std::fs::read_to_string(&included).unwrap(), included_text);
}
}
#[tokio::test]
async fn http_preview_watcher_and_reload_keep_diagnostics_pending_until_activation() {
let mut fixture = Fixture::new(false);
std::fs::write(
&fixture.shared.config_path,
format!("[general]\nlog_level='normal'\n{API}"),
)
.unwrap();
let config = Arc::new(ProxyConfig::load(&fixture.shared.config_path).unwrap());
let (control, mut commands) = ReloadControl::channel(1);
Arc::get_mut(&mut fixture.shared).unwrap().reload_control = control;
let cancellation = CancellationToken::new();
let _cancel_on_drop = cancellation.clone().drop_guard();
let (activation, activated) = watch::channel(false);
let (mut receiver, _log_receiver, watcher) = spawn_config_watcher(
fixture.shared.config_path.clone(),
config.clone(),
None,
None,
cancellation.clone(),
None,
Some(activated),
Some(fixture.shared.mutation_lock.clone()),
None,
);
let mut runtime = test_runtime_generation(1, (*config).clone());
Arc::get_mut(&mut runtime).unwrap().config_rx = receiver.clone();
fixture.shared.active_runtime.store(runtime);
let task = tokio::spawn(watcher);
let body = json!({"general": {"log_level": "debug"}, "server": {"api": {
"minimal_runtime_enabled":true, "runtime_edge_enabled":true,
"minimal_runtime_cache_ttl_ms":0, "runtime_edge_cache_ttl_ms":0, "runtime_edge_top_n":1,
}}});
let original = std::fs::read(&fixture.shared.config_path).unwrap();
let preview = fixture
.request_body("POST", "/v1/config/validate", true, Some(&body), &[])
.await;
assert_eq!(preview.0, 200, "{}", preview.2);
let data = &preview.2["data"];
assert_eq!(data["apply"]["watcher_paths"], json!(["general.log_level"]));
assert_eq!(
data["apply"]["generation_paths"].as_array().unwrap().len(),
5
);
assert_eq!(data["apply"]["process_restart_paths"], json!([]));
assert_eq!(
std::fs::read(&fixture.shared.config_path).unwrap(),
original
);
assert_eq!(
fixture
.request_body("PATCH", "/v1/config", false, Some(&body), &[])
.await
.0,
401
);
let response = fixture
.request_body(
"PATCH",
"/v1/config",
true,
Some(&body),
&[
("If-Match", data["base_revision"].as_str().unwrap()),
(
"If-Runtime-Match",
data["runtime"]["runtime_match"].as_str().unwrap(),
),
],
)
.await;
assert_eq!(response.0, 200, "{}", response.2);
assert_eq!(response.2["data"]["runtime_reload_required"], true);
assert_eq!(response.2["data"]["process_restart_required"], false);
activation.send_replace(true);
tokio::time::timeout(
Duration::from_secs(5),
receiver.wait_for(|cfg| cfg.general.log_level == crate::config::LogLevel::Debug),
)
.await
.unwrap()
.unwrap();
assert_eq!(receiver.borrow().server.api, config.server.api);
assert_diagnostics(&fixture, false).await;
// Even an unchanged desired patch can explicitly request the pending reload.
let accepted = fixture
.request_body(
"PATCH",
"/v1/config?reload=instant",
true,
Some(&patch(body["server"]["api"].clone())),
&[],
)
.await;
assert_eq!(accepted.0, 202, "{}", accepted.2);
let command = tokio::time::timeout(Duration::from_secs(5), commands.recv())
.await
.unwrap()
.unwrap();
assert_diagnostics(&fixture, false).await;
let resolved = resolve_reload_config(&config, &command.config).unwrap();
assert_eq!(
resolved.effective.server.api.listen,
config.server.api.listen
);
assert_eq!(
resolved.effective.server.api.runtime_edge_events_capacity,
64
);
fixture
.shared
.active_runtime
.store(test_runtime_generation(2, resolved.effective));
assert_diagnostics(&fixture, true).await;
assert_eq!(
fixture
.request("GET", "/v1/runtime/events/recent", true)
.await
.2["data"]["data"]["capacity"],
64
);
fixture
.shared
.reload_control
.fail(command.reload_id, "fixture completed")
.await;
let disabled = patch(json!({"minimal_runtime_enabled": false, "runtime_edge_enabled": false}));
let accepted = fixture
.request_body(
"PATCH",
"/v1/config?reload=drain&timeout_secs=30",
true,
Some(&disabled),
&[],
)
.await;
assert_eq!(accepted.0, 202, "{}", accepted.2);
assert_diagnostics(&fixture, true).await;
let command = commands.recv().await.unwrap();
fixture
.shared
.active_runtime
.store(test_runtime_generation(3, (*command.config).clone()));
assert_diagnostics(&fixture, false).await;
assert_eq!(fixture.request("GET", "/v1/health", true).await.0, 200);
cancellation.cancel();
tokio::time::timeout(Duration::from_secs(5), task)
.await
.unwrap()
.unwrap();
}
async fn assert_diagnostics(fixture: &Fixture, enabled: bool) {
for path in [
"/v1/stats/minimal/all",
"/v1/runtime/connections/summary",
"/v1/runtime/events/recent",
] {
let (status, _, body) = fixture.request("GET", path, true).await;
assert_eq!(status, 200, "{body}");
assert_eq!(body["data"]["enabled"], enabled, "{body}");
if !enabled {
assert_eq!(body["data"]["reason"], "feature_disabled");
}
}
let (_, _, caps) = fixture.request("GET", "/v1/capabilities", true).await;
for path in ["/v1/stats/minimal/all", "/v1/runtime/connections/summary"] {
let operation = caps["data"]["operations"]
.as_array()
.unwrap()
.iter()
.find(|op| op["path"] == path)
.unwrap();
assert_eq!(operation["enabled"], enabled);
}
}
#[tokio::test]
async fn observability_patch_obeys_read_only_and_source_revision_fences() {
let fixture = Fixture::new(false);
let body = patch(json!({"runtime_edge_enabled": true}));
let original = std::fs::read(&fixture.shared.config_path).unwrap();
let preview = fixture
.request_body("POST", "/v1/config/validate", true, Some(&body), &[])
.await;
assert_eq!(preview.0, 200, "{}", preview.2);
let rejected = fixture
.request_body("PATCH", "/v1/config", true, Some(&body), &[])
.await;
assert_eq!(rejected.0, 403);
assert_eq!(rejected.2["error"]["code"], "read_only");
assert_eq!(
std::fs::read(&fixture.shared.config_path).unwrap(),
original
);
let prepared = prepare_patch_to_path(&fixture.shared.config_path, &body, None)
.await
.unwrap();
let external = "# external source change\n";
std::fs::write(&fixture.shared.config_path, external).unwrap();
let conflict = write_atomic_if_unchanged(
prepared.config_path,
prepared.expected_revision,
prepared.owner_path,
prepared.expected_owner_contents,
prepared.owner_contents,
)
.await
.unwrap_err();
assert_eq!(conflict.code, "revision_conflict");
assert_eq!(
std::fs::read_to_string(&fixture.shared.config_path).unwrap(),
external
);
}
#[tokio::test]
async fn listeners_and_observability_in_one_owner_preserve_each_other() {
let (path, _directory) = tests::temp_config(API);
apply_patch_to_path(
&path,
&json!({"server": {
"listeners": [{"ip": "127.0.0.1", "port": 8443}],
"api": {"runtime_edge_enabled": true},
}}),
None,
)
.await
.unwrap();
let config = ProxyConfig::load(&path).unwrap();
assert_eq!(config.server.listeners.len(), 1);
assert_eq!(config.server.listeners[0].port, Some(8443));
assert!(config.server.api.runtime_edge_enabled);
assert_eq!(config.server.api.auth_header, "Bearer discovery-test");
}
+79 -11
View File
@@ -1,5 +1,6 @@
//! One source-preserving candidate pipeline shared by preview and commit.
use super::*;
use crate::api::config_store::{FieldEdit, prepare_field_edits};
use crate::config::{LoadedConfig, ParsedConfigSource};
use tokio::sync::OwnedSemaphorePermit;
@@ -82,7 +83,7 @@ fn parse_patch(
"access.* is managed via the users API, not editable here",
));
}
for (key, value) in patch_table {
for key in patch_table.keys() {
if !is_editable_section(key.as_str()) {
return Err(ApiFailure::new(
hyper::StatusCode::BAD_REQUEST,
@@ -91,7 +92,7 @@ fn parse_patch(
));
}
if key == "server" {
validate_server_patch(value)?;
validate_server_patch(&patch_json["server"])?;
} else if key == "network" {
validate_network_patch(&patch_json["network"])?;
}
@@ -127,30 +128,59 @@ fn parse_patch(
.validate()
.map_err(|error| diagnostics::validation_error("constraint_violation", "$", error))?;
let api_edits = api_field_edits(&loaded, &patch_toml)?;
let api_source = prepare_field_edits(&loaded, config_path, &api_edits)
.map_err(diagnostics::source_conflict)?;
let patch_listeners = patch_toml
.get("server")
.is_some_and(|server| server.get("listeners").is_some());
let ownership_targets: Vec<&str> = touched
.iter()
.map(|section| {
.filter_map(|section| {
if section == "server" {
"server.listeners"
patch_listeners.then_some("server.listeners")
} else if section == "network" {
"network.dns_overrides"
Some("network.dns_overrides")
} else {
section.as_str()
Some(section.as_str())
}
})
.collect();
let owner_path = resolve_single_source_owner(&loaded, config_path, &ownership_targets)
.map_err(diagnostics::source_conflict)?;
let section_owner = if ownership_targets.is_empty() {
None
} else {
Some(
resolve_single_source_owner(&loaded, config_path, &ownership_targets)
.map_err(diagnostics::source_conflict)?,
)
};
if let (Some(owner), Some(api)) = (&section_owner, &api_source) {
if *owner != api.owner_path {
return Err(ApiFailure::new(
hyper::StatusCode::CONFLICT,
"config_patch_not_atomic",
"Mutation requires one independently writable config source",
));
}
}
let owner_path = section_owner
.or_else(|| api_source.as_ref().map(|source| source.owner_path.clone()))
.ok_or_else(|| ApiFailure::bad_request("empty patch: no editable fields"))?;
let mut owner_contents = loaded
.source_contents
.get(&owner_path)
.cloned()
.ok_or_else(|| ApiFailure::internal("config source owner is missing from snapshot"))?;
let expected_owner_contents = owner_contents.clone();
if let Some(api) = api_source {
owner_contents = api.owner_contents;
}
for section in &touched {
if *section == "server" {
let rendered = render_server_listeners(&requested_cfg)?;
owner_contents = upsert_toml_table(&owner_contents, "server.listeners", &rendered);
if patch_listeners {
let rendered = render_server_listeners(&requested_cfg)?;
owner_contents = upsert_toml_table(&owner_contents, "server.listeners", &rendered);
}
} else if *section == "network" {
owner_contents =
upsert_dns_overrides(&owner_contents, &requested_cfg.network.dns_overrides)?;
@@ -164,7 +194,7 @@ fn parse_patch(
overrides.insert(owner_path.clone(), owner_contents.clone());
let candidate = ProxyConfig::parse_source_with_overrides(config_path, &overrides)
.map_err(|error| diagnostics::validation_error("constraint_violation", "$", error))?;
if touched.iter().any(|section| section == "server")
if patch_listeners
&& serde_json::to_value(&candidate.config.server.listeners).ok()
!= serde_json::to_value(&requested_cfg.server.listeners).ok()
{
@@ -191,6 +221,44 @@ fn parse_patch(
))
}
fn api_field_edits(
loaded: &ParsedConfigSource,
patch: &Toml,
) -> Result<Vec<FieldEdit>, ApiFailure> {
let Some(fields) = patch
.get("server")
.and_then(|server| server.get("api"))
.and_then(Toml::as_table)
else {
return Ok(Vec::new());
};
if fields.is_empty() {
return Ok(Vec::new());
}
// PATCH uses canonical paths, while an authored legacy table keeps its name.
let mut table_name = "api";
for source in loaded.source_contents.values() {
let document: Toml = toml::from_str(source).map_err(|_| {
ApiFailure::new(
hyper::StatusCode::CONFLICT,
"config_patch_not_atomic",
"Mutation requires one independently writable config source",
)
})?;
if document
.get("server")
.and_then(|server| server.get("admin_api"))
.is_some()
{
table_name = "admin_api";
}
}
Ok(fields
.iter()
.map(|(field, value)| FieldEdit::new(&["server", table_name, field], Some(value.clone())))
.collect())
}
fn finish(
source: PatchSource,
candidate: LoadedConfig,
+6 -5
View File
@@ -228,8 +228,7 @@ async fn invalid_web_patch_does_not_modify_the_source() {
#[tokio::test]
async fn read_managed_config_returns_only_editable_sections() {
// Server identity and network siblings must not leak. Listeners-only server
// is returned via the nested allowlist (covered in a dedicated test).
// Server identity and network siblings must not leak through nested allowlists.
let (path, _d) = temp_config(concat!(
"[censorship]\ntls_domain = \"a\"\n",
"[server]\nport = 443\n[server.api]\nauth_header = \"SECRET\"\n",
@@ -241,7 +240,8 @@ async fn read_managed_config_returns_only_editable_sections() {
assert!(table.contains_key("censorship"));
let server = table["server"].as_table().unwrap();
assert!(server.contains_key("listeners"));
assert!(!server.contains_key("api"));
assert_eq!(server["api"].as_table().unwrap().len(), 5);
assert!(server["api"].get("auth_header").is_none());
assert!(!server.contains_key("port"));
assert_eq!(
serde_json::to_value(&table["network"]).unwrap(),
@@ -251,7 +251,7 @@ async fn read_managed_config_returns_only_editable_sections() {
}
#[tokio::test]
async fn read_managed_config_returns_server_listeners_only() {
async fn read_managed_config_returns_server_listeners_and_api_observability() {
let (path, _d) = temp_config(concat!(
"[censorship]\ntls_domain = \"a\"\n",
"[server]\nport = 443\n",
@@ -266,7 +266,8 @@ async fn read_managed_config_returns_server_listeners_only() {
.as_table()
.unwrap();
assert!(server.contains_key("listeners"));
assert!(!server.contains_key("api"));
assert_eq!(server["api"].as_table().unwrap().len(), 5);
assert!(server["api"].get("auth_header").is_none());
assert!(!server.contains_key("port"));
let listeners = server["listeners"].as_array().unwrap();
assert_eq!(listeners.len(), 1);
+13 -4
View File
@@ -326,9 +326,9 @@ pub(super) async fn save_config_to_disk(
/// located or replaced safely, so it is edited via `general`.
///
/// `server` is partially editable: only the nested fields listed in
/// [`EDITABLE_SERVER_FIELDS`] (currently `listeners`) may appear in GET/PATCH.
/// Secrets and bind identity (`api`/`admin_api`, `port`, unix sockets, …) stay
/// blocked. `network` is also partial: only [`EDITABLE_NETWORK_FIELDS`] are
/// [`EDITABLE_SERVER_FIELDS`] may appear in GET/PATCH; `api` is further restricted
/// to [`EDITABLE_SERVER_API_FIELDS`]. Secrets and bind identity stay blocked.
/// `network` is also partial: only [`EDITABLE_NETWORK_FIELDS`] are
/// exposed, retaining the boundary around per-node identity and routing policy.
pub(super) const EDITABLE_SECTIONS: &[&str] = &[
"general",
@@ -343,7 +343,16 @@ pub(super) const EDITABLE_SECTIONS: &[&str] = &[
///
/// Arrays (e.g. `listeners`) replace wholesale on PATCH, matching the existing
/// merge semantics for non-table values.
pub(super) const EDITABLE_SERVER_FIELDS: &[&str] = &["listeners"];
pub(super) const EDITABLE_SERVER_FIELDS: &[&str] = &["listeners", "api"];
/// Diagnostics settings applied by explicit generation reload, not the watcher.
pub(super) const EDITABLE_SERVER_API_FIELDS: &[&str] = &[
"minimal_runtime_enabled",
"minimal_runtime_cache_ttl_ms",
"runtime_edge_enabled",
"runtime_edge_cache_ttl_ms",
"runtime_edge_top_n",
];
/// Only the generation resolver's overrides may be read/patched under `network`.
pub(super) const EDITABLE_NETWORK_FIELDS: &[&str] = &["dns_overrides"];
+1
View File
@@ -38,6 +38,7 @@ impl Fixture {
}];
let runtime = test_runtime_generation(1, config.clone());
let shared = ApiShared {
runtime_generation: runtime.id,
stats: runtime.stats.clone(),
ip_tracker: runtime.ip_tracker.clone(),
me_pool: runtime.me_pool_runtime.clone(),
+13 -7
View File
@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::super::config_store::{
EDITABLE_NETWORK_FIELDS, EDITABLE_SECTIONS, EDITABLE_SERVER_FIELDS,
EDITABLE_NETWORK_FIELDS, EDITABLE_SECTIONS, EDITABLE_SERVER_API_FIELDS, EDITABLE_SERVER_FIELDS,
};
use super::super::model::ApiFailure;
use super::SCHEMA_VERSION;
@@ -204,13 +204,19 @@ fn field_api(path: &str) -> FieldApi {
|| path
.strip_prefix("network.")
.is_some_and(|field| EDITABLE_NETWORK_FIELDS.contains(&field))
|| path
.strip_prefix("server.api.")
.is_some_and(|field| EDITABLE_SERVER_API_FIELDS.contains(&field))
|| (top == "server"
&& EDITABLE_SERVER_FIELDS.iter().any(|field| {
let prefix = format!("server.{field}");
path == prefix
|| path.starts_with(&format!("{prefix}["))
|| path.starts_with(&format!("{prefix}."))
}));
&& EDITABLE_SERVER_FIELDS
.iter()
.filter(|field| **field != "api")
.any(|field| {
let prefix = format!("server.{field}");
path == prefix
|| path.starts_with(&format!("{prefix}["))
|| path.starts_with(&format!("{prefix}."))
}));
let policy_field = path.strip_prefix("access.").and_then(|path| {
let field = path.split('.').next()?;
crate::api::access_policy::FIELDS
+7 -5
View File
@@ -133,11 +133,13 @@ fn api_ownership_aliases_and_null_contracts_match_existing_surfaces() {
let schema = config_schema().unwrap();
assert!(field(schema, "web.vhosts[].base_path").api.config_write);
assert!(field(schema, "server.listeners[].port").api.config_write);
assert!(
!field(schema, "server.api.runtime_edge_enabled")
.api
.config_write
);
for entry in &schema.fields {
if let Some(key) = entry.spec.path.strip_prefix("server.api.") {
let editable = EDITABLE_SERVER_API_FIELDS.contains(&key);
assert_eq!(entry.api.config_read, editable, "{key}");
assert_eq!(entry.api.config_write, editable, "{key}");
}
}
assert!(field(schema, "network.dns_overrides").api.config_read);
assert!(field(schema, "network.dns_overrides").api.config_write);
assert_eq!(
+1
View File
@@ -15,6 +15,7 @@ pub(super) async fn handle(
let runtime = shared.active_runtime.load_full();
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
if previous_cache_generation != runtime.id {
// Entry generation checks also reject late writes from in-flight requests.
*shared.minimal_cache.lock().await = None;
*shared.runtime_edge_connections_cache.lock().await = None;
}
+6
View File
@@ -47,6 +47,8 @@ mod http_utils;
// Authenticated request admission and route dispatch.
mod handler;
mod model;
#[cfg(test)]
mod observability_cache_tests;
mod patch;
#[cfg(test)]
mod reload_tests;
@@ -116,6 +118,8 @@ pub(super) struct ApiRuntimeState {
#[derive(Clone)]
pub(super) struct ApiShared {
/// Generation pinned by this request, including cache reads and publication.
pub(super) runtime_generation: u64,
pub(super) stats: Arc<Stats>,
pub(super) ip_tracker: Arc<UserIpTracker>,
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
@@ -154,6 +158,7 @@ impl ApiShared {
fn for_runtime(&self, runtime: &RuntimeGeneration) -> Self {
Self {
runtime_generation: runtime.id,
stats: runtime.stats.clone(),
ip_tracker: runtime.ip_tracker.clone(),
me_pool: runtime.me_pool_runtime.clone(),
@@ -377,6 +382,7 @@ pub(crate) async fn serve(
});
let shared = Arc::new(ApiShared {
runtime_generation: active_runtime.load().id,
stats,
ip_tracker,
me_pool,
+206
View File
@@ -0,0 +1,206 @@
use super::*;
use crate::api::discovery::http_tests::Fixture;
use crate::maestro::generation::test_runtime_generation;
use std::time::Instant;
fn enabled_config(fixture: &Fixture) -> ProxyConfig {
let mut cfg = (*fixture
.shared
.active_runtime
.load()
.config_rx
.borrow()
.clone())
.clone();
cfg.server.api.minimal_runtime_enabled = true;
cfg.server.api.minimal_runtime_cache_ttl_ms = 60_000;
cfg.server.api.runtime_edge_enabled = true;
cfg.server.api.runtime_edge_cache_ttl_ms = 60_000;
cfg.server.api.runtime_edge_top_n = 7;
cfg
}
#[tokio::test]
async fn late_edge_recompute_cannot_contaminate_new_generation_reads() {
let fixture = Fixture::new(false);
let old_cfg = enabled_config(&fixture);
let old_runtime = test_runtime_generation(1, old_cfg.clone());
let old_request = fixture.shared.for_runtime(&old_runtime);
let (release, resume) = tokio::sync::oneshot::channel();
// A request already pinned to generation 1 finishes after generation 2's
// handler has invalidated the shared caches.
let pending = tokio::spawn(async move {
resume.await.unwrap();
build_runtime_connections_summary_data(&old_request, &old_cfg).await
});
let mut new_cfg = enabled_config(&fixture);
new_cfg.server.api.runtime_edge_top_n = 1;
fixture
.shared
.active_runtime
.store(test_runtime_generation(2, new_cfg));
assert_eq!(fixture.request("GET", "/v1/health", true).await.0, 200);
release.send(()).unwrap();
assert_eq!(pending.await.unwrap().data.unwrap().top.limit, 7);
let mut reads = tokio::task::JoinSet::new();
let fixture = Arc::new(fixture);
for _ in 0..16 {
let fixture = fixture.clone();
reads.spawn(async move {
let (status, _, body) = fixture
.request("GET", "/v1/runtime/connections/summary", true)
.await;
assert_eq!(status, 200);
assert_eq!(body["data"]["enabled"], true);
if let Some(payload) = body["data"].get("data") {
assert_eq!(payload["top"]["limit"], 1, "{body}");
assert_eq!(payload["cache"]["ttl_ms"], 60_000);
} else {
assert_eq!(body["data"]["reason"], "source_unavailable");
}
});
}
while let Some(result) = reads.join_next().await {
result.unwrap();
}
}
#[tokio::test]
async fn late_minimal_cache_publication_is_not_served_by_new_generation() {
let fixture = Fixture::new(false);
let cfg = enabled_config(&fixture);
let old_request = fixture
.shared
.for_runtime(&test_runtime_generation(1, cfg.clone()));
let payload = build_minimal_all_data(&old_request, &cfg.server.api)
.await
.data
.unwrap();
fixture
.shared
.active_runtime
.store(test_runtime_generation(2, cfg));
assert_eq!(fixture.request("GET", "/v1/health", true).await.0, 200);
// Model the publication of a completed old-generation pool snapshot after
// invalidation. The new generation has no pool and must report that fact.
*fixture.shared.minimal_cache.lock().await = Some(MinimalCacheEntry {
generation: 1,
expires_at: Instant::now() + Duration::from_secs(60),
payload,
generated_at_epoch_secs: 42,
});
let (status, _, body) = fixture.request("GET", "/v1/stats/minimal/all", true).await;
assert_eq!(status, 200);
assert_eq!(body["data"]["reason"], "source_unavailable", "{body}");
assert_ne!(body["data"]["generated_at_epoch_secs"], 42);
}
#[tokio::test]
async fn edge_cache_respects_ttl_zero_expiry_top_n_and_recompute_contention() {
let fixture = Fixture::new(false);
let mut cfg = enabled_config(&fixture);
cfg.server.api.runtime_edge_top_n = 1;
let runtime = test_runtime_generation(1, cfg.clone());
let request = fixture.shared.for_runtime(&runtime);
request.stats.add_user_octets_from("alice", 20);
request.stats.add_user_octets_from("bob", 10);
let fresh = build_runtime_connections_summary_data(&request, &cfg)
.await
.data
.unwrap();
assert!(!fresh.cache.served_from_cache);
assert_eq!(fresh.top.by_throughput.len(), 1);
assert_eq!(fresh.top.by_throughput[0].username, "alice");
request.stats.add_user_octets_from("bob", 100);
let cached = build_runtime_connections_summary_data(&request, &cfg)
.await
.data
.unwrap();
assert!(cached.cache.served_from_cache);
assert!(!cached.cache.stale_cache_used);
assert_eq!(cached.top.by_throughput[0].username, "alice");
request
.runtime_edge_connections_cache
.lock()
.await
.as_mut()
.unwrap()
.expires_at = Instant::now();
let guard = request.runtime_edge_recompute_lock.lock().await;
let stale = build_runtime_connections_summary_data(&request, &cfg)
.await
.data
.unwrap();
assert!(stale.cache.stale_cache_used);
let new_request = fixture
.shared
.for_runtime(&test_runtime_generation(2, cfg.clone()));
let unavailable = build_runtime_connections_summary_data(&new_request, &cfg).await;
assert_eq!(unavailable.reason, Some("source_unavailable"));
assert!(unavailable.data.is_none());
cfg.server.api.runtime_edge_cache_ttl_ms = 0;
let zero = build_runtime_connections_summary_data(&request, &cfg).await;
assert_eq!(zero.reason, Some("source_unavailable"));
assert!(zero.data.is_none());
drop(guard);
for _ in 0..2 {
let uncached = build_runtime_connections_summary_data(&request, &cfg)
.await
.data
.unwrap();
assert_eq!(uncached.cache.ttl_ms, 0);
assert!(!uncached.cache.served_from_cache);
assert_eq!(uncached.top.by_throughput[0].username, "bob");
}
cfg.server.api.runtime_edge_cache_ttl_ms = 60_000;
let refreshed = build_runtime_connections_summary_data(&request, &cfg)
.await
.data
.unwrap();
assert!(!refreshed.cache.served_from_cache);
assert_eq!(refreshed.top.by_throughput[0].username, "bob");
}
#[tokio::test]
async fn minimal_cache_checks_generation_expiry_and_zero_ttl() {
let fixture = Fixture::new(false);
let mut cfg = enabled_config(&fixture);
let request = fixture
.shared
.for_runtime(&test_runtime_generation(1, cfg.clone()));
let payload = build_minimal_all_data(&request, &cfg.server.api)
.await
.data
.unwrap();
*request.minimal_cache.lock().await = Some(MinimalCacheEntry {
generation: 1,
expires_at: Instant::now() + Duration::from_secs(60),
payload,
generated_at_epoch_secs: 42,
});
let cached = build_minimal_all_data(&request, &cfg.server.api).await;
assert_eq!(cached.generated_at_epoch_secs, 42);
assert!(cached.reason.is_none());
cfg.server.api.minimal_runtime_cache_ttl_ms = 0;
assert_eq!(
build_minimal_all_data(&request, &cfg.server.api)
.await
.reason,
Some("source_unavailable")
);
cfg.server.api.minimal_runtime_cache_ttl_ms = 60_000;
request
.minimal_cache
.lock()
.await
.as_mut()
.unwrap()
.expires_at = Instant::now();
assert_eq!(
build_minimal_all_data(&request, &cfg.server.api)
.await
.reason,
Some("source_unavailable")
);
}
+7 -1
View File
@@ -70,6 +70,7 @@ pub(super) struct RuntimeEdgeConnectionsSummaryData {
#[derive(Clone)]
pub(crate) struct EdgeConnectionsCacheEntry {
pub(super) generation: u64,
pub(super) expires_at: Instant,
pub(super) payload: RuntimeEdgeConnectionsSummaryPayload,
pub(super) generated_at_epoch_secs: u64,
@@ -270,6 +271,7 @@ async fn get_connections_payload_cached(
let now = Instant::now();
let cached = shared.runtime_edge_connections_cache.lock().await.clone();
if let Some(entry) = cached
&& entry.generation == shared.runtime_generation
&& now < entry.expires_at
{
let mut payload = entry.payload;
@@ -281,7 +283,10 @@ async fn get_connections_payload_cached(
let Ok(_guard) = shared.runtime_edge_recompute_lock.try_lock() else {
let cached = shared.runtime_edge_connections_cache.lock().await.clone();
if let Some(entry) = cached {
if let Some(entry) = cached
&& cache_ttl_ms > 0
&& entry.generation == shared.runtime_generation
{
let mut payload = entry.payload;
payload.cache.served_from_cache = true;
payload.cache.stale_cache_used = true;
@@ -295,6 +300,7 @@ async fn get_connections_payload_cached(
if cache_ttl_ms > 0 {
let entry = EdgeConnectionsCacheEntry {
generation: shared.runtime_generation,
expires_at: Instant::now() + Duration::from_millis(cache_ttl_ms),
payload: payload.clone(),
generated_at_epoch_secs,
+3
View File
@@ -19,6 +19,7 @@ const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable";
#[derive(Clone)]
pub(crate) struct MinimalCacheEntry {
pub(super) generation: u64,
pub(super) expires_at: Instant,
pub(super) payload: MinimalAllPayload,
pub(super) generated_at_epoch_secs: u64,
@@ -342,6 +343,7 @@ async fn get_minimal_payload_cached(
let now = Instant::now();
let cached = shared.minimal_cache.lock().await.clone();
if let Some(entry) = cached
&& entry.generation == shared.runtime_generation
&& now < entry.expires_at
{
return Some((entry.generated_at_epoch_secs, entry.payload));
@@ -522,6 +524,7 @@ async fn get_minimal_payload_cached(
if cache_ttl_ms > 0 {
let entry = MinimalCacheEntry {
generation: shared.runtime_generation,
expires_at: Instant::now() + Duration::from_millis(cache_ttl_ms),
payload: payload.clone(),
generated_at_epoch_secs,
+1
View File
@@ -49,6 +49,7 @@ impl Fixture {
mutable.proxy_shared = proxy_shared.clone();
mutable.config_rx = config_rx;
let shared = ApiShared {
runtime_generation: runtime.id,
stats,
ip_tracker: runtime.ip_tracker.clone(),
me_pool: runtime.me_pool_runtime.clone(),