API: expose observability settings and fix cross-generation cache races

This commit is contained in:
Alexey
2026-11-07 08:05:22 +03:00
parent b1502b629b
commit 982c4e4a34
19 changed files with 1024 additions and 52 deletions
@@ -0,0 +1,508 @@
use super::*;
use crate::api::discovery::http_tests::Fixture;
use crate::config::hot_reload::spawn_config_watcher;
use crate::maestro::{generation::test_runtime_generation, reload::ReloadControl};
use serde_json::json;
use std::time::Duration;
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
const API: &str = "[server.api] # keep API perimeter\nenabled=true\nlisten='127.0.0.1:9092'\nread_only=false\nauth_header='Bearer discovery-test' # keep secret\nwhitelist=['127.0.0.1/32']\ngray_action='api'\nrequest_body_limit_bytes=4096\nruntime_edge_events_capacity=64\nminimal_runtime_enabled=false\nruntime_edge_enabled=false\n";
fn patch(fields: Json) -> Json {
json!({"server": {"api": fields}})
}
#[tokio::test]
async fn get_exposes_exactly_five_settings_and_defaults_without_api_secrets() {
for source in ["", API] {
let (path, _directory) = tests::temp_config(source);
let (managed, _) = read_managed_config(&path).await.unwrap();
let fields = managed["server"]["api"].as_table().unwrap();
let config = ProxyConfig::load(&path).unwrap();
let typed = toml::Value::try_from(&config.server.api).unwrap();
assert_eq!(fields.len(), 5);
for (key, value) in fields {
assert!(EDITABLE_SERVER_API_FIELDS.contains(&key.as_str()));
assert_eq!(*value, typed[key]);
}
assert!(!managed.to_string().contains("discovery-test"));
}
}
#[tokio::test]
async fn every_forbidden_api_sibling_and_alias_is_rejected_even_when_null() {
let (path, _directory) = tests::temp_config(API);
let original = std::fs::read(&path).unwrap();
let api = serde_json::to_value(ProxyConfig::default().server.api).unwrap();
for (key, value) in api.as_object().unwrap() {
if EDITABLE_SERVER_API_FIELDS.contains(&key.as_str()) {
continue;
}
for value in [value.clone(), Json::Null] {
let error = apply_patch_to_path(
&path,
&patch(json!({key: value, "runtime_edge_enabled": true})),
None,
)
.await
.unwrap_err();
assert_eq!(error.code, "field_not_editable", "{key}: {error:?}");
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
for body in [
patch(json!({"typo": null, "runtime_edge_enabled": true})),
json!({"server": {"admin_api": null, "api": {"runtime_edge_enabled": true}}}),
json!({"server": {"port": null, "api": {"runtime_edge_enabled": true}}}),
] {
assert_eq!(
apply_patch_to_path(&path, &body, None)
.await
.unwrap_err()
.code,
"field_not_editable"
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
#[tokio::test]
async fn observability_bounds_and_types_are_checked_before_persistence() {
let (path, _directory) = tests::temp_config(API);
for (key, good, bad) in [
(
"minimal_runtime_cache_ttl_ms",
vec![json!(0), json!(60_000)],
vec![json!(-1), json!(60_001), json!(1.5), json!(false)],
),
(
"runtime_edge_cache_ttl_ms",
vec![json!(0), json!(60_000)],
vec![json!(-1), json!(60_001), json!("0")],
),
(
"runtime_edge_top_n",
vec![json!(1), json!(1000)],
vec![json!(0), json!(1001), json!(-1), json!([])],
),
(
"minimal_runtime_enabled",
vec![json!(true), json!(false)],
vec![json!(0), json!("true")],
),
(
"runtime_edge_enabled",
vec![json!(true), json!(false)],
vec![json!(1), json!({})],
),
] {
for value in good {
apply_patch_to_path(&path, &patch(json!({key: value})), None)
.await
.unwrap();
assert_eq!(
serde_json::to_value(ProxyConfig::load(&path).unwrap().server.api).unwrap()[key],
value
);
}
let original = std::fs::read(&path).unwrap();
for value in bad {
let error = apply_patch_to_path(&path, &patch(json!({key: value})), None)
.await
.unwrap_err();
assert_eq!(
error.status,
hyper::StatusCode::BAD_REQUEST,
"{key}: {error:?}"
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
let original = std::fs::read(&path).unwrap();
for value in [
json!(null),
json!({}),
json!([]),
json!(false),
json!({"runtime_edge_top_n": null}),
] {
assert!(
apply_patch_to_path(&path, &patch(value), None)
.await
.is_err()
);
assert_eq!(std::fs::read(&path).unwrap(), original);
}
}
#[tokio::test]
async fn sparse_observability_patch_preserves_perimeter_listeners_and_comments() {
let source = format!("{API}\n[[server.listeners]]\nip='127.0.0.1'\nport=443\n");
let (path, _directory) = tests::temp_config(&source);
let before = ProxyConfig::load(&path).unwrap();
let mut response = apply_patch_to_path(
&path,
&patch(json!({
"minimal_runtime_enabled": true, "minimal_runtime_cache_ttl_ms": 0,
"runtime_edge_enabled": true, "runtime_edge_cache_ttl_ms": 60_000,
"runtime_edge_top_n": 1,
})),
None,
)
.await
.unwrap();
let after = ProxyConfig::load(&path).unwrap();
reconcile_runtime_effect(&mut response, &before, &after).unwrap();
assert!(response.restart_required);
assert!(response.runtime_reload_required);
assert!(!response.process_restart_required);
assert!(response.deferred_process_fields.is_empty());
assert_eq!(response.changed_paths.len(), 5);
let mut restored = after.clone();
for field in EDITABLE_SERVER_API_FIELDS {
let mut api = serde_json::to_value(&restored.server.api).unwrap();
api[*field] = serde_json::to_value(&before.server.api).unwrap()[*field].clone();
restored.server.api = serde_json::from_value(api).unwrap();
}
assert_eq!(
serde_json::to_value(restored).unwrap(),
serde_json::to_value(&before).unwrap()
);
let written = std::fs::read_to_string(&path).unwrap();
for line in source.lines().filter(|line| {
!line.starts_with("minimal_runtime_enabled=") && !line.starts_with("runtime_edge_enabled=")
}) {
assert!(written.contains(line), "{line}: {written}");
}
let response = apply_patch_to_path(
&path,
&patch(json!({"runtime_edge_top_n": 2, "runtime_edge_enabled": null})),
None,
)
.await
.unwrap();
assert_eq!(response.changed_paths, ["server.api.runtime_edge_top_n"]);
assert!(
ProxyConfig::load(&path)
.unwrap()
.server
.api
.runtime_edge_enabled
);
}
#[tokio::test]
async fn precise_api_edits_preserve_inline_dotted_quoted_and_legacy_source_shapes() {
for source in [
"server = { api = { auth_header = 'secret', runtime_edge_top_n = 10 } }\n",
"server.api.auth_header = 'secret'\nserver.api.runtime_edge_top_n = 10\n",
"['server'.'api']\nauth_header='secret'\nruntime_edge_top_n=10 # retain\n",
"[server.admin_api]\nauth_header='secret'\nruntime_edge_top_n=10 # retain\n",
] {
let (path, _directory) = tests::temp_config(source);
let before = ProxyConfig::load(&path).unwrap();
apply_patch_to_path(
&path,
&patch(json!({"runtime_edge_top_n": 3, "minimal_runtime_cache_ttl_ms": 0})),
None,
)
.await
.unwrap();
let after = ProxyConfig::load(&path).unwrap();
assert_eq!(after.server.api.auth_header, before.server.api.auth_header);
assert_eq!(after.server.api.runtime_edge_top_n, 3);
assert_eq!(after.server.api.minimal_runtime_cache_ttl_ms, 0);
let written = std::fs::read_to_string(&path).unwrap();
if source.contains("admin_api") {
assert!(written.contains("[server.admin_api]"));
assert!(!written.contains("[server.api]"));
}
if source.contains("# retain") {
assert!(written.contains("# retain"));
}
}
}
#[tokio::test]
async fn included_api_owner_is_updated_and_multiple_owners_are_rejected_atomically() {
let directory = tempfile::tempdir().unwrap();
let root = directory.path().join("config.toml");
let included = directory.path().join("api.toml");
let root_text = "include=\"api.toml\"\n[general]\nlog_level='normal'\n[server]\nport=443\n";
std::fs::write(&root, root_text).unwrap();
std::fs::write(&included, API).unwrap();
apply_patch_to_path(&root, &patch(json!({"runtime_edge_top_n": 3})), None)
.await
.unwrap();
assert_eq!(std::fs::read_to_string(&root).unwrap(), root_text);
assert_eq!(
ProxyConfig::load(&root)
.unwrap()
.server
.api
.runtime_edge_top_n,
3
);
for (root_text, included_text, body) in [
(
root_text.to_owned(),
std::fs::read_to_string(&included).unwrap(),
json!({"server":{"api":{"runtime_edge_top_n": 4}}, "general":{"log_level":"debug"}}),
),
(
"include=\"api.toml\"\nserver.api.minimal_runtime_enabled=false\n".to_owned(),
"server.api.runtime_edge_enabled=false\n".to_owned(),
patch(json!({"minimal_runtime_enabled":true,"runtime_edge_enabled":true})),
),
(
"include=\"api.toml\"\nserver.api.minimal_runtime_enabled=false\n".to_owned(),
"server.api.runtime_edge_enabled=false\n".to_owned(),
patch(json!({"runtime_edge_top_n":3})),
),
] {
std::fs::write(&root, &root_text).unwrap();
std::fs::write(&included, &included_text).unwrap();
let error = apply_patch_to_path(&root, &body, None).await.unwrap_err();
assert_eq!(error.code, "config_patch_not_atomic", "{error:?}");
assert_eq!(std::fs::read_to_string(&root).unwrap(), root_text);
assert_eq!(std::fs::read_to_string(&included).unwrap(), included_text);
}
}
#[tokio::test]
async fn http_preview_watcher_and_reload_keep_diagnostics_pending_until_activation() {
let mut fixture = Fixture::new(false);
std::fs::write(
&fixture.shared.config_path,
format!("[general]\nlog_level='normal'\n{API}"),
)
.unwrap();
let config = Arc::new(ProxyConfig::load(&fixture.shared.config_path).unwrap());
let (control, mut commands) = ReloadControl::channel(1);
Arc::get_mut(&mut fixture.shared).unwrap().reload_control = control;
let cancellation = CancellationToken::new();
let _cancel_on_drop = cancellation.clone().drop_guard();
let (activation, activated) = watch::channel(false);
let (mut receiver, _log_receiver, watcher) = spawn_config_watcher(
fixture.shared.config_path.clone(),
config.clone(),
None,
None,
cancellation.clone(),
None,
Some(activated),
Some(fixture.shared.mutation_lock.clone()),
None,
);
let mut runtime = test_runtime_generation(1, (*config).clone());
Arc::get_mut(&mut runtime).unwrap().config_rx = receiver.clone();
fixture.shared.active_runtime.store(runtime);
let task = tokio::spawn(watcher);
let body = json!({"general": {"log_level": "debug"}, "server": {"api": {
"minimal_runtime_enabled":true, "runtime_edge_enabled":true,
"minimal_runtime_cache_ttl_ms":0, "runtime_edge_cache_ttl_ms":0, "runtime_edge_top_n":1,
}}});
let original = std::fs::read(&fixture.shared.config_path).unwrap();
let preview = fixture
.request_body("POST", "/v1/config/validate", true, Some(&body), &[])
.await;
assert_eq!(preview.0, 200, "{}", preview.2);
let data = &preview.2["data"];
assert_eq!(data["apply"]["watcher_paths"], json!(["general.log_level"]));
assert_eq!(
data["apply"]["generation_paths"].as_array().unwrap().len(),
5
);
assert_eq!(data["apply"]["process_restart_paths"], json!([]));
assert_eq!(
std::fs::read(&fixture.shared.config_path).unwrap(),
original
);
assert_eq!(
fixture
.request_body("PATCH", "/v1/config", false, Some(&body), &[])
.await
.0,
401
);
let response = fixture
.request_body(
"PATCH",
"/v1/config",
true,
Some(&body),
&[
("If-Match", data["base_revision"].as_str().unwrap()),
(
"If-Runtime-Match",
data["runtime"]["runtime_match"].as_str().unwrap(),
),
],
)
.await;
assert_eq!(response.0, 200, "{}", response.2);
assert_eq!(response.2["data"]["runtime_reload_required"], true);
assert_eq!(response.2["data"]["process_restart_required"], false);
activation.send_replace(true);
tokio::time::timeout(
Duration::from_secs(5),
receiver.wait_for(|cfg| cfg.general.log_level == crate::config::LogLevel::Debug),
)
.await
.unwrap()
.unwrap();
assert_eq!(receiver.borrow().server.api, config.server.api);
assert_diagnostics(&fixture, false).await;
// Even an unchanged desired patch can explicitly request the pending reload.
let accepted = fixture
.request_body(
"PATCH",
"/v1/config?reload=instant",
true,
Some(&patch(body["server"]["api"].clone())),
&[],
)
.await;
assert_eq!(accepted.0, 202, "{}", accepted.2);
let command = tokio::time::timeout(Duration::from_secs(5), commands.recv())
.await
.unwrap()
.unwrap();
assert_diagnostics(&fixture, false).await;
let resolved = resolve_reload_config(&config, &command.config).unwrap();
assert_eq!(
resolved.effective.server.api.listen,
config.server.api.listen
);
assert_eq!(
resolved.effective.server.api.runtime_edge_events_capacity,
64
);
fixture
.shared
.active_runtime
.store(test_runtime_generation(2, resolved.effective));
assert_diagnostics(&fixture, true).await;
assert_eq!(
fixture
.request("GET", "/v1/runtime/events/recent", true)
.await
.2["data"]["data"]["capacity"],
64
);
fixture
.shared
.reload_control
.fail(command.reload_id, "fixture completed")
.await;
let disabled = patch(json!({"minimal_runtime_enabled": false, "runtime_edge_enabled": false}));
let accepted = fixture
.request_body(
"PATCH",
"/v1/config?reload=drain&timeout_secs=30",
true,
Some(&disabled),
&[],
)
.await;
assert_eq!(accepted.0, 202, "{}", accepted.2);
assert_diagnostics(&fixture, true).await;
let command = commands.recv().await.unwrap();
fixture
.shared
.active_runtime
.store(test_runtime_generation(3, (*command.config).clone()));
assert_diagnostics(&fixture, false).await;
assert_eq!(fixture.request("GET", "/v1/health", true).await.0, 200);
cancellation.cancel();
tokio::time::timeout(Duration::from_secs(5), task)
.await
.unwrap()
.unwrap();
}
async fn assert_diagnostics(fixture: &Fixture, enabled: bool) {
for path in [
"/v1/stats/minimal/all",
"/v1/runtime/connections/summary",
"/v1/runtime/events/recent",
] {
let (status, _, body) = fixture.request("GET", path, true).await;
assert_eq!(status, 200, "{body}");
assert_eq!(body["data"]["enabled"], enabled, "{body}");
if !enabled {
assert_eq!(body["data"]["reason"], "feature_disabled");
}
}
let (_, _, caps) = fixture.request("GET", "/v1/capabilities", true).await;
for path in ["/v1/stats/minimal/all", "/v1/runtime/connections/summary"] {
let operation = caps["data"]["operations"]
.as_array()
.unwrap()
.iter()
.find(|op| op["path"] == path)
.unwrap();
assert_eq!(operation["enabled"], enabled);
}
}
#[tokio::test]
async fn observability_patch_obeys_read_only_and_source_revision_fences() {
let fixture = Fixture::new(false);
let body = patch(json!({"runtime_edge_enabled": true}));
let original = std::fs::read(&fixture.shared.config_path).unwrap();
let preview = fixture
.request_body("POST", "/v1/config/validate", true, Some(&body), &[])
.await;
assert_eq!(preview.0, 200, "{}", preview.2);
let rejected = fixture
.request_body("PATCH", "/v1/config", true, Some(&body), &[])
.await;
assert_eq!(rejected.0, 403);
assert_eq!(rejected.2["error"]["code"], "read_only");
assert_eq!(
std::fs::read(&fixture.shared.config_path).unwrap(),
original
);
let prepared = prepare_patch_to_path(&fixture.shared.config_path, &body, None)
.await
.unwrap();
let external = "# external source change\n";
std::fs::write(&fixture.shared.config_path, external).unwrap();
let conflict = write_atomic_if_unchanged(
prepared.config_path,
prepared.expected_revision,
prepared.owner_path,
prepared.expected_owner_contents,
prepared.owner_contents,
)
.await
.unwrap_err();
assert_eq!(conflict.code, "revision_conflict");
assert_eq!(
std::fs::read_to_string(&fixture.shared.config_path).unwrap(),
external
);
}
#[tokio::test]
async fn listeners_and_observability_in_one_owner_preserve_each_other() {
let (path, _directory) = tests::temp_config(API);
apply_patch_to_path(
&path,
&json!({"server": {
"listeners": [{"ip": "127.0.0.1", "port": 8443}],
"api": {"runtime_edge_enabled": true},
}}),
None,
)
.await
.unwrap();
let config = ProxyConfig::load(&path).unwrap();
assert_eq!(config.server.listeners.len(), 1);
assert_eq!(config.server.listeners[0].port, Some(8443));
assert!(config.server.api.runtime_edge_enabled);
assert_eq!(config.server.api.auth_header, "Bearer discovery-test");
}
+79 -11
View File
@@ -1,5 +1,6 @@
//! One source-preserving candidate pipeline shared by preview and commit.
use super::*;
use crate::api::config_store::{FieldEdit, prepare_field_edits};
use crate::config::{LoadedConfig, ParsedConfigSource};
use tokio::sync::OwnedSemaphorePermit;
@@ -82,7 +83,7 @@ fn parse_patch(
"access.* is managed via the users API, not editable here",
));
}
for (key, value) in patch_table {
for key in patch_table.keys() {
if !is_editable_section(key.as_str()) {
return Err(ApiFailure::new(
hyper::StatusCode::BAD_REQUEST,
@@ -91,7 +92,7 @@ fn parse_patch(
));
}
if key == "server" {
validate_server_patch(value)?;
validate_server_patch(&patch_json["server"])?;
} else if key == "network" {
validate_network_patch(&patch_json["network"])?;
}
@@ -127,30 +128,59 @@ fn parse_patch(
.validate()
.map_err(|error| diagnostics::validation_error("constraint_violation", "$", error))?;
let api_edits = api_field_edits(&loaded, &patch_toml)?;
let api_source = prepare_field_edits(&loaded, config_path, &api_edits)
.map_err(diagnostics::source_conflict)?;
let patch_listeners = patch_toml
.get("server")
.is_some_and(|server| server.get("listeners").is_some());
let ownership_targets: Vec<&str> = touched
.iter()
.map(|section| {
.filter_map(|section| {
if section == "server" {
"server.listeners"
patch_listeners.then_some("server.listeners")
} else if section == "network" {
"network.dns_overrides"
Some("network.dns_overrides")
} else {
section.as_str()
Some(section.as_str())
}
})
.collect();
let owner_path = resolve_single_source_owner(&loaded, config_path, &ownership_targets)
.map_err(diagnostics::source_conflict)?;
let section_owner = if ownership_targets.is_empty() {
None
} else {
Some(
resolve_single_source_owner(&loaded, config_path, &ownership_targets)
.map_err(diagnostics::source_conflict)?,
)
};
if let (Some(owner), Some(api)) = (&section_owner, &api_source) {
if *owner != api.owner_path {
return Err(ApiFailure::new(
hyper::StatusCode::CONFLICT,
"config_patch_not_atomic",
"Mutation requires one independently writable config source",
));
}
}
let owner_path = section_owner
.or_else(|| api_source.as_ref().map(|source| source.owner_path.clone()))
.ok_or_else(|| ApiFailure::bad_request("empty patch: no editable fields"))?;
let mut owner_contents = loaded
.source_contents
.get(&owner_path)
.cloned()
.ok_or_else(|| ApiFailure::internal("config source owner is missing from snapshot"))?;
let expected_owner_contents = owner_contents.clone();
if let Some(api) = api_source {
owner_contents = api.owner_contents;
}
for section in &touched {
if *section == "server" {
let rendered = render_server_listeners(&requested_cfg)?;
owner_contents = upsert_toml_table(&owner_contents, "server.listeners", &rendered);
if patch_listeners {
let rendered = render_server_listeners(&requested_cfg)?;
owner_contents = upsert_toml_table(&owner_contents, "server.listeners", &rendered);
}
} else if *section == "network" {
owner_contents =
upsert_dns_overrides(&owner_contents, &requested_cfg.network.dns_overrides)?;
@@ -164,7 +194,7 @@ fn parse_patch(
overrides.insert(owner_path.clone(), owner_contents.clone());
let candidate = ProxyConfig::parse_source_with_overrides(config_path, &overrides)
.map_err(|error| diagnostics::validation_error("constraint_violation", "$", error))?;
if touched.iter().any(|section| section == "server")
if patch_listeners
&& serde_json::to_value(&candidate.config.server.listeners).ok()
!= serde_json::to_value(&requested_cfg.server.listeners).ok()
{
@@ -191,6 +221,44 @@ fn parse_patch(
))
}
fn api_field_edits(
loaded: &ParsedConfigSource,
patch: &Toml,
) -> Result<Vec<FieldEdit>, ApiFailure> {
let Some(fields) = patch
.get("server")
.and_then(|server| server.get("api"))
.and_then(Toml::as_table)
else {
return Ok(Vec::new());
};
if fields.is_empty() {
return Ok(Vec::new());
}
// PATCH uses canonical paths, while an authored legacy table keeps its name.
let mut table_name = "api";
for source in loaded.source_contents.values() {
let document: Toml = toml::from_str(source).map_err(|_| {
ApiFailure::new(
hyper::StatusCode::CONFLICT,
"config_patch_not_atomic",
"Mutation requires one independently writable config source",
)
})?;
if document
.get("server")
.and_then(|server| server.get("admin_api"))
.is_some()
{
table_name = "admin_api";
}
}
Ok(fields
.iter()
.map(|(field, value)| FieldEdit::new(&["server", table_name, field], Some(value.clone())))
.collect())
}
fn finish(
source: PatchSource,
candidate: LoadedConfig,
+6 -5
View File
@@ -228,8 +228,7 @@ async fn invalid_web_patch_does_not_modify_the_source() {
#[tokio::test]
async fn read_managed_config_returns_only_editable_sections() {
// Server identity and network siblings must not leak. Listeners-only server
// is returned via the nested allowlist (covered in a dedicated test).
// Server identity and network siblings must not leak through nested allowlists.
let (path, _d) = temp_config(concat!(
"[censorship]\ntls_domain = \"a\"\n",
"[server]\nport = 443\n[server.api]\nauth_header = \"SECRET\"\n",
@@ -241,7 +240,8 @@ async fn read_managed_config_returns_only_editable_sections() {
assert!(table.contains_key("censorship"));
let server = table["server"].as_table().unwrap();
assert!(server.contains_key("listeners"));
assert!(!server.contains_key("api"));
assert_eq!(server["api"].as_table().unwrap().len(), 5);
assert!(server["api"].get("auth_header").is_none());
assert!(!server.contains_key("port"));
assert_eq!(
serde_json::to_value(&table["network"]).unwrap(),
@@ -251,7 +251,7 @@ async fn read_managed_config_returns_only_editable_sections() {
}
#[tokio::test]
async fn read_managed_config_returns_server_listeners_only() {
async fn read_managed_config_returns_server_listeners_and_api_observability() {
let (path, _d) = temp_config(concat!(
"[censorship]\ntls_domain = \"a\"\n",
"[server]\nport = 443\n",
@@ -266,7 +266,8 @@ async fn read_managed_config_returns_server_listeners_only() {
.as_table()
.unwrap();
assert!(server.contains_key("listeners"));
assert!(!server.contains_key("api"));
assert_eq!(server["api"].as_table().unwrap().len(), 5);
assert!(server["api"].get("auth_header").is_none());
assert!(!server.contains_key("port"));
let listeners = server["listeners"].as_array().unwrap();
assert_eq!(listeners.len(), 1);