Hardswap Invariants in tests + Quota fixes

src/api/config_edit.rs

@@ -62,6 +62,21 @@ pub(super) async fn patch_config(
     expected_revision: Option<String>,
     reload_request: Option<ReloadRequest>,
     shared: &ApiShared,
+) -> Result<PatchConfigResponse, ApiFailure> {
+    let shared = shared.clone();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            patch_config_to_completion(patch_json, expected_revision, reload_request, &shared).await
+        })
+        .await
+}
+
+async fn patch_config_to_completion(
+    patch_json: Json,
+    expected_revision: Option<String>,
+    reload_request: Option<ReloadRequest>,
+    shared: &ApiShared,
 ) -> Result<PatchConfigResponse, ApiFailure> {
     let _guard = shared.mutation_lock.lock().await;
     let active_config = shared.active_runtime.load_full().config();
@@ -83,7 +98,7 @@ pub(super) async fn patch_config(
     } else {
         None
     };
-    write_atomic_if_unchanged(
+    prepared.response.revision = write_atomic_if_unchanged(
         prepared.config_path,
         prepared.expected_revision,
         prepared.owner_path,
@@ -123,8 +138,8 @@ pub(super) async fn apply_patch_to_path(
     patch_json: &Json,
     expected_revision: Option<String>,
 ) -> Result<PatchConfigResponse, ApiFailure> {
-    let prepared = prepare_patch_to_path(config_path, patch_json, expected_revision).await?;
-    write_atomic_if_unchanged(
+    let mut prepared = prepare_patch_to_path(config_path, patch_json, expected_revision).await?;
+    let revision = write_atomic_if_unchanged(
         prepared.config_path,
         prepared.expected_revision,
         prepared.owner_path,
@@ -132,6 +147,7 @@ pub(super) async fn apply_patch_to_path(
         prepared.owner_contents,
     )
     .await?;
+    prepared.response.revision = revision;
     Ok(prepared.response)
 }
 

src/api/config_store/atomic.rs

@@ -31,6 +31,11 @@ struct ExistingTarget {
     metadata: std::fs::Metadata,
 }
 
+struct GraphFence<'a> {
+    config_path: &'a Path,
+    expected_revision: &'a str,
+}
+
 struct ConfigWriteLock {
     #[cfg(unix)]
     _file: Flock<File>,
@@ -38,9 +43,10 @@ struct ConfigWriteLock {
 
 impl ConfigWriteLock {
     fn acquire(path: &Path) -> std::io::Result<Self> {
+        let path = normalize_path(path);
         #[cfg(unix)]
         {
-            let lock_path = sibling_lock_path(path);
+            let lock_path = sibling_lock_path(&path);
             let anchored = AnchoredPath::open_creating_parents(&lock_path, 0o750)?;
             let descriptor = openat(
                 anchored.parent(),
@@ -76,7 +82,7 @@ pub(in crate::api) async fn write_atomic(
 ) -> Result<(), ApiFailure> {
     tokio::task::spawn_blocking(move || {
         let _lock = ConfigWriteLock::acquire(&path)?;
-        write_atomic_sync(&path, None, &contents)
+        write_atomic_sync(&path, None, &contents, None).map(|_| ())
     })
         .await
         .map_err(|error| ApiFailure::internal(format!("failed to join writer: {error}")))?
@@ -90,8 +96,10 @@ pub(in crate::api) async fn write_atomic_if_unchanged(
     path: PathBuf,
     expected_contents: String,
     contents: String,
-) -> Result<(), ApiFailure> {
+) -> Result<String, ApiFailure> {
     tokio::task::spawn_blocking(move || {
+        let config_path = normalize_path(&config_path);
+        let path = normalize_path(&path);
         // Every API mutation locks the root source so writes to different includes serialize.
         let _lock = ConfigWriteLock::acquire(&config_path).map_err(AtomicWriteError::Io)?;
         let graph = ProxyConfig::read_source_graph(&config_path)
@@ -99,12 +107,26 @@ pub(in crate::api) async fn write_atomic_if_unchanged(
         if compute_source_revision(&graph) != expected_revision {
             return Err(AtomicWriteError::Conflict);
         }
-        write_atomic_sync(&path, Some(&expected_contents), &contents).map_err(|error| {
+        write_atomic_sync(
+            &path,
+            Some(&expected_contents),
+            &contents,
+            Some(GraphFence {
+                config_path: &config_path,
+                expected_revision: &expected_revision,
+            }),
+        )
+        .map_err(|error| {
             if error.kind() == std::io::ErrorKind::AlreadyExists {
                 AtomicWriteError::Conflict
             } else {
                 AtomicWriteError::Io(error)
             }
+        })?
+        .ok_or_else(|| {
+            AtomicWriteError::Io(std::io::Error::other(
+                "config graph fence did not produce a committed revision",
+            ))
         })
     })
     .await
@@ -137,6 +159,51 @@ fn sibling_lock_path(path: &Path) -> PathBuf {
     path.parent().unwrap_or_else(|| Path::new(".")).join(name)
 }
 
+fn normalize_path(path: &Path) -> PathBuf {
+    let absolute = if path.is_absolute() {
+        path.to_path_buf()
+    } else {
+        std::env::current_dir()
+            .map(|current| current.join(path))
+            .unwrap_or_else(|_| path.to_path_buf())
+    };
+    let mut normalized = PathBuf::new();
+    for component in absolute.components() {
+        match component {
+            std::path::Component::CurDir => {}
+            std::path::Component::ParentDir => {
+                normalized.pop();
+            }
+            component => normalized.push(component.as_os_str()),
+        }
+    }
+    normalized
+}
+
+fn fenced_post_commit_revision(
+    fence: GraphFence<'_>,
+    path: &Path,
+    contents: &str,
+) -> std::io::Result<String> {
+    let mut graph = ProxyConfig::read_source_graph(fence.config_path)
+        .map_err(|error| std::io::Error::other(error.to_string()))?;
+    if compute_source_revision(&graph) != fence.expected_revision {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::AlreadyExists,
+            "config graph changed during persistence",
+        ));
+    }
+    let path = normalize_path(path);
+    let Some(owner) = graph.source_contents.get_mut(&path) else {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "config source owner left the source graph during persistence",
+        ));
+    };
+    *owner = contents.to_string();
+    Ok(compute_source_revision(&graph))
+}
+
 #[cfg(unix)]
 fn open_existing_target(anchored: &AnchoredPath) -> std::io::Result<Option<ExistingTarget>> {
     let descriptor = match openat(
@@ -210,7 +277,8 @@ fn write_atomic_sync(
     path: &Path,
     expected_contents: Option<&str>,
     contents: &str,
-) -> std::io::Result<()> {
+    graph_fence: Option<GraphFence<'_>>,
+) -> std::io::Result<Option<String>> {
     let anchored = AnchoredPath::open_creating_parents(path, 0o750)?;
     let existing = open_existing_target(&anchored)?;
     validate_expected_contents(existing.as_ref(), expected_contents)?;
@@ -234,10 +302,12 @@ fn write_atomic_sync(
     .map_err(errno_to_io)?;
     let write_result = write_and_publish(
         descriptor,
+        path,
         &anchored,
         &temp_name,
         existing.as_ref(),
         contents,
+        graph_fence,
     );
     if write_result.is_err() {
         let _ = unlinkat(
@@ -252,11 +322,13 @@ fn write_atomic_sync(
 #[cfg(unix)]
 fn write_and_publish(
     descriptor: std::os::fd::OwnedFd,
+    path: &Path,
     anchored: &AnchoredPath,
     temp_name: &str,
     existing: Option<&ExistingTarget>,
     contents: &str,
-) -> std::io::Result<()> {
+    graph_fence: Option<GraphFence<'_>>,
+) -> std::io::Result<Option<String>> {
     let mut file = File::from(descriptor);
     if let Some(existing) = existing {
         use nix::unistd::{Gid, Uid, fchown};
@@ -280,6 +352,9 @@ fn write_and_publish(
             "config target changed during persistence",
         ));
     }
+    let committed_revision = graph_fence
+        .map(|fence| fenced_post_commit_revision(fence, path, contents))
+        .transpose()?;
     renameat(
         anchored.parent(),
         temp_name,
@@ -287,7 +362,8 @@ fn write_and_publish(
         anchored.name(),
     )
     .map_err(errno_to_io)?;
-    fsync(anchored.parent()).map_err(errno_to_io)
+    fsync(anchored.parent()).map_err(errno_to_io)?;
+    Ok(committed_revision)
 }
 
 #[cfg(not(unix))]
@@ -295,7 +371,8 @@ fn write_atomic_sync(
     path: &Path,
     expected_contents: Option<&str>,
     contents: &str,
-) -> std::io::Result<()> {
+    graph_fence: Option<GraphFence<'_>>,
+) -> std::io::Result<Option<String>> {
     let parent = path.parent().unwrap_or_else(|| Path::new("."));
     std::fs::create_dir_all(parent)?;
     let existing = open_existing_target(path)?;
@@ -310,7 +387,11 @@ fn write_atomic_sync(
             "config target changed during persistence",
         ));
     }
-    std::fs::rename(temp, path)
+    let committed_revision = graph_fence
+        .map(|fence| fenced_post_commit_revision(fence, path, contents))
+        .transpose()?;
+    std::fs::rename(temp, path)?;
+    Ok(committed_revision)
 }
 
 fn validate_expected_contents(

src/api/config_store/persistence.rs

@@ -159,8 +159,8 @@ pub(in crate::api) async fn save_access_sections_to_disk_if_revision(
         owner_contents.clone(),
     )
     .await?;
-    let revision = compute_snapshot_revision(&candidate);
-    write_atomic_if_unchanged(
+    let _candidate_revision = compute_snapshot_revision(&candidate);
+    let revision = write_atomic_if_unchanged(
         config_path.to_path_buf(),
         loaded_revision,
         owner_path,

src/api/handler/user_routes.rs

@@ -133,38 +133,55 @@ pub(super) async fn handle(
             ));
         }
         let expected_revision = parse_if_match(req.headers());
-        let _mutation_guard = shared.mutation_lock.lock().await;
-        let (disk_cfg, _) =
-            load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
-        if !disk_cfg.access.users.contains_key(user) {
-            return Ok(error_response(
-                request_id,
-                ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
-            ));
-        }
-        let configured_users = disk_cfg
-            .access
-            .users
-            .keys()
-            .cloned()
-            .collect::<BTreeSet<_>>();
-        let snapshot = match shared.quota_state.reset_user(&configured_users, user).await {
-            Ok(snapshot) => snapshot,
-            Err(error) => {
-                shared.runtime_events.record(
-                    "api.user.reset_quota.failed",
-                    format!("username={} error={}", user, error),
+        let completion_shared = shared.as_ref().clone();
+        let user_owned = user.to_string();
+        let completion = shared
+            .run_mutation_completion(async move {
+                let _mutation_guard = completion_shared.mutation_lock.lock().await;
+                let (disk_cfg, _) = load_config_for_mutation(
+                    &completion_shared.config_path,
+                    expected_revision.as_deref(),
+                )
+                .await?;
+                if !disk_cfg.access.users.contains_key(&user_owned) {
+                    return Err(ApiFailure::new(
+                        StatusCode::NOT_FOUND,
+                        "not_found",
+                        "User not found",
+                    ));
+                }
+                let configured_users = disk_cfg
+                    .access
+                    .users
+                    .keys()
+                    .cloned()
+                    .collect::<BTreeSet<_>>();
+                let snapshot = completion_shared
+                    .quota_state
+                    .reset_user(&configured_users, &user_owned)
+                    .await
+                    .map_err(|error| {
+                        completion_shared.runtime_events.record(
+                            "api.user.reset_quota.failed",
+                            format!("username={} error={}", user_owned, error),
+                        );
+                        ApiFailure::internal(format!("Failed to reset user quota: {}", error))
+                    })?;
+                completion_shared.runtime_events.record(
+                    "api.user.reset_quota.ok",
+                    format!("username={}", user_owned),
                 );
-                return Err(ApiFailure::internal(format!(
-                    "Failed to reset user quota: {}",
-                    error
-                )));
+                let revision = current_revision(&completion_shared.config_path).await?;
+                Ok((snapshot, revision))
+            })
+            .await;
+        let (snapshot, revision) = match completion {
+            Ok(result) => result,
+            Err(error) if error.code == "not_found" => {
+                return Ok(error_response(request_id, error));
             }
+            Err(error) => return Err(error),
         };
-        shared
-            .runtime_events
-            .record("api.user.reset_quota.ok", format!("username={}", user));
-        let revision = current_revision(&shared.config_path).await?;
         return Ok(success_response(
             StatusCode::OK,
             ResetUserQuotaResponse {

src/api/mod.rs

@@ -17,7 +17,7 @@ use hyper::service::service_fn;
 use hyper::{Method, Request, Response, StatusCode};
 use subtle::ConstantTimeEq;
 use tokio::net::TcpListener;
-use tokio::sync::{Mutex, RwLock, Semaphore, watch};
+use tokio::sync::{Mutex, RwLock, Semaphore, oneshot, watch};
 use tokio::time::timeout;
 use tracing::{debug, info, warn};
 
@@ -134,6 +134,7 @@ pub(super) struct ApiShared {
     pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
     pub(super) web_trace: Arc<WebTraceStore>,
     pub(super) web_runtime_rx: watch::Receiver<WebRuntimePublication>,
+    pub(super) control_plane: ProcessControlPlane,
 }
 
 impl ApiShared {
@@ -169,8 +170,32 @@ impl ApiShared {
             active_runtime: self.active_runtime.clone(),
             web_trace: self.web_trace.clone(),
             web_runtime_rx: self.web_runtime_rx.clone(),
+            control_plane: self.control_plane.clone(),
         }
     }
+
+    /// Keeps an accepted mutation alive until persistence and mandatory publication finish.
+    async fn run_mutation_completion<T, F>(&self, future: F) -> Result<T, ApiFailure>
+    where
+        T: Send + 'static,
+        F: std::future::Future<Output = Result<T, ApiFailure>> + Send + 'static,
+    {
+        let (result_tx, result_rx) = oneshot::channel();
+        self.control_plane
+            .spawn_completion(async move {
+                let _ = result_tx.send(future.await);
+            })
+            .map_err(|_| {
+                ApiFailure::new(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "control_plane_shutting_down",
+                    "Control plane is shutting down",
+                )
+            })?;
+        result_rx.await.map_err(|_| {
+            ApiFailure::internal("accepted config mutation did not report completion")
+        })?
+    }
 }
 
 fn auth_header_matches(actual: &str, expected: &str) -> bool {
@@ -364,6 +389,7 @@ pub(crate) async fn serve(
         active_runtime,
         web_trace,
         web_runtime_rx,
+        control_plane: control_plane.clone(),
     });
 
     spawn_runtime_watchers(

src/api/users.rs

@@ -5,6 +5,7 @@ use hyper::StatusCode;
 use crate::config::ProxyConfig;
 use crate::config::RateLimitBps;
 use crate::ip_tracker::UserIpTracker;
+use crate::proxy::user_admission::credential_id_from_hex;
 use crate::stats::Stats;
 
 use super::ApiShared;

src/api/users/create.rs

@@ -4,6 +4,20 @@ pub(in crate::api) async fn create_user(
     body: CreateUserRequest,
     expected_revision: Option<String>,
     shared: &ApiShared,
+) -> Result<(CreateUserResponse, String), ApiFailure> {
+    let shared = shared.clone();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            create_user_to_completion(body, expected_revision, &shared).await
+        })
+        .await
+}
+
+async fn create_user_to_completion(
+    body: CreateUserRequest,
+    expected_revision: Option<String>,
+    shared: &ApiShared,
 ) -> Result<(CreateUserResponse, String), ApiFailure> {
     let touches_user_ad_tags = body.user_ad_tag.is_some();
     let touches_user_max_tcp_conns = body.max_tcp_conns.is_some();
@@ -41,6 +55,8 @@ pub(in crate::api) async fn create_user(
     }
 
     let expiration = parse_optional_expiration(body.expiration_rfc3339.as_deref())?;
+    let credential_id = credential_id_from_hex(&secret)
+        .ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
     let _guard = shared.mutation_lock.lock().await;
     let (mut cfg, base_revision) =
         load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
@@ -131,12 +147,11 @@ pub(in crate::api) async fn create_user(
     .await?;
     shared
         .proxy_shared
-        .stage_user(
+        .stage_user_credential(
             &body.username,
-            &secret,
+            credential_id,
             cfg.access.is_user_enabled(&body.username),
-        )
-        .ok_or_else(|| ApiFailure::internal("failed to stage user admission policy"))?;
+        );
 
     if let Some(limit) = updated_limit {
         shared

src/api/users/lifecycle.rs

@@ -6,6 +6,22 @@ pub(in crate::api) async fn rotate_secret(
     body: RotateSecretRequest,
     expected_revision: Option<String>,
     shared: &ApiShared,
+) -> Result<(CreateUserResponse, String), ApiFailure> {
+    let shared = shared.clone();
+    let user = user.to_string();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            rotate_secret_to_completion(&user, body, expected_revision, &shared).await
+        })
+        .await
+}
+
+async fn rotate_secret_to_completion(
+    user: &str,
+    body: RotateSecretRequest,
+    expected_revision: Option<String>,
+    shared: &ApiShared,
 ) -> Result<(CreateUserResponse, String), ApiFailure> {
     let secret = body.secret.unwrap_or_else(random_user_secret);
     if !is_valid_user_secret(&secret) {
@@ -13,6 +29,8 @@ pub(in crate::api) async fn rotate_secret(
             "secret must be exactly 32 hex characters",
         ));
     }
+    let credential_id = credential_id_from_hex(&secret)
+        .ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
 
     let _guard = shared.mutation_lock.lock().await;
     let (mut cfg, base_revision) =
@@ -38,8 +56,7 @@ pub(in crate::api) async fn rotate_secret(
     .await?;
     shared
         .proxy_shared
-        .stage_user(user, &secret, cfg.access.is_user_enabled(user))
-        .ok_or_else(|| ApiFailure::internal("failed to stage rotated user credential"))?;
+        .stage_user_credential(user, credential_id, cfg.access.is_user_enabled(user));
     drop(_guard);
 
     let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
@@ -70,6 +87,21 @@ pub(in crate::api) async fn delete_user(
     user: &str,
     expected_revision: Option<String>,
     shared: &ApiShared,
+) -> Result<(String, String), ApiFailure> {
+    let shared = shared.clone();
+    let user = user.to_string();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            delete_user_to_completion(&user, expected_revision, &shared).await
+        })
+        .await
+}
+
+async fn delete_user_to_completion(
+    user: &str,
+    expected_revision: Option<String>,
+    shared: &ApiShared,
 ) -> Result<(String, String), ApiFailure> {
     let _guard = shared.mutation_lock.lock().await;
     let (mut cfg, base_revision) =

src/api/users/update.rs

@@ -5,6 +5,22 @@ pub(in crate::api) async fn patch_user(
     body: PatchUserRequest,
     expected_revision: Option<String>,
     shared: &ApiShared,
+) -> Result<(UserInfo, String), ApiFailure> {
+    let shared = shared.clone();
+    let user = user.to_string();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            patch_user_to_completion(&user, body, expected_revision, &shared).await
+        })
+        .await
+}
+
+async fn patch_user_to_completion(
+    user: &str,
+    body: PatchUserRequest,
+    expected_revision: Option<String>,
+    shared: &ApiShared,
 ) -> Result<(UserInfo, String), ApiFailure> {
     let touches_users = body.secret.is_some();
     let touches_user_ad_tags = !matches!(&body.user_ad_tag, Patch::Unchanged);
@@ -138,6 +154,19 @@ pub(in crate::api) async fn patch_user(
 
     cfg.validate()
         .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
+    let staged_credential = if touches_users || touches_user_enabled {
+        let secret = cfg
+            .access
+            .users
+            .get(user)
+            .ok_or_else(|| ApiFailure::internal("updated user secret is missing"))?;
+        Some(
+            credential_id_from_hex(secret)
+                .ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?,
+        )
+    } else {
+        None
+    };
 
     let mut touched_sections = Vec::new();
     if touches_users {
@@ -176,16 +205,10 @@ pub(in crate::api) async fn patch_user(
         )
         .await?
     };
-    if touches_users || touches_user_enabled {
-        let secret = cfg
-            .access
-            .users
-            .get(user)
-            .ok_or_else(|| ApiFailure::internal("updated user secret is missing"))?;
+    if let Some(credential_id) = staged_credential {
         shared
             .proxy_shared
-            .stage_user(user, secret, cfg.access.is_user_enabled(user))
-            .ok_or_else(|| ApiFailure::internal("failed to stage user admission policy"))?;
+            .stage_user_credential(user, credential_id, cfg.access.is_user_enabled(user));
     }
     match max_unique_ips_change {
         Some(Some(limit)) => shared.ip_tracker.set_user_limit(user, limit).await,
@@ -216,6 +239,22 @@ pub(in crate::api) async fn set_user_enabled(
     enabled: bool,
     expected_revision: Option<String>,
     shared: &ApiShared,
+) -> Result<(UserInfo, String), ApiFailure> {
+    let shared = shared.clone();
+    let user = user.to_string();
+    shared
+        .clone()
+        .run_mutation_completion(async move {
+            set_user_enabled_to_completion(&user, enabled, expected_revision, &shared).await
+        })
+        .await
+}
+
+async fn set_user_enabled_to_completion(
+    user: &str,
+    enabled: bool,
+    expected_revision: Option<String>,
+    shared: &ApiShared,
 ) -> Result<(UserInfo, String), ApiFailure> {
     let _guard = shared.mutation_lock.lock().await;
     let (mut cfg, base_revision) =
@@ -237,6 +276,12 @@ pub(in crate::api) async fn set_user_enabled(
 
     cfg.validate()
         .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
+    let credential_id = cfg
+        .access
+        .users
+        .get(user)
+        .and_then(|secret| credential_id_from_hex(secret))
+        .ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
     let revision = save_access_sections_to_disk_if_revision(
         &shared.config_path,
         &cfg,
@@ -244,15 +289,9 @@ pub(in crate::api) async fn set_user_enabled(
         Some(&base_revision),
     )
     .await?;
-    let secret = cfg
-        .access
-        .users
-        .get(user)
-        .ok_or_else(|| ApiFailure::internal("updated user secret is missing"))?;
     shared
         .proxy_shared
-        .stage_user(user, secret, enabled)
-        .ok_or_else(|| ApiFailure::internal("failed to stage user admission policy"))?;
+        .stage_user_credential(user, credential_id, enabled);
     drop(_guard);
 
     let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();