mirror of
https://github.com/telemt/telemt.git
synced 2026-10-03 07:55:58 +03:00
Compare commits
10 Commits
3.5.9
...
carrier-methods
| Author | SHA1 | Date | |
|---|---|---|---|
| d65efcc479 | |||
| fca6506449 | |||
| 94f4f7d540 | |||
| c10677ed97 | |||
| 86912e34b2 | |||
| e5bfeafa2f | |||
| 7635d1fdef | |||
| 698fd315ea | |||
| 6fa8da3c43 | |||
| 53f131e34d |
Generated
+1
-1
@@ -2780,7 +2780,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "telemt"
|
||||
version = "3.5.9"
|
||||
version = "3.5.11"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"anyhow",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.5.9"
|
||||
version = "3.5.11"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
|
||||
|
||||
@@ -1,5 +1,54 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_api_defaults_and_patches_are_hot() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
let (value, _) = read_managed_config(&path).await.unwrap();
|
||||
assert_eq!(value["web"]["carrier_method"].as_str(), Some("post"));
|
||||
for token in ["put", "post"] {
|
||||
let active = ProxyConfig::load(&path).unwrap();
|
||||
let patch = serde_json::json!({"web": {"carrier_method": token}});
|
||||
let mut response = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
let desired = ProxyConfig::load(&path).unwrap();
|
||||
reconcile_runtime_effect(&mut response, &active, &desired).unwrap();
|
||||
assert!(!response.restart_required);
|
||||
assert!(response.runtime_reload_required);
|
||||
assert!(!response.process_restart_required);
|
||||
assert!(response.deferred_process_fields.is_empty());
|
||||
let (value, revision) = read_managed_config(&path).await.unwrap();
|
||||
assert_eq!(value["web"]["carrier_method"].as_str(), Some(token));
|
||||
assert_eq!(revision, response.revision);
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
assert!(written.contains(&format!("carrier_method = \"{token}\"")));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_api_rejects_invalid_values_without_writing() {
|
||||
let (path, _directory) = temp_config("[web]\ncarrier_method = \"put\"\n");
|
||||
let original = tokio::fs::read(&path).await.unwrap();
|
||||
let revision = crate::api::config_store::current_revision(&path)
|
||||
.await
|
||||
.unwrap();
|
||||
for value in [
|
||||
serde_json::json!("PUT"),
|
||||
serde_json::json!("patch"),
|
||||
serde_json::json!(true),
|
||||
serde_json::json!(42),
|
||||
] {
|
||||
let patch = serde_json::json!({"web": {"carrier_method": value}});
|
||||
let error = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||
assert_eq!(tokio::fs::read(&path).await.unwrap(), original);
|
||||
assert_eq!(
|
||||
crate::api::config_store::current_revision(&path)
|
||||
.await
|
||||
.unwrap(),
|
||||
revision
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_object_converts_to_toml_table() {
|
||||
let j: Json = serde_json::json!({"censorship": {"tls_domain": "a.com"}, "default_dc": 2});
|
||||
|
||||
+19
-6
@@ -2,9 +2,9 @@
|
||||
//!
|
||||
//! Subcommands:
|
||||
//! - `start [OPTIONS] [config.toml]` - Start the daemon
|
||||
//! - `stop [--pid-file PATH]` - Stop a running daemon
|
||||
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
|
||||
//! - `status [--pid-file PATH]` - Check daemon status
|
||||
//! - `stop [--pid-file PATH] [--strict-runtime-paths]` - Stop a running daemon
|
||||
//! - `reload [--pid-file PATH] [--strict-runtime-paths]` - Reload configuration (SIGHUP)
|
||||
//! - `status [--pid-file PATH] [--strict-runtime-paths]` - Check daemon status
|
||||
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
||||
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
|
||||
|
||||
@@ -137,6 +137,10 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
|
||||
#[cfg(unix)]
|
||||
"--strict-runtime-paths" => {
|
||||
cmd.daemon_opts.strict_runtime_paths = true;
|
||||
}
|
||||
"--mode" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
@@ -199,9 +203,18 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
||||
#[cfg(unix)]
|
||||
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
||||
match cmd.subcommand {
|
||||
Subcommand::Stop => Some(daemon_commands::stop(&cmd.pid_file)),
|
||||
Subcommand::Reload => Some(daemon_commands::reload(&cmd.pid_file)),
|
||||
Subcommand::Status => Some(daemon_commands::status(&cmd.pid_file)),
|
||||
Subcommand::Stop => Some(daemon_commands::stop(
|
||||
&cmd.pid_file,
|
||||
cmd.daemon_opts.strict_runtime_paths,
|
||||
)),
|
||||
Subcommand::Reload => Some(daemon_commands::reload(
|
||||
&cmd.pid_file,
|
||||
cmd.daemon_opts.strict_runtime_paths,
|
||||
)),
|
||||
Subcommand::Status => Some(daemon_commands::status(
|
||||
&cmd.pid_file,
|
||||
cmd.daemon_opts.strict_runtime_paths,
|
||||
)),
|
||||
Subcommand::Healthcheck => {
|
||||
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
|
||||
if invalid_mode.is_empty() {
|
||||
|
||||
+165
-8
@@ -15,6 +15,9 @@ pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
|
||||
"--foreground" | "-f" => {
|
||||
opts.foreground = true;
|
||||
}
|
||||
"--strict-runtime-paths" => {
|
||||
opts.strict_runtime_paths = true;
|
||||
}
|
||||
"--pid-file" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
@@ -60,19 +63,21 @@ pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
|
||||
}
|
||||
|
||||
/// Sends SIGTERM and waits briefly for graceful PID-file cleanup.
|
||||
pub(super) fn stop(pid_file: &Path) -> i32 {
|
||||
pub(super) fn stop(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||
use nix::sys::signal::Signal;
|
||||
|
||||
println!("Stopping telemt daemon...");
|
||||
|
||||
match daemon::signal_pid_file(pid_file, Signal::SIGTERM) {
|
||||
match daemon::signal_pid_file(pid_file, Signal::SIGTERM, strict_runtime_paths) {
|
||||
Ok(()) => {
|
||||
println!("Stop signal sent successfully");
|
||||
|
||||
// Wait for process to exit for up to ten seconds.
|
||||
for _ in 0..20 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
if let daemon::DaemonStatus::NotRunning = daemon::check_status(pid_file) {
|
||||
if let daemon::DaemonStatus::NotRunning =
|
||||
daemon::check_status(pid_file, strict_runtime_paths)
|
||||
{
|
||||
println!("Daemon stopped");
|
||||
return 0;
|
||||
}
|
||||
@@ -88,12 +93,12 @@ pub(super) fn stop(pid_file: &Path) -> i32 {
|
||||
}
|
||||
|
||||
/// Sends SIGHUP to trigger configuration reload.
|
||||
pub(super) fn reload(pid_file: &Path) -> i32 {
|
||||
pub(super) fn reload(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||
use nix::sys::signal::Signal;
|
||||
|
||||
println!("Reloading telemt configuration...");
|
||||
|
||||
match daemon::signal_pid_file(pid_file, Signal::SIGHUP) {
|
||||
match daemon::signal_pid_file(pid_file, Signal::SIGHUP, strict_runtime_paths) {
|
||||
Ok(()) => {
|
||||
println!("Reload signal sent successfully");
|
||||
0
|
||||
@@ -106,8 +111,8 @@ pub(super) fn reload(pid_file: &Path) -> i32 {
|
||||
}
|
||||
|
||||
/// Reports daemon status without mutating PID lifecycle state.
|
||||
pub(super) fn status(pid_file: &Path) -> i32 {
|
||||
match daemon::check_status(pid_file) {
|
||||
pub(super) fn status(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||
match daemon::check_status(pid_file, strict_runtime_paths) {
|
||||
daemon::DaemonStatus::Running(pid) => {
|
||||
println!("telemt is running (pid {})", pid);
|
||||
0
|
||||
@@ -126,16 +131,168 @@ pub(super) fn status(pid_file: &Path) -> i32 {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::*;
|
||||
|
||||
const CONTROL_PID: &str = "TELEMT_RUNTIME_PATH_TEST_PID";
|
||||
const CONTROL_READY: &str = "TELEMT_RUNTIME_PATH_TEST_READY";
|
||||
const CONTROL_RELOADED: &str = "TELEMT_RUNTIME_PATH_TEST_RELOADED";
|
||||
|
||||
struct ControlChild(Child);
|
||||
|
||||
impl Drop for ControlChild {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_for_file(path: &Path) -> bool {
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
while Instant::now() < deadline {
|
||||
if path.exists() {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_path_policy_defaults_to_compatibility_for_all_commands() {
|
||||
for command in ["run", "start", "stop", "reload", "status"] {
|
||||
let args = vec![command.to_string(), "config.toml".to_string()];
|
||||
assert!(
|
||||
!crate::cli::parse_command(&args)
|
||||
.daemon_opts
|
||||
.strict_runtime_paths
|
||||
);
|
||||
assert!(!parse_daemon_args(&args).strict_runtime_paths);
|
||||
|
||||
for position in [1, args.len()] {
|
||||
let mut strict_args = args.clone();
|
||||
strict_args.insert(position, "--strict-runtime-paths".to_string());
|
||||
let parsed = crate::cli::parse_command(&strict_args);
|
||||
assert!(parsed.daemon_opts.strict_runtime_paths);
|
||||
assert!(parse_daemon_args(&strict_args).strict_runtime_paths);
|
||||
assert_eq!(parsed.config_path, "config.toml");
|
||||
}
|
||||
}
|
||||
let args = vec![
|
||||
"--strict-runtime-paths".to_string(),
|
||||
"config.toml".to_string(),
|
||||
];
|
||||
let parsed = crate::cli::parse_command(&args);
|
||||
assert_eq!(parsed.subcommand, crate::cli::Subcommand::Run);
|
||||
assert!(parsed.daemon_opts.strict_runtime_paths);
|
||||
assert_eq!(parsed.config_path, "config.toml");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn daemon_control_subprocess() {
|
||||
let Some(pid_path) = std::env::var_os(CONTROL_PID) else {
|
||||
return;
|
||||
};
|
||||
let ready = PathBuf::from(std::env::var_os(CONTROL_READY).unwrap());
|
||||
let reloaded = PathBuf::from(std::env::var_os(CONTROL_RELOADED).unwrap());
|
||||
let mut terminate =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap();
|
||||
let mut reload =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()).unwrap();
|
||||
let mut owner = daemon::PidFile::new(PathBuf::from(pid_path), false);
|
||||
owner.acquire().unwrap();
|
||||
fs::write(&ready, b"ready").unwrap();
|
||||
let deadline = tokio::time::sleep(Duration::from_secs(15));
|
||||
tokio::pin!(deadline);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = terminate.recv() => break,
|
||||
_ = reload.recv() => fs::write(&reloaded, b"reloaded").unwrap(),
|
||||
_ = &mut deadline => panic!("daemon control subprocess timed out"),
|
||||
}
|
||||
}
|
||||
owner.release().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_commands_follow_runtime_path_policy() {
|
||||
for (mode, linked_parent, trusted_parent) in [
|
||||
(0o777, false, false),
|
||||
(0o777, true, false),
|
||||
(0o755, false, true),
|
||||
] {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let real = root.path().join("run");
|
||||
let linked = root.path().join("linked");
|
||||
fs::create_dir(&real).unwrap();
|
||||
fs::set_permissions(&real, fs::Permissions::from_mode(mode)).unwrap();
|
||||
symlink(&real, &linked).unwrap();
|
||||
let pid_path = if linked_parent { &linked } else { &real }.join("telemt.pid");
|
||||
let ready = root.path().join("ready");
|
||||
let reloaded = root.path().join("reloaded");
|
||||
let mut child = ControlChild(
|
||||
Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"cli::daemon_commands::tests::daemon_control_subprocess",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CONTROL_PID, &pid_path)
|
||||
.env(CONTROL_READY, &ready)
|
||||
.env(CONTROL_RELOADED, &reloaded)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(
|
||||
wait_for_file(&ready),
|
||||
"daemon control subprocess did not become ready"
|
||||
);
|
||||
let command = |name: &str, strict: bool| {
|
||||
let mut args = vec![
|
||||
name.to_string(),
|
||||
"--pid-file".to_string(),
|
||||
pid_path.to_str().unwrap().to_string(),
|
||||
];
|
||||
if strict {
|
||||
args.push("--strict-runtime-paths".to_string());
|
||||
}
|
||||
crate::cli::execute_subcommand(&crate::cli::parse_command(&args))
|
||||
};
|
||||
|
||||
assert_eq!(command("status", false), Some(0));
|
||||
if trusted_parent {
|
||||
assert_eq!(command("status", true), Some(0));
|
||||
assert_eq!(command("reload", true), Some(0));
|
||||
assert!(wait_for_file(&reloaded));
|
||||
assert_eq!(command("stop", true), Some(0));
|
||||
} else {
|
||||
assert_eq!(command("status", true), Some(1));
|
||||
assert_eq!(command("reload", true), Some(1));
|
||||
assert!(!reloaded.exists());
|
||||
assert_eq!(command("stop", true), Some(1));
|
||||
assert_eq!(command("status", false), Some(0));
|
||||
assert_eq!(command("reload", false), Some(0));
|
||||
assert!(wait_for_file(&reloaded));
|
||||
assert_eq!(command("stop", false), Some(0));
|
||||
}
|
||||
assert!(child.0.wait().unwrap().success());
|
||||
assert!(!pid_path.exists());
|
||||
assert_eq!(command("status", false), Some(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_does_not_remove_stale_pid_file() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_file = directory.path().join("telemt.pid");
|
||||
fs::write(&pid_file, b"2000000000\n").unwrap();
|
||||
|
||||
assert_eq!(status(&pid_file), 1);
|
||||
assert_eq!(status(&pid_file, false), 1);
|
||||
assert!(pid_file.exists());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 16;
|
||||
const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 1000;
|
||||
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
||||
const DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS: u64 = 250;
|
||||
const DEFAULT_CONNTRACK_CONTROL_ENABLED: bool = true;
|
||||
const DEFAULT_CONNTRACK_CONTROL_ENABLED: bool = false;
|
||||
const DEFAULT_CONNTRACK_PRESSURE_HIGH_WATERMARK_PCT: u8 = 85;
|
||||
const DEFAULT_CONNTRACK_PRESSURE_LOW_WATERMARK_PCT: u8 = 70;
|
||||
const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096;
|
||||
@@ -244,6 +244,7 @@ pub(crate) fn default_accept_permit_timeout_ms() -> u64 {
|
||||
DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS
|
||||
}
|
||||
|
||||
/// Keeps privileged conntrack control disabled unless explicitly requested.
|
||||
pub(crate) fn default_conntrack_control_enabled() -> bool {
|
||||
DEFAULT_CONNTRACK_CONTROL_ENABLED
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! | `network` | `dns_overrides` | Applied immediately |
|
||||
//! | `access` | All user/quota fields | Effective immediately |
|
||||
//! | `web` | Carrier, timing, and debug policy | Applied to newly issued sessions |
|
||||
//! | `web` | `carrier_method` | Applied to newly rendered bridge pages |
|
||||
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
||||
@@ -67,3 +68,7 @@ use watcher::{ReloadState, reload_config};
|
||||
mod base_path_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
// Carrier method reloads preserve page-owned requests and process-owned limits.
|
||||
#[cfg(test)]
|
||||
#[path = "hot_reload/carrier_method_tests.rs"]
|
||||
mod carrier_method_tests;
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
use super::*;
|
||||
|
||||
use crate::config::WebCarrierMethod;
|
||||
|
||||
#[test]
|
||||
fn carrier_method_reload_is_hot_and_preserves_process_limits() {
|
||||
let mut active = ProxyConfig::default();
|
||||
for method in [WebCarrierMethod::Put, WebCarrierMethod::Post] {
|
||||
let mut desired = active.clone();
|
||||
desired.web.carrier_method = method;
|
||||
assert_eq!(classify_config_changes(&active, &desired).changed, ["web"]);
|
||||
assert!(!classify_config_changes(&active, &desired).restart_required);
|
||||
desired.web.limits.max_http_connections += 1;
|
||||
let applied = overlay_hot_fields(&active, &desired);
|
||||
assert_eq!(applied.web.carrier_method, method);
|
||||
assert_eq!(
|
||||
applied.web.limits.max_http_connections,
|
||||
active.web.limits.max_http_connections
|
||||
);
|
||||
active = applied;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrier_method_reload_publishes_both_directions_and_keeps_last_good_value() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.toml");
|
||||
let source = |method| format!("[web]\ncarrier_method = \"{method}\"\n");
|
||||
std::fs::write(&path, source("post")).unwrap();
|
||||
let initial = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||
.unwrap()
|
||||
.rendered_hash;
|
||||
let (config_tx, _config_rx) = watch::channel(Arc::clone(&initial));
|
||||
let (log_tx, _log_rx) = watch::channel(initial.general.log_level.clone());
|
||||
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||
for (token, method) in [
|
||||
("put", WebCarrierMethod::Put),
|
||||
("post", WebCarrierMethod::Post),
|
||||
] {
|
||||
std::fs::write(&path, source(token)).unwrap();
|
||||
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||
let applied = config_tx.borrow().clone();
|
||||
assert_eq!(applied.web.carrier_method, method);
|
||||
std::fs::write(&path, source("PATCH")).unwrap();
|
||||
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state);
|
||||
let unchanged = config_tx.borrow().clone();
|
||||
assert!(Arc::ptr_eq(&unchanged, &applied));
|
||||
}
|
||||
}
|
||||
@@ -263,6 +263,7 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
||||
const WEB_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled",
|
||||
"carrier",
|
||||
"carrier_method",
|
||||
"carriers",
|
||||
"carrier_learning",
|
||||
"carrier_negotiation_aggressiveness",
|
||||
|
||||
@@ -1,5 +1,71 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn conntrack_control_is_opt_in_by_default() {
|
||||
assert!(!default_conntrack_control_enabled());
|
||||
assert!(
|
||||
!ProxyConfig::default()
|
||||
.server
|
||||
.conntrack_control
|
||||
.inline_conntrack_control
|
||||
);
|
||||
let cfg = load_config_from_temp_toml(
|
||||
r#"
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#,
|
||||
);
|
||||
assert!(!cfg.server.conntrack_control.inline_conntrack_control);
|
||||
assert!(
|
||||
!cfg.server
|
||||
.conntrack_control
|
||||
.inline_conntrack_control_explicit
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&cfg.server.conntrack_control).unwrap()["inline_conntrack_control"],
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conntrack_control_preserves_explicit_enablement() {
|
||||
for enabled in [false, true] {
|
||||
let cfg = load_config_from_temp_toml(&format!(
|
||||
r#"
|
||||
[server.conntrack_control]
|
||||
inline_conntrack_control = {enabled}
|
||||
backend = "nftables"
|
||||
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#,
|
||||
));
|
||||
assert_eq!(
|
||||
cfg.server.conntrack_control.inline_conntrack_control,
|
||||
enabled
|
||||
);
|
||||
assert!(
|
||||
cfg.server
|
||||
.conntrack_control
|
||||
.inline_conntrack_control_explicit
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.server.conntrack_control.backend,
|
||||
ConntrackBackend::Nftables
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&cfg.server.conntrack_control).unwrap()["inline_conntrack_control"],
|
||||
enabled,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conntrack_pressure_high_watermark_out_of_range_is_rejected() {
|
||||
let toml = r#"
|
||||
|
||||
@@ -3,6 +3,10 @@ use super::*;
|
||||
#[path = "web_tests/base_path_tests.rs"]
|
||||
mod base_path_tests;
|
||||
|
||||
// Method selection must not alter capability identity or accepted config literals.
|
||||
#[path = "web_tests/carrier_method_tests.rs"]
|
||||
mod carrier_method_tests;
|
||||
|
||||
const WEB_CONFIG: &str = r#"
|
||||
[access.users]
|
||||
alice = "000102030405060708090a0b0c0d0e0f"
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn carrier_method_defaults_and_roundtrips_without_changing_capabilities() {
|
||||
let default = load_config_from_temp_toml(WEB_CONFIG);
|
||||
assert_eq!(default.web.carrier_method, WebCarrierMethod::Post);
|
||||
let capabilities = default.web.runtime.as_ref().unwrap().capabilities.clone();
|
||||
for (token, method) in [
|
||||
("post", WebCarrierMethod::Post),
|
||||
("put", WebCarrierMethod::Put),
|
||||
] {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
&format!("carrier = \"https-lanes\"\ncarrier_method = \"{token}\""),
|
||||
);
|
||||
let source = format!("[general]\nconfig_strict = true\n{configured}");
|
||||
let config = load_config_from_temp_toml(&source);
|
||||
assert_eq!(config.web.carrier_method, method);
|
||||
assert_eq!(
|
||||
config.web.runtime.as_ref().unwrap().capabilities,
|
||||
capabilities
|
||||
);
|
||||
let json = serde_json::to_value(&config.web).unwrap();
|
||||
assert_eq!(json["carrier_method"], token);
|
||||
let decoded: WebConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(decoded.carrier_method, method);
|
||||
let serialized = toml::to_string(&config.web).unwrap();
|
||||
let decoded: WebConfig = toml::from_str(&serialized).unwrap();
|
||||
assert_eq!(decoded.carrier_method, method);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrier_method_rejects_unknown_tokens_types_and_aliases() {
|
||||
for value in ["\"POST\"", "\"PUT\"", "\"patch\"", "true", "42", "[]"] {
|
||||
let source = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
&format!("carrier = \"https-lanes\"\ncarrier_method = {value}"),
|
||||
);
|
||||
assert!(load_config_error_from_temp_toml(&source).contains("carrier_method"));
|
||||
}
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\nhttp_method = \"put\"",
|
||||
);
|
||||
let source = format!("[general]\nconfig_strict = true\n{configured}");
|
||||
assert!(load_config_error_from_temp_toml(&source).contains("http_method"));
|
||||
}
|
||||
+1
-1
@@ -63,7 +63,7 @@ pub(crate) use web::{
|
||||
};
|
||||
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
||||
#[allow(unused_imports)]
|
||||
pub use web_carrier::{WebCarrier, WebCarriers};
|
||||
pub use web_carrier::{WebCarrier, WebCarrierMethod, WebCarriers};
|
||||
pub(crate) use web_debug::web_debug_fits_limits;
|
||||
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use std::sync::Arc;
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::web_carrier::{WebCarrier, WebCarriers};
|
||||
use super::web_carrier::{WebCarrier, WebCarrierMethod, WebCarriers};
|
||||
use super::web_debug::WebDebugConfig;
|
||||
|
||||
// Serialized WEB defaults remain separate from the runtime data model.
|
||||
@@ -413,6 +413,9 @@ pub struct WebConfig {
|
||||
/// Sole carrier when negotiation is disabled and final fallback when enabled.
|
||||
#[serde(default)]
|
||||
pub carrier: WebCarrier,
|
||||
/// Page-owned HTTP method for HTTPS uplink and downlink, including retries.
|
||||
#[serde(default)]
|
||||
pub carrier_method: WebCarrierMethod,
|
||||
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
||||
#[serde(default)]
|
||||
pub carriers: WebCarriers,
|
||||
@@ -471,6 +474,7 @@ impl Default for WebConfig {
|
||||
Self {
|
||||
enabled: false,
|
||||
carrier: WebCarrier::default(),
|
||||
carrier_method: WebCarrierMethod::default(),
|
||||
carriers: WebCarriers::default(),
|
||||
carrier_learning: default_web_carrier_learning(),
|
||||
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||
|
||||
@@ -3,6 +3,27 @@ use serde::{Deserialize, Serialize};
|
||||
/// Minimum restart-owned entries required for one complete learning sample.
|
||||
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
|
||||
|
||||
/// HTTP method emitted by newly rendered HTTPS carrier bridges.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum WebCarrierMethod {
|
||||
/// Preserve the original bridge request method.
|
||||
#[default]
|
||||
Post,
|
||||
/// Allow idempotent-method connection reuse in Linux WebKitGTK.
|
||||
Put,
|
||||
}
|
||||
|
||||
impl WebCarrierMethod {
|
||||
/// Returns the HTTP token embedded in the page-owned carrier policy.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Post => "POST",
|
||||
Self::Put => "PUT",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Carrier selected for one newly issued WEB relay session.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
|
||||
@@ -26,15 +26,21 @@ const RETRY_DELAYS: [Duration; 6] = [
|
||||
Duration::from_secs(30),
|
||||
];
|
||||
|
||||
/// Reports whether a generation's desired firewall policy was confirmed.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(super) enum ReconcileOutcome {
|
||||
/// The applied policy matches the accepted generation's desired policy.
|
||||
Applied,
|
||||
/// The attempt failed without confirming the desired policy.
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Associates a reconciliation result with its accepted runtime generation.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(super) struct ReconcileStatus {
|
||||
/// Runtime generation whose desired policy was attempted.
|
||||
pub(super) generation: u64,
|
||||
/// Result of the latest completed attempt for this generation.
|
||||
pub(super) outcome: ReconcileOutcome,
|
||||
}
|
||||
|
||||
@@ -51,25 +57,14 @@ pub(crate) struct FirewallAuthority {
|
||||
}
|
||||
|
||||
impl FirewallAuthority {
|
||||
/// Starts the single process-owned firewall reconciler.
|
||||
pub(crate) fn spawn(control_plane: &ProcessControlPlane) -> Result<Self, String> {
|
||||
let (desired_tx, desired_rx) = watch::channel(None);
|
||||
let (status_tx, status_rx) = watch::channel(None);
|
||||
let terminal = CancellationToken::new();
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let completed_flag = Arc::new(AtomicBool::new(false));
|
||||
let cleanup_succeeded = Arc::new(AtomicBool::new(false));
|
||||
let completed = Arc::new(Notify::new());
|
||||
let actor = FirewallReconciler::new(
|
||||
SystemCommandRunner,
|
||||
desired_rx,
|
||||
status_tx,
|
||||
terminal.clone(),
|
||||
Arc::clone(&closed),
|
||||
Arc::clone(&completed_flag),
|
||||
Arc::clone(&cleanup_succeeded),
|
||||
Arc::clone(&completed),
|
||||
);
|
||||
/// Starts reconciliation only when enabled and CAP_NET_ADMIN is available.
|
||||
pub(crate) fn spawn(
|
||||
control_plane: &ProcessControlPlane,
|
||||
config: &ProxyConfig,
|
||||
) -> Result<Option<Self>, String> {
|
||||
let Some((authority, actor)) = Self::prepare(config, SystemCommandRunner) else {
|
||||
return Ok(None);
|
||||
};
|
||||
control_plane
|
||||
.spawn_cooperative(move |process_cancellation| async move {
|
||||
actor.run(process_cancellation).await;
|
||||
@@ -78,7 +73,37 @@ impl FirewallAuthority {
|
||||
"process control-plane admission closed before conntrack firewall startup"
|
||||
.to_string()
|
||||
})?;
|
||||
Ok(Self {
|
||||
Ok(Some(authority))
|
||||
}
|
||||
|
||||
// Admission precedes allocation so an opted-out process never owns firewall cleanup.
|
||||
// Keeping construction separate permits fake runners without changing async Send bounds.
|
||||
fn prepare<R>(config: &ProxyConfig, runner: R) -> Option<(Self, FirewallReconciler<R>)>
|
||||
where
|
||||
R: FirewallCommandRunner + 'static,
|
||||
{
|
||||
if !config.server.conntrack_control.inline_conntrack_control || !runner.has_cap_net_admin()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let (desired_tx, desired_rx) = watch::channel(None);
|
||||
let (status_tx, status_rx) = watch::channel(None);
|
||||
let terminal = CancellationToken::new();
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let completed_flag = Arc::new(AtomicBool::new(false));
|
||||
let cleanup_succeeded = Arc::new(AtomicBool::new(false));
|
||||
let completed = Arc::new(Notify::new());
|
||||
let actor = FirewallReconciler::new(
|
||||
runner,
|
||||
desired_rx,
|
||||
status_tx,
|
||||
terminal.clone(),
|
||||
Arc::clone(&closed),
|
||||
Arc::clone(&completed_flag),
|
||||
Arc::clone(&cleanup_succeeded),
|
||||
Arc::clone(&completed),
|
||||
);
|
||||
let authority = Self {
|
||||
desired_tx,
|
||||
status_rx,
|
||||
terminal,
|
||||
@@ -86,7 +111,8 @@ impl FirewallAuthority {
|
||||
completed_flag,
|
||||
cleanup_succeeded,
|
||||
completed,
|
||||
})
|
||||
};
|
||||
Some((authority, actor))
|
||||
}
|
||||
|
||||
/// Publishes policy only after its runtime generation becomes active.
|
||||
@@ -169,6 +195,7 @@ impl Drop for CompletionGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes confirmed-state transitions and retains cleanup ownership until shutdown.
|
||||
pub(super) struct FirewallReconciler<R> {
|
||||
runner: R,
|
||||
desired_rx: watch::Receiver<Option<DesiredState>>,
|
||||
@@ -186,6 +213,7 @@ impl<R> FirewallReconciler<R>
|
||||
where
|
||||
R: FirewallCommandRunner + 'static,
|
||||
{
|
||||
/// Constructs an admitted actor with unknown state pending startup recovery.
|
||||
pub(super) fn new(
|
||||
runner: R,
|
||||
desired_rx: watch::Receiver<Option<DesiredState>>,
|
||||
@@ -214,6 +242,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconciles accepted generations and completes bounded cleanup on cancellation.
|
||||
pub(super) async fn run(mut self, process_cancellation: CancellationToken) {
|
||||
let mut current = None;
|
||||
let mut retry_index = 0usize;
|
||||
@@ -337,6 +366,7 @@ where
|
||||
let _completion = &self.completion;
|
||||
}
|
||||
|
||||
/// Accepts fenced publications without replacing telemetry with stale desired state.
|
||||
pub(super) fn take_latest_desired(&mut self) -> Option<DesiredState> {
|
||||
let next = self.desired_rx.borrow_and_update().clone()?;
|
||||
if next.generation < self.last_generation {
|
||||
@@ -369,3 +399,8 @@ where
|
||||
Some(next)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercises startup admission and cleanup ownership with fake helper processes.
|
||||
#[cfg(test)]
|
||||
#[path = "tests/startup_admission.rs"]
|
||||
mod startup_admission_tests;
|
||||
|
||||
@@ -10,14 +10,19 @@ use crate::util::trusted_command::{resolve_trusted_helper, trusted_helper_comman
|
||||
|
||||
const COMMAND_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// A privileged helper invocation with arguments kept separate from shell syntax.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(super) struct CommandSpec {
|
||||
/// Logical helper name resolved by the trusted executable policy.
|
||||
pub(super) binary: &'static str,
|
||||
/// Arguments passed directly to the helper process.
|
||||
pub(super) args: Vec<String>,
|
||||
/// Optional restore script supplied on standard input.
|
||||
pub(super) stdin: Option<String>,
|
||||
}
|
||||
|
||||
impl CommandSpec {
|
||||
/// Creates a helper invocation without an input script.
|
||||
pub(super) fn new(binary: &'static str, args: impl IntoIterator<Item = &'static str>) -> Self {
|
||||
Self {
|
||||
binary,
|
||||
@@ -26,6 +31,7 @@ impl CommandSpec {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a helper invocation that receives a restore script.
|
||||
pub(super) fn with_stdin(
|
||||
binary: &'static str,
|
||||
args: impl IntoIterator<Item = &'static str>,
|
||||
@@ -39,22 +45,32 @@ impl CommandSpec {
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinguishes idempotent absence from transaction and execution failures.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(super) enum CommandErrorKind {
|
||||
/// The trusted helper executable is unavailable.
|
||||
Missing,
|
||||
/// The requested firewall object or rule is absent.
|
||||
NotFound,
|
||||
/// A terminal or process cancellation interrupted the invocation.
|
||||
Cancelled,
|
||||
/// The helper exceeded its execution deadline.
|
||||
Timeout,
|
||||
/// A failure that must not be treated as successful cleanup.
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// A classified helper failure retaining its diagnostic for reconciliation.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(super) struct CommandError {
|
||||
/// Recovery semantics associated with the failure.
|
||||
pub(super) kind: CommandErrorKind,
|
||||
/// Original helper diagnostic or an execution failure description.
|
||||
pub(super) message: String,
|
||||
}
|
||||
|
||||
impl CommandError {
|
||||
/// Creates the cancellation failure used by interruptible transactions.
|
||||
pub(super) fn cancelled() -> Self {
|
||||
Self {
|
||||
kind: CommandErrorKind::Cancelled,
|
||||
@@ -62,6 +78,7 @@ impl CommandError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a failure that prevents idempotent cleanup from claiming success.
|
||||
pub(super) fn failed(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: CommandErrorKind::Failed,
|
||||
@@ -76,14 +93,19 @@ impl std::fmt::Display for CommandError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes firewall commands through a production or deterministic test runner.
|
||||
pub(super) trait FirewallCommandRunner: Send + Sync {
|
||||
/// Reports whether a helper can be resolved under the runner's trust policy.
|
||||
fn available(&self, binary: &str) -> bool;
|
||||
|
||||
/// Reports whether the process has the capability required to alter firewall rules.
|
||||
fn has_cap_net_admin(&self) -> bool;
|
||||
|
||||
/// Executes one invocation while preserving classified failure semantics.
|
||||
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError>;
|
||||
}
|
||||
|
||||
/// Runs trusted system helpers with bounded execution and captured diagnostics.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(super) struct SystemCommandRunner;
|
||||
|
||||
@@ -187,18 +209,95 @@ impl FirewallCommandRunner for SystemCommandRunner {
|
||||
} else {
|
||||
stderr
|
||||
};
|
||||
let kind = if is_not_found_error(&message) {
|
||||
CommandErrorKind::NotFound
|
||||
} else {
|
||||
CommandErrorKind::Failed
|
||||
};
|
||||
let kind = classify_command_error(binary, &spec.args, &message);
|
||||
Err(CommandError { kind, message })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Recognizes the existing legacy iptables and native nftables absence formats.
|
||||
pub(super) fn is_not_found_error(message: &str) -> bool {
|
||||
message.contains("No chain/target/match by that name")
|
||||
|| message.contains("Bad rule (does a matching rule exist in that chain?)")
|
||||
|| message.contains("Could not process rule: No such file or directory")
|
||||
}
|
||||
|
||||
/// Bounds additional iptables-nft absence diagnostics to owned cleanup and checks.
|
||||
pub(super) fn classify_command_error(
|
||||
binary: &str,
|
||||
args: &[String],
|
||||
message: &str,
|
||||
) -> CommandErrorKind {
|
||||
if is_not_found_error(message) || is_missing_owned_iptables_chain(binary, args, message) {
|
||||
CommandErrorKind::NotFound
|
||||
} else {
|
||||
CommandErrorKind::Failed
|
||||
}
|
||||
}
|
||||
|
||||
fn is_missing_owned_iptables_chain(binary: &str, args: &[String], message: &str) -> bool {
|
||||
if !matches!(binary, "iptables" | "ip6tables") {
|
||||
return false;
|
||||
}
|
||||
// An absent jump target is harmless for deletion, but not for installation.
|
||||
let chain = match args {
|
||||
[flag, table, operation, source, jump, target]
|
||||
if flag == "-t"
|
||||
&& table == "raw"
|
||||
&& matches!(operation.as_str(), "-C" | "-D")
|
||||
&& source == "PREROUTING"
|
||||
&& jump == "-j"
|
||||
&& target == "TELEMT_NOTRACK" =>
|
||||
{
|
||||
target.as_str()
|
||||
}
|
||||
[flag, table, operation, target]
|
||||
if flag == "-t"
|
||||
&& table == "raw"
|
||||
&& matches!(operation.as_str(), "-F" | "-X")
|
||||
&& matches!(
|
||||
target.as_str(),
|
||||
"TELEMT_NOTRACK" | "TELEMT_NT_A" | "TELEMT_NT_B"
|
||||
) =>
|
||||
{
|
||||
target.as_str()
|
||||
}
|
||||
_ => return false,
|
||||
};
|
||||
let mut lines = message.lines();
|
||||
let Some(first) = lines.next() else {
|
||||
return false;
|
||||
};
|
||||
let diagnostic = if first.starts_with("Chain '") {
|
||||
first
|
||||
} else {
|
||||
let Some((version, diagnostic)) = first
|
||||
.strip_prefix(binary)
|
||||
.and_then(|line| line.strip_prefix(" v"))
|
||||
.and_then(|line| line.split_once(" (nf_tables): "))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if version.is_empty() || version.bytes().any(|byte| byte.is_ascii_whitespace()) {
|
||||
return false;
|
||||
}
|
||||
diagnostic
|
||||
};
|
||||
if diagnostic
|
||||
.strip_prefix("Chain '")
|
||||
.and_then(|line| line.strip_suffix("' does not exist"))
|
||||
!= Some(chain)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// Reject mixed diagnostics instead of hiding another failure after an absence message.
|
||||
let Some(help) = lines.next() else {
|
||||
return true;
|
||||
};
|
||||
help.strip_prefix("Try `")
|
||||
.and_then(|line| line.strip_prefix(binary))
|
||||
.and_then(|line| line.strip_prefix(" -h' or '"))
|
||||
.and_then(|line| line.strip_prefix(binary))
|
||||
== Some(" --help' for more information.")
|
||||
&& lines.next().is_none()
|
||||
}
|
||||
|
||||
@@ -20,6 +20,10 @@ use super::transaction::{InterruptibleRunner, reconcile_once};
|
||||
#[path = "tests/model_tests.rs"]
|
||||
mod model_tests;
|
||||
|
||||
// Replays helper diagnostics through command classification and recovery.
|
||||
#[path = "tests/recovery_errors.rs"]
|
||||
mod recovery_errors;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FailureRule {
|
||||
binary: &'static str,
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
use super::super::command::classify_command_error;
|
||||
use super::super::transaction::recover_to_empty;
|
||||
use super::*;
|
||||
|
||||
const OWNED_CHAINS: [&str; 3] = ["TELEMT_NOTRACK", "TELEMT_NT_A", "TELEMT_NT_B"];
|
||||
type ChainKey = (&'static str, String, String);
|
||||
type ChainRules = BTreeMap<ChainKey, BTreeSet<String>>;
|
||||
|
||||
fn missing_chain(binary: &str, chain: &str) -> String {
|
||||
format!(
|
||||
"{binary} v1.8.10 (nf_tables): Chain '{chain}' does not exist\n\
|
||||
Try `{binary} -h' or '{binary} --help' for more information."
|
||||
)
|
||||
}
|
||||
|
||||
fn classified_error(spec: &CommandSpec, message: &str) -> CommandError {
|
||||
CommandError {
|
||||
kind: classify_command_error(spec.binary, &spec.args, message),
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FixtureState {
|
||||
calls: Vec<CommandSpec>,
|
||||
chains: ChainRules,
|
||||
nft_tables: BTreeSet<String>,
|
||||
failure: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixtureRunner {
|
||||
state: Arc<Mutex<FixtureState>>,
|
||||
}
|
||||
|
||||
impl FixtureRunner {
|
||||
fn new(failure: Option<&'static str>) -> Self {
|
||||
let mut state = FixtureState {
|
||||
failure,
|
||||
..FixtureState::default()
|
||||
};
|
||||
for binary in ["iptables", "ip6tables"] {
|
||||
for (table, chain, rules) in [
|
||||
("raw", "PREROUTING", vec!["-j FOREIGN_RAW"]),
|
||||
("raw", "FOREIGN_RAW", vec!["-j ACCEPT"]),
|
||||
("filter", "INPUT", vec!["-j MTPR_SYNFIX", "-j TMT_SYN_TEST"]),
|
||||
("filter", "MTPR_SYNFIX", vec!["-p tcp --syn -j DROP"]),
|
||||
("filter", "TMT_SYN_TEST", vec!["-p tcp --syn -j DROP"]),
|
||||
] {
|
||||
state.chains.insert(
|
||||
(binary, table.to_string(), chain.to_string()),
|
||||
rules.into_iter().map(str::to_string).collect(),
|
||||
);
|
||||
}
|
||||
}
|
||||
state.nft_tables.extend([
|
||||
"mtpr_synfix".to_string(),
|
||||
"telemt_synlimit_test".to_string(),
|
||||
"foreign_table".to_string(),
|
||||
]);
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(state)),
|
||||
}
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<CommandSpec> {
|
||||
self.state.lock().unwrap().calls.clone()
|
||||
}
|
||||
|
||||
fn foreign_snapshot(&self) -> (ChainRules, BTreeSet<String>) {
|
||||
let state = self.state.lock().unwrap();
|
||||
let mut chains = state.chains.clone();
|
||||
chains.retain(|(_, table, chain), _| {
|
||||
table != "raw" || !OWNED_CHAINS.contains(&chain.as_str())
|
||||
});
|
||||
for ((_, table, chain), rules) in &mut chains {
|
||||
if table == "raw" && chain == "PREROUTING" {
|
||||
rules.remove("-j TELEMT_NOTRACK");
|
||||
}
|
||||
}
|
||||
(chains, state.nft_tables.clone())
|
||||
}
|
||||
|
||||
fn assert_owned_policy(&self, enabled: bool) {
|
||||
let state = self.state.lock().unwrap();
|
||||
for (binary, ip) in [("iptables", "192.0.2.10"), ("ip6tables", "2001:db8::10")] {
|
||||
for chain in OWNED_CHAINS {
|
||||
let key = (binary, "raw".to_string(), chain.to_string());
|
||||
if !enabled {
|
||||
assert!(!state.chains.contains_key(&key));
|
||||
continue;
|
||||
}
|
||||
let expected = match chain {
|
||||
"TELEMT_NOTRACK" => BTreeSet::from(["-j TELEMT_NT_A".to_string()]),
|
||||
"TELEMT_NT_A" => {
|
||||
BTreeSet::from([format!("-p tcp --dport 443 -d {ip} -j CT --notrack")])
|
||||
}
|
||||
"TELEMT_NT_B" => BTreeSet::new(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert_eq!(state.chains.get(&key), Some(&expected));
|
||||
}
|
||||
let prerouting = (binary, "raw".to_string(), "PREROUTING".to_string());
|
||||
assert_eq!(
|
||||
state.chains[&prerouting].contains("-j TELEMT_NOTRACK"),
|
||||
enabled
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FirewallCommandRunner for FixtureRunner {
|
||||
fn available(&self, _binary: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn has_cap_net_admin(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.calls.push(spec.clone());
|
||||
if spec.binary == "nft" {
|
||||
assert_eq!(&spec.args[..3], ["delete", "table", "inet"]);
|
||||
return if state.nft_tables.remove(&spec.args[3]) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(classified_error(
|
||||
&spec,
|
||||
"Error: Could not process rule: No such file or directory",
|
||||
))
|
||||
};
|
||||
}
|
||||
if let Some(script) = &spec.stdin {
|
||||
let binary = match spec.binary {
|
||||
"iptables-restore" => "iptables",
|
||||
"ip6tables-restore" => "ip6tables",
|
||||
_ => panic!("unexpected restore helper"),
|
||||
};
|
||||
assert_eq!(spec.args, ["--noflush"]);
|
||||
for line in script.lines().filter(|line| line.starts_with('-')) {
|
||||
let fields = line.split_whitespace().collect::<Vec<_>>();
|
||||
let key = (binary, "raw".to_string(), fields[1].to_string());
|
||||
let rules = state.chains.get_mut(&key).expect("restore target exists");
|
||||
match fields[0] {
|
||||
"-F" => rules.clear(),
|
||||
"-A" => {
|
||||
rules.insert(fields[2..].join(" "));
|
||||
}
|
||||
_ => panic!("unexpected restore operation"),
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
assert!(matches!(spec.binary, "iptables" | "ip6tables"));
|
||||
assert_eq!(spec.args[0], "-t");
|
||||
let operation = spec.args[2].as_str();
|
||||
let key = (spec.binary, spec.args[1].clone(), spec.args[3].clone());
|
||||
match operation {
|
||||
"-N" => {
|
||||
if state.chains.contains_key(&key) {
|
||||
Err(classified_error(&spec, "iptables: Chain already exists."))
|
||||
} else {
|
||||
state.chains.insert(key, BTreeSet::new());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
"-C" | "-D" => {
|
||||
if operation == "-D"
|
||||
&& let Some(message) = state.failure.take()
|
||||
{
|
||||
return Err(classified_error(&spec, message));
|
||||
}
|
||||
let target = (spec.binary, spec.args[1].clone(), spec.args[5].clone());
|
||||
if !state.chains.contains_key(&target) {
|
||||
return Err(classified_error(
|
||||
&spec,
|
||||
&missing_chain(spec.binary, &spec.args[5]),
|
||||
));
|
||||
}
|
||||
let rules = state.chains.get_mut(&key).expect("builtin chain exists");
|
||||
let rule = spec.args[4..].join(" ");
|
||||
if !rules.contains(&rule) {
|
||||
return Err(classified_error(
|
||||
&spec,
|
||||
"Bad rule (does a matching rule exist in that chain?).",
|
||||
));
|
||||
}
|
||||
if operation == "-D" {
|
||||
rules.remove(&rule);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
"-I" => {
|
||||
let target = (spec.binary, spec.args[1].clone(), spec.args[6].clone());
|
||||
if !state.chains.contains_key(&target) {
|
||||
return Err(classified_error(
|
||||
&spec,
|
||||
&missing_chain(spec.binary, &spec.args[6]),
|
||||
));
|
||||
}
|
||||
state
|
||||
.chains
|
||||
.get_mut(&key)
|
||||
.unwrap()
|
||||
.insert(spec.args[5..].join(" "));
|
||||
Ok(())
|
||||
}
|
||||
"-F" | "-X" => {
|
||||
let Some(rules) = state.chains.get_mut(&key) else {
|
||||
return Err(classified_error(
|
||||
&spec,
|
||||
"iptables: No chain/target/match by that name.",
|
||||
));
|
||||
};
|
||||
if operation == "-F" {
|
||||
rules.clear();
|
||||
} else {
|
||||
assert!(rules.is_empty());
|
||||
state.chains.remove(&key);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => panic!("unexpected iptables operation"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoted_missing_chains_are_scoped_to_owned_commands() {
|
||||
for binary in ["iptables", "ip6tables"] {
|
||||
for operation in ["-C", "-D"] {
|
||||
let spec = CommandSpec::new(
|
||||
binary,
|
||||
["-t", "raw", operation, "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||
);
|
||||
let message = missing_chain(binary, "TELEMT_NOTRACK");
|
||||
let error = classified_error(&spec, &message);
|
||||
assert_eq!(error.kind, CommandErrorKind::NotFound, "{spec:?}");
|
||||
assert_eq!(error.message, message);
|
||||
}
|
||||
for operation in ["-F", "-X"] {
|
||||
for chain in OWNED_CHAINS {
|
||||
let spec = CommandSpec::new(binary, ["-t", "raw", operation, chain]);
|
||||
assert_eq!(
|
||||
classified_error(&spec, &missing_chain(binary, chain)).kind,
|
||||
CommandErrorKind::NotFound,
|
||||
);
|
||||
}
|
||||
}
|
||||
let spec = CommandSpec::new(binary, ["-t", "raw", "-F", "TELEMT_NOTRACK"]);
|
||||
for message in [
|
||||
"Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||
format!("{binary} v1.8.11 (nf_tables): Chain 'TELEMT_NOTRACK' does not exist"),
|
||||
] {
|
||||
assert_eq!(
|
||||
classified_error(&spec, &message).kind,
|
||||
CommandErrorKind::NotFound
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_commands_and_diagnostics_remain_failures() {
|
||||
let message = missing_chain("iptables", "TELEMT_NOTRACK");
|
||||
for spec in [
|
||||
CommandSpec::new("iptables", ["-t", "raw", "-N", "TELEMT_NOTRACK"]),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-A", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||
),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-I", "PREROUTING", "1", "-j", "TELEMT_NOTRACK"],
|
||||
),
|
||||
CommandSpec::new("iptables", ["-t", "filter", "-F", "TELEMT_NOTRACK"]),
|
||||
CommandSpec::new("iptables", ["-t", "raw", "-F", "TELEMT_NT_A"]),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NT_A"],
|
||||
),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-D", "INPUT", "-j", "TELEMT_NOTRACK"],
|
||||
),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-D", "PREROUTING", "-g", "TELEMT_NOTRACK"],
|
||||
),
|
||||
CommandSpec::new("iptables", ["-F", "TELEMT_NOTRACK"]),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-F", "TELEMT_NOTRACK", "--invalid"],
|
||||
),
|
||||
CommandSpec::new("iptables-restore", ["--noflush"]),
|
||||
CommandSpec::new("ip6tables-restore", ["--noflush"]),
|
||||
CommandSpec::new("nft", ["delete", "table", "inet", "telemt_conntrack"]),
|
||||
CommandSpec::new("conntrack", ["-D"]),
|
||||
] {
|
||||
assert_eq!(
|
||||
classified_error(&spec, &message).kind,
|
||||
CommandErrorKind::Failed,
|
||||
"{spec:?}",
|
||||
);
|
||||
}
|
||||
for chain in [
|
||||
"TELEMT_NT_C",
|
||||
"TELEMT_NOTRACK_EXTRA",
|
||||
"MTPR_SYNFIX",
|
||||
"PREROUTING",
|
||||
] {
|
||||
let spec = CommandSpec::new("iptables", ["-t", "raw", "-F", chain]);
|
||||
assert_eq!(
|
||||
classified_error(&spec, &missing_chain("iptables", chain)).kind,
|
||||
CommandErrorKind::Failed,
|
||||
);
|
||||
}
|
||||
let spec = CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||
);
|
||||
for message in [
|
||||
"Permission denied",
|
||||
"can't initialize iptables table `raw': Table does not exist",
|
||||
"Another app is currently holding the xtables lock",
|
||||
"Chain is not empty",
|
||||
"Directory not empty",
|
||||
"Can't delete chain with references left",
|
||||
"unknown option --invalid",
|
||||
"command timed out",
|
||||
] {
|
||||
assert_eq!(
|
||||
classified_error(&spec, message).kind,
|
||||
CommandErrorKind::Failed
|
||||
);
|
||||
}
|
||||
for message in [
|
||||
format!("{message}\nPermission denied"),
|
||||
format!("Permission denied\n{message}"),
|
||||
"Chain 'TELEMT_NOTRACK' does not exist\nPermission denied".to_string(),
|
||||
"Chain 'TELEMT_NOTRACK' does not exist\n\
|
||||
Try `ip6tables -h' or 'ip6tables --help' for more information."
|
||||
.to_string(),
|
||||
"iptables v1.8.10 (legacy): Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||
"iptables v (nf_tables): Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||
missing_chain("ip6tables", "TELEMT_NOTRACK"),
|
||||
missing_chain("iptables", "TELEMT_NT_A"),
|
||||
] {
|
||||
assert_eq!(
|
||||
classified_error(&spec, &message).kind,
|
||||
CommandErrorKind::Failed
|
||||
);
|
||||
}
|
||||
for binary in ["nft", "conntrack", "iptables-restore", "ip6tables-restore"] {
|
||||
let spec = CommandSpec::new(binary, ["-t", "raw", "-F", "TELEMT_NOTRACK"]);
|
||||
assert_eq!(
|
||||
classified_error(&spec, "Chain 'TELEMT_NOTRACK' does not exist").kind,
|
||||
CommandErrorKind::Failed,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_recovery_converges_without_touching_foreign_rules() {
|
||||
for policy in [DesiredPolicy::Empty, dual_stack_policy(443)] {
|
||||
let runner = FixtureRunner::new(None);
|
||||
let foreign = runner.foreign_snapshot();
|
||||
let mut applied = AppliedState::Unknown;
|
||||
let requested = desired(1, policy);
|
||||
reconcile_once(&runner, &runner, &mut applied, &requested)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
&applied,
|
||||
AppliedState::Known(plan) if plan.matches_policy(&requested.policy)
|
||||
));
|
||||
runner.assert_owned_policy(matches!(requested.policy, DesiredPolicy::Rules { .. }));
|
||||
assert_eq!(runner.foreign_snapshot(), foreign);
|
||||
for binary in ["iptables", "ip6tables"] {
|
||||
let calls = runner.calls();
|
||||
assert_eq!(
|
||||
calls
|
||||
.iter()
|
||||
.filter(|call| call.binary == binary && call.args[2] == "-D")
|
||||
.count(),
|
||||
1,
|
||||
);
|
||||
assert_eq!(
|
||||
calls
|
||||
.iter()
|
||||
.filter(|call| {
|
||||
call.binary == binary && matches!(call.args[2].as_str(), "-F" | "-X")
|
||||
})
|
||||
.count(),
|
||||
6,
|
||||
);
|
||||
}
|
||||
let calls = runner.calls().len();
|
||||
reconcile_once(
|
||||
&runner,
|
||||
&runner,
|
||||
&mut applied,
|
||||
&desired(2, requested.policy.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(runner.calls().len(), calls);
|
||||
recover_to_empty(&runner).await.unwrap();
|
||||
recover_to_empty(&runner).await.unwrap();
|
||||
runner.assert_owned_policy(false);
|
||||
assert_eq!(runner.foreign_snapshot(), foreign);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn genuine_recovery_failures_leave_state_unknown_and_do_not_install() {
|
||||
for message in [
|
||||
"Permission denied",
|
||||
"can't initialize iptables table `raw': Table does not exist",
|
||||
"Another app is currently holding the xtables lock",
|
||||
"unknown firewall failure",
|
||||
] {
|
||||
let runner = FixtureRunner::new(Some(message));
|
||||
let mut applied = AppliedState::Unknown;
|
||||
let error = reconcile_once(
|
||||
&runner,
|
||||
&runner,
|
||||
&mut applied,
|
||||
&desired(1, dual_stack_policy(443)),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.message.contains(message));
|
||||
assert_eq!(applied, AppliedState::Unknown);
|
||||
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn actor_converges_then_idles_and_preserves_foreign_rules_on_shutdown() {
|
||||
for failure in [None, Some("Permission denied")] {
|
||||
let runner = FixtureRunner::new(failure);
|
||||
let observed = runner.clone();
|
||||
let foreign = observed.foreign_snapshot();
|
||||
let (desired_tx, desired_rx) = watch::channel(None);
|
||||
let (status_tx, mut status_rx) = watch::channel(None);
|
||||
let terminal = CancellationToken::new();
|
||||
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cleanup = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let reconciler = FirewallReconciler::new(
|
||||
runner,
|
||||
desired_rx,
|
||||
status_tx,
|
||||
terminal.clone(),
|
||||
closed.clone(),
|
||||
completed.clone(),
|
||||
cleanup.clone(),
|
||||
Arc::new(Notify::new()),
|
||||
);
|
||||
let task = tokio::spawn(reconciler.run(CancellationToken::new()));
|
||||
let requested = desired(1, dual_stack_policy(443));
|
||||
let stats = requested.stats.clone();
|
||||
desired_tx.send_replace(Some(requested));
|
||||
status_rx.changed().await.unwrap();
|
||||
if failure.is_some() {
|
||||
assert_eq!(
|
||||
status_rx.borrow().as_ref().unwrap().outcome,
|
||||
ReconcileOutcome::Failed,
|
||||
);
|
||||
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 1);
|
||||
let calls = observed.calls().len();
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(observed.calls().len(), calls);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
status_rx.changed().await.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
status_rx.borrow().as_ref().unwrap().outcome,
|
||||
ReconcileOutcome::Applied,
|
||||
);
|
||||
assert_eq!(status_rx.borrow().as_ref().unwrap().generation, 1);
|
||||
assert!(stats.get_conntrack_rule_apply_ok());
|
||||
assert_eq!(stats.get_conntrack_rule_reconcile_success_total(), 1);
|
||||
assert_eq!(
|
||||
stats.get_conntrack_rule_reconcile_error_total(),
|
||||
u64::from(failure.is_some()),
|
||||
);
|
||||
let calls = observed.calls().len();
|
||||
tokio::time::advance(Duration::from_secs(90)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(observed.calls().len(), calls);
|
||||
assert!(!status_rx.has_changed().unwrap());
|
||||
observed.assert_owned_policy(true);
|
||||
assert_eq!(observed.foreign_snapshot(), foreign);
|
||||
terminal.cancel();
|
||||
tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(closed.load(Ordering::Acquire));
|
||||
assert!(completed.load(Ordering::Acquire));
|
||||
assert!(cleanup.load(Ordering::Acquire));
|
||||
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||
observed.assert_owned_policy(false);
|
||||
assert_eq!(observed.foreign_snapshot(), foreign);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
use crate::config::{ConntrackBackend, ConntrackMode};
|
||||
|
||||
use super::super::command::{CommandErrorKind, CommandSpec, classify_command_error};
|
||||
use super::super::transaction::reconcile_once;
|
||||
use super::*;
|
||||
|
||||
const NETLINK_ERROR: &str =
|
||||
"src/mnl.c:68: Unable to initialize Netlink socket: Address family not supported by protocol";
|
||||
const PERMISSION_ERROR: &str = "iptables v1.8.11 (nf_tables): Could not fetch rule set generation id: Permission denied (you must be root)";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct StartupRunner {
|
||||
has_cap_net_admin: bool,
|
||||
cap_probes: Arc<AtomicUsize>,
|
||||
calls: Arc<Mutex<Vec<CommandSpec>>>,
|
||||
failure: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl StartupRunner {
|
||||
fn new(has_cap_net_admin: bool, failure: Option<&'static str>) -> Self {
|
||||
Self {
|
||||
has_cap_net_admin,
|
||||
cap_probes: Arc::new(AtomicUsize::new(0)),
|
||||
calls: Arc::new(Mutex::new(Vec::new())),
|
||||
failure,
|
||||
}
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<CommandSpec> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl FirewallCommandRunner for StartupRunner {
|
||||
fn available(&self, _binary: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn has_cap_net_admin(&self) -> bool {
|
||||
self.cap_probes.fetch_add(1, Ordering::Relaxed);
|
||||
self.has_cap_net_admin
|
||||
}
|
||||
|
||||
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
|
||||
self.calls.lock().unwrap().push(spec.clone());
|
||||
if let Some(message) = self.failure {
|
||||
return Err(CommandError {
|
||||
kind: classify_command_error(spec.binary, &spec.args, message),
|
||||
message: message.to_string(),
|
||||
});
|
||||
}
|
||||
if (spec.binary == "nft" && spec.args.first().map(String::as_str) == Some("delete"))
|
||||
|| (matches!(spec.binary, "iptables" | "ip6tables")
|
||||
&& matches!(
|
||||
spec.args.get(2).map(String::as_str),
|
||||
Some("-C" | "-D" | "-F" | "-X")
|
||||
))
|
||||
{
|
||||
return Err(CommandError {
|
||||
kind: CommandErrorKind::NotFound,
|
||||
message: "injected absent owned object".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn start_with_runner(
|
||||
config: &ProxyConfig,
|
||||
runner: StartupRunner,
|
||||
scope: &ProcessControlPlane,
|
||||
) -> Option<FirewallAuthority> {
|
||||
let (authority, actor) = FirewallAuthority::prepare(config, runner)?;
|
||||
assert!(
|
||||
scope
|
||||
.spawn_cooperative(move |cancellation| actor.run(cancellation))
|
||||
.is_ok()
|
||||
);
|
||||
Some(authority)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disabled_startup_never_acquires_cleanup_ownership() {
|
||||
for explicit in [false, true] {
|
||||
for has_cap in [true, false] {
|
||||
for backend in [
|
||||
ConntrackBackend::Auto,
|
||||
ConntrackBackend::Nftables,
|
||||
ConntrackBackend::Iptables,
|
||||
] {
|
||||
for mode in [
|
||||
ConntrackMode::Tracked,
|
||||
ConntrackMode::Notrack,
|
||||
ConntrackMode::Hybrid,
|
||||
] {
|
||||
for message in [NETLINK_ERROR, PERMISSION_ERROR] {
|
||||
let mut config = ProxyConfig::default();
|
||||
if explicit {
|
||||
config.server.conntrack_control.inline_conntrack_control = false;
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.inline_conntrack_control_explicit = true;
|
||||
}
|
||||
config.server.conntrack_control.backend = backend;
|
||||
config.server.conntrack_control.mode = mode;
|
||||
let runner = StartupRunner::new(has_cap, Some(message));
|
||||
let scope = ProcessControlPlane::new();
|
||||
let authority = start_with_runner(&config, runner.clone(), &scope);
|
||||
assert!(
|
||||
authority.is_none(),
|
||||
"{explicit} {has_cap} {backend:?} {mode:?}"
|
||||
);
|
||||
assert_eq!(runner.cap_probes.load(Ordering::Relaxed), 0);
|
||||
tokio::time::advance(Duration::from_secs(90)).await;
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(runner.calls().is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn enabled_startup_without_capability_creates_no_firewall_work() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.server.conntrack_control.inline_conntrack_control = true;
|
||||
for mode in [ConntrackMode::Tracked, ConntrackMode::Notrack] {
|
||||
config.server.conntrack_control.mode = mode;
|
||||
let runner = StartupRunner::new(false, Some(PERMISSION_ERROR));
|
||||
let scope = ProcessControlPlane::new();
|
||||
assert!(start_with_runner(&config, runner.clone(), &scope).is_none());
|
||||
assert_eq!(runner.cap_probes.load(Ordering::Relaxed), 1);
|
||||
tokio::time::advance(Duration::from_secs(90)).await;
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(runner.calls().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn admitted_startup_preserves_recovery_idle_and_shutdown_cleanup() {
|
||||
for mode in [ConntrackMode::Tracked, ConntrackMode::Notrack] {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.server.conntrack_control.inline_conntrack_control = true;
|
||||
config.server.conntrack_control.mode = mode;
|
||||
config.server.conntrack_control.backend = ConntrackBackend::Nftables;
|
||||
let runner = StartupRunner::new(true, None);
|
||||
let scope = ProcessControlPlane::new();
|
||||
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||
let stats = Arc::new(Stats::new());
|
||||
assert!(
|
||||
authority
|
||||
.publish_initial(1, Arc::new(config), stats.clone())
|
||||
.await
|
||||
);
|
||||
assert!(stats.get_conntrack_rule_apply_ok());
|
||||
for binary in ["nft", "iptables", "ip6tables"] {
|
||||
assert!(runner.calls().iter().any(|call| call.binary == binary));
|
||||
}
|
||||
let calls = runner.calls().len();
|
||||
tokio::time::advance(Duration::from_secs(90)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(runner.calls().len(), calls);
|
||||
assert!(authority.shutdown_and_clear().await);
|
||||
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||
assert!(runner.calls().len() > calls);
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn admitted_process_cancellation_preserves_cleanup() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.server.conntrack_control.inline_conntrack_control = true;
|
||||
let runner = StartupRunner::new(true, None);
|
||||
let scope = ProcessControlPlane::new();
|
||||
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(authority.shutdown_and_clear().await);
|
||||
assert!(!runner.calls().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn netlink_and_permission_errors_are_not_classified_as_absence() {
|
||||
for spec in [
|
||||
CommandSpec::new("nft", ["delete", "table", "inet", "telemt_conntrack"]),
|
||||
CommandSpec::new("nft", ["-f", "-"]),
|
||||
CommandSpec::new(
|
||||
"iptables",
|
||||
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||
),
|
||||
CommandSpec::new("iptables", ["-t", "raw", "-F", "TELEMT_NT_A"]),
|
||||
CommandSpec::new("ip6tables", ["-t", "raw", "-X", "TELEMT_NT_B"]),
|
||||
CommandSpec::new("iptables-restore", ["--noflush"]),
|
||||
CommandSpec::new("ip6tables-restore", ["--noflush"]),
|
||||
] {
|
||||
for message in [
|
||||
NETLINK_ERROR,
|
||||
PERMISSION_ERROR,
|
||||
"iptables: Failed to initialize nft: Address family not supported by protocol",
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_command_error(spec.binary, &spec.args, message),
|
||||
CommandErrorKind::Failed,
|
||||
"{spec:?} {message}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn admitted_genuine_failures_remain_unknown_and_retry() {
|
||||
for message in [NETLINK_ERROR, PERMISSION_ERROR] {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.server.conntrack_control.inline_conntrack_control = true;
|
||||
config.server.conntrack_control.mode = ConntrackMode::Notrack;
|
||||
config.server.conntrack_control.backend = ConntrackBackend::Nftables;
|
||||
let runner = StartupRunner::new(true, Some(message));
|
||||
let mut applied = AppliedState::Unknown;
|
||||
let requested = DesiredState {
|
||||
generation: 1,
|
||||
policy: DesiredPolicy::from_config(&config),
|
||||
stats: Arc::new(Stats::new()),
|
||||
};
|
||||
let error = reconcile_once(&runner, &runner, &mut applied, &requested)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.message.contains(message));
|
||||
assert_eq!(applied, AppliedState::Unknown);
|
||||
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||
|
||||
let scope = ProcessControlPlane::new();
|
||||
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||
let stats = Arc::new(Stats::new());
|
||||
assert!(
|
||||
!authority
|
||||
.publish_initial(1, Arc::new(config), stats.clone())
|
||||
.await
|
||||
);
|
||||
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 1);
|
||||
let mut status = authority.status_rx.clone();
|
||||
assert_eq!(
|
||||
status.borrow_and_update().as_ref().unwrap().outcome,
|
||||
ReconcileOutcome::Failed,
|
||||
);
|
||||
let calls = runner.calls().len();
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(runner.calls().len(), calls);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
status.changed().await.unwrap();
|
||||
assert_eq!(
|
||||
status.borrow().as_ref().unwrap().outcome,
|
||||
ReconcileOutcome::Failed,
|
||||
);
|
||||
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 2);
|
||||
assert!(!authority.shutdown_and_clear().await);
|
||||
assert!(authority.completed_flag.load(Ordering::Acquire));
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||
}
|
||||
}
|
||||
+8
-3
@@ -9,7 +9,7 @@ use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use nix::errno::Errno;
|
||||
use nix::unistd::{self, ForkResult, Gid, Uid, chdir, close, fork, getpid, setsid};
|
||||
use nix::unistd::{self, ForkResult, Gid, Uid, chdir, fork, getpid, setsid};
|
||||
use tracing::info;
|
||||
|
||||
// PID file ownership and process-control helpers.
|
||||
@@ -29,6 +29,8 @@ pub struct DaemonOptions {
|
||||
pub daemonize: bool,
|
||||
/// Path to PID file.
|
||||
pub pid_file: Option<PathBuf>,
|
||||
/// Require trusted, symlink-free PID and log parents. Disabled by default for compatibility.
|
||||
pub strict_runtime_paths: bool,
|
||||
/// User to run as after binding sockets.
|
||||
pub user: Option<String>,
|
||||
/// Group to run as after binding sockets.
|
||||
@@ -174,8 +176,10 @@ fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
|
||||
}
|
||||
}
|
||||
|
||||
if devnull_fd > 2 {
|
||||
let _ = close(devnull_fd);
|
||||
// Keep stdio descriptors open; other source descriptors are closed once by File's Drop.
|
||||
// Transfer ownership only after all dup2 calls succeed so errors retain RAII cleanup.
|
||||
if devnull_fd <= 2 {
|
||||
let _ = std::os::unix::io::IntoRawFd::into_raw_fd(devnull);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -332,6 +336,7 @@ mod tests {
|
||||
fn test_daemon_options_default() {
|
||||
let opts = DaemonOptions::default();
|
||||
assert!(!opts.daemonize);
|
||||
assert!(!opts.strict_runtime_paths);
|
||||
assert!(!opts.should_daemonize());
|
||||
assert_eq!(opts.pid_file_path(), Path::new(DEFAULT_PID_FILE));
|
||||
}
|
||||
|
||||
+34
-21
@@ -17,6 +17,7 @@ use crate::util::secure_fs::AnchoredPath;
|
||||
/// PID file manager backed by a persistent sibling lock file.
|
||||
pub struct PidFile {
|
||||
path: PathBuf,
|
||||
strict_runtime_paths: bool,
|
||||
lock_path: PathBuf,
|
||||
pid_file: Option<File>,
|
||||
pid_identity: Option<FileIdentity>,
|
||||
@@ -40,12 +41,13 @@ impl FileIdentity {
|
||||
}
|
||||
|
||||
impl PidFile {
|
||||
/// Creates a new PID file manager for the given path.
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
||||
/// Creates a PID manager with explicit parent-path policy; `false` allows legacy parents.
|
||||
pub fn new<P: AsRef<Path>>(path: P, strict_runtime_paths: bool) -> Self {
|
||||
let path = normalize_pid_path(path.as_ref());
|
||||
let lock_path = sibling_lock_path(&path);
|
||||
Self {
|
||||
path,
|
||||
strict_runtime_paths,
|
||||
lock_path,
|
||||
pid_file: None,
|
||||
pid_identity: None,
|
||||
@@ -56,7 +58,7 @@ impl PidFile {
|
||||
|
||||
/// Checks whether the PID file names a running process without modifying either file.
|
||||
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
|
||||
let Some(pid) = read_pid_file_if_exists(&self.path)? else {
|
||||
let Some(pid) = read_pid_file_if_exists(&self.path, self.strict_runtime_paths)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(is_process_running(pid).then_some(pid))
|
||||
@@ -67,9 +69,10 @@ impl PidFile {
|
||||
/// Fails if another owner holds the lock or the existing PID names a running process.
|
||||
pub fn acquire(&mut self) -> Result<(), DaemonError> {
|
||||
let anchor =
|
||||
AnchoredPath::open_trusted_parent_or_create(&self.path, 0o755).map_err(|error| {
|
||||
AnchoredPath::open_runtime_parent(&self.path, Some(0o755), self.strict_runtime_paths)
|
||||
.map_err(|error| {
|
||||
DaemonError::PidFile(format!(
|
||||
"cannot open trusted parent for {}: {}",
|
||||
"cannot open PID parent for {}: {}",
|
||||
self.path.display(),
|
||||
error
|
||||
))
|
||||
@@ -245,13 +248,16 @@ fn open_file_at(anchor: &AnchoredPath, name: &OsStr, flags: OFlag, mode: u32) ->
|
||||
Ok(File::from(descriptor))
|
||||
}
|
||||
|
||||
fn read_pid_file_if_exists(path: &Path) -> Result<Option<i32>, DaemonError> {
|
||||
let anchor = match AnchoredPath::open_trusted_parent(path) {
|
||||
fn read_pid_file_if_exists(
|
||||
path: &Path,
|
||||
strict_runtime_paths: bool,
|
||||
) -> Result<Option<i32>, DaemonError> {
|
||||
let anchor = match AnchoredPath::open_runtime_parent(path, None, strict_runtime_paths) {
|
||||
Ok(anchor) => anchor,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => {
|
||||
return Err(DaemonError::PidFile(format!(
|
||||
"cannot open trusted parent for {}: {}",
|
||||
"cannot open PID parent for {}: {}",
|
||||
path.display(),
|
||||
error
|
||||
)));
|
||||
@@ -352,11 +358,14 @@ fn validate_regular_single_link(file: &File, path: &Path) -> Result<fs::Metadata
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// Reads a PID from a PID file.
|
||||
/// Reads a PID using the selected parent-path policy; `false` allows legacy parents.
|
||||
#[allow(dead_code)]
|
||||
pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
|
||||
pub fn read_pid_file<P: AsRef<Path>>(
|
||||
path: P,
|
||||
strict_runtime_paths: bool,
|
||||
) -> Result<i32, DaemonError> {
|
||||
let path = normalize_pid_path(path.as_ref());
|
||||
read_pid_file_if_exists(&path)?.ok_or_else(|| {
|
||||
read_pid_file_if_exists(&path, strict_runtime_paths)?.ok_or_else(|| {
|
||||
DaemonError::PidFile(format!(
|
||||
"cannot read {}: file does not exist",
|
||||
path.display()
|
||||
@@ -364,17 +373,18 @@ pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Sends a signal to the process specified in a PID file.
|
||||
/// Signals a lock-owning process using the same parent-path policy for PID and lock files.
|
||||
#[allow(dead_code)]
|
||||
pub fn signal_pid_file<P: AsRef<Path>>(
|
||||
path: P,
|
||||
signal: nix::sys::signal::Signal,
|
||||
strict_runtime_paths: bool,
|
||||
) -> Result<(), DaemonError> {
|
||||
let path = normalize_pid_path(path.as_ref());
|
||||
let pid = read_pid_file(&path)?;
|
||||
let pid = read_pid_file(&path, strict_runtime_paths)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
let pidfd = open_pidfd(pid)?;
|
||||
if !daemon_lock_is_held(&path)? {
|
||||
if !daemon_lock_is_held(&path, strict_runtime_paths)? {
|
||||
return Err(DaemonError::PidFile(format!(
|
||||
"refusing to signal unlocked or stale PID file {}",
|
||||
path.display()
|
||||
@@ -399,12 +409,15 @@ pub enum DaemonStatus {
|
||||
NotRunning,
|
||||
}
|
||||
|
||||
/// Checks daemon status without modifying the PID or lock file.
|
||||
/// Checks daemon status read-only, applying the selected policy to both parent lookups.
|
||||
#[allow(dead_code)]
|
||||
pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
|
||||
pub fn check_status<P: AsRef<Path>>(path: P, strict_runtime_paths: bool) -> DaemonStatus {
|
||||
let path = normalize_pid_path(path.as_ref());
|
||||
match read_pid_file_if_exists(&path) {
|
||||
Ok(Some(pid)) if daemon_lock_is_held(&path).unwrap_or(false) && is_process_running(pid) => {
|
||||
match read_pid_file_if_exists(&path, strict_runtime_paths) {
|
||||
Ok(Some(pid))
|
||||
if daemon_lock_is_held(&path, strict_runtime_paths).unwrap_or(false)
|
||||
&& is_process_running(pid) =>
|
||||
{
|
||||
DaemonStatus::Running(pid)
|
||||
}
|
||||
Ok(Some(pid)) => DaemonStatus::Stale(pid),
|
||||
@@ -412,14 +425,14 @@ pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
|
||||
}
|
||||
}
|
||||
|
||||
fn daemon_lock_is_held(path: &Path) -> Result<bool, DaemonError> {
|
||||
fn daemon_lock_is_held(path: &Path, strict_runtime_paths: bool) -> Result<bool, DaemonError> {
|
||||
let lock_path = sibling_lock_path(path);
|
||||
let anchor = match AnchoredPath::open_trusted_parent(path) {
|
||||
let anchor = match AnchoredPath::open_runtime_parent(path, None, strict_runtime_paths) {
|
||||
Ok(anchor) => anchor,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => {
|
||||
return Err(DaemonError::PidFile(format!(
|
||||
"cannot open trusted parent for {}: {}",
|
||||
"cannot open PID parent for {}: {}",
|
||||
path.display(),
|
||||
error
|
||||
)));
|
||||
|
||||
+132
-23
@@ -38,6 +38,107 @@ fn pid_file_remains_send_and_sync() {
|
||||
assert_send_sync::<PidFile>();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compatibility_pid_lifecycle_accepts_writable_parent_directories() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
for mode in [0o770, 0o777, 0o1777] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
fs::set_permissions(directory.path(), fs::Permissions::from_mode(mode)).unwrap();
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
let mut strict_owner = PidFile::new(&pid_path, true);
|
||||
assert!(strict_owner.acquire().is_err());
|
||||
assert!(!pid_path.exists());
|
||||
assert!(!sibling_lock_path(&pid_path).exists());
|
||||
let mut owner = PidFile::new(&pid_path, false);
|
||||
owner.acquire().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
read_pid_file(&pid_path, false).unwrap(),
|
||||
std::process::id() as i32
|
||||
);
|
||||
assert_eq!(
|
||||
owner.check_running().unwrap(),
|
||||
Some(std::process::id() as i32)
|
||||
);
|
||||
assert_eq!(
|
||||
check_status(&pid_path, false),
|
||||
DaemonStatus::Running(std::process::id() as i32)
|
||||
);
|
||||
assert!(read_pid_file(&pid_path, true).is_err());
|
||||
assert!(strict_owner.check_running().is_err());
|
||||
assert_eq!(check_status(&pid_path, true), DaemonStatus::NotRunning);
|
||||
|
||||
owner.release().unwrap();
|
||||
assert!(!pid_path.exists());
|
||||
assert!(sibling_lock_path(&pid_path).exists());
|
||||
assert_eq!(check_status(&pid_path, false), DaemonStatus::NotRunning);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compatibility_pid_lifecycle_follows_symlinked_parents() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let real = directory.path().join("tmp");
|
||||
let linked = directory.path().join("var");
|
||||
fs::create_dir(&real).unwrap();
|
||||
symlink("tmp", &linked).unwrap();
|
||||
let pid_path = linked.join("run/telemt.pid");
|
||||
let mut strict_owner = PidFile::new(&pid_path, true);
|
||||
assert!(strict_owner.acquire().is_err());
|
||||
assert!(!real.join("run").exists());
|
||||
let mut owner = PidFile::new(&pid_path, false);
|
||||
owner.acquire().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
read_pid_file(&pid_path, false).unwrap(),
|
||||
std::process::id() as i32
|
||||
);
|
||||
assert_eq!(
|
||||
check_status(&pid_path, false),
|
||||
DaemonStatus::Running(std::process::id() as i32)
|
||||
);
|
||||
assert!(real.join("run/telemt.pid.lock").exists());
|
||||
assert!(read_pid_file(&pid_path, true).is_err());
|
||||
assert_eq!(check_status(&pid_path, true), DaemonStatus::NotRunning);
|
||||
|
||||
owner.release().unwrap();
|
||||
assert!(!real.join("run/telemt.pid").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_pid_parent_supports_both_policies() {
|
||||
for strict_runtime_paths in [false, true] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_path = directory.path().join("nested/run/telemt.pid");
|
||||
let mut owner = PidFile::new(&pid_path, strict_runtime_paths);
|
||||
owner.acquire().unwrap();
|
||||
|
||||
for read_strict in [false, true] {
|
||||
assert_eq!(
|
||||
read_pid_file(&pid_path, read_strict).unwrap(),
|
||||
std::process::id() as i32
|
||||
);
|
||||
assert_eq!(
|
||||
check_status(&pid_path, read_strict),
|
||||
DaemonStatus::Running(std::process::id() as i32)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
fs::metadata(pid_path.parent().unwrap()).unwrap().mode() & 0o777,
|
||||
0o755
|
||||
);
|
||||
let mut contender = PidFile::new(&pid_path, strict_runtime_paths);
|
||||
assert!(matches!(
|
||||
contender.acquire(),
|
||||
Err(DaemonError::AlreadyRunning(_))
|
||||
));
|
||||
owner.release().unwrap();
|
||||
contender.acquire().unwrap();
|
||||
contender.release().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_var_run_alias_keeps_the_default_pid_path_usable() {
|
||||
let Ok(metadata) = fs::symlink_metadata("/var/run") else {
|
||||
@@ -52,7 +153,7 @@ fn system_var_run_alias_keeps_the_default_pid_path_usable() {
|
||||
return;
|
||||
}
|
||||
|
||||
let pid_file = PidFile::new("/var/run/telemt.pid");
|
||||
let pid_file = PidFile::new("/var/run/telemt.pid", false);
|
||||
|
||||
assert_eq!(pid_file.path(), Path::new("/run/telemt.pid"));
|
||||
}
|
||||
@@ -64,7 +165,7 @@ fn lock_holder_subprocess() {
|
||||
};
|
||||
let ready_path = PathBuf::from(std::env::var_os(HELPER_READY_PATH).unwrap());
|
||||
let stop_path = PathBuf::from(std::env::var_os(HELPER_STOP_PATH).unwrap());
|
||||
let mut pid_file = PidFile::new(PathBuf::from(pid_path));
|
||||
let mut pid_file = PidFile::new(PathBuf::from(pid_path), false);
|
||||
pid_file.acquire().unwrap();
|
||||
fs::write(&ready_path, b"ready").unwrap();
|
||||
assert!(wait_for_path(&stop_path, Duration::from_secs(10)));
|
||||
@@ -100,7 +201,7 @@ fn persistent_sibling_lock_serializes_processes_after_pid_unlink() {
|
||||
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
||||
fs::remove_file(&pid_path).unwrap();
|
||||
|
||||
let mut contender = PidFile::new(&pid_path);
|
||||
let mut contender = PidFile::new(&pid_path, false);
|
||||
assert!(contender.acquire().is_err());
|
||||
|
||||
fs::write(&stop_path, b"stop").unwrap();
|
||||
@@ -123,10 +224,13 @@ fn stale_pid_checks_are_read_only() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
fs::write(&pid_path, b"2000000000\n").unwrap();
|
||||
let pid_file = PidFile::new(&pid_path);
|
||||
let pid_file = PidFile::new(&pid_path, false);
|
||||
|
||||
assert_eq!(pid_file.check_running().unwrap(), None);
|
||||
assert_eq!(check_status(&pid_path), DaemonStatus::Stale(2_000_000_000));
|
||||
assert_eq!(
|
||||
check_status(&pid_path, false),
|
||||
DaemonStatus::Stale(2_000_000_000)
|
||||
);
|
||||
assert!(pid_path.exists());
|
||||
}
|
||||
|
||||
@@ -136,15 +240,15 @@ fn status_requires_live_lock_ownership() {
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
fs::write(&pid_path, format!("{}\n", std::process::id())).unwrap();
|
||||
assert_eq!(
|
||||
check_status(&pid_path),
|
||||
check_status(&pid_path, false),
|
||||
DaemonStatus::Stale(std::process::id() as i32)
|
||||
);
|
||||
|
||||
fs::remove_file(&pid_path).unwrap();
|
||||
let mut owner = PidFile::new(&pid_path);
|
||||
let mut owner = PidFile::new(&pid_path, false);
|
||||
owner.acquire().unwrap();
|
||||
assert_eq!(
|
||||
check_status(&pid_path),
|
||||
check_status(&pid_path, false),
|
||||
DaemonStatus::Running(std::process::id() as i32)
|
||||
);
|
||||
owner.release().unwrap();
|
||||
@@ -155,7 +259,7 @@ fn unowned_release_does_not_remove_pid_file() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
fs::write(&pid_path, b"2000000000\n").unwrap();
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path, false);
|
||||
|
||||
pid_file.release().unwrap();
|
||||
|
||||
@@ -169,10 +273,11 @@ fn acquire_rejects_pid_symlink_without_truncating_target() {
|
||||
let target_path = directory.path().join("target");
|
||||
fs::write(&target_path, b"preserve\n").unwrap();
|
||||
symlink(&target_path, &pid_path).unwrap();
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
|
||||
assert!(pid_file.acquire().is_err());
|
||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||
for strict_runtime_paths in [false, true] {
|
||||
let mut pid_file = PidFile::new(&pid_path, strict_runtime_paths);
|
||||
assert!(pid_file.acquire().is_err());
|
||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -182,10 +287,11 @@ fn acquire_rejects_pid_hard_link_without_truncating_target() {
|
||||
let target_path = directory.path().join("target");
|
||||
fs::write(&target_path, b"preserve\n").unwrap();
|
||||
fs::hard_link(&target_path, &pid_path).unwrap();
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
|
||||
assert!(pid_file.acquire().is_err());
|
||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||
for strict_runtime_paths in [false, true] {
|
||||
let mut pid_file = PidFile::new(&pid_path, strict_runtime_paths);
|
||||
assert!(pid_file.acquire().is_err());
|
||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -196,7 +302,7 @@ fn acquire_rejects_symlinked_parent_without_publishing_outside() {
|
||||
fs::create_dir(&real_parent).unwrap();
|
||||
symlink(&real_parent, &linked_parent).unwrap();
|
||||
let pid_path = linked_parent.join("telemt.pid");
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path, true);
|
||||
|
||||
assert!(pid_file.acquire().is_err());
|
||||
assert!(!real_parent.join("telemt.pid").exists());
|
||||
@@ -210,7 +316,7 @@ fn release_remains_anchored_after_parent_path_replacement() {
|
||||
let moved_parent = directory.path().join("moved");
|
||||
fs::create_dir(&active_parent).unwrap();
|
||||
let pid_path = active_parent.join("telemt.pid");
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path, false);
|
||||
pid_file.acquire().unwrap();
|
||||
|
||||
fs::rename(&active_parent, &moved_parent).unwrap();
|
||||
@@ -231,7 +337,7 @@ fn release_does_not_remove_replacement_path() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
let owned_path = directory.path().join("owned.pid");
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path, false);
|
||||
pid_file.acquire().unwrap();
|
||||
fs::rename(&pid_path, &owned_path).unwrap();
|
||||
fs::write(&pid_path, b"replacement\n").unwrap();
|
||||
@@ -253,7 +359,7 @@ fn pid_parser_rejects_process_group_values() {
|
||||
|
||||
for value in ["-1\n", "0\n", "1\n"] {
|
||||
fs::write(&pid_path, value).unwrap();
|
||||
assert!(read_pid_file(&pid_path).is_err());
|
||||
assert!(read_pid_file(&pid_path, false).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,7 +368,7 @@ fn pid_file_release_keeps_lock_inode() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let pid_path = directory.path().join("telemt.pid");
|
||||
let lock_path = sibling_lock_path(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path);
|
||||
let mut pid_file = PidFile::new(&pid_path, false);
|
||||
|
||||
pid_file.acquire().unwrap();
|
||||
assert!(
|
||||
@@ -271,7 +377,10 @@ fn pid_file_release_keeps_lock_inode() {
|
||||
.into_iter()
|
||||
.all(|file| file.is_some())
|
||||
);
|
||||
assert_eq!(read_pid_file(&pid_path).unwrap(), std::process::id() as i32);
|
||||
assert_eq!(
|
||||
read_pid_file(&pid_path, false).unwrap(),
|
||||
std::process::id() as i32
|
||||
);
|
||||
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
||||
pid_file.release().unwrap();
|
||||
|
||||
|
||||
+7
-11
@@ -77,6 +77,8 @@ pub struct LoggingOptions {
|
||||
pub destination: LogDestination,
|
||||
/// Disable ANSI colors.
|
||||
pub disable_colors: bool,
|
||||
/// Require trusted, symlink-free log parents on Unix. Disabled by default for compatibility.
|
||||
pub strict_runtime_paths: bool,
|
||||
}
|
||||
|
||||
/// Guard that must be held to keep file logging active.
|
||||
@@ -125,7 +127,6 @@ pub fn init_logging(
|
||||
|
||||
#[cfg(unix)]
|
||||
LogDestination::Syslog => {
|
||||
// Use a custom fmt layer that writes to syslog
|
||||
let fmt_layer = fmt::Layer::default()
|
||||
.with_ansi(false)
|
||||
.with_target(false)
|
||||
@@ -143,7 +144,8 @@ pub fn init_logging(
|
||||
|
||||
LogDestination::File { options } => {
|
||||
let file_appender =
|
||||
file::BoundedFileAppender::new(options.clone()).expect("Failed to open log file");
|
||||
file::BoundedFileAppender::new(options.clone(), opts.strict_runtime_paths)
|
||||
.expect("Failed to open log file");
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
let fmt_layer = fmt::Layer::default()
|
||||
@@ -175,14 +177,10 @@ struct SyslogWriter {
|
||||
#[cfg(unix)]
|
||||
impl SyslogMakeWriter {
|
||||
fn new() -> Self {
|
||||
// Open syslog connection on first use
|
||||
static INIT: std::sync::Once = std::sync::Once::new();
|
||||
INIT.call_once(|| {
|
||||
unsafe {
|
||||
// Open syslog with ident "telemt", LOG_PID, LOG_DAEMON facility
|
||||
let ident = b"telemt\0".as_ptr() as *const libc::c_char;
|
||||
libc::openlog(ident, libc::LOG_PID | libc::LOG_NDELAY, libc::LOG_DAEMON);
|
||||
}
|
||||
INIT.call_once(|| unsafe {
|
||||
let ident = b"telemt\0".as_ptr() as *const libc::c_char;
|
||||
libc::openlog(ident, libc::LOG_PID | libc::LOG_NDELAY, libc::LOG_DAEMON);
|
||||
});
|
||||
Self
|
||||
}
|
||||
@@ -202,7 +200,6 @@ fn syslog_priority_for_level(level: &tracing::Level) -> libc::c_int {
|
||||
#[cfg(unix)]
|
||||
impl std::io::Write for SyslogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
// Convert to C string, stripping newlines
|
||||
let msg = String::from_utf8_lossy(buf);
|
||||
let msg = msg.trim_end();
|
||||
|
||||
@@ -210,7 +207,6 @@ impl std::io::Write for SyslogWriter {
|
||||
return Ok(buf.len());
|
||||
}
|
||||
|
||||
// Write to syslog
|
||||
let c_msg = std::ffi::CString::new(msg.as_bytes())
|
||||
.unwrap_or_else(|_| std::ffi::CString::new("(invalid utf8)").unwrap());
|
||||
|
||||
|
||||
+11
-3
@@ -44,13 +44,15 @@ pub(crate) struct BoundedFileAppender {
|
||||
}
|
||||
|
||||
impl BoundedFileAppender {
|
||||
pub(crate) fn new(options: FileLogOptions) -> io::Result<Self> {
|
||||
Self::with_now(options, Box::new(Utc::now))
|
||||
/// Opens the appender using the process-level Unix parent-path policy.
|
||||
pub(crate) fn new(options: FileLogOptions, strict_runtime_paths: bool) -> io::Result<Self> {
|
||||
Self::with_now(options, Box::new(Utc::now), strict_runtime_paths)
|
||||
}
|
||||
|
||||
fn with_now(
|
||||
options: FileLogOptions,
|
||||
now: Box<dyn Fn() -> DateTime<Utc> + Send + Sync>,
|
||||
strict_runtime_paths: bool,
|
||||
) -> io::Result<Self> {
|
||||
let path = Path::new(&options.path);
|
||||
let dir = path
|
||||
@@ -67,7 +69,13 @@ impl BoundedFileAppender {
|
||||
let start = now();
|
||||
let current_path = active_path_for(&dir, &base_name, options.rotation, &start);
|
||||
#[cfg(unix)]
|
||||
let dir_fd = crate::util::secure_fs::open_trusted_dir_nofollow_or_create(&dir, 0o750)?;
|
||||
let dir_fd = if strict_runtime_paths {
|
||||
crate::util::secure_fs::open_trusted_dir_nofollow_or_create(&dir, 0o750)?
|
||||
} else {
|
||||
crate::util::secure_fs::open_compatible_dir(&dir, Some(0o750))?
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let _ = strict_runtime_paths;
|
||||
#[cfg(unix)]
|
||||
let (file, current_size) = open_append_file(&dir_fd, ¤t_path)?;
|
||||
#[cfg(not(unix))]
|
||||
|
||||
+119
-6
@@ -34,6 +34,115 @@ fn matching_logs(dir: &Path) -> Vec<PathBuf> {
|
||||
files
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn compatibility_appender_accepts_writable_log_directories() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
for mode in [0o770, 0o777, 0o1777] {
|
||||
let dir = tempdir().unwrap();
|
||||
fs::set_permissions(dir.path(), fs::Permissions::from_mode(mode)).unwrap();
|
||||
let path = dir.path().join("telemt.log");
|
||||
assert!(
|
||||
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), true)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!path.exists());
|
||||
let mut appender =
|
||||
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), false)
|
||||
.unwrap();
|
||||
appender.write_all(b"compatibility\n").unwrap();
|
||||
appender.flush().unwrap();
|
||||
|
||||
assert_eq!(fs::read(&path).unwrap(), b"compatibility\n");
|
||||
assert_eq!(
|
||||
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
|
||||
0o640
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn compatibility_appender_follows_symlinked_parents() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let real = dir.path().join("real");
|
||||
let linked = dir.path().join("linked");
|
||||
fs::create_dir(&real).unwrap();
|
||||
symlink(&real, &linked).unwrap();
|
||||
let path = linked.join("nested/telemt.log");
|
||||
assert!(
|
||||
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), true).is_err()
|
||||
);
|
||||
assert!(!real.join("nested").exists());
|
||||
let mut appender =
|
||||
BoundedFileAppender::with_now(options(path), Box::new(fixed_now), false).unwrap();
|
||||
appender.write_all(b"compatibility\n").unwrap();
|
||||
appender.flush().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fs::read(real.join("nested/telemt.log")).unwrap(),
|
||||
b"compatibility\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn appender_rejects_final_symlinks_and_hard_links_in_both_modes() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
for strict_runtime_paths in [false, true] {
|
||||
for hard_link in [false, true] {
|
||||
let dir = tempdir().unwrap();
|
||||
let target = dir.path().join("sentinel");
|
||||
let path = dir.path().join("telemt.log");
|
||||
fs::write(&target, b"preserve\n").unwrap();
|
||||
if hard_link {
|
||||
fs::hard_link(&target, &path).unwrap();
|
||||
} else {
|
||||
symlink(&target, &path).unwrap();
|
||||
}
|
||||
|
||||
assert!(
|
||||
BoundedFileAppender::with_now(
|
||||
options(path),
|
||||
Box::new(fixed_now),
|
||||
strict_runtime_paths
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(fs::read(&target).unwrap(), b"preserve\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn time_rotation_and_retention_work_through_compatible_parent_alias() {
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
|
||||
let root = tempdir().unwrap();
|
||||
let real = root.path().join("logs");
|
||||
let linked = root.path().join("linked");
|
||||
fs::create_dir(&real).unwrap();
|
||||
fs::set_permissions(&real, fs::Permissions::from_mode(0o777)).unwrap();
|
||||
symlink(&real, &linked).unwrap();
|
||||
let mut options = options(linked.join("telemt.log"));
|
||||
options.rotation = LogRotation::Daily;
|
||||
options.max_files = 1;
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||
appender.write_all(b"first\n").unwrap();
|
||||
appender.now = Box::new(|| fixed_now() + ChronoDuration::days(1));
|
||||
appender.write_all(b"second\n").unwrap();
|
||||
appender.flush().unwrap();
|
||||
|
||||
let remaining = matching_logs(&real);
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert_eq!(fs::read(&remaining[0]).unwrap(), b"second\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_rotation_keeps_latest_write_in_active_file() {
|
||||
let dir = tempdir().unwrap();
|
||||
@@ -41,7 +150,7 @@ fn size_rotation_keeps_latest_write_in_active_file() {
|
||||
let mut options = options(path.clone());
|
||||
options.max_size_bytes = 6;
|
||||
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||
appender.write_all(b"abc\n").unwrap();
|
||||
appender.write_all(b"def\n").unwrap();
|
||||
appender.flush().unwrap();
|
||||
@@ -58,7 +167,7 @@ fn max_files_retention_removes_oldest_archives() {
|
||||
options.max_size_bytes = 4;
|
||||
options.max_files = 2;
|
||||
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||
for line in [b"aa\n", b"bb\n", b"cc\n", b"dd\n"] {
|
||||
appender.write_all(line).unwrap();
|
||||
}
|
||||
@@ -94,7 +203,7 @@ fn max_age_retention_removes_old_archives() {
|
||||
|
||||
let mut options = options(path);
|
||||
options.max_age_secs = 1;
|
||||
let _appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
let _appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||
|
||||
assert!(!old_archive.exists());
|
||||
}
|
||||
@@ -112,7 +221,7 @@ fn rotation_stays_bound_to_opened_directory_after_path_replacement() {
|
||||
fs::create_dir(&redirect).unwrap();
|
||||
let mut options = options(original.join("telemt.log"));
|
||||
options.max_size_bytes = 4;
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||
appender.write_all(b"aa\n").unwrap();
|
||||
fs::rename(&original, &moved).unwrap();
|
||||
symlink(&redirect, &original).unwrap();
|
||||
@@ -137,7 +246,11 @@ fn appender_rejects_group_writable_log_directory() {
|
||||
fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o770)).unwrap();
|
||||
|
||||
assert!(
|
||||
BoundedFileAppender::with_now(options(dir.path().join("telemt.log")), Box::new(fixed_now),)
|
||||
.is_err()
|
||||
BoundedFileAppender::with_now(
|
||||
options(dir.path().join("telemt.log")),
|
||||
Box::new(fixed_now),
|
||||
true
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::*;
|
||||
fn test_parse_log_cli_options_default() {
|
||||
let args: Vec<String> = vec![];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
assert!(!LoggingOptions::default().strict_runtime_paths);
|
||||
assert_eq!(
|
||||
resolve_log_destination(&LoggingConfig::default(), &options).unwrap(),
|
||||
LogDestination::Stderr
|
||||
|
||||
@@ -13,6 +13,11 @@ use super::generation::RuntimeTaskScope;
|
||||
const STARTUP_FALLBACK_AFTER: Duration = Duration::from_secs(80);
|
||||
const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6);
|
||||
|
||||
// Admission regressions cover notification loss without network startup.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// Keeps generation admission and routing synchronized with periodic ME readiness.
|
||||
pub(crate) async fn configure_admission_gate(
|
||||
config: &Arc<ProxyConfig>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
@@ -76,11 +81,18 @@ pub(crate) async fn configure_admission_gate(
|
||||
} else {
|
||||
Some(Instant::now())
|
||||
};
|
||||
let mut config_watch_open = true;
|
||||
let mut me_ready_watch_open = true;
|
||||
loop {
|
||||
tokio::select! {
|
||||
changed = config_rx_gate.changed() => {
|
||||
changed = config_rx_gate.changed(), if config_watch_open => {
|
||||
if changed.is_err() {
|
||||
break;
|
||||
config_watch_open = false;
|
||||
warn!(
|
||||
watch_channel = "config",
|
||||
"Admission config watch closed; continuing readiness polling with last configuration"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let cfg = config_rx_gate.borrow_and_update().clone();
|
||||
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
||||
@@ -88,9 +100,13 @@ pub(crate) async fn configure_admission_gate(
|
||||
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
||||
continue;
|
||||
}
|
||||
changed = me_ready_rx_gate.changed() => {
|
||||
changed = me_ready_rx_gate.changed(), if me_ready_watch_open => {
|
||||
if changed.is_err() {
|
||||
break;
|
||||
me_ready_watch_open = false;
|
||||
warn!(
|
||||
watch_channel = "me_ready",
|
||||
"Admission ME readiness watch closed; continuing periodic polling"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use tracing::{Event, Subscriber};
|
||||
use tracing_subscriber::Layer;
|
||||
use tracing_subscriber::layer::{Context, SubscriberExt};
|
||||
|
||||
use super::*;
|
||||
use crate::transport::middle_proxy::admission_test_support::IdlePoolFixture;
|
||||
|
||||
const POLL_MS: u64 = 20;
|
||||
|
||||
struct ClosureWarnings(Arc<AtomicUsize>);
|
||||
|
||||
impl<S: Subscriber> Layer<S> for ClosureWarnings {
|
||||
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
|
||||
struct ChannelField(bool);
|
||||
impl tracing::field::Visit for ChannelField {
|
||||
fn record_debug(
|
||||
&mut self,
|
||||
field: &tracing::field::Field,
|
||||
_value: &dyn std::fmt::Debug,
|
||||
) {
|
||||
self.0 |= field.name() == "watch_channel";
|
||||
}
|
||||
}
|
||||
if event.metadata().level() == &tracing::Level::WARN {
|
||||
let mut visitor = ChannelField(false);
|
||||
event.record(&mut visitor);
|
||||
if visitor.0 {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct GateFixture {
|
||||
pool: IdlePoolFixture,
|
||||
route: Arc<RouteRuntimeController>,
|
||||
admission: watch::Receiver<bool>,
|
||||
config_tx: Option<watch::Sender<Arc<ProxyConfig>>>,
|
||||
ready_tx: Option<watch::Sender<u64>>,
|
||||
config: Arc<ProxyConfig>,
|
||||
scope: RuntimeTaskScope,
|
||||
}
|
||||
|
||||
impl GateFixture {
|
||||
async fn new() -> Self {
|
||||
let pool = IdlePoolFixture::new(false).await;
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.use_middle_proxy = true;
|
||||
cfg.general.me2dc_fallback = true;
|
||||
cfg.general.me2dc_fast = true;
|
||||
cfg.general.me_admission_poll_ms = POLL_MS;
|
||||
let config = Arc::new(cfg);
|
||||
let (config_tx, config_rx) = watch::channel(config.clone());
|
||||
let (ready_tx, ready_rx) = watch::channel(0);
|
||||
let (admission_tx, mut admission) = watch::channel(false);
|
||||
let route = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let scope = RuntimeTaskScope::new();
|
||||
configure_admission_gate(
|
||||
&config,
|
||||
Some(pool.pool()),
|
||||
Arc::new(RwLock::new(Some(pool.pool()))),
|
||||
route.clone(),
|
||||
&admission_tx,
|
||||
config_rx,
|
||||
ready_rx,
|
||||
scope.clone(),
|
||||
)
|
||||
.await;
|
||||
drop(admission_tx);
|
||||
admission.borrow_and_update();
|
||||
tokio::task::yield_now().await;
|
||||
Self {
|
||||
pool,
|
||||
route,
|
||||
admission,
|
||||
config_tx: Some(config_tx),
|
||||
ready_tx: Some(ready_tx),
|
||||
config,
|
||||
scope,
|
||||
}
|
||||
}
|
||||
|
||||
async fn poll(&self) {
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_millis(POLL_MS)).await;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
async fn expect_route(&self, mode: RelayRouteMode) {
|
||||
let mut rx = self.route.subscribe();
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if rx.borrow_and_update().mode == mode {
|
||||
return;
|
||||
}
|
||||
rx.changed().await.unwrap();
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the generation must keep sampling readiness without client traffic");
|
||||
}
|
||||
}
|
||||
|
||||
async fn recovery_after_watch_closure(close_config: bool, close_ready: bool) {
|
||||
let warnings = Arc::new(AtomicUsize::new(0));
|
||||
let subscriber = tracing_subscriber::registry().with(ClosureWarnings(warnings.clone()));
|
||||
let _default = tracing::subscriber::set_default(subscriber);
|
||||
let mut fixture = GateFixture::new().await;
|
||||
assert!(*fixture.admission.borrow());
|
||||
if close_config {
|
||||
fixture.config_tx.take();
|
||||
}
|
||||
if close_ready {
|
||||
fixture.ready_tx.take();
|
||||
}
|
||||
for _ in 0..4 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let expected_warnings = usize::from(close_config) + usize::from(close_ready);
|
||||
fixture.pool.set_ready(true);
|
||||
for _ in 0..4 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(fixture.route.snapshot().mode, RelayRouteMode::Direct);
|
||||
fixture.poll().await;
|
||||
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||
fixture.pool.set_ready(false);
|
||||
fixture.poll().await;
|
||||
fixture.expect_route(RelayRouteMode::Direct).await;
|
||||
fixture.pool.set_ready(true);
|
||||
fixture.poll().await;
|
||||
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||
for _ in 0..5 {
|
||||
fixture.poll().await;
|
||||
}
|
||||
assert_eq!(warnings.load(Ordering::Relaxed), expected_warnings);
|
||||
fixture.scope.stop().await;
|
||||
assert!(fixture.admission.has_changed().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_gate_recovers_after_readiness_watch_closes() {
|
||||
recovery_after_watch_closure(false, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_gate_recovers_after_config_watch_closes() {
|
||||
recovery_after_watch_closure(true, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_gate_recovers_after_both_watches_close_without_spinning() {
|
||||
recovery_after_watch_closure(true, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn closed_config_watch_preserves_last_effective_fallback_policy() {
|
||||
let mut fixture = GateFixture::new().await;
|
||||
let mut cfg = fixture.config.as_ref().clone();
|
||||
cfg.general.me2dc_fallback = false;
|
||||
fixture
|
||||
.config_tx
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.send_replace(Arc::new(cfg));
|
||||
fixture.poll().await;
|
||||
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||
assert!(!*fixture.admission.borrow_and_update());
|
||||
fixture.config_tx.take();
|
||||
fixture.ready_tx.take();
|
||||
fixture.poll().await;
|
||||
fixture.pool.set_ready(true);
|
||||
fixture.poll().await;
|
||||
tokio::time::timeout(Duration::from_secs(1), fixture.admission.changed())
|
||||
.await
|
||||
.expect("restored readiness must open admission after notification loss")
|
||||
.unwrap();
|
||||
assert!(*fixture.admission.borrow_and_update());
|
||||
fixture.pool.set_ready(false);
|
||||
fixture.poll().await;
|
||||
tokio::time::timeout(Duration::from_secs(1), fixture.admission.changed())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!*fixture.admission.borrow_and_update());
|
||||
assert_eq!(fixture.route.snapshot().mode, RelayRouteMode::Middle);
|
||||
fixture.scope.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn generation_cancellation_stops_gate_after_notification_loss() {
|
||||
let mut fixture = GateFixture::new().await;
|
||||
fixture.config_tx.take();
|
||||
fixture.ready_tx.take();
|
||||
fixture.poll().await;
|
||||
fixture.pool.set_ready(true);
|
||||
fixture.poll().await;
|
||||
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||
fixture.scope.stop().await;
|
||||
assert!(fixture.admission.has_changed().is_err());
|
||||
let state = fixture.route.snapshot();
|
||||
fixture.pool.set_ready(false);
|
||||
for _ in 0..3 {
|
||||
fixture.poll().await;
|
||||
}
|
||||
assert_eq!(fixture.route.snapshot(), state);
|
||||
assert!(*fixture.admission.borrow());
|
||||
}
|
||||
@@ -15,20 +15,32 @@ use super::helpers::{
|
||||
use super::runtime_tasks;
|
||||
use super::validate_synlimit_privilege_drop;
|
||||
|
||||
/// Process-level configuration and logging resources retained across runtime generations.
|
||||
pub(super) struct BootstrapState {
|
||||
/// Monotonic process startup time.
|
||||
pub(super) process_started_at: Instant,
|
||||
/// Process startup time as Unix epoch seconds.
|
||||
pub(super) process_started_at_epoch_secs: u64,
|
||||
/// Startup component progress shared with the control plane.
|
||||
pub(super) startup_tracker: Arc<StartupTracker>,
|
||||
/// Validated initial runtime configuration.
|
||||
pub(super) config: ProxyConfig,
|
||||
/// Resolved source path used by reload operations.
|
||||
pub(super) config_path: PathBuf,
|
||||
/// Whether the environment owns the log filter policy.
|
||||
pub(super) has_rust_log: bool,
|
||||
/// Initial verbosity after CLI overrides.
|
||||
pub(super) effective_log_level: LogLevel,
|
||||
/// Process-owned dynamic tracing filter.
|
||||
pub(super) runtime_log_filter: runtime_tasks::RuntimeLogFilter,
|
||||
/// Keeps the file logging worker alive until process shutdown.
|
||||
pub(super) logging_guard: Option<crate::logging::LoggingGuard>,
|
||||
}
|
||||
|
||||
/// Loads configuration and initializes process logging with the startup parent-path policy.
|
||||
pub(super) async fn bootstrap(
|
||||
privilege_drop_requested: bool,
|
||||
strict_runtime_paths: bool,
|
||||
) -> std::result::Result<BootstrapState, Box<dyn std::error::Error>> {
|
||||
let process_started_at = Instant::now();
|
||||
let process_started_at_epoch_secs = SystemTime::now()
|
||||
@@ -268,6 +280,7 @@ pub(super) async fn bootstrap(
|
||||
let logging_opts = crate::logging::LoggingOptions {
|
||||
destination: log_destination,
|
||||
disable_colors: true,
|
||||
strict_runtime_paths,
|
||||
};
|
||||
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||
logging_guard = Some(guard);
|
||||
@@ -276,6 +289,7 @@ pub(super) async fn bootstrap(
|
||||
let logging_opts = crate::logging::LoggingOptions {
|
||||
destination: log_destination,
|
||||
disable_colors: true,
|
||||
strict_runtime_paths,
|
||||
};
|
||||
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||
logging_guard = Some(guard);
|
||||
|
||||
@@ -30,6 +30,7 @@ pub(crate) fn print_maestro_line(message: impl AsRef<str>) {
|
||||
);
|
||||
}
|
||||
|
||||
/// Resolves the config source against startup cwd while retaining symlink components.
|
||||
pub(crate) fn resolve_runtime_config_path(
|
||||
config_path_cli: &str,
|
||||
startup_cwd: &Path,
|
||||
@@ -75,6 +76,7 @@ pub(crate) fn resolve_runtime_config_path(
|
||||
startup_cwd.join("config.toml")
|
||||
}
|
||||
|
||||
/// Selects the runtime directory from CLI, startup cwd, and explicit config location.
|
||||
pub(crate) fn resolve_runtime_base_dir(
|
||||
config_path: &Path,
|
||||
startup_cwd: &Path,
|
||||
@@ -120,14 +122,21 @@ fn normalize_runtime_dir(path: &Path, startup_cwd: &Path) -> PathBuf {
|
||||
|
||||
/// Parsed CLI arguments.
|
||||
pub(crate) struct CliArgs {
|
||||
/// Config source selected by positional arguments.
|
||||
pub config_path: String,
|
||||
/// Whether the config source was explicitly provided.
|
||||
pub config_path_explicit: bool,
|
||||
/// Runtime directory override from CLI.
|
||||
pub data_path: Option<PathBuf>,
|
||||
/// Whether CLI requests minimal logging output.
|
||||
pub silent: bool,
|
||||
/// Verbosity override from CLI.
|
||||
pub log_level: Option<String>,
|
||||
/// Logging destination, rotation, and retention overrides.
|
||||
pub log_cli_options: LogCliOptions,
|
||||
}
|
||||
|
||||
/// Parses runtime arguments after early daemon and control-command handling.
|
||||
pub(crate) fn parse_cli() -> CliArgs {
|
||||
let mut config_path = "config.toml".to_string();
|
||||
let mut config_path_explicit = false;
|
||||
@@ -222,6 +231,8 @@ pub(crate) fn parse_cli() -> CliArgs {
|
||||
}
|
||||
// Skip daemon-related flags (already parsed)
|
||||
"--daemon" | "-d" | "--foreground" | "-f" => {}
|
||||
#[cfg(unix)]
|
||||
"--strict-runtime-paths" => {}
|
||||
s if s.starts_with("--pid-file") => {
|
||||
if !s.contains('=') {
|
||||
// Skip the pid-file value consumed by daemon argument parsing.
|
||||
@@ -300,6 +311,15 @@ fn print_help() {
|
||||
eprintln!(" --daemon, -d Fork to background (daemonize)");
|
||||
eprintln!(" --foreground, -f Explicit foreground mode (for systemd)");
|
||||
eprintln!(" --pid-file <PATH> PID file path (default: /var/run/telemt.pid)");
|
||||
eprintln!(
|
||||
" --strict-runtime-paths Require trusted, symlink-free PID/log parents (default: off)"
|
||||
);
|
||||
eprintln!(
|
||||
" Applies to run/start/stop/reload/status; parent symlinks and"
|
||||
);
|
||||
eprintln!(
|
||||
" writable directories are allowed when this flag is absent"
|
||||
);
|
||||
eprintln!(" --run-as-user <USER> Drop privileges to this user after binding");
|
||||
eprintln!(" --run-as-group <GROUP> Drop privileges to this group after binding");
|
||||
eprintln!(" --working-dir <DIR> Working directory for daemon mode");
|
||||
|
||||
+18
-10
@@ -96,7 +96,10 @@ async fn run_inner(
|
||||
// Acquire PID file if daemonizing or if explicitly requested.
|
||||
// Keep it alive until shutdown for RAII cleanup.
|
||||
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
||||
let mut pf = PidFile::new(daemon_opts.pid_file_path());
|
||||
let mut pf = PidFile::new(
|
||||
daemon_opts.pid_file_path(),
|
||||
daemon_opts.strict_runtime_paths,
|
||||
);
|
||||
if let Err(e) = pf.acquire() {
|
||||
eprintln!("[telemt] {}", e);
|
||||
std::process::exit(1);
|
||||
@@ -109,20 +112,25 @@ async fn run_inner(
|
||||
let user = daemon_opts.user.clone();
|
||||
let group = daemon_opts.group.clone();
|
||||
|
||||
orchestrator::run_telemt_core(user.is_some() || group.is_some(), || {
|
||||
if (user.is_some() || group.is_some())
|
||||
&& let Err(e) = drop_privileges(user.as_deref(), group.as_deref(), _pid_file.as_ref())
|
||||
{
|
||||
error!(error = %e, "Failed to drop privileges");
|
||||
std::process::exit(1);
|
||||
}
|
||||
})
|
||||
orchestrator::run_telemt_core(
|
||||
user.is_some() || group.is_some(),
|
||||
daemon_opts.strict_runtime_paths,
|
||||
|| {
|
||||
if (user.is_some() || group.is_some())
|
||||
&& let Err(e) =
|
||||
drop_privileges(user.as_deref(), group.as_deref(), _pid_file.as_ref())
|
||||
{
|
||||
error!(error = %e, "Failed to drop privileges");
|
||||
std::process::exit(1);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn run_inner() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
orchestrator::run_telemt_core(false, || {}).await
|
||||
orchestrator::run_telemt_core(false, false, || {}).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -30,10 +30,11 @@ use super::{
|
||||
runtime_tasks, shutdown, tls_bootstrap,
|
||||
};
|
||||
|
||||
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
||||
// and privileged firewall setup completes; it is a no-op on other platforms.
|
||||
/// Runs startup and the main loop with explicit runtime-path policy.
|
||||
/// `drop_after_bind` runs after listeners and privileged firewall setup are ready.
|
||||
pub(super) async fn run_telemt_core(
|
||||
privilege_drop_requested: bool,
|
||||
strict_runtime_paths: bool,
|
||||
drop_after_bind: impl FnOnce(),
|
||||
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
let bootstrap::BootstrapState {
|
||||
@@ -46,7 +47,7 @@ pub(super) async fn run_telemt_core(
|
||||
effective_log_level,
|
||||
runtime_log_filter,
|
||||
logging_guard: _logging_guard,
|
||||
} = bootstrap::bootstrap(privilege_drop_requested).await?;
|
||||
} = bootstrap::bootstrap(privilege_drop_requested, strict_runtime_paths).await?;
|
||||
|
||||
if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control {
|
||||
warn!("Inline conntrack control is disabled when process privileges are dropped");
|
||||
@@ -374,17 +375,21 @@ pub(super) async fn run_telemt_core(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let conntrack_firewall = {
|
||||
let authority = crate::conntrack_control::FirewallAuthority::spawn(&process_control_plane)
|
||||
.map_err(std::io::Error::other)?;
|
||||
if !authority
|
||||
.publish_initial(1, runtime.config.clone(), stats.clone())
|
||||
.await
|
||||
let authority = crate::conntrack_control::FirewallAuthority::spawn(
|
||||
&process_control_plane,
|
||||
&runtime.config,
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
if let Some(authority) = &authority
|
||||
&& !authority
|
||||
.publish_initial(1, runtime.config.clone(), stats.clone())
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
"Initial conntrack firewall reconciliation failed; background retries remain active"
|
||||
);
|
||||
}
|
||||
Some(authority)
|
||||
authority
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let conntrack_firewall = None::<crate::conntrack_control::FirewallAuthority>;
|
||||
|
||||
+32
-7
@@ -104,11 +104,12 @@ pub fn generate_service_file(init_system: InitSystem, opts: &ServiceOptions) ->
|
||||
InitSystem::Systemd => generate_systemd_unit(opts),
|
||||
InitSystem::OpenRC => generate_openrc_script(opts),
|
||||
InitSystem::FreeBSDRc => generate_freebsd_rc_script(opts),
|
||||
InitSystem::Unknown => generate_systemd_unit(opts), // Default to systemd format
|
||||
// Preserve systemd generation when init detection is unavailable.
|
||||
InitSystem::Unknown => generate_systemd_unit(opts),
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates an enhanced systemd unit file.
|
||||
/// Generates a hardened systemd unit with Netlink access for optional firewall helpers.
|
||||
fn generate_systemd_unit(opts: &ServiceOptions) -> String {
|
||||
let user_line = opts.user.map(|u| format!("User={}", u)).unwrap_or_default();
|
||||
let group_line = opts
|
||||
@@ -151,14 +152,14 @@ PrivateDevices=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
||||
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
|
||||
RestrictNamespaces=true
|
||||
RestrictRealtime=true
|
||||
RestrictSUIDSGID=true
|
||||
MemoryDenyWriteExecute=true
|
||||
LockPersonality=true
|
||||
|
||||
# Allow binding to privileged ports and writing to specific paths
|
||||
# Allow privileged port binding, optional firewall helpers, and runtime state writes
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
||||
CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
||||
ReadWritePaths=/etc/telemt /var/run /var/lib/telemt
|
||||
@@ -233,9 +234,11 @@ fn generate_freebsd_rc_script(opts: &ServiceOptions) -> String {
|
||||
# Add the following lines to /etc/rc.conf to enable telemt:
|
||||
#
|
||||
# telemt_enable="YES"
|
||||
# telemt_config="/etc/telemt/config.toml" # optional
|
||||
# telemt_user="telemt" # optional
|
||||
# telemt_group="telemt" # optional
|
||||
# The configuration path can be overridden in rc.conf.
|
||||
# telemt_config="/etc/telemt/config.toml"
|
||||
# The service user and group can be overridden in rc.conf.
|
||||
# telemt_user="telemt"
|
||||
# telemt_group="telemt"
|
||||
#
|
||||
|
||||
. /etc/rc.subr
|
||||
@@ -355,6 +358,28 @@ mod tests {
|
||||
assert!(unit.contains("PIDFile="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn systemd_unit_allows_netlink_without_removing_hardening() {
|
||||
let unit = generate_systemd_unit(&ServiceOptions::default());
|
||||
let families = unit
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("RestrictAddressFamilies="))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
families.split_whitespace().collect::<Vec<_>>(),
|
||||
["AF_INET", "AF_INET6", "AF_UNIX", "AF_NETLINK"],
|
||||
);
|
||||
for directive in [
|
||||
"NoNewPrivileges=true",
|
||||
"ProtectSystem=strict",
|
||||
"RestrictNamespaces=true",
|
||||
"AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN",
|
||||
"CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_NET_ADMIN",
|
||||
] {
|
||||
assert!(unit.lines().any(|line| line == directive), "{directive}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_openrc_script_generation() {
|
||||
let opts = ServiceOptions::default();
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
use super::*;
|
||||
|
||||
// Outage regressions use loopback sockets and explicit family-state snapshots.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// Reports whether replacing an endpoint could disrupt a bound client.
|
||||
pub(super) fn has_bound_clients_on_endpoint(
|
||||
writer_ids: &[u64],
|
||||
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||
@@ -9,6 +14,7 @@ pub(super) fn has_bound_clients_on_endpoint(
|
||||
.any(|writer_id| bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0)
|
||||
}
|
||||
|
||||
/// Restores missing single-endpoint coverage under the bounded outage retry policy.
|
||||
pub(super) async fn recover_single_endpoint_outage(
|
||||
pool: &Arc<MePool>,
|
||||
rng: &Arc<SecureRandom>,
|
||||
@@ -53,13 +59,21 @@ pub(super) async fn recover_single_endpoint_outage(
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||
|
||||
let generation = pool.current_generation();
|
||||
let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine();
|
||||
let attempt_ok = if bypass_quarantine {
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||
match tokio::time::timeout(
|
||||
pool.reconnect_runtime.me_one_timeout,
|
||||
pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()),
|
||||
pool.connect_one_with_generation_contour_for_dc_with_intent(
|
||||
endpoint,
|
||||
rng.as_ref(),
|
||||
generation,
|
||||
WriterContour::Active,
|
||||
key.0,
|
||||
WriterOpenIntent::Coverage,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -88,7 +102,14 @@ pub(super) async fn recover_single_endpoint_outage(
|
||||
let one_endpoint = [endpoint];
|
||||
match tokio::time::timeout(
|
||||
pool.reconnect_runtime.me_one_timeout,
|
||||
pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()),
|
||||
pool.connect_endpoints_round_robin_with_generation_contour(
|
||||
key.0,
|
||||
&one_endpoint,
|
||||
rng.as_ref(),
|
||||
generation,
|
||||
WriterContour::Active,
|
||||
WriterOpenIntent::Coverage,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -142,6 +163,7 @@ pub(super) async fn recover_single_endpoint_outage(
|
||||
);
|
||||
}
|
||||
|
||||
/// Rotates an idle single-endpoint shadow without removing the existing coverage first.
|
||||
pub(super) async fn maybe_rotate_single_endpoint_shadow(
|
||||
pool: &Arc<MePool>,
|
||||
rng: &Arc<SecureRandom>,
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use super::*;
|
||||
use crate::transport::middle_proxy::admission_test_support::unregistered_writer;
|
||||
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||
|
||||
async fn single_endpoint_reaches_tcp_with_a_full_cap(bypass_quarantine: bool) {
|
||||
let pool = make_pool().await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let endpoint = listener.local_addr().unwrap();
|
||||
let donor = SocketAddr::new(endpoint.ip(), 1);
|
||||
pool.update_proxy_maps(
|
||||
HashMap::from([
|
||||
(2, vec![(endpoint.ip(), endpoint.port())]),
|
||||
(3, vec![(donor.ip(), donor.port())]),
|
||||
]),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_cpu_cores_override
|
||||
.store(1, Ordering::Relaxed);
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_max_active_writers_per_core
|
||||
.store(1, Ordering::Relaxed);
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_max_active_writers_global
|
||||
.store(1, Ordering::Relaxed);
|
||||
pool.single_endpoint_runtime
|
||||
.me_single_endpoint_outage_disable_quarantine
|
||||
.store(bypass_quarantine, Ordering::Relaxed);
|
||||
let (writer, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
1,
|
||||
3,
|
||||
donor,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
pool.writers.write().await.push(writer);
|
||||
if bypass_quarantine {
|
||||
pool.endpoint_quarantine
|
||||
.lock()
|
||||
.await
|
||||
.insert(endpoint, Instant::now() + Duration::from_secs(60));
|
||||
}
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let key = (2, IpFamily::V4);
|
||||
let mut backoff = HashMap::new();
|
||||
let mut next_attempt = HashMap::new();
|
||||
let semaphore = Arc::new(Semaphore::new(1));
|
||||
let required = pool.required_writers_for_dc(1);
|
||||
// TCP acceptance proves capacity admission; closing the peer deliberately rejects the handshake.
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
tokio::join!(
|
||||
recover_single_endpoint_outage(
|
||||
&pool,
|
||||
&rng,
|
||||
key,
|
||||
endpoint,
|
||||
required,
|
||||
&mut backoff,
|
||||
&mut next_attempt,
|
||||
&semaphore,
|
||||
),
|
||||
async {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
drop(stream);
|
||||
},
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("missing-DC recovery must reach TCP even when another DC fills the cap");
|
||||
assert_eq!(
|
||||
pool.stats
|
||||
.get_me_single_endpoint_outage_reconnect_attempt_total(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
pool.stats
|
||||
.get_me_single_endpoint_outage_reconnect_success_total(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
pool.stats.get_me_single_endpoint_quarantine_bypass_total(),
|
||||
u64::from(bypass_quarantine)
|
||||
);
|
||||
assert!(next_attempt.contains_key(&key));
|
||||
assert!(backoff.contains_key(&key));
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_endpoint_quarantine_bypass_recovers_coverage_at_cap() {
|
||||
single_endpoint_reaches_tcp_with_a_full_cap(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_endpoint_round_robin_recovers_coverage_at_cap() {
|
||||
single_endpoint_reaches_tcp_with_a_full_cap(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restored_coverage_preserves_family_suppression_hysteresis() {
|
||||
let pool = make_pool().await;
|
||||
let now = MePool::now_epoch_secs();
|
||||
let until = now + 3600;
|
||||
pool.set_family_runtime_state(
|
||||
IpFamily::V4,
|
||||
MeFamilyRuntimeState::Suppressed,
|
||||
now,
|
||||
until,
|
||||
5,
|
||||
0,
|
||||
);
|
||||
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||
assert_eq!(
|
||||
pool.family_runtime_state(IpFamily::V4),
|
||||
MeFamilyRuntimeState::Suppressed
|
||||
);
|
||||
assert_eq!(pool.family_suppressed_until_epoch_secs(IpFamily::V4), until);
|
||||
assert_eq!(pool.family_recover_success_streak(IpFamily::V4), 0);
|
||||
|
||||
pool.set_family_runtime_state(
|
||||
IpFamily::V4,
|
||||
MeFamilyRuntimeState::Suppressed,
|
||||
now,
|
||||
now - 1,
|
||||
5,
|
||||
0,
|
||||
);
|
||||
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||
assert_eq!(
|
||||
pool.family_runtime_state(IpFamily::V4),
|
||||
MeFamilyRuntimeState::Recovering
|
||||
);
|
||||
assert_eq!(pool.family_recover_success_streak(IpFamily::V4), 1);
|
||||
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||
assert_eq!(
|
||||
pool.family_runtime_state(IpFamily::V4),
|
||||
MeFamilyRuntimeState::Healthy
|
||||
);
|
||||
assert_eq!(pool.family_fail_streak(IpFamily::V4), 0);
|
||||
}
|
||||
@@ -49,6 +49,11 @@ mod send;
|
||||
mod send_adversarial_tests;
|
||||
mod wire;
|
||||
|
||||
/// Network-free writer fixtures shared by admission and recovery regressions.
|
||||
#[cfg(test)]
|
||||
#[path = "tests/admission_test_support.rs"]
|
||||
pub(crate) mod admission_test_support;
|
||||
|
||||
use bytes::Bytes;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
|
||||
@@ -61,13 +61,13 @@ impl Drop for WriterOpenReservation<'_> {
|
||||
}
|
||||
|
||||
impl MePool {
|
||||
/// Computes the authoritative active-writer floor across enabled families and DCs.
|
||||
/// Computes reservation coverage across configured families, including suppressed ones.
|
||||
pub(in crate::transport::middle_proxy) async fn active_coverage_required_total(&self) -> usize {
|
||||
let now_epoch_secs = Self::now_epoch_secs();
|
||||
let mut required_total = 0usize;
|
||||
let endpoint_snapshot = self.endpoint_snapshot.load_full();
|
||||
|
||||
if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch_secs) {
|
||||
// Suppression must not remove the capacity needed to recover active coverage.
|
||||
if self.decision.ipv4_me {
|
||||
for addrs in endpoint_snapshot.map_v4.values() {
|
||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||
for (ip, port) in addrs.iter().copied() {
|
||||
@@ -79,7 +79,7 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
|
||||
if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch_secs) {
|
||||
if self.decision.ipv6_me {
|
||||
for addrs in endpoint_snapshot.map_v6.values() {
|
||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||
for (ip, port) in addrs.iter().copied() {
|
||||
|
||||
@@ -33,6 +33,10 @@ mod publication;
|
||||
mod replacement;
|
||||
mod runtime;
|
||||
|
||||
// Recovery regressions exercise the serialized production publication boundary.
|
||||
#[cfg(test)]
|
||||
mod recovery_tests;
|
||||
|
||||
struct PreparedWriter<'a> {
|
||||
writer: MeWriter,
|
||||
tx: mpsc::Sender<WriterCommand>,
|
||||
|
||||
@@ -84,7 +84,17 @@ impl MePool {
|
||||
crate::network::IpFamily::V6
|
||||
};
|
||||
let now_epoch_secs = Self::now_epoch_secs();
|
||||
if !self.family_enabled_for_drain_coverage(family, now_epoch_secs) {
|
||||
let configured = match family {
|
||||
crate::network::IpFamily::V4 => self.decision.ipv4_me,
|
||||
crate::network::IpFamily::V6 => self.decision.ipv6_me,
|
||||
};
|
||||
// Recovery coverage must remain publishable during suppression; the serialized
|
||||
// per-group floor check below still bounds it and generation authority is unchanged.
|
||||
let recovery_coverage =
|
||||
contour == WriterContour::Active && intent == WriterOpenIntent::Coverage;
|
||||
if !configured
|
||||
|| (self.is_family_temporarily_suppressed(family, now_epoch_secs) && !recovery_coverage)
|
||||
{
|
||||
return Err(ProxyError::Proxy(
|
||||
"ME writer family lost publication authority".into(),
|
||||
));
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
|
||||
use super::*;
|
||||
use crate::network::{IpFamily, probe::NetworkDecision};
|
||||
use crate::transport::middle_proxy::admission_test_support::unregistered_writer;
|
||||
use crate::transport::middle_proxy::pool::MeFamilyRuntimeState;
|
||||
use crate::transport::middle_proxy::pool_writer_security_tests::{
|
||||
make_pool, make_pool_with_decision,
|
||||
};
|
||||
|
||||
fn cap_at_one(pool: &MePool) {
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_cpu_cores_override
|
||||
.store(1, Ordering::Relaxed);
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_max_active_writers_per_core
|
||||
.store(1, Ordering::Relaxed);
|
||||
pool.floor_runtime
|
||||
.me_adaptive_floor_max_active_writers_global
|
||||
.store(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn suppress(pool: &MePool, family: IpFamily) {
|
||||
let now = MePool::now_epoch_secs();
|
||||
pool.set_family_runtime_state(
|
||||
family,
|
||||
MeFamilyRuntimeState::Suppressed,
|
||||
now,
|
||||
now + 3600,
|
||||
5,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
fn prepared<'a>(
|
||||
pool: &'a Arc<MePool>,
|
||||
writer: MeWriter,
|
||||
intent: WriterOpenIntent,
|
||||
reservation: WriterOpenReservation<'a>,
|
||||
task_started: Arc<AtomicBool>,
|
||||
) -> PreparedWriter<'a> {
|
||||
let cancel = writer.cancel.clone();
|
||||
PreparedWriter {
|
||||
tx: writer.tx.clone(),
|
||||
byte_budget: writer.byte_budget.clone(),
|
||||
writer,
|
||||
task_registration: pool.lifecycle.try_register().unwrap(),
|
||||
writer_task: Box::pin(async move {
|
||||
task_started.store(true, Ordering::Release);
|
||||
cancel.cancelled().await;
|
||||
}),
|
||||
intent,
|
||||
_open_reservation: reservation,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suppressed_active_coverage_publishes_only_the_missing_floor() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
cap_at_one(&pool);
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let required = pool.required_writers_for_dc(1);
|
||||
let mut receivers = Vec::new();
|
||||
for id in 1..=required as u64 {
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.expect("missing coverage must have a reservation despite suppression");
|
||||
let (writer, receiver) = unregistered_writer(
|
||||
&pool,
|
||||
id,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
receivers.push(receiver);
|
||||
pool.publish_connected_writer(prepared(
|
||||
&pool,
|
||||
writer,
|
||||
WriterOpenIntent::Coverage,
|
||||
reservation,
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
))
|
||||
.await
|
||||
.expect("authenticated coverage must publish without waiting for suppression expiry");
|
||||
assert_eq!(pool.conn_count.load(Ordering::Relaxed), id as usize);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
let (extra, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
99,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
assert!(
|
||||
pool.authorize_writer_publication_capacity(
|
||||
&extra,
|
||||
WriterContour::Active,
|
||||
WriterOpenIntent::Coverage,
|
||||
&pool.writers.read().await,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
pool.family_runtime_state(IpFamily::V4),
|
||||
MeFamilyRuntimeState::Suppressed
|
||||
);
|
||||
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_suppressed_coverage_publications_cannot_overfill_a_group() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
cap_at_one(&pool);
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let required = pool.required_writers_for_dc(1);
|
||||
let mut receivers = Vec::new();
|
||||
let mut publications = Vec::new();
|
||||
for id in 1..=(required + 1) as u64 {
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.unwrap();
|
||||
let (writer, receiver) = unregistered_writer(
|
||||
&pool,
|
||||
id,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
receivers.push(receiver);
|
||||
publications.push(pool.publish_connected_writer(prepared(
|
||||
&pool,
|
||||
writer,
|
||||
WriterOpenIntent::Coverage,
|
||||
reservation,
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
)));
|
||||
}
|
||||
let guard = pool.writers.write().await;
|
||||
let results = futures::future::join_all(publications);
|
||||
tokio::pin!(results);
|
||||
tokio::select! {
|
||||
biased;
|
||||
results = &mut results => panic!("publication bypassed the writer barrier: {results:?}"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
drop(guard);
|
||||
let results = results.await;
|
||||
assert_eq!(
|
||||
results.iter().filter(|result| result.is_ok()).count(),
|
||||
required
|
||||
);
|
||||
assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1);
|
||||
assert_eq!(pool.writers.read().await.len(), required);
|
||||
assert_eq!(pool.conn_count.load(Ordering::Relaxed), required);
|
||||
assert_eq!(
|
||||
pool.registry.writer_idle_since_snapshot().await.len(),
|
||||
required
|
||||
);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_warm_writer_uses_resolved_active_coverage_authority() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Warm, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.unwrap();
|
||||
let (writer, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
1,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Warm,
|
||||
);
|
||||
pool.publish_connected_writer(prepared(
|
||||
&pool,
|
||||
writer,
|
||||
WriterOpenIntent::Coverage,
|
||||
reservation,
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
))
|
||||
.await
|
||||
.expect("a committed generation must resolve warm coverage to its active role");
|
||||
assert_eq!(
|
||||
pool.writers.read().await[0].contour.load(Ordering::Acquire),
|
||||
WriterContour::Active.as_u8()
|
||||
);
|
||||
assert_eq!(pool.writer_connect_warm_reserved.load(Ordering::Acquire), 0);
|
||||
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suppressed_families_keep_their_full_coverage_reservation_budget() {
|
||||
let pool = make_pool_with_decision(NetworkDecision {
|
||||
ipv4_me: true,
|
||||
ipv6_me: true,
|
||||
..NetworkDecision::default()
|
||||
})
|
||||
.await;
|
||||
let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(
|
||||
HashMap::from([(2, vec![(v4.ip(), v4.port())])]),
|
||||
Some(HashMap::from([(2, vec![(v6.ip(), v6.port())])])),
|
||||
)
|
||||
.await;
|
||||
cap_at_one(&pool);
|
||||
suppress(&pool, IpFamily::V4);
|
||||
suppress(&pool, IpFamily::V6);
|
||||
let required = pool.required_writers_for_dc(1);
|
||||
assert_eq!(pool.active_coverage_required_total().await, required * 4);
|
||||
|
||||
let mut receivers = Vec::new();
|
||||
let mut writers = pool.writers.write().await;
|
||||
for (index, (dc, addr)) in [(2, v4), (-2, v4), (-2, v6)].into_iter().enumerate() {
|
||||
for offset in 0..required {
|
||||
let (writer, receiver) = unregistered_writer(
|
||||
&pool,
|
||||
(index * required + offset + 1) as u64,
|
||||
dc,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
writers.push(writer);
|
||||
receivers.push(receiver);
|
||||
}
|
||||
}
|
||||
drop(writers);
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, v6)
|
||||
.await
|
||||
.expect("other covered groups must not consume the suppressed family's floor budget");
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
1
|
||||
);
|
||||
drop(reservation);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coverage_reservations_remain_bounded_and_release_on_drop() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
cap_at_one(&pool);
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let limit = pool.required_writers_for_dc(1) * 2
|
||||
+ pool
|
||||
.reconnect_runtime
|
||||
.me_reconnect_max_concurrent_per_dc
|
||||
.max(1) as usize;
|
||||
let mut reservations = Vec::new();
|
||||
for _ in 0..limit {
|
||||
reservations.push(
|
||||
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.expect("the configured coverage budget must remain available"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr,)
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
limit
|
||||
);
|
||||
drop(reservations);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suppression_exception_does_not_admit_growth_replacement_or_pending_warm() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
suppress(&pool, IpFamily::V4);
|
||||
for (contour, intent) in [
|
||||
(WriterContour::Active, WriterOpenIntent::Normal),
|
||||
(WriterContour::Active, WriterOpenIntent::Replacement),
|
||||
(WriterContour::Warm, WriterOpenIntent::Coverage),
|
||||
(WriterContour::Warm, WriterOpenIntent::Normal),
|
||||
] {
|
||||
let (writer, _receiver) =
|
||||
unregistered_writer(&pool, 1, 2, addr, pool.current_generation(), contour);
|
||||
assert!(
|
||||
pool.authorize_writer_publication_capacity(&writer, contour, intent, &[])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coverage_exception_never_enables_a_disabled_family() {
|
||||
let pool = make_pool().await;
|
||||
let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(
|
||||
HashMap::from([(2, vec![(v4.ip(), v4.port())])]),
|
||||
Some(HashMap::from([(2, vec![(v6.ip(), v6.port())])])),
|
||||
)
|
||||
.await;
|
||||
suppress(&pool, IpFamily::V6);
|
||||
assert_eq!(
|
||||
pool.active_coverage_required_total().await,
|
||||
pool.required_writers_for_dc(1) * 2
|
||||
);
|
||||
let (writer, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
1,
|
||||
2,
|
||||
v6,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
assert!(
|
||||
pool.authorize_writer_publication_capacity(
|
||||
&writer,
|
||||
WriterContour::Active,
|
||||
WriterOpenIntent::Coverage,
|
||||
&[],
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suppressed_coverage_cannot_publish_a_stale_generation() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.unwrap();
|
||||
let (writer, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
1,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation().saturating_sub(1),
|
||||
WriterContour::Active,
|
||||
);
|
||||
let task_started = Arc::new(AtomicBool::new(false));
|
||||
let error = pool
|
||||
.publish_connected_writer(prepared(
|
||||
&pool,
|
||||
writer,
|
||||
WriterOpenIntent::Coverage,
|
||||
reservation,
|
||||
task_started.clone(),
|
||||
))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("generation lost publication authority")
|
||||
);
|
||||
assert!(pool.writers.read().await.is_empty());
|
||||
assert!(pool.registry.writer_idle_since_snapshot().await.is_empty());
|
||||
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
assert!(!task_started.load(Ordering::Acquire));
|
||||
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suppressed_coverage_rechecks_endpoints_after_publication_waits() {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
suppress(&pool, IpFamily::V4);
|
||||
let reservation = pool
|
||||
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||
.await
|
||||
.unwrap();
|
||||
let (writer, _receiver) = unregistered_writer(
|
||||
&pool,
|
||||
1,
|
||||
2,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
let task_started = Arc::new(AtomicBool::new(false));
|
||||
let publication = pool.publish_connected_writer(prepared(
|
||||
&pool,
|
||||
writer,
|
||||
WriterOpenIntent::Coverage,
|
||||
reservation,
|
||||
task_started.clone(),
|
||||
));
|
||||
tokio::pin!(publication);
|
||||
let guard = pool.writers.write().await;
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut publication => panic!("publication bypassed the writer barrier: {result:?}"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), 444)])]), None)
|
||||
.await;
|
||||
drop(guard);
|
||||
let error = publication.await.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("target changed before publication")
|
||||
);
|
||||
assert!(pool.writers.read().await.is_empty());
|
||||
assert!(pool.registry.writer_idle_since_snapshot().await.is_empty());
|
||||
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(
|
||||
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||
0
|
||||
);
|
||||
assert!(!task_started.load(Ordering::Acquire));
|
||||
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::codec::WriterCommand;
|
||||
use super::pool::{MePool, MeWriter, WriterContour};
|
||||
use super::pool_writer_security_tests::make_pool;
|
||||
|
||||
/// Owns idle writer visibility without spawning ME transports or client sessions.
|
||||
pub(crate) struct IdlePoolFixture {
|
||||
pool: Arc<MePool>,
|
||||
draining: Vec<Arc<AtomicBool>>,
|
||||
_receivers: Vec<mpsc::Receiver<WriterCommand>>,
|
||||
}
|
||||
|
||||
impl IdlePoolFixture {
|
||||
/// Builds both signed DC groups with a controllable readiness quorum.
|
||||
pub(crate) async fn new(ready: bool) -> Self {
|
||||
let pool = make_pool().await;
|
||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||
.await;
|
||||
let mut draining = Vec::new();
|
||||
let mut receivers = Vec::new();
|
||||
{
|
||||
let mut writers = pool.writers.write().await;
|
||||
for (id, dc) in [(1, 2), (2, -2)] {
|
||||
let (writer, receiver) = unregistered_writer(
|
||||
&pool,
|
||||
id,
|
||||
dc,
|
||||
addr,
|
||||
pool.current_generation(),
|
||||
WriterContour::Active,
|
||||
);
|
||||
writer.draining.store(!ready, Ordering::Release);
|
||||
draining.push(writer.draining.clone());
|
||||
receivers.push(receiver);
|
||||
writers.push(writer);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
pool,
|
||||
draining,
|
||||
_receivers: receivers,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the pool sampled by the production admission gate.
|
||||
pub(crate) fn pool(&self) -> Arc<MePool> {
|
||||
self.pool.clone()
|
||||
}
|
||||
|
||||
/// Changes the fixture quorum without notifying the admission gate.
|
||||
pub(crate) fn set_ready(&self, ready: bool) {
|
||||
for draining in &self.draining {
|
||||
draining.store(!ready, Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a writer whose command receiver remains owned by the test.
|
||||
pub(super) fn unregistered_writer(
|
||||
pool: &Arc<MePool>,
|
||||
id: u64,
|
||||
dc: i32,
|
||||
addr: SocketAddr,
|
||||
generation: u64,
|
||||
contour: WriterContour,
|
||||
) -> (MeWriter, mpsc::Receiver<WriterCommand>) {
|
||||
let (tx, rx) = mpsc::channel(8);
|
||||
(
|
||||
MeWriter {
|
||||
id,
|
||||
addr,
|
||||
source_ip: addr.ip(),
|
||||
writer_dc: dc,
|
||||
generation,
|
||||
contour: Arc::new(AtomicU8::new(contour.as_u8())),
|
||||
created_at: Instant::now(),
|
||||
tx,
|
||||
byte_budget: pool.new_writer_byte_budget(),
|
||||
cancel: CancellationToken::new(),
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)),
|
||||
draining: Arc::new(AtomicBool::new(false)),
|
||||
draining_started_at_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||
drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||
allow_drain_fallback: Arc::new(AtomicBool::new(false)),
|
||||
},
|
||||
rx,
|
||||
)
|
||||
}
|
||||
@@ -1,14 +1,15 @@
|
||||
//! Descriptor-anchored filesystem operations for privileged runtime paths.
|
||||
//!
|
||||
//! Submodules:
|
||||
//! - `path`: symlink-free directory traversal and anchored path ownership
|
||||
//! - `path`: strict or compatible directory traversal and anchored path ownership
|
||||
//! - `write`: regular-file opening and durable atomic replacement
|
||||
|
||||
mod path;
|
||||
mod write;
|
||||
|
||||
pub(crate) use path::{
|
||||
AnchoredPath, chdir_nofollow_or_create, open_dir_nofollow, open_trusted_dir_nofollow_or_create,
|
||||
AnchoredPath, chdir_nofollow_or_create, open_compatible_dir, open_dir_nofollow,
|
||||
open_trusted_dir_nofollow_or_create,
|
||||
};
|
||||
pub(crate) use write::{
|
||||
atomic_replace, atomic_replace_async, open_append_regular, open_append_regular_at,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::io;
|
||||
use std::os::fd::OwnedFd;
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt};
|
||||
use std::path::{Component, Path};
|
||||
|
||||
use nix::fcntl::{OFlag, open, openat};
|
||||
@@ -53,6 +53,28 @@ impl AnchoredPath {
|
||||
Ok(Self { parent, name })
|
||||
}
|
||||
|
||||
/// Anchors a runtime parent, applying trusted traversal only when explicitly requested.
|
||||
pub(crate) fn open_runtime_parent(
|
||||
path: &Path,
|
||||
create_mode: Option<u32>,
|
||||
strict_runtime_paths: bool,
|
||||
) -> io::Result<Self> {
|
||||
if strict_runtime_paths {
|
||||
return match create_mode {
|
||||
Some(mode) => Self::open_trusted_parent_or_create(path, mode),
|
||||
None => Self::open_trusted_parent(path),
|
||||
};
|
||||
}
|
||||
let name = path
|
||||
.file_name()
|
||||
.filter(|name| !name.is_empty())
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "path has no file name"))?
|
||||
.to_os_string();
|
||||
let parent =
|
||||
open_compatible_dir(path.parent().unwrap_or_else(|| Path::new(".")), create_mode)?;
|
||||
Ok(Self { parent, name })
|
||||
}
|
||||
|
||||
fn open_with_parent_creation(path: &Path, create_mode: Option<u32>) -> io::Result<Self> {
|
||||
let name = path
|
||||
.file_name()
|
||||
@@ -97,6 +119,23 @@ pub(crate) fn open_trusted_dir_nofollow_or_create(path: &Path, mode: u32) -> io:
|
||||
open_dir_components(path, Some(mode), true)
|
||||
}
|
||||
|
||||
/// Follows parent-directory symlinks without imposing ownership or permission policy.
|
||||
pub(crate) fn open_compatible_dir(path: &Path, create_mode: Option<u32>) -> io::Result<OwnedFd> {
|
||||
let path = if path.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
path
|
||||
};
|
||||
if let Some(mode) = create_mode {
|
||||
std::fs::DirBuilder::new()
|
||||
.recursive(true)
|
||||
.mode(mode)
|
||||
.create(path)?;
|
||||
}
|
||||
// Retain the resolved directory inode so subsequent file operations do not rewalk the path.
|
||||
open(path, DIRECTORY_FLAGS & !OFlag::O_NOFOLLOW, Mode::empty()).map_err(errno_to_io)
|
||||
}
|
||||
|
||||
/// Opens a directory only when its entire path is owned by root or the effective user.
|
||||
fn open_trusted_dir_nofollow(path: &Path) -> io::Result<OwnedFd> {
|
||||
open_dir_components(path, None, true)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::path::Path;
|
||||
|
||||
use super::path::AnchoredPath;
|
||||
use super::write::atomic_replace_after_anchor;
|
||||
@@ -15,6 +16,50 @@ fn directory_walk_rejects_intermediate_symlink() {
|
||||
assert!(open_dir_nofollow(&link).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_parent_policy_creates_and_reads_relative_paths() {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
let current = std::env::current_dir().unwrap();
|
||||
let directory = tempfile::tempdir_in(¤t).unwrap();
|
||||
let relative = directory.path().strip_prefix(¤t).unwrap();
|
||||
for strict_runtime_paths in [false, true] {
|
||||
let mode = if strict_runtime_paths {
|
||||
"strict"
|
||||
} else {
|
||||
"compatible"
|
||||
};
|
||||
let path = relative.join(mode).join("nested/telemt.pid");
|
||||
let created =
|
||||
AnchoredPath::open_runtime_parent(&path, Some(0o750), strict_runtime_paths).unwrap();
|
||||
let opened = AnchoredPath::open_runtime_parent(&path, None, strict_runtime_paths).unwrap();
|
||||
let created = std::fs::File::from(created.parent().try_clone().unwrap())
|
||||
.metadata()
|
||||
.unwrap();
|
||||
let opened = std::fs::File::from(opened.parent().try_clone().unwrap())
|
||||
.metadata()
|
||||
.unwrap();
|
||||
assert_eq!((created.dev(), created.ino()), (opened.dev(), opened.ino()));
|
||||
assert_eq!(created.permissions().mode() & 0o777, 0o750);
|
||||
}
|
||||
assert!(AnchoredPath::open_runtime_parent(Path::new("telemt.pid"), None, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_parent_policy_only_requires_trusted_ownership_when_strict() {
|
||||
if !nix::unistd::Uid::effective().is_root() {
|
||||
return;
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let parent = directory.path().join("foreign-owner");
|
||||
std::fs::create_dir(&parent).unwrap();
|
||||
nix::unistd::chown(&parent, Some(nix::unistd::Uid::from_raw(65534)), None).unwrap();
|
||||
let path = parent.join("telemt.pid");
|
||||
|
||||
assert!(AnchoredPath::open_runtime_parent(&path, None, false).is_ok());
|
||||
assert!(AnchoredPath::open_runtime_parent(&path, None, true).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_replace_does_not_follow_final_symlink() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use base64::Engine as _;
|
||||
|
||||
use crate::config::WebCarrierMethod;
|
||||
use crate::crypto::SecureRandom;
|
||||
|
||||
/// Browser security policy for the transient Telegram Desktop bridge page.
|
||||
@@ -34,6 +35,7 @@ pub(crate) fn render(
|
||||
reconnect_grace_secs: u64,
|
||||
carrier_probe_coalesce_ms: u64,
|
||||
bridge_diagnostics_enabled: bool,
|
||||
carrier_method: WebCarrierMethod,
|
||||
rng: &SecureRandom,
|
||||
) -> BridgePage {
|
||||
let mut nonce = [0u8; 18];
|
||||
@@ -124,6 +126,7 @@ pub(crate) fn render(
|
||||
.replace("__HOST__", host)
|
||||
.replace("__BASE_PREFIX__", base_prefix)
|
||||
.replace("__BOOTSTRAP__", bootstrap)
|
||||
.replace("__CARRIER_METHOD__", carrier_method.as_str())
|
||||
.replace("__BATCH_LIMIT__", &batch_limit.to_string())
|
||||
.replace("__QUEUE_LIMIT__", &queue_limit.to_string())
|
||||
.replace("__QUEUE_ITEMS__", &queue_items.to_string())
|
||||
|
||||
@@ -7,6 +7,8 @@ const responseBody=globalThis.TelemtBridgeResponse;if(!responseBody)throw new Er
|
||||
const requestSupport=globalThis.TelemtBridgeRequest;if(!requestSupport)throw new Error('missing request runtime');
|
||||
const bufferSupport=globalThis.TelemtBridgeBuffers;if(!bufferSupport)throw new Error('missing buffer runtime');
|
||||
const recoverySupport=globalThis.TelemtBridgeRecovery;if(!recoverySupport)throw new Error('missing recovery runtime');
|
||||
// Keep the method page-owned so recovery and config rollback cannot change frozen retries.
|
||||
const carrierMethod='__CARRIER_METHOD__';
|
||||
let negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
||||
let longPollMs=__LONG_POLL_SECS__*1000,bridgeRequestMs=__BRIDGE_REQUEST_SECS__*1000,bridgeRetryMs=__BRIDGE_RETRY_SECS__*1000;
|
||||
let bridgeRecoveryMs=__BRIDGE_RECOVERY_SECS__*1000,websocketOpenMs=__WEBSOCKET_OPEN_SECS__*1000,reconnectGraceMs=__RECONNECT_GRACE_SECS__*1000;
|
||||
@@ -226,7 +228,7 @@ async function createSession(epoch){
|
||||
async function probeHttp(probe,laneID,epoch){
|
||||
try{
|
||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
||||
const response=await request('/api/v1/up',options('POST',token,body,headers,controller.signal));
|
||||
const response=await request('/api/v1/up',options(carrierMethod,token,body,headers,controller.signal));
|
||||
if(closed||epoch!==attemptEpoch)return
|
||||
if(response.status!==204){advanceCarrier('http',epoch);return}
|
||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
||||
@@ -263,14 +265,14 @@ async function runUp(){
|
||||
lease=takeBatch(upPending,null);upLease=lease;lease.controller=new AbortController();const sequence=String(upSequence),token=sessionToken;
|
||||
for(;;){
|
||||
try{
|
||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence},lease.controller.signal),null,1);
|
||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence},lease.controller.signal),null,1);
|
||||
if(response.status!==204)throw failure('http','uplink rejected');
|
||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink acknowledgement rejected');
|
||||
break;
|
||||
}catch(error){
|
||||
let replayed=false;
|
||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence},signal),remaining,2);
|
||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence},signal),remaining,2);
|
||||
if(response.status!==204)throw failure('http','uplink replay rejected');
|
||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink replay acknowledgement rejected');
|
||||
replayed=true;
|
||||
@@ -343,7 +345,7 @@ async function poll(){
|
||||
const token=sessionToken,cursor=downCursor;
|
||||
try{
|
||||
pollController=new AbortController();
|
||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor},pollController.signal),null,1);
|
||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor},pollController.signal),null,1);
|
||||
if(closed||sessionToken!==token)return;
|
||||
if(response.status===204){status('connected');continue}
|
||||
if(response.status!==200)throw failure('http','downlink rejected');
|
||||
@@ -354,7 +356,7 @@ async function poll(){
|
||||
}catch(error){
|
||||
if(closed)return;
|
||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor},signal),remaining,2);
|
||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor},signal),remaining,2);
|
||||
if(response.status===204)return;
|
||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','downlink replay rejected');
|
||||
});
|
||||
@@ -422,14 +424,14 @@ async function runLaneUp(lane){
|
||||
const sequence=String(lane.sequence),laneID=String(lane.id),token=sessionToken;
|
||||
for(;;){
|
||||
try{
|
||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},lease.controller.signal),null,1);
|
||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},lease.controller.signal),null,1);
|
||||
if(response.status!==204)throw failure('http','lane uplink rejected');
|
||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink acknowledgement rejected');
|
||||
break;
|
||||
}catch(error){
|
||||
let replayed=false;
|
||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},signal),remaining,2);
|
||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},signal),remaining,2);
|
||||
if(response.status!==204)throw failure('http','lane uplink replay rejected');
|
||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink replay acknowledgement rejected');
|
||||
replayed=true;
|
||||
@@ -451,7 +453,7 @@ async function pollLane(lane){
|
||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
||||
const controller=new AbortController(),laneID=String(lane.id),token=sessionToken,cursor=lane.cursor;lane.controller=controller;
|
||||
failedToken=token;failedCursor=cursor;failedLaneID=laneID;
|
||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor,'X-Lane-ID':laneID},controller.signal),null,1);
|
||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor,'X-Lane-ID':laneID},controller.signal),null,1);
|
||||
if(closed||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
||||
if(response.status===204){
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
||||
@@ -467,7 +469,7 @@ async function pollLane(lane){
|
||||
}catch(error){
|
||||
if(!closed&&lanes.get(lane.id)===lane){
|
||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
||||
const response=await request('/api/v1/down',options('POST',failedToken,null,{'X-Down-Cursor':failedCursor,'X-Lane-ID':failedLaneID},signal),remaining,2);
|
||||
const response=await request('/api/v1/down',options(carrierMethod,failedToken,null,{'X-Down-Cursor':failedCursor,'X-Lane-ID':failedLaneID},signal),remaining,2);
|
||||
if(response.status===204)return;
|
||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','lane downlink replay rejected');
|
||||
});
|
||||
|
||||
@@ -1,5 +1,73 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn carrier_method_is_page_owned_and_used_by_every_https_request() {
|
||||
for method in [WebCarrierMethod::Post, WebCarrierMethod::Put] {
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
"/telegram/web/",
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
1024,
|
||||
true,
|
||||
4,
|
||||
[3, 5, 8, 12],
|
||||
25,
|
||||
10,
|
||||
90,
|
||||
15,
|
||||
15,
|
||||
120,
|
||||
0,
|
||||
true,
|
||||
method,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(!page.body.contains("__"));
|
||||
assert!(
|
||||
page.body
|
||||
.contains(&format!("const carrierMethod='{}';", method.as_str()))
|
||||
);
|
||||
assert_eq!(page.body.matches("carrierMethod=").count(), 1);
|
||||
assert_eq!(page.body.matches("options(carrierMethod,").count(), 9);
|
||||
let requests: Vec<_> = page
|
||||
.body
|
||||
.lines()
|
||||
.filter(|line| {
|
||||
line.contains("request('/api/v1/up'") || line.contains("request('/api/v1/down'")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(requests.len(), 9);
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.all(|line| line.contains("options(carrierMethod,"))
|
||||
);
|
||||
assert_eq!(page.body.matches("options('POST',bootstrap,").count(), 2);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("fetch(relayBase+'/api/v1/diagnostic',{method:'POST'")
|
||||
);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("options('DELETE',token,null,headers,undefined,true)")
|
||||
);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("method:'GET',signal:requestController.signal")
|
||||
);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("exactKeys(value,['v','bootstrap','limits','timeouts','negotiation'])")
|
||||
);
|
||||
assert!(!page.body.contains("policy.carrier_method"));
|
||||
assert!(page.body.contains("port.postMessage({t:'status',state})"));
|
||||
assert!(!page.body.contains("port.postMessage({t:'status',state,"));
|
||||
}
|
||||
}
|
||||
|
||||
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||
render(
|
||||
"proxy.example.com",
|
||||
@@ -20,6 +88,7 @@ fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||
120,
|
||||
0,
|
||||
false,
|
||||
WebCarrierMethod::Post,
|
||||
&SecureRandom::new(),
|
||||
)
|
||||
}
|
||||
@@ -44,6 +113,7 @@ fn render_diagnostic_page(bootstrap: &str) -> BridgePage {
|
||||
120,
|
||||
0,
|
||||
true,
|
||||
WebCarrierMethod::Post,
|
||||
&SecureRandom::new(),
|
||||
)
|
||||
}
|
||||
@@ -95,6 +165,7 @@ fn rendered_page_resolves_carriers_against_the_exact_base_path() {
|
||||
120,
|
||||
0,
|
||||
true,
|
||||
WebCarrierMethod::Post,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
|
||||
@@ -148,6 +219,7 @@ fn rendered_page_embeds_the_configured_bridge_timing_policy() {
|
||||
119,
|
||||
4,
|
||||
false,
|
||||
WebCarrierMethod::Post,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
|
||||
@@ -201,6 +273,7 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
||||
120,
|
||||
0,
|
||||
false,
|
||||
WebCarrierMethod::Post,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(page.body.contains(
|
||||
|
||||
+3
-3
@@ -44,9 +44,9 @@ mod recovery;
|
||||
mod response;
|
||||
// Session creation and replacement negotiation remain separate from request routing.
|
||||
mod session;
|
||||
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
||||
mod websocket;
|
||||
// Enabled-debug integration coverage remains separate from carrier behavior tests.
|
||||
#[cfg(test)]
|
||||
@@ -389,6 +389,7 @@ async fn handle_root(
|
||||
config.web.timeouts.reconnect_grace_secs,
|
||||
config.web.timeouts.carrier_probe_coalesce_ms,
|
||||
config.web.debug.bridge_diagnostics_enabled(),
|
||||
config.web.carrier_method,
|
||||
&generation.rng,
|
||||
);
|
||||
let mut response = full_response(StatusCode::OK, Bytes::from(page.body));
|
||||
@@ -456,14 +457,13 @@ async fn handle_api(
|
||||
_ => serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_up(
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: crate::web::manager::TokenHash,
|
||||
) -> HttpResponse {
|
||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
||||
if !matches!(*request.method(), Method::POST | Method::PUT) || !binary_content_type(&request) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some(sequence) = canonical_u64_header(&request, "x-up-seq").filter(|value| *value != 0)
|
||||
|
||||
@@ -0,0 +1,480 @@
|
||||
use super::super::session_policy_tests::{open_keepalive, read_http_response};
|
||||
use super::super::*;
|
||||
use crate::config::WebCarrierMethod;
|
||||
|
||||
fn carrier_request(method: &str, path: &str, token: &str, headers: &str, body: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = format!(
|
||||
"{method} {path} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nAuthorization: Bearer {token}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
)
|
||||
.into_bytes();
|
||||
bytes.extend_from_slice(body);
|
||||
bytes
|
||||
}
|
||||
|
||||
async fn bridge_page(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
capability: [u8; 32],
|
||||
base: &str,
|
||||
) -> Vec<u8> {
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
request(
|
||||
listener,
|
||||
runtime,
|
||||
format!(
|
||||
"GET {base}?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn bootstrap_from(page: &[u8]) -> &str {
|
||||
let (headers, body) = split_response(page);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
std::str::from_utf8(body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.unwrap()
|
||||
.1
|
||||
.split_once('"')
|
||||
.unwrap()
|
||||
.0
|
||||
}
|
||||
|
||||
async fn create_session(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
bootstrap: &str,
|
||||
base: &str,
|
||||
) -> String {
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let bytes = carrier_request(
|
||||
"POST",
|
||||
&format!("{base}api/v1/session"),
|
||||
bootstrap,
|
||||
"Content-Type: application/octet-stream\r\n",
|
||||
&hello,
|
||||
);
|
||||
let response = request(listener, runtime, bytes).await;
|
||||
let (headers, _) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
response_header(headers, "x-session-token").to_string()
|
||||
}
|
||||
|
||||
async fn session_token(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
capability: [u8; 32],
|
||||
base: &str,
|
||||
) -> String {
|
||||
let page = bridge_page(listener, runtime, capability, base).await;
|
||||
create_session(listener, runtime, bootstrap_from(&page), base).await
|
||||
}
|
||||
|
||||
async fn assert_put_accepted(path: &str) {
|
||||
for (carrier, base, method) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||
.into_iter()
|
||||
.flat_map(|carrier| {
|
||||
["/", "/telegram/web/"].into_iter().flat_map(move |base| {
|
||||
[WebCarrierMethod::Post, WebCarrierMethod::Put]
|
||||
.map(|method| (carrier, base, method))
|
||||
})
|
||||
})
|
||||
{
|
||||
let capability = [60; 32];
|
||||
let mut config = runtime_config_with_base(capability, carrier, base);
|
||||
config.web.carrier_method = method;
|
||||
config.web.timeouts.long_poll_secs = 0;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let token = session_token(&listener, &runtime, capability, base).await;
|
||||
let lane = if carrier.uses_lanes() {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let (headers, body) = if path.ends_with("/up") {
|
||||
(
|
||||
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}"),
|
||||
pong.as_ref(),
|
||||
)
|
||||
} else {
|
||||
(format!("X-Down-Cursor: 0\r\n{lane}"), &[][..])
|
||||
};
|
||||
let path = format!("{base}{}", path.trim_start_matches('/'));
|
||||
let bytes = carrier_request("PUT", &path, &token, &headers, body);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(
|
||||
response.starts_with(b"HTTP/1.1 204"),
|
||||
"{carrier:?}: {}",
|
||||
String::from_utf8_lossy(&response)
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_uplink_is_accepted_for_both_https_carriers() {
|
||||
assert_put_accepted("/api/v1/up").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_downlink_is_accepted_for_both_https_carriers() {
|
||||
assert_put_accepted("/api/v1/down").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_mixed_retries_share_sequence_and_cursor_state() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
for method in [WebCarrierMethod::Post, WebCarrierMethod::Put] {
|
||||
let capability = [61; 32];
|
||||
let mut config = runtime_config(capability, carrier);
|
||||
config.web.carrier_method = method;
|
||||
config.web.timeouts.long_poll_secs = 0;
|
||||
// Exhausted admission queues a deterministic CLOSE without starting a relay task.
|
||||
config.web.limits.max_streams_global = 0;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime =
|
||||
WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||
for (index, stream_id) in [7, 8].into_iter().enumerate() {
|
||||
let lane = if carrier.uses_lanes() {
|
||||
format!("X-Lane-ID: {stream_id}\r\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let sequence = if carrier.uses_lanes() { 1 } else { index + 1 };
|
||||
let cursor = if carrier.uses_lanes() { 0 } else { index };
|
||||
let open = frame::encode(FrameType::Open, stream_id, &[]);
|
||||
let up_headers = format!(
|
||||
"Content-Type: application/octet-stream\r\nX-Up-Seq: {sequence}\r\n{lane}"
|
||||
);
|
||||
let methods = if index == 0 {
|
||||
["POST", "PUT"]
|
||||
} else {
|
||||
["PUT", "POST"]
|
||||
};
|
||||
for verb in methods {
|
||||
let bytes = carrier_request(verb, "/api/v1/up", &token, &up_headers, &open);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
let (headers, body) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert_eq!(response_header(headers, "x-up-ack"), sequence.to_string());
|
||||
assert!(body.is_empty());
|
||||
}
|
||||
for verb in methods {
|
||||
let headers = format!("X-Down-Cursor: {cursor}\r\n{lane}");
|
||||
let bytes = carrier_request(verb, "/api/v1/down", &token, &headers, &[]);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
let (headers, body) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(headers, "x-down-cursor"),
|
||||
(cursor + 1).to_string()
|
||||
);
|
||||
assert_eq!(body, frame::encode(FrameType::Close, stream_id, &[]));
|
||||
}
|
||||
let headers = format!("X-Down-Cursor: {}\r\n{lane}", cursor + 1);
|
||||
let bytes = carrier_request("PUT", "/api/v1/down", &token, &headers, &[]);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||
}
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_changed_body_retry_keeps_protocol_failure() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
let capability = [62; 32];
|
||||
let generation = test_runtime_generation(1, runtime_config(capability, carrier));
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||
let lane = if carrier.uses_lanes() {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let bytes = carrier_request("POST", "/api/v1/up", &token, &headers, &pong);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||
let mut changed = pong.to_vec();
|
||||
changed.extend_from_slice(&pong);
|
||||
let bytes = carrier_request("PUT", "/api/v1/up", &token, &headers, &changed);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert_private_decoy(&response);
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_private_decoy(response: &[u8]) {
|
||||
let (headers, body) = split_response(response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 404"));
|
||||
assert_eq!(response_header(headers, "cache-control"), "no-store");
|
||||
assert_eq!(body, b"not found\n");
|
||||
let headers = std::str::from_utf8(headers).unwrap().to_ascii_lowercase();
|
||||
for name in ["x-up-ack:", "x-down-cursor:", "x-session-token:", "allow:"] {
|
||||
assert!(!headers.contains(name));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_put_preserves_authenticated_request_shape_checks() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
let capability = [63; 32];
|
||||
let base = "/telegram/web/";
|
||||
let generation =
|
||||
test_runtime_generation(1, runtime_config_with_base(capability, carrier, base));
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let token = session_token(&listener, &runtime, capability, base).await;
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let lane = if carrier.uses_lanes() {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let up = format!("{base}api/v1/up");
|
||||
let down = format!("{base}api/v1/down");
|
||||
let up_headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||
let down_headers = format!("X-Down-Cursor: 0\r\n{lane}");
|
||||
let mut invalid = Vec::new();
|
||||
for media in [
|
||||
"",
|
||||
"Content-Type: text/plain\r\n",
|
||||
"Content-Type: application/octet-stream\r\nContent-Type: application/octet-stream\r\n",
|
||||
] {
|
||||
let headers = format!("{media}X-Up-Seq: 1\r\n{lane}");
|
||||
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||
}
|
||||
for sequence in ["0", "01", ""] {
|
||||
let sequence = if sequence.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("X-Up-Seq: {sequence}\r\n")
|
||||
};
|
||||
let headers = format!("Content-Type: application/octet-stream\r\n{sequence}{lane}");
|
||||
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||
}
|
||||
let invalid_lane = if carrier.uses_lanes() {
|
||||
""
|
||||
} else {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
};
|
||||
let headers =
|
||||
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{invalid_lane}");
|
||||
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||
let headers = format!("{down_headers}Content-Type: application/octet-stream\r\n");
|
||||
invalid.push(carrier_request("PUT", &down, &token, &headers, &[]));
|
||||
invalid.push(carrier_request("PUT", &down, &token, &down_headers, &[0]));
|
||||
let headers = format!("X-Down-Cursor: 01\r\n{lane}");
|
||||
invalid.push(carrier_request("PUT", &down, &token, &headers, &[]));
|
||||
for verb in ["GET", "PATCH"] {
|
||||
invalid.push(carrier_request(verb, &up, &token, &up_headers, &pong));
|
||||
invalid.push(carrier_request(verb, &down, &token, &down_headers, &[]));
|
||||
}
|
||||
for path in [
|
||||
"/api/v1/up",
|
||||
"/telegram/web/api/v1/up/",
|
||||
"/telegram/web/api/v1/up?q=1",
|
||||
"/telegram/web/api/v1//up",
|
||||
"/telegram/web/api/v1/%75p",
|
||||
] {
|
||||
invalid.push(carrier_request("PUT", path, &token, &up_headers, &pong));
|
||||
}
|
||||
let wrong_host = String::from_utf8(carrier_request("PUT", &up, &token, &up_headers, &pong))
|
||||
.unwrap()
|
||||
.replace("Host: proxy.example.com", "Host: other.example.com")
|
||||
.into_bytes();
|
||||
invalid.push(wrong_host);
|
||||
let path = format!("{base}api/v1/session");
|
||||
invalid.push(carrier_request("PUT", &path, &token, &up_headers, &pong));
|
||||
for bytes in invalid {
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert_private_decoy(&response);
|
||||
}
|
||||
// Rejected shapes must not consume the first sequence or close the valid session.
|
||||
let bytes = carrier_request("PUT", &up, &token, &up_headers, &pong);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_post_and_put_reuse_one_private_http_connection() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
let capability = [64; 32];
|
||||
let mut config = runtime_config(capability, carrier);
|
||||
config.web.timeouts.long_poll_secs = 0;
|
||||
config.web.limits.max_streams_global = 0;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||
let (mut client, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||
let lane = if carrier.uses_lanes() {
|
||||
"X-Lane-ID: 7\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
let close = frame::encode(FrameType::Close, 7, &[]);
|
||||
for verb in ["POST", "PUT", "PUT", "POST"] {
|
||||
for (path, headers, body, ack_header) in [
|
||||
(
|
||||
"/api/v1/up",
|
||||
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}"),
|
||||
open.as_ref(),
|
||||
"x-up-ack",
|
||||
),
|
||||
(
|
||||
"/api/v1/down",
|
||||
format!("X-Down-Cursor: 0\r\n{lane}"),
|
||||
&[][..],
|
||||
"x-down-cursor",
|
||||
),
|
||||
] {
|
||||
let bytes = carrier_request(verb, path, &token, &headers, body);
|
||||
let bytes = String::from_utf8(bytes)
|
||||
.unwrap()
|
||||
.replace("Connection: close\r\n", "Connection: keep-alive\r\n")
|
||||
.into_bytes();
|
||||
client.write_all(&bytes).await.unwrap();
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
read_http_response(&mut client),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (headers, body) = split_response(&response);
|
||||
assert_eq!(response_header(headers, ack_header), "1");
|
||||
assert!(
|
||||
!std::str::from_utf8(headers)
|
||||
.unwrap()
|
||||
.to_ascii_lowercase()
|
||||
.contains("connection: close")
|
||||
);
|
||||
if path.ends_with("/up") {
|
||||
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||
assert!(body.is_empty());
|
||||
} else {
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(headers, "content-length"),
|
||||
close.len().to_string()
|
||||
);
|
||||
assert_eq!(body, close);
|
||||
}
|
||||
}
|
||||
}
|
||||
cancellation.cancel();
|
||||
task.await.unwrap();
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn carrier_method_reload_preserves_old_pages_bootstraps_sessions_and_recovery() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
let capability = [65; 32];
|
||||
let mut config = runtime_config(capability, carrier);
|
||||
config.web.limits.max_bootstraps_per_ip = 6;
|
||||
let generation = test_runtime_generation(1, config.clone());
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let post_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||
let old_session =
|
||||
create_session(&listener, &runtime, bootstrap_from(&post_page), "/").await;
|
||||
let unused_post_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||
config.web.carrier_method = WebCarrierMethod::Put;
|
||||
let put_generation = test_runtime_generation(2, config.clone());
|
||||
active_runtime.store(Arc::clone(&put_generation));
|
||||
let post_session =
|
||||
create_session(&listener, &runtime, bootstrap_from(&unused_post_page), "/").await;
|
||||
let put_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||
assert!(String::from_utf8_lossy(&put_page).contains("const carrierMethod='PUT';"));
|
||||
config.web.carrier_method = WebCarrierMethod::Post;
|
||||
let rollback_generation = test_runtime_generation(3, config);
|
||||
active_runtime.store(Arc::clone(&rollback_generation));
|
||||
let put_session = create_session(&listener, &runtime, bootstrap_from(&put_page), "/").await;
|
||||
let rollback_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||
assert!(String::from_utf8_lossy(&rollback_page).contains("const carrierMethod='POST';"));
|
||||
assert!(String::from_utf8_lossy(&post_page).contains("const carrierMethod='POST';"));
|
||||
let lane = if carrier.uses_lanes() {
|
||||
"X-Lane-ID: 0\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||
for token in [&old_session, &post_session, &put_session] {
|
||||
for verb in ["PUT", "POST"] {
|
||||
let bytes = carrier_request(verb, "/api/v1/up", token, &headers, &pong);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||
}
|
||||
}
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let bytes = carrier_request(
|
||||
"GET",
|
||||
&format!("/?bridge={encoded}"),
|
||||
&put_session,
|
||||
"Accept: application/vnd.telemt.web-recovery+json\r\n",
|
||||
&[],
|
||||
);
|
||||
let recovery = request(&listener, &runtime, bytes).await;
|
||||
let (recovery_headers, body) = split_response(&recovery);
|
||||
assert!(recovery_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(recovery_headers, "content-type"),
|
||||
"application/vnd.telemt.web-recovery+json"
|
||||
);
|
||||
let document: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||
let keys: std::collections::BTreeSet<_> = document
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
keys,
|
||||
["v", "bootstrap", "limits", "timeouts", "negotiation"]
|
||||
.into_iter()
|
||||
.collect()
|
||||
);
|
||||
let bootstrap = document["bootstrap"].as_str().unwrap();
|
||||
let recovered = create_session(&listener, &runtime, bootstrap, "/").await;
|
||||
let bytes = carrier_request("PUT", "/api/v1/up", &recovered, &headers, &pong);
|
||||
let response = request(&listener, &runtime, bytes).await;
|
||||
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||
let bytes = carrier_request("PUT", "/api/v1/up", &put_session, &headers, &pong);
|
||||
let retired = request(&listener, &runtime, bytes).await;
|
||||
assert_private_decoy(&retired);
|
||||
runtime.shutdown().await;
|
||||
for generation in [generation, put_generation, rollback_generation] {
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ pub(super) async fn handle_down(
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: TokenHash,
|
||||
) -> HttpResponse {
|
||||
if request.method() != hyper::Method::POST
|
||||
if !matches!(*request.method(), hyper::Method::POST | hyper::Method::PUT)
|
||||
|| request.headers().contains_key(header::CONTENT_TYPE)
|
||||
{
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
use super::*;
|
||||
|
||||
// HTTP method compatibility keeps platform workarounds outside session state.
|
||||
#[path = "carrier_method_tests.rs"]
|
||||
mod carrier_method_tests;
|
||||
|
||||
#[tokio::test]
|
||||
async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
||||
for (index, carrier) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::*;
|
||||
|
||||
async fn open_keepalive(
|
||||
/// Opens one reusable private HTTP connection for integration scenarios.
|
||||
pub(super) async fn open_keepalive(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
||||
@@ -21,7 +22,8 @@ async fn open_keepalive(
|
||||
(client.unwrap(), cancellation, task)
|
||||
}
|
||||
|
||||
async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
||||
/// Reads one length-delimited response without waiting for connection closure.
|
||||
pub(super) async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
||||
let mut response = Vec::new();
|
||||
while !response.ends_with(b"\r\n\r\n") {
|
||||
assert!(response.len() < 16 * 1024);
|
||||
|
||||
Reference in New Issue
Block a user