mirror of
https://github.com/telemt/telemt.git
synced 2026-10-29 20:17:16 +03:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9d5b896bb6 | |||
| 556affc63a | |||
| 9c31e970e6 | |||
| 3b924ab40c | |||
| bc1c130f38 | |||
| d3de9865cf | |||
| e2f1c3f3a9 | |||
| dcce897d8b | |||
| 823c6cbc7f |
Generated
+1
-1
@@ -2780,7 +2780,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.5.12"
|
version = "3.5.14"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.5.12"
|
version = "3.5.14"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.88"
|
rust-version = "1.88"
|
||||||
|
|
||||||
|
|||||||
+13
-3
@@ -158,7 +158,7 @@ async fn prepare_patch_to_path(
|
|||||||
) -> Result<PreparedConfigPatch, ApiFailure> {
|
) -> Result<PreparedConfigPatch, ApiFailure> {
|
||||||
// 1. optimistic concurrency
|
// 1. optimistic concurrency
|
||||||
let loaded = load_config_snapshot(config_path, false).await?;
|
let loaded = load_config_snapshot(config_path, false).await?;
|
||||||
let current = compute_snapshot_revision(&loaded);
|
let current = compute_snapshot_revision(&loaded.source_contents);
|
||||||
if expected_revision.is_some_and(|expected| expected != current) {
|
if expected_revision.is_some_and(|expected| expected != current) {
|
||||||
return Err(ApiFailure::new(
|
return Err(ApiFailure::new(
|
||||||
hyper::StatusCode::CONFLICT,
|
hyper::StatusCode::CONFLICT,
|
||||||
@@ -208,6 +208,8 @@ async fn prepare_patch_to_path(
|
|||||||
let mut merged = Toml::try_from(&old_cfg)
|
let mut merged = Toml::try_from(&old_cfg)
|
||||||
.map_err(|e| ApiFailure::internal(format!("failed to serialize config: {}", e)))?;
|
.map_err(|e| ApiFailure::internal(format!("failed to serialize config: {}", e)))?;
|
||||||
deep_merge(&mut merged, &patch_toml);
|
deep_merge(&mut merged, &patch_toml);
|
||||||
|
ProxyConfig::validate_decoy_source_keys(&merged)
|
||||||
|
.map_err(|error| ApiFailure::bad_request(format!("invalid patched config: {error}")))?;
|
||||||
|
|
||||||
let requested_cfg: ProxyConfig = merged
|
let requested_cfg: ProxyConfig = merged
|
||||||
.clone()
|
.clone()
|
||||||
@@ -263,7 +265,7 @@ async fn prepare_patch_to_path(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 4. classify the validated, normalized candidate.
|
// 4. classify the validated, normalized candidate.
|
||||||
let revision = compute_snapshot_revision(&candidate);
|
let revision = compute_snapshot_revision(&candidate.source_contents);
|
||||||
let new_cfg = candidate.config;
|
let new_cfg = candidate.config;
|
||||||
let class = classify_config_changes(&old_cfg, &new_cfg);
|
let class = classify_config_changes(&old_cfg, &new_cfg);
|
||||||
let deferred_process_fields =
|
let deferred_process_fields =
|
||||||
@@ -306,7 +308,7 @@ fn reload_submit_failure(error: ReloadSubmitError) -> ApiFailure {
|
|||||||
/// Returns only the editable config sections and current revision.
|
/// Returns only the editable config sections and current revision.
|
||||||
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
||||||
let loaded = load_config_snapshot(config_path, false).await?;
|
let loaded = load_config_snapshot(config_path, false).await?;
|
||||||
let revision = compute_snapshot_revision(&loaded);
|
let revision = compute_snapshot_revision(&loaded.source_contents);
|
||||||
let parsed = Toml::try_from(&loaded.config)
|
let parsed = Toml::try_from(&loaded.config)
|
||||||
.map_err(|error| ApiFailure::internal(format!("failed to serialize config: {error}")))?;
|
.map_err(|error| ApiFailure::internal(format!("failed to serialize config: {error}")))?;
|
||||||
|
|
||||||
@@ -453,3 +455,11 @@ mod base_path_tests;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "config_edit/tests.rs"]
|
#[path = "config_edit/tests.rs"]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "config_edit/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "config_edit/decoy_dns_tests.rs"]
|
||||||
|
mod decoy_dns_tests;
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn conveyor_api_is_boolean_hot_and_rejects_invalid_patch_atomically() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(&path, "[web]\nconveyor = true\n").unwrap();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let active = ProxyConfig::load(&path).unwrap();
|
||||||
|
let patch = serde_json::json!({"web": {"conveyor": enabled}});
|
||||||
|
let mut response = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||||
|
let desired = ProxyConfig::load(&path).unwrap();
|
||||||
|
let resolved = reconcile_runtime_effect(&mut response, &active, &desired).unwrap();
|
||||||
|
assert_eq!(resolved.effective.web.conveyor, enabled);
|
||||||
|
assert!(response.runtime_reload_required);
|
||||||
|
assert!(!response.process_restart_required);
|
||||||
|
let (managed, _) = read_managed_config(&path).await.unwrap();
|
||||||
|
assert_eq!(managed["web"]["conveyor"].as_bool(), Some(enabled));
|
||||||
|
let previous = std::fs::read(&path).unwrap();
|
||||||
|
let invalid = serde_json::json!({"web": {"conveyor": "false"}});
|
||||||
|
let error = apply_patch_to_path(&path, &invalid, None)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), previous);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const SOURCE: &str = r#"
|
||||||
|
[access.users]
|
||||||
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
[[server.listeners]]
|
||||||
|
ip = "127.0.0.1"
|
||||||
|
port = 18080
|
||||||
|
transport = "web"
|
||||||
|
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://localhost:18081"
|
||||||
|
resolve = "startup"
|
||||||
|
[[web.vhosts.profiles]]
|
||||||
|
user = "alice"
|
||||||
|
secret_mode = "dd"
|
||||||
|
"#;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_api_get_and_mutation_base_need_no_prepared_runtime() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
SOURCE.replace("localhost", "unresolvable.example.invalid"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (managed, revision) = read_managed_config(&path).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
managed["web"]["vhosts"][0]["decoy"]["resolve"].as_str(),
|
||||||
|
Some("startup")
|
||||||
|
);
|
||||||
|
assert!(managed["web"].get("decoy_dns").is_none());
|
||||||
|
let (base, base_revision) = crate::api::config_store::load_config_for_mutation(&path, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(revision, base_revision);
|
||||||
|
assert!(base.web.runtime.is_none());
|
||||||
|
assert!(base.web.decoy_dns.origins.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_api_patch_without_reload_prepares_before_disk_write() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(&path, SOURCE).unwrap();
|
||||||
|
// This current-thread runtime exercises the real Tokio resolver without nested block_on.
|
||||||
|
let active = ProxyConfig::load_prepared(path.clone())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.config;
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
let mut vhosts = serde_json::to_value(&active.web.vhosts).unwrap();
|
||||||
|
vhosts[0]["decoy"]["upstream"] = Json::String("http://localhost:18080".to_string());
|
||||||
|
let error = apply_patch_to_path(&path, &serde_json::json!({"web": {"vhosts": vhosts}}), None)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.message.contains("overlaps WEB listener"));
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), before);
|
||||||
|
assert!(active.web.runtime.is_some());
|
||||||
|
|
||||||
|
let prepared = prepare_patch_to_path(
|
||||||
|
&path,
|
||||||
|
&serde_json::json!({"web": {"conveyor": false}}),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(prepared.desired_config.web.decoy_dns.origins.len(), 1);
|
||||||
|
assert!(prepared.desired_config.web.runtime.is_some());
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), before);
|
||||||
|
apply_patch_to_path(
|
||||||
|
&path,
|
||||||
|
&serde_json::json!({"web": {"conveyor": false}}),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let reloaded = ProxyConfig::load_prepared(path).await.unwrap();
|
||||||
|
assert!(!reloaded.config.web.conveyor);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_api_rejects_resolve_on_static_mode_before_serde_drops_it() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(&path, SOURCE).unwrap();
|
||||||
|
let base = ProxyConfig::parse_source(&path).unwrap();
|
||||||
|
let mut vhosts = serde_json::to_value(&base.config.web.vhosts).unwrap();
|
||||||
|
vhosts[0]["decoy"] = serde_json::json!({
|
||||||
|
"mode": "static_directory", "directory": directory.path(), "resolve": "startup"
|
||||||
|
});
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
let error = apply_patch_to_path(&path, &serde_json::json!({"web": {"vhosts": vhosts}}), None)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.message.contains("only valid for mode=http_upstream"));
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), before);
|
||||||
|
}
|
||||||
@@ -176,7 +176,8 @@ async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
|||||||
assert!(desired.web.debug.sideband);
|
assert!(desired.web.debug.sideband);
|
||||||
|
|
||||||
let limits_patch: Json = serde_json::json!({
|
let limits_patch: Json = serde_json::json!({
|
||||||
"web": {"limits": {"max_http_connections": 2049}}
|
// Extra HTTP heads must fit alongside the default-on conveyor metadata reservation.
|
||||||
|
"web": {"limits": {"max_http_connections": 2049, "memory_envelope_bytes": 1610612736u64}}
|
||||||
});
|
});
|
||||||
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
||||||
.await
|
.await
|
||||||
|
|||||||
+27
-15
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
|
|||||||
use hyper::header::IF_MATCH;
|
use hyper::header::IF_MATCH;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use crate::config::{ConfigSourceGraph, LoadedConfig, ProxyConfig};
|
use crate::config::{ConfigSourceGraph, LoadedConfig, ParsedConfigSource, ProxyConfig};
|
||||||
|
|
||||||
use super::model::ApiFailure;
|
use super::model::ApiFailure;
|
||||||
|
|
||||||
@@ -63,7 +63,7 @@ pub(super) async fn load_config_for_mutation(
|
|||||||
expected_revision: Option<&str>,
|
expected_revision: Option<&str>,
|
||||||
) -> Result<(ProxyConfig, String), ApiFailure> {
|
) -> Result<(ProxyConfig, String), ApiFailure> {
|
||||||
let loaded = load_config_snapshot(config_path, false).await?;
|
let loaded = load_config_snapshot(config_path, false).await?;
|
||||||
let revision = compute_snapshot_revision(&loaded);
|
let revision = compute_snapshot_revision(&loaded.source_contents);
|
||||||
if expected_revision.is_some_and(|expected| expected != revision) {
|
if expected_revision.is_some_and(|expected| expected != revision) {
|
||||||
return Err(ApiFailure::new(
|
return Err(ApiFailure::new(
|
||||||
hyper::StatusCode::CONFLICT,
|
hyper::StatusCode::CONFLICT,
|
||||||
@@ -98,9 +98,10 @@ pub(super) fn compute_revision(content: &str) -> String {
|
|||||||
hex::encode(hasher.finalize())
|
hex::encode(hasher.finalize())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn compute_snapshot_revision(loaded: &LoadedConfig) -> String {
|
/// Hashes source documents only; prepared DNS answers never change a revision.
|
||||||
|
pub(super) fn compute_snapshot_revision(source_contents: &BTreeMap<PathBuf, String>) -> String {
|
||||||
compute_source_revision(&ConfigSourceGraph {
|
compute_source_revision(&ConfigSourceGraph {
|
||||||
source_contents: loaded.source_contents.clone(),
|
source_contents: source_contents.clone(),
|
||||||
rendered: String::new(),
|
rendered: String::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -118,12 +119,13 @@ pub(super) fn compute_source_revision(graph: &ConfigSourceGraph) -> String {
|
|||||||
hex::encode(hasher.finalize())
|
hex::encode(hasher.finalize())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads a normalized source snapshot without resolving decoys or loading static assets.
|
||||||
pub(super) async fn load_config_snapshot(
|
pub(super) async fn load_config_snapshot(
|
||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
invalid_is_bad_request: bool,
|
invalid_is_bad_request: bool,
|
||||||
) -> Result<LoadedConfig, ApiFailure> {
|
) -> Result<ParsedConfigSource, ApiFailure> {
|
||||||
let config_path = config_path.to_path_buf();
|
let config_path = config_path.to_path_buf();
|
||||||
tokio::task::spawn_blocking(move || ProxyConfig::load_with_metadata(config_path))
|
tokio::task::spawn_blocking(move || ProxyConfig::parse_source(config_path))
|
||||||
.await
|
.await
|
||||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {error}")))?
|
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {error}")))?
|
||||||
.map_err(|error| {
|
.map_err(|error| {
|
||||||
@@ -135,8 +137,9 @@ pub(super) async fn load_config_snapshot(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Selects the single writable source document owning all requested sections.
|
||||||
pub(super) fn resolve_single_source_owner(
|
pub(super) fn resolve_single_source_owner(
|
||||||
loaded: &LoadedConfig,
|
loaded: &ParsedConfigSource,
|
||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
targets: &[&str],
|
targets: &[&str],
|
||||||
) -> Result<PathBuf, ApiFailure> {
|
) -> Result<PathBuf, ApiFailure> {
|
||||||
@@ -227,6 +230,7 @@ fn has_include_inside_table(content: &str) -> bool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Prepares the entire mutation candidate before any source file may be replaced.
|
||||||
pub(super) async fn load_candidate_snapshot(
|
pub(super) async fn load_candidate_snapshot(
|
||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
base_sources: &BTreeMap<PathBuf, String>,
|
base_sources: &BTreeMap<PathBuf, String>,
|
||||||
@@ -236,12 +240,16 @@ pub(super) async fn load_candidate_snapshot(
|
|||||||
let config_path = config_path.to_path_buf();
|
let config_path = config_path.to_path_buf();
|
||||||
let mut overrides = base_sources.clone();
|
let mut overrides = base_sources.clone();
|
||||||
overrides.insert(owner_path, owner_contents);
|
overrides.insert(owner_path, owner_contents);
|
||||||
tokio::task::spawn_blocking(move || {
|
let parsed = tokio::task::spawn_blocking(move || {
|
||||||
ProxyConfig::load_with_source_overrides(config_path, &overrides)
|
ProxyConfig::parse_source_with_overrides(config_path, &overrides)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {error}")))?
|
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {error}")))?
|
||||||
.map_err(|error| ApiFailure::bad_request(format!("invalid patched config: {error}")))
|
.map_err(|error| ApiFailure::bad_request(format!("invalid patched config: {error}")))?;
|
||||||
|
parsed
|
||||||
|
.prepare()
|
||||||
|
.await
|
||||||
|
.map_err(|error| ApiFailure::bad_request(format!("invalid patched config: {error}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_source_path(path: &Path) -> PathBuf {
|
fn normalize_source_path(path: &Path) -> PathBuf {
|
||||||
@@ -265,19 +273,23 @@ fn normalize_source_path(path: &Path) -> PathBuf {
|
|||||||
normalized
|
normalized
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads desired user and quota policy without acquiring runtime decoy state.
|
||||||
pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
||||||
let config_path = config_path.to_path_buf();
|
load_config_snapshot(config_path, false)
|
||||||
tokio::task::spawn_blocking(move || ProxyConfig::load(config_path))
|
|
||||||
.await
|
.await
|
||||||
.map_err(|e| ApiFailure::internal(format!("failed to join config loader: {}", e)))?
|
.map(|source| source.config)
|
||||||
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquires fresh decoy evidence for an explicit API reload, even at the same revision.
|
||||||
pub(super) async fn load_config_for_reload(
|
pub(super) async fn load_config_for_reload(
|
||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
) -> Result<(ProxyConfig, String), ApiFailure> {
|
) -> Result<(ProxyConfig, String), ApiFailure> {
|
||||||
let loaded = load_config_snapshot(config_path, true).await?;
|
let loaded = load_config_snapshot(config_path, true).await?;
|
||||||
let revision = compute_snapshot_revision(&loaded);
|
let revision = compute_snapshot_revision(&loaded.source_contents);
|
||||||
|
let loaded = loaded
|
||||||
|
.prepare()
|
||||||
|
.await
|
||||||
|
.map_err(|error| ApiFailure::bad_request(format!("invalid runtime config: {error}")))?;
|
||||||
Ok((loaded.config, revision))
|
Ok((loaded.config, revision))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ pub(in crate::api) async fn save_access_sections_to_disk_if_revision(
|
|||||||
expected_revision: Option<&str>,
|
expected_revision: Option<&str>,
|
||||||
) -> Result<String, ApiFailure> {
|
) -> Result<String, ApiFailure> {
|
||||||
let loaded = load_config_snapshot(config_path, false).await?;
|
let loaded = load_config_snapshot(config_path, false).await?;
|
||||||
let loaded_revision = compute_snapshot_revision(&loaded);
|
let loaded_revision = compute_snapshot_revision(&loaded.source_contents);
|
||||||
if expected_revision.is_some_and(|expected| expected != loaded_revision) {
|
if expected_revision.is_some_and(|expected| expected != loaded_revision) {
|
||||||
return Err(revision_conflict());
|
return Err(revision_conflict());
|
||||||
}
|
}
|
||||||
@@ -159,7 +159,7 @@ pub(in crate::api) async fn save_access_sections_to_disk_if_revision(
|
|||||||
owner_contents.clone(),
|
owner_contents.clone(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let _candidate_revision = compute_snapshot_revision(&candidate);
|
let _candidate_revision = compute_snapshot_revision(&candidate.source_contents);
|
||||||
let revision = write_atomic_if_unchanged(
|
let revision = write_atomic_if_unchanged(
|
||||||
config_path.to_path_buf(),
|
config_path.to_path_buf(),
|
||||||
loaded_revision,
|
loaded_revision,
|
||||||
|
|||||||
@@ -72,3 +72,7 @@ mod tests;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "hot_reload/carrier_method_tests.rs"]
|
#[path = "hot_reload/carrier_method_tests.rs"]
|
||||||
mod carrier_method_tests;
|
mod carrier_method_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "hot_reload/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_reload_is_hot_in_both_directions_and_preserves_process_limits() {
|
||||||
|
let mut active = ProxyConfig::default();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let mut desired = active.clone();
|
||||||
|
desired.web.conveyor = enabled;
|
||||||
|
assert_eq!(classify_config_changes(&active, &desired).changed, ["web"]);
|
||||||
|
assert!(!classify_config_changes(&active, &desired).restart_required);
|
||||||
|
desired.web.limits.max_body_readers += 1;
|
||||||
|
let applied = overlay_hot_fields(&active, &desired);
|
||||||
|
assert_eq!(applied.web.conveyor, enabled);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.max_body_readers,
|
||||||
|
active.web.limits.max_body_readers
|
||||||
|
);
|
||||||
|
active = applied;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,8 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// Source polling and explicit reloads have distinct DNS preparation lifecycles.
|
||||||
|
mod preparation;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
pub(super) struct WatchManifest {
|
pub(super) struct WatchManifest {
|
||||||
files: BTreeSet<PathBuf>,
|
files: BTreeSet<PathBuf>,
|
||||||
@@ -132,9 +135,21 @@ fn apply_watch_manifest<W1: Watcher, W2: Watcher>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load config, validate, diff against current, and broadcast if changed.
|
/// Builds the effective WEB snapshot without changing live state.
|
||||||
|
fn prepare_effective_config(
|
||||||
|
old_cfg: &ProxyConfig,
|
||||||
|
new_cfg: &ProxyConfig,
|
||||||
|
) -> crate::error::Result<ProxyConfig> {
|
||||||
|
new_cfg.validate()?;
|
||||||
|
let mut applied_cfg = overlay_hot_fields(old_cfg, new_cfg);
|
||||||
|
applied_cfg.validate_effective_web()?;
|
||||||
|
applied_cfg.rebuild_runtime_web()?;
|
||||||
|
Ok(applied_cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Diffs and publishes a fully prepared configuration on the watcher owner task.
|
||||||
fn reload_config_with_resolver(
|
fn reload_config_with_resolver(
|
||||||
config_path: &PathBuf,
|
prepared: (LoadedConfig, ProxyConfig),
|
||||||
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||||
log_tx: &watch::Sender<LogLevel>,
|
log_tx: &watch::Sender<LogLevel>,
|
||||||
detected_ip_v4: Option<IpAddr>,
|
detected_ip_v4: Option<IpAddr>,
|
||||||
@@ -142,13 +157,7 @@ fn reload_config_with_resolver(
|
|||||||
reload_state: &mut ReloadState,
|
reload_state: &mut ReloadState,
|
||||||
dns_resolver: Option<&crate::network::dns_overrides::GenerationDnsResolver>,
|
dns_resolver: Option<&crate::network::dns_overrides::GenerationDnsResolver>,
|
||||||
) -> Option<WatchManifest> {
|
) -> Option<WatchManifest> {
|
||||||
let loaded = match ProxyConfig::load_with_metadata(config_path) {
|
let (loaded, applied_cfg) = prepared;
|
||||||
Ok(loaded) => loaded,
|
|
||||||
Err(e) => {
|
|
||||||
error!("config reload: failed to parse {:?}: {}", config_path, e);
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let LoadedConfig {
|
let LoadedConfig {
|
||||||
config: new_cfg,
|
config: new_cfg,
|
||||||
source_files,
|
source_files,
|
||||||
@@ -157,34 +166,12 @@ fn reload_config_with_resolver(
|
|||||||
} = loaded;
|
} = loaded;
|
||||||
let next_manifest = WatchManifest::from_source_files(&source_files);
|
let next_manifest = WatchManifest::from_source_files(&source_files);
|
||||||
|
|
||||||
if let Err(e) = new_cfg.validate() {
|
|
||||||
error!(
|
|
||||||
"config reload: validation failed: {}; keeping old config",
|
|
||||||
e
|
|
||||||
);
|
|
||||||
return Some(next_manifest);
|
|
||||||
}
|
|
||||||
|
|
||||||
if reload_state.is_applied(rendered_hash) {
|
|
||||||
return Some(next_manifest);
|
|
||||||
}
|
|
||||||
|
|
||||||
let old_cfg = config_tx.borrow().clone();
|
let old_cfg = config_tx.borrow().clone();
|
||||||
let mut applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
|
|
||||||
if let Err(error) = applied_cfg
|
|
||||||
.validate_effective_web()
|
|
||||||
.and_then(|_| applied_cfg.rebuild_runtime_web())
|
|
||||||
{
|
|
||||||
error!(
|
|
||||||
"config reload: effective WEB validation failed: {}; keeping old config",
|
|
||||||
error
|
|
||||||
);
|
|
||||||
return Some(next_manifest);
|
|
||||||
}
|
|
||||||
let old_hot = HotFields::from_config(&old_cfg);
|
let old_hot = HotFields::from_config(&old_cfg);
|
||||||
let applied_hot = HotFields::from_config(&applied_cfg);
|
let applied_hot = HotFields::from_config(&applied_cfg);
|
||||||
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
||||||
let hot_changed = !config_equal(&old_cfg, &applied_cfg);
|
let hot_changed =
|
||||||
|
!config_equal(&old_cfg, &applied_cfg) || !old_cfg.web_decoy_endpoints_equal(&applied_cfg);
|
||||||
|
|
||||||
if non_hot_changed {
|
if non_hot_changed {
|
||||||
warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed);
|
warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed);
|
||||||
@@ -228,8 +215,13 @@ pub(super) fn reload_config(
|
|||||||
detected_ip_v6: Option<IpAddr>,
|
detected_ip_v6: Option<IpAddr>,
|
||||||
reload_state: &mut ReloadState,
|
reload_state: &mut ReloadState,
|
||||||
) -> Option<WatchManifest> {
|
) -> Option<WatchManifest> {
|
||||||
|
let loaded = ProxyConfig::load_with_metadata(config_path).ok()?;
|
||||||
|
if reload_state.is_applied(loaded.rendered_hash) {
|
||||||
|
return Some(WatchManifest::from_source_files(&loaded.source_files));
|
||||||
|
}
|
||||||
|
let applied = prepare_effective_config(&config_tx.borrow(), &loaded.config).ok()?;
|
||||||
reload_config_with_resolver(
|
reload_config_with_resolver(
|
||||||
config_path,
|
(loaded, applied),
|
||||||
config_tx,
|
config_tx,
|
||||||
log_tx,
|
log_tx,
|
||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
@@ -282,7 +274,7 @@ pub fn spawn_config_watcher(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
let initial_loaded = preparation::read_source(&config_path).await;
|
||||||
let initial_manifest = initial_loaded
|
let initial_manifest = initial_loaded
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||||
@@ -291,7 +283,7 @@ pub fn spawn_config_watcher(
|
|||||||
});
|
});
|
||||||
let initial_matches_disk = initial_loaded
|
let initial_matches_disk = initial_loaded
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|loaded| config_equal(config_tx.borrow().as_ref(), &loaded.config));
|
.is_some_and(|source| preparation::source_matches_active(&config_tx.borrow(), source));
|
||||||
let initial_snapshot_hash = initial_loaded
|
let initial_snapshot_hash = initial_loaded
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.filter(|_| initial_matches_disk)
|
.filter(|_| initial_matches_disk)
|
||||||
@@ -384,12 +376,14 @@ pub fn spawn_config_watcher(
|
|||||||
};
|
};
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
let mut force = false;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
msg = notify_rx.recv() => {
|
msg = notify_rx.recv() => {
|
||||||
if msg.is_none() { break; }
|
if msg.is_none() { break; }
|
||||||
}
|
}
|
||||||
_ = sighup.recv() => {
|
_ = sighup.recv() => {
|
||||||
|
force = true;
|
||||||
info!("SIGHUP received — reloading {:?}", config_path);
|
info!("SIGHUP received — reloading {:?}", config_path);
|
||||||
}
|
}
|
||||||
_ = cancellation.cancelled() => break,
|
_ = cancellation.cancelled() => break,
|
||||||
@@ -406,27 +400,31 @@ pub fn spawn_config_watcher(
|
|||||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||||
while notify_rx.try_recv().is_ok() {}
|
while notify_rx.try_recv().is_ok() {}
|
||||||
|
|
||||||
let mut next_manifest = reload_config_with_resolver(
|
let mut next_manifest = preparation::reload(
|
||||||
&config_path,
|
&config_path,
|
||||||
&config_tx,
|
&config_tx,
|
||||||
&log_tx,
|
&log_tx,
|
||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
detected_ip_v6,
|
detected_ip_v6,
|
||||||
&mut reload_state,
|
&mut reload_state,
|
||||||
dns_resolver.as_deref(),
|
dns_resolver.clone(),
|
||||||
);
|
force,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if next_manifest.is_none() {
|
if next_manifest.is_none() {
|
||||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||||
while notify_rx.try_recv().is_ok() {}
|
while notify_rx.try_recv().is_ok() {}
|
||||||
next_manifest = reload_config_with_resolver(
|
next_manifest = preparation::reload(
|
||||||
&config_path,
|
&config_path,
|
||||||
&config_tx,
|
&config_tx,
|
||||||
&log_tx,
|
&log_tx,
|
||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
detected_ip_v6,
|
detected_ip_v6,
|
||||||
&mut reload_state,
|
&mut reload_state,
|
||||||
dns_resolver.as_deref(),
|
dns_resolver.clone(),
|
||||||
);
|
force,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(next_manifest) = next_manifest {
|
if let Some(next_manifest) = next_manifest {
|
||||||
@@ -460,3 +458,6 @@ mod path_tests {
|
|||||||
assert_eq!(normalize_watch_path(&linked), linked);
|
assert_eq!(normalize_watch_path(&linked), linked);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod decoy_dns_tests;
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const SOURCE: &str = r#"
|
||||||
|
[access.users]
|
||||||
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
[[server.listeners]]
|
||||||
|
ip = "127.0.0.1"
|
||||||
|
port = 18080
|
||||||
|
transport = "web"
|
||||||
|
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://localhost:18081"
|
||||||
|
resolve = "startup"
|
||||||
|
[[web.vhosts.profiles]]
|
||||||
|
user = "alice"
|
||||||
|
secret_mode = "dd"
|
||||||
|
"#;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_watcher_publishes_dns_only_changes_once() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.toml");
|
||||||
|
std::fs::write(&path, SOURCE).unwrap();
|
||||||
|
let mut loaded = ProxyConfig::load_prepared(path.clone()).await.unwrap();
|
||||||
|
let (config_tx, mut config_rx) = watch::channel(Arc::new(loaded.config.clone()));
|
||||||
|
let (log_tx, _) = watch::channel(loaded.config.general.log_level.clone());
|
||||||
|
let old = config_rx.borrow().clone();
|
||||||
|
let mut state = ReloadState::new(Some(loaded.rendered_hash));
|
||||||
|
Arc::make_mut(&mut loaded.config.web.decoy_dns)
|
||||||
|
.origins
|
||||||
|
.insert(
|
||||||
|
("localhost".to_string(), 18081),
|
||||||
|
vec!["127.0.0.2:18081".parse().unwrap()],
|
||||||
|
);
|
||||||
|
loaded.config.rebuild_runtime_web().unwrap();
|
||||||
|
let applied = prepare_effective_config(&old, &loaded.config).unwrap();
|
||||||
|
reload_config_with_resolver(
|
||||||
|
(loaded.clone(), applied),
|
||||||
|
&config_tx,
|
||||||
|
&log_tx,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
&mut state,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(config_rx.has_changed().unwrap());
|
||||||
|
let published = config_rx.borrow_and_update().clone();
|
||||||
|
assert!(!old.web_decoy_endpoints_equal(&published));
|
||||||
|
let applied = prepare_effective_config(&published, &loaded.config).unwrap();
|
||||||
|
reload_config_with_resolver(
|
||||||
|
(loaded, applied),
|
||||||
|
&config_tx,
|
||||||
|
&log_tx,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
&mut state,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(!config_rx.has_changed().unwrap());
|
||||||
|
assert!(Arc::ptr_eq(&published, &config_rx.borrow()));
|
||||||
|
|
||||||
|
preparation::reload(
|
||||||
|
&path, &config_tx, &log_tx, None, None, &mut state, None, false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!config_rx.has_changed().unwrap());
|
||||||
|
assert!(Arc::ptr_eq(&published, &config_rx.borrow()));
|
||||||
|
|
||||||
|
// Explicit reload bypasses the unchanged source hash and replaces the injected endpoint.
|
||||||
|
preparation::reload(
|
||||||
|
&path, &config_tx, &log_tx, None, None, &mut state, None, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(config_rx.has_changed().unwrap());
|
||||||
|
assert!(!published.web_decoy_endpoints_equal(&config_rx.borrow()));
|
||||||
|
|
||||||
|
let current = config_rx.borrow_and_update().clone();
|
||||||
|
std::fs::write(&path, SOURCE.replace("localhost:18081", "localhost:18080")).unwrap();
|
||||||
|
preparation::reload(
|
||||||
|
&path, &config_tx, &log_tx, None, None, &mut state, None, false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!config_rx.has_changed().unwrap());
|
||||||
|
assert!(Arc::ptr_eq(¤t, &config_rx.borrow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_unchanged_watcher_source_skips_resolution() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.toml");
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
SOURCE.replace("localhost", "unresolvable.example.invalid"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let source = preparation::read_source(&path).await.unwrap();
|
||||||
|
let initial = Arc::new(source.config.clone());
|
||||||
|
let (config_tx, config_rx) = watch::channel(initial.clone());
|
||||||
|
let (log_tx, _) = watch::channel(initial.general.log_level.clone());
|
||||||
|
let mut state = ReloadState::new(Some(source.rendered_hash));
|
||||||
|
assert!(preparation::source_matches_active(&initial, &source));
|
||||||
|
preparation::reload(
|
||||||
|
&path, &config_tx, &log_tx, None, None, &mut state, None, false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!config_rx.has_changed().unwrap());
|
||||||
|
assert!(Arc::ptr_eq(&initial, &config_rx.borrow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_watcher_init_ignores_retained_restart_fields() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.toml");
|
||||||
|
std::fs::write(&path, SOURCE).unwrap();
|
||||||
|
let loaded = ProxyConfig::load_prepared(path.clone()).await.unwrap();
|
||||||
|
let mut source = ProxyConfig::parse_source(&path).unwrap();
|
||||||
|
source.config.web.limits.max_static_file_bytes += 1;
|
||||||
|
assert!(preparation::source_matches_active(&loaded.config, &source));
|
||||||
|
source.config.web.conveyor = !source.config.web.conveyor;
|
||||||
|
assert!(!preparation::source_matches_active(&loaded.config, &source));
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Restart-only differences must not enqueue another DNS preparation after activation.
|
||||||
|
pub(super) fn source_matches_active(
|
||||||
|
active: &ProxyConfig,
|
||||||
|
source: &crate::config::ParsedConfigSource,
|
||||||
|
) -> bool {
|
||||||
|
config_equal(active, &overlay_hot_fields(active, &source.config))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads only source state; watcher initialization and unchanged events never need DNS.
|
||||||
|
pub(super) async fn read_source(path: &PathBuf) -> Option<crate::config::ParsedConfigSource> {
|
||||||
|
let path = path.clone();
|
||||||
|
match tokio::task::spawn_blocking(move || ProxyConfig::parse_source(path)).await {
|
||||||
|
Ok(Ok(source)) => Some(source),
|
||||||
|
Ok(Err(error)) => {
|
||||||
|
error!("config reload: failed to parse: {}", error);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
error!("config reload: source reader failed: {}", error);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prepares fresh evidence only for changed sources or an explicit reload request.
|
||||||
|
pub(super) async fn reload(
|
||||||
|
path: &PathBuf,
|
||||||
|
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||||
|
log_tx: &watch::Sender<LogLevel>,
|
||||||
|
detected_ip_v4: Option<IpAddr>,
|
||||||
|
detected_ip_v6: Option<IpAddr>,
|
||||||
|
reload_state: &mut ReloadState,
|
||||||
|
dns_resolver: Option<Arc<crate::network::dns_overrides::GenerationDnsResolver>>,
|
||||||
|
force: bool,
|
||||||
|
) -> Option<WatchManifest> {
|
||||||
|
let source = read_source(path).await?;
|
||||||
|
let manifest = WatchManifest::from_source_files(&source.source_files);
|
||||||
|
if !force && reload_state.is_applied(source.rendered_hash) {
|
||||||
|
return Some(manifest);
|
||||||
|
}
|
||||||
|
let loaded = match source.prepare().await {
|
||||||
|
Ok(loaded) => loaded,
|
||||||
|
Err(error) => {
|
||||||
|
error!(
|
||||||
|
"config reload: preparation failed: {}; keeping old config",
|
||||||
|
error
|
||||||
|
);
|
||||||
|
return Some(manifest);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let old = config_tx.borrow().clone();
|
||||||
|
// Static-site reconstruction after the listener overlay must not block Tokio workers.
|
||||||
|
// The worker only builds data: cancellation cannot leave a detached publisher behind.
|
||||||
|
let result = tokio::task::spawn_blocking(move || {
|
||||||
|
prepare_effective_config(&old, &loaded.config).map(|applied| (loaded, applied))
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
match result {
|
||||||
|
Ok(Ok(prepared)) => reload_config_with_resolver(
|
||||||
|
prepared,
|
||||||
|
config_tx,
|
||||||
|
log_tx,
|
||||||
|
detected_ip_v4,
|
||||||
|
detected_ip_v6,
|
||||||
|
reload_state,
|
||||||
|
dns_resolver.as_deref(),
|
||||||
|
),
|
||||||
|
Ok(Err(error)) => {
|
||||||
|
error!(
|
||||||
|
"config reload: effective WEB validation failed: {}; keeping old config",
|
||||||
|
error
|
||||||
|
);
|
||||||
|
Some(manifest)
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
error!(
|
||||||
|
"config reload: runtime builder failed: {}; keeping old config",
|
||||||
|
error
|
||||||
|
);
|
||||||
|
Some(manifest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,6 +29,8 @@ mod runtime_web;
|
|||||||
mod decode;
|
mod decode;
|
||||||
mod effective;
|
mod effective;
|
||||||
mod pipeline;
|
mod pipeline;
|
||||||
|
// Candidate-owned DNS preparation is separate from pure runtime reconstruction.
|
||||||
|
mod decoy_dns;
|
||||||
mod validate_core;
|
mod validate_core;
|
||||||
mod validate_me;
|
mod validate_me;
|
||||||
mod validate_runtime;
|
mod validate_runtime;
|
||||||
@@ -74,6 +76,19 @@ pub(crate) struct LoadedConfig {
|
|||||||
pub(crate) rendered_hash: u64,
|
pub(crate) rendered_hash: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Normalized source configuration that has not acquired runtime DNS or static snapshots.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct ParsedConfigSource {
|
||||||
|
/// Source-only configuration; must be prepared before runtime publication.
|
||||||
|
pub(crate) config: ProxyConfig,
|
||||||
|
/// Normalized paths in the captured include graph.
|
||||||
|
pub(crate) source_files: Vec<PathBuf>,
|
||||||
|
/// Exact source documents used for revisions and atomic mutation checks.
|
||||||
|
pub(crate) source_contents: BTreeMap<PathBuf, String>,
|
||||||
|
/// Source-only hash, independent of DNS answers.
|
||||||
|
pub(crate) rendered_hash: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Raw recursive source graph captured before typed deserialization.
|
/// Raw recursive source graph captured before typed deserialization.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct ConfigSourceGraph {
|
pub(crate) struct ConfigSourceGraph {
|
||||||
@@ -229,6 +244,33 @@ impl ProxyConfig {
|
|||||||
pipeline::load_source_graph(graph)
|
pipeline::load_source_graph(graph)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Parses source without resolving decoys or creating runtime WEB snapshots.
|
||||||
|
pub(crate) fn parse_source<P: AsRef<Path>>(path: P) -> Result<ParsedConfigSource> {
|
||||||
|
Self::parse_source_with_overrides(path, &BTreeMap::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Checks decoy mode-specific keys before a source or API patch loses unknown fields.
|
||||||
|
pub(crate) fn validate_decoy_source_keys(document: &toml::Value) -> Result<()> {
|
||||||
|
decoy_dns::validate_mode_keys(document)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses an atomic mutation candidate without performing DNS.
|
||||||
|
pub(crate) fn parse_source_with_overrides<P: AsRef<Path>>(
|
||||||
|
path: P,
|
||||||
|
overrides: &BTreeMap<PathBuf, String>,
|
||||||
|
) -> Result<ParsedConfigSource> {
|
||||||
|
pipeline::parse_source_graph(Self::read_source_graph_with_overrides(path, overrides)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Loads and prepares a complete runtime candidate without blocking async workers.
|
||||||
|
pub(crate) async fn load_prepared(path: PathBuf) -> Result<LoadedConfig> {
|
||||||
|
let parsed = tokio::task::spawn_blocking(move || Self::parse_source(path))
|
||||||
|
.await
|
||||||
|
.map_err(|error| ProxyError::Config(format!("config reader failed: {error}")))??;
|
||||||
|
parsed.prepare().await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rebuilds immutable credential lookup data for this configuration.
|
||||||
pub(crate) fn rebuild_runtime_user_auth(&mut self) -> Result<()> {
|
pub(crate) fn rebuild_runtime_user_auth(&mut self) -> Result<()> {
|
||||||
let snapshot = UserAuthSnapshot::from_users(&self.access.users)?;
|
let snapshot = UserAuthSnapshot::from_users(&self.access.users)?;
|
||||||
self.runtime_user_auth = Some(Arc::new(snapshot));
|
self.runtime_user_auth = Some(Arc::new(snapshot));
|
||||||
@@ -250,6 +292,7 @@ impl ProxyConfig {
|
|||||||
validate_web::validate_decoy_listener_separation(self)
|
validate_web::validate_decoy_listener_separation(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns authentication data owned by this configuration generation.
|
||||||
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
|
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
|
||||||
self.runtime_user_auth.as_deref()
|
self.runtime_user_auth.as_deref()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Typed source together with its exact include graph and rendered input.
|
||||||
pub(super) type DecodedSourceGraph = (
|
pub(super) type DecodedSourceGraph = (
|
||||||
ProxyConfig,
|
ProxyConfig,
|
||||||
BTreeSet<PathBuf>,
|
BTreeSet<PathBuf>,
|
||||||
@@ -7,6 +8,7 @@ pub(super) type DecodedSourceGraph = (
|
|||||||
String,
|
String,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/// Decodes source while retaining the metadata needed for atomic mutations.
|
||||||
pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSourceGraph> {
|
pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSourceGraph> {
|
||||||
let ConfigSourceGraph {
|
let ConfigSourceGraph {
|
||||||
source_contents,
|
source_contents,
|
||||||
@@ -17,6 +19,7 @@ pub(super) fn decode_source_graph(graph: ConfigSourceGraph) -> Result<DecodedSou
|
|||||||
let parsed_toml: toml::Value =
|
let parsed_toml: toml::Value =
|
||||||
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
|
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||||
handle_unknown_config_keys(&parsed_toml)?;
|
handle_unknown_config_keys(&parsed_toml)?;
|
||||||
|
ProxyConfig::validate_decoy_source_keys(&parsed_toml)?;
|
||||||
let general_table = parsed_toml
|
let general_table = parsed_toml
|
||||||
.get("general")
|
.get("general")
|
||||||
.and_then(|value| value.as_table());
|
.and_then(|value| value.as_table());
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
use std::future::Future;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Rejects mode-specific keys even when unknown-key handling is permissive.
|
||||||
|
pub(super) fn validate_mode_keys(document: &toml::Value) -> Result<()> {
|
||||||
|
if let Some(vhosts) = document
|
||||||
|
.get("web")
|
||||||
|
.and_then(|web| web.get("vhosts"))
|
||||||
|
.and_then(toml::Value::as_array)
|
||||||
|
{
|
||||||
|
for (idx, vhost) in vhosts.iter().enumerate() {
|
||||||
|
if let Some(decoy) = vhost.get("decoy")
|
||||||
|
&& decoy.get("resolve").is_some()
|
||||||
|
&& decoy.get("mode").and_then(toml::Value::as_str) != Some("http_upstream")
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.resolve is only valid for mode=http_upstream"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses one origin and preserves the existing literal-address containment policy.
|
||||||
|
pub(super) fn parse_origin(
|
||||||
|
idx: usize,
|
||||||
|
upstream: &str,
|
||||||
|
resolve: WebDecoyResolve,
|
||||||
|
) -> Result<url::Url> {
|
||||||
|
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream is invalid: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
if parsed.scheme() != "http"
|
||||||
|
|| parsed.host_str().is_none()
|
||||||
|
|| !parsed.username().is_empty()
|
||||||
|
|| parsed.password().is_some()
|
||||||
|
|| parsed.query().is_some()
|
||||||
|
|| parsed.fragment().is_some()
|
||||||
|
|| parsed.path() != "/"
|
||||||
|
|| parsed.port() == Some(0)
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
match parsed.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => validate_address(idx, IpAddr::V4(ip))?,
|
||||||
|
Some(url::Host::Ipv6(ip)) => validate_address(idx, IpAddr::V6(ip))?,
|
||||||
|
Some(url::Host::Domain(_)) if resolve == WebDecoyResolve::Startup => {}
|
||||||
|
_ => {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream host must be a loopback or private IP literal unless resolve = \"startup\""
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(parsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_address(idx: usize, ip: IpAddr) -> Result<()> {
|
||||||
|
let private = match ip {
|
||||||
|
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||||
|
IpAddr::V6(ip) => ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local(),
|
||||||
|
};
|
||||||
|
if !private {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream must remain inside loopback or a private network"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns all captured answers so non-selected addresses cannot hide a listener loop.
|
||||||
|
pub(super) fn addresses(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
idx: usize,
|
||||||
|
parsed: &url::Url,
|
||||||
|
require_resolved: bool,
|
||||||
|
) -> Result<Vec<SocketAddr>> {
|
||||||
|
let port = parsed
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| ProxyError::Config("WEB decoy port cannot be resolved".to_string()))?;
|
||||||
|
let answers = match parsed.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => vec![SocketAddr::new(IpAddr::V4(ip), port)],
|
||||||
|
Some(url::Host::Ipv6(ip)) => vec![SocketAddr::new(IpAddr::V6(ip), port)],
|
||||||
|
Some(url::Host::Domain(host)) => {
|
||||||
|
let key = (host.to_string(), port);
|
||||||
|
match config.web.decoy_dns.origins.get(&key) {
|
||||||
|
Some(answers) if !answers.is_empty() => answers.clone(),
|
||||||
|
_ if !require_resolved => return Ok(Vec::new()),
|
||||||
|
_ => {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream requires async preparation for resolve = \"startup\""
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => return Err(ProxyError::Config("WEB decoy host is missing".to_string())),
|
||||||
|
};
|
||||||
|
for addr in &answers {
|
||||||
|
validate_address(idx, addr.ip())?;
|
||||||
|
if addr.port() != port {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"WEB decoy DNS snapshot port mismatch".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(answers)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captures fresh resolver evidence without publishing any partially prepared configuration.
|
||||||
|
pub(super) async fn prepare<F, Fut>(config: &mut ProxyConfig, mut lookup: F) -> Result<()>
|
||||||
|
where
|
||||||
|
F: FnMut(String, u16) -> Fut,
|
||||||
|
Fut: Future<Output = std::io::Result<Vec<SocketAddr>>>,
|
||||||
|
{
|
||||||
|
let mut snapshot = WebDecoyDnsSnapshot::default();
|
||||||
|
for (idx, vhost) in config.web.vhosts.iter().enumerate() {
|
||||||
|
let WebDecoyConfig::HttpUpstream { upstream, resolve } = &vhost.decoy else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let parsed = parse_origin(idx, upstream, *resolve)?;
|
||||||
|
let Some(url::Host::Domain(host)) = parsed.host() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let port = parsed
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| ProxyError::Config("WEB decoy port cannot be resolved".to_string()))?;
|
||||||
|
let key = (host.to_string(), port);
|
||||||
|
if snapshot.origins.contains_key(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Dropping this wait does not forcibly cancel a running OS resolver call.
|
||||||
|
let answers = tokio::time::timeout(
|
||||||
|
Duration::from_secs(config.web.timeouts.decoy_resolve_secs),
|
||||||
|
lookup(key.0.clone(), port),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream DNS resolution timed out after {} seconds",
|
||||||
|
config.web.timeouts.decoy_resolve_secs
|
||||||
|
))
|
||||||
|
})?
|
||||||
|
.map_err(|error| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream DNS resolution failed: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
if answers.is_empty() {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{idx}].decoy.upstream DNS resolution returned no addresses"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
for addr in &answers {
|
||||||
|
validate_address(idx, addr.ip())?;
|
||||||
|
}
|
||||||
|
snapshot.origins.insert(key, answers);
|
||||||
|
}
|
||||||
|
config.web.decoy_dns = Arc::new(snapshot);
|
||||||
|
validate_web::validate_decoy_listener_separation(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the unchanged request-time tuple from immutable configuration evidence.
|
||||||
|
pub(super) fn build_upstream(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
idx: usize,
|
||||||
|
upstream: &str,
|
||||||
|
resolve: WebDecoyResolve,
|
||||||
|
) -> Result<WebRuntimeDecoy> {
|
||||||
|
let parsed = parse_origin(idx, upstream, resolve)?;
|
||||||
|
let addr = addresses(config, idx, &parsed, true)?
|
||||||
|
.first()
|
||||||
|
.copied()
|
||||||
|
.ok_or_else(|| ProxyError::Config("WEB decoy DNS snapshot is empty".to_string()))?;
|
||||||
|
// URL authority keeps the configured hostname and brackets IPv6 literals.
|
||||||
|
let authority = parsed[url::Position::BeforeHost..url::Position::AfterPort].to_string();
|
||||||
|
Ok(WebRuntimeDecoy::HttpUpstream { addr, authority })
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProxyConfig {
|
||||||
|
/// Compares selected endpoints separately from source-only configuration equality.
|
||||||
|
pub(crate) fn web_decoy_endpoints_equal(&self, other: &Self) -> bool {
|
||||||
|
fn selected(config: &ProxyConfig) -> impl Iterator<Item = (&String, SocketAddr)> {
|
||||||
|
config
|
||||||
|
.web
|
||||||
|
.runtime
|
||||||
|
.iter()
|
||||||
|
.flat_map(|runtime| runtime.vhosts.iter())
|
||||||
|
.filter_map(|(host, vhost)| match vhost.decoy {
|
||||||
|
WebRuntimeDecoy::HttpUpstream { addr, .. } => Some((host, addr)),
|
||||||
|
WebRuntimeDecoy::StaticDirectory(_) => None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
selected(self).eq(selected(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const SOURCE: &str = r#"
|
||||||
|
[access.users]
|
||||||
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
[[server.listeners]]
|
||||||
|
ip = "127.0.0.1"
|
||||||
|
port = 18080
|
||||||
|
transport = "web"
|
||||||
|
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://Example.COM:18081"
|
||||||
|
resolve = "startup"
|
||||||
|
[[web.vhosts.profiles]]
|
||||||
|
user = "alice"
|
||||||
|
secret_mode = "dd"
|
||||||
|
"#;
|
||||||
|
|
||||||
|
fn source(text: &str) -> ParsedConfigSource {
|
||||||
|
pipeline::parse_source_graph(ConfigSourceGraph {
|
||||||
|
rendered: text.to_string(),
|
||||||
|
source_contents: BTreeMap::new(),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepared(answers: &[&str]) -> LoadedConfig {
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
let answers: Vec<SocketAddr> = answers.iter().map(|addr| addr.parse().unwrap()).collect();
|
||||||
|
prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(answers.clone()))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
pipeline::finish(parsed).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn endpoint(config: &ProxyConfig) -> (SocketAddr, &str) {
|
||||||
|
let runtime = config.web.runtime.as_ref().unwrap();
|
||||||
|
match &runtime.vhosts["proxy.example.com"].decoy {
|
||||||
|
WebRuntimeDecoy::HttpUpstream { addr, authority } => (*addr, authority),
|
||||||
|
_ => panic!("expected HTTP decoy"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_snapshot_deduplicates_normalized_origins_and_preserves_order() {
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
let mut second = parsed.config.web.vhosts[0].clone();
|
||||||
|
second.host = "bsi.bund.de".to_string();
|
||||||
|
second.decoy = WebDecoyConfig::HttpUpstream {
|
||||||
|
upstream: "http://example.com:18081/".to_string(),
|
||||||
|
resolve: WebDecoyResolve::Startup,
|
||||||
|
};
|
||||||
|
parsed.config.web.vhosts.push(second);
|
||||||
|
let calls = AtomicUsize::new(0);
|
||||||
|
prepare(&mut parsed.config, |host, port| {
|
||||||
|
assert_eq!((host.as_str(), port), ("example.com", 18081));
|
||||||
|
calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
std::future::ready(Ok(vec![
|
||||||
|
"[fd00::2]:18081".parse().unwrap(),
|
||||||
|
"10.0.0.2:18081".parse().unwrap(),
|
||||||
|
]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut loaded = pipeline::finish(parsed).unwrap();
|
||||||
|
for _ in 0..3 {
|
||||||
|
loaded.config.validate_effective_web().unwrap();
|
||||||
|
loaded.config.rebuild_runtime_web().unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(
|
||||||
|
endpoint(&loaded.config),
|
||||||
|
("[fd00::2]:18081".parse().unwrap(), "example.com:18081")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
loaded
|
||||||
|
.config
|
||||||
|
.web
|
||||||
|
.decoy_dns
|
||||||
|
.origins
|
||||||
|
.values()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&loaded.config)
|
||||||
|
.unwrap()
|
||||||
|
.contains("fd00::2")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_rejects_every_public_answer_including_non_selected() {
|
||||||
|
for disallowed in [
|
||||||
|
"8.8.8.8:18081",
|
||||||
|
"[2001:4860:4860::8888]:18081",
|
||||||
|
"[::ffff:127.0.0.1]:18081",
|
||||||
|
] {
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
let error = prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(vec![
|
||||||
|
"127.0.0.2:18081".parse().unwrap(),
|
||||||
|
disallowed.parse().unwrap(),
|
||||||
|
]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("must remain inside"));
|
||||||
|
assert!(parsed.config.web.runtime.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_rejects_non_selected_direct_and_wildcard_listener_loops() {
|
||||||
|
for listener in ["127.0.0.1", "0.0.0.0"] {
|
||||||
|
let mut parsed = source(&SOURCE.replace("Example.COM:18081", "example.com:18080"));
|
||||||
|
parsed.config.server.listeners[0].ip = listener.parse().unwrap();
|
||||||
|
let error = prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(vec![
|
||||||
|
"[fd00::2]:18080".parse().unwrap(),
|
||||||
|
"127.0.0.1:18080".parse().unwrap(),
|
||||||
|
]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("overlaps WEB listener"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_empty_answers_and_resolver_errors_fail_closed() {
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
let error = prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(Vec::new()))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("returned no addresses"));
|
||||||
|
let error = prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::NotFound,
|
||||||
|
"test resolver failure",
|
||||||
|
)))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("DNS resolution failed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_timeout_is_bounded() {
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
parsed.config.web.timeouts.decoy_resolve_secs = 1;
|
||||||
|
let error = tokio::time::timeout(
|
||||||
|
Duration::from_secs(3),
|
||||||
|
prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::pending::<std::io::Result<Vec<SocketAddr>>>()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("timed out after 1 seconds"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_literal_startup_never_looks_up_and_keeps_ipv6_authority() {
|
||||||
|
let mut parsed = source(&SOURCE.replace("Example.COM:18081", "[::1]:18081"));
|
||||||
|
prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Err(std::io::Error::other("IP literals must not reach DNS")))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let loaded = pipeline::finish(parsed).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
endpoint(&loaded.config),
|
||||||
|
("[::1]:18081".parse().unwrap(), "[::1]:18081")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_source_parse_is_not_a_prepared_runtime() {
|
||||||
|
let parsed = source(SOURCE);
|
||||||
|
assert!(parsed.config.web.runtime.is_none());
|
||||||
|
assert!(parsed.config.web.decoy_dns.origins.is_empty());
|
||||||
|
assert!(
|
||||||
|
pipeline::finish(parsed)
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string()
|
||||||
|
.contains("requires async preparation")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_evidence_is_bound_to_origin_and_port() {
|
||||||
|
let loaded = prepared(&["10.0.0.2:18081"]).await;
|
||||||
|
for changed in ["http://telekom.com:18081", "http://example.com:18082"] {
|
||||||
|
let mut config = loaded.config.clone();
|
||||||
|
config.web.vhosts[0].decoy = WebDecoyConfig::HttpUpstream {
|
||||||
|
upstream: changed.to_string(),
|
||||||
|
resolve: WebDecoyResolve::Startup,
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
config
|
||||||
|
.validate_effective_web()
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string()
|
||||||
|
.contains("requires async preparation")
|
||||||
|
);
|
||||||
|
assert!(config.rebuild_runtime_web().is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_reload_pins_old_generation_and_detects_dns_only_changes() {
|
||||||
|
let old = prepared(&["10.0.0.2:18081"]).await;
|
||||||
|
let new = prepared(&["10.0.0.3:18081"]).await;
|
||||||
|
assert_eq!(old.rendered_hash, new.rendered_hash);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&old.config).unwrap(),
|
||||||
|
serde_json::to_value(&new.config).unwrap()
|
||||||
|
);
|
||||||
|
let resolved =
|
||||||
|
crate::maestro::runtime_build::resolve_reload_config(&old.config, &new.config).unwrap();
|
||||||
|
assert!(resolved.runtime_changed);
|
||||||
|
assert!(!old.config.web_decoy_endpoints_equal(&new.config));
|
||||||
|
assert_eq!(endpoint(&old.config).0, "10.0.0.2:18081".parse().unwrap());
|
||||||
|
let same = prepared(&["10.0.0.3:18081", "10.0.0.4:18081"]).await;
|
||||||
|
assert!(same.config.web_decoy_endpoints_equal(&new.config));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_effective_listener_overlay_rechecks_every_answer() {
|
||||||
|
let old = prepared(&["10.0.0.2:18081"]).await;
|
||||||
|
let mut parsed = source(SOURCE);
|
||||||
|
parsed.config.server.listeners[0].port = Some(18082);
|
||||||
|
parsed.config.web.vhosts[0].decoy = WebDecoyConfig::HttpUpstream {
|
||||||
|
upstream: "http://example.com:18080".to_string(),
|
||||||
|
resolve: WebDecoyResolve::Startup,
|
||||||
|
};
|
||||||
|
prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(vec![
|
||||||
|
"10.0.0.2:18080".parse().unwrap(),
|
||||||
|
"127.0.0.1:18080".parse().unwrap(),
|
||||||
|
]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let desired = pipeline::finish(parsed).unwrap();
|
||||||
|
assert!(
|
||||||
|
crate::maestro::runtime_build::resolve_reload_config(&old.config, &desired.config)
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.contains("overlaps WEB listener")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_rejects_ipv6_direct_and_wildcard_listener_loops() {
|
||||||
|
for listener in ["::1", "::"] {
|
||||||
|
let mut parsed = source(&SOURCE.replace("Example.COM:18081", "example.com:18080"));
|
||||||
|
parsed.config.network.ipv6 = Some(true);
|
||||||
|
parsed.config.server.listeners[0].ip = listener.parse().unwrap();
|
||||||
|
let error = prepare(&mut parsed.config, |_, _| {
|
||||||
|
std::future::ready(Ok(vec![
|
||||||
|
"10.0.0.2:18080".parse().unwrap(),
|
||||||
|
"[::1]:18080".parse().unwrap(),
|
||||||
|
]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("overlaps WEB listener"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_startup_does_not_relax_origin_url_rules() {
|
||||||
|
for url in [
|
||||||
|
"https://example.com",
|
||||||
|
"http://user@example.com",
|
||||||
|
"http://user:pass@example.com",
|
||||||
|
"http://example.com/path",
|
||||||
|
"http://example.com/?q=1",
|
||||||
|
"http://example.com/#fragment",
|
||||||
|
"http://example.com:0",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
parse_origin(0, url, WebDecoyResolve::Startup).is_err(),
|
||||||
|
"accepted {url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_default_port_authority_and_evidence_port_fence() {
|
||||||
|
let mut parsed = source(&SOURCE.replace("Example.COM:18081", "example.com"));
|
||||||
|
prepare(&mut parsed.config, |_, port| {
|
||||||
|
assert_eq!(port, 80);
|
||||||
|
std::future::ready(Ok(vec!["127.0.0.2:80".parse().unwrap()]))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let loaded = pipeline::finish(parsed).unwrap();
|
||||||
|
assert_eq!(endpoint(&loaded.config).1, "example.com");
|
||||||
|
let mut config = loaded.config.clone();
|
||||||
|
Arc::make_mut(&mut config.web.decoy_dns).origins.insert(
|
||||||
|
("example.com".to_string(), 80),
|
||||||
|
vec!["127.0.0.2:81".parse().unwrap()],
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
config
|
||||||
|
.validate_effective_web()
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string()
|
||||||
|
.contains("port mismatch")
|
||||||
|
);
|
||||||
|
assert!(config.rebuild_runtime_web().is_err());
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Normalizes source policy and authentication data before external decoy preparation.
|
||||||
pub(super) fn apply(config: &mut ProxyConfig) -> Result<()> {
|
pub(super) fn apply(config: &mut ProxyConfig) -> Result<()> {
|
||||||
// Normalize optional TLS fetch scope: whitespace-only values disable scoped routing.
|
// Normalize optional TLS fetch scope: whitespace-only values disable scoped routing.
|
||||||
config.censorship.tls_fetch_scope = config.censorship.tls_fetch_scope.trim().to_string();
|
config.censorship.tls_fetch_scope = config.censorship.tls_fetch_scope.trim().to_string();
|
||||||
@@ -217,6 +218,5 @@ pub(super) fn apply(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
validate_logging_config(&config.logging)?;
|
validate_logging_config(&config.logging)?;
|
||||||
validate_upstreams(config)?;
|
validate_upstreams(config)?;
|
||||||
config.rebuild_runtime_user_auth()?;
|
config.rebuild_runtime_user_auth()?;
|
||||||
config.rebuild_runtime_web()?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,18 +1,51 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Completes legacy synchronous loads without performing hostname resolution.
|
||||||
pub(super) fn load_source_graph(graph: ConfigSourceGraph) -> Result<LoadedConfig> {
|
pub(super) fn load_source_graph(graph: ConfigSourceGraph) -> Result<LoadedConfig> {
|
||||||
|
finish(parse_source_graph(graph)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Normalizes source while leaving DNS and immutable site acquisition to preparation.
|
||||||
|
pub(super) fn parse_source_graph(graph: ConfigSourceGraph) -> Result<ParsedConfigSource> {
|
||||||
let (mut config, source_files, source_contents, processed) =
|
let (mut config, source_files, source_contents, processed) =
|
||||||
decode::decode_source_graph(graph)?;
|
decode::decode_source_graph(graph)?;
|
||||||
validate_core::validate(&mut config)?;
|
validate_core::validate(&mut config)?;
|
||||||
validate_runtime::validate(&mut config)?;
|
validate_runtime::validate(&mut config)?;
|
||||||
validate_me::validate(&mut config)?;
|
validate_me::validate(&mut config)?;
|
||||||
validate_server::validate(&mut config)?;
|
validate_server::validate(&mut config)?;
|
||||||
validate_web::validate(&mut config)?;
|
validate_web::validate_source(&mut config)?;
|
||||||
effective::apply(&mut config)?;
|
effective::apply(&mut config)?;
|
||||||
Ok(LoadedConfig {
|
Ok(ParsedConfigSource {
|
||||||
config,
|
config,
|
||||||
source_files: source_files.into_iter().collect(),
|
source_files: source_files.into_iter().collect(),
|
||||||
source_contents,
|
source_contents,
|
||||||
rendered_hash: hash_rendered_snapshot(&processed),
|
rendered_hash: hash_rendered_snapshot(&processed),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Completes synchronous validation and static acquisition after DNS is captured.
|
||||||
|
pub(super) fn finish(mut parsed: ParsedConfigSource) -> Result<LoadedConfig> {
|
||||||
|
parsed.config.validate_effective_web()?;
|
||||||
|
parsed.config.rebuild_runtime_web()?;
|
||||||
|
Ok(LoadedConfig {
|
||||||
|
config: parsed.config,
|
||||||
|
source_files: parsed.source_files,
|
||||||
|
source_contents: parsed.source_contents,
|
||||||
|
rendered_hash: parsed.rendered_hash,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ParsedConfigSource {
|
||||||
|
/// Resolves each unique decoy origin once, then builds a fully validated candidate.
|
||||||
|
pub(crate) async fn prepare(mut self) -> Result<LoadedConfig> {
|
||||||
|
decoy_dns::prepare(&mut self.config, |host, port| async move {
|
||||||
|
tokio::net::lookup_host((host.as_str(), port))
|
||||||
|
.await
|
||||||
|
.map(|answers| answers.collect())
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
tokio::task::spawn_blocking(move || finish(self))
|
||||||
|
.await
|
||||||
|
.map_err(|error| ProxyError::Config(format!("config preparation failed: {error}")))?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -47,8 +47,10 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
let mut static_bytes = 0usize;
|
let mut static_bytes = 0usize;
|
||||||
|
|
||||||
let carrier_candidates: Arc<[WebCarrier]> = config.web.carrier_candidates().into();
|
let carrier_candidates: Arc<[WebCarrier]> = config.web.carrier_candidates().into();
|
||||||
for vhost in &config.web.vhosts {
|
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
|
||||||
let decoy = build_decoy(
|
let decoy = build_decoy(
|
||||||
|
config,
|
||||||
|
vhost_idx,
|
||||||
vhost,
|
vhost,
|
||||||
&config.web.limits,
|
&config.web.limits,
|
||||||
&mut static_files,
|
&mut static_files,
|
||||||
@@ -182,42 +184,16 @@ fn client_secret(secret: [u8; 16], mode: WebSecretMode) -> ([u8; 17], usize) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn build_decoy(
|
fn build_decoy(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
vhost_idx: usize,
|
||||||
vhost: &WebVhostConfig,
|
vhost: &WebVhostConfig,
|
||||||
limits: &WebLimitsConfig,
|
limits: &WebLimitsConfig,
|
||||||
static_files: &mut usize,
|
static_files: &mut usize,
|
||||||
static_bytes: &mut usize,
|
static_bytes: &mut usize,
|
||||||
) -> Result<WebRuntimeDecoy> {
|
) -> Result<WebRuntimeDecoy> {
|
||||||
match &vhost.decoy {
|
match &vhost.decoy {
|
||||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
WebDecoyConfig::HttpUpstream { upstream, resolve } => {
|
||||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
decoy_dns::build_upstream(config, vhost_idx, upstream, *resolve)
|
||||||
ProxyError::Config(format!(
|
|
||||||
"WEB decoy upstream for `{}` is invalid: {error}",
|
|
||||||
vhost.host
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
let ip = match parsed.host() {
|
|
||||||
Some(url::Host::Ipv4(ip)) => std::net::IpAddr::V4(ip),
|
|
||||||
Some(url::Host::Ipv6(ip)) => std::net::IpAddr::V6(ip),
|
|
||||||
_ => {
|
|
||||||
return Err(ProxyError::Config(
|
|
||||||
"WEB decoy host must be an IP literal".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let host = ip.to_string();
|
|
||||||
let port = parsed.port_or_known_default().ok_or_else(|| {
|
|
||||||
ProxyError::Config("WEB decoy port cannot be resolved".to_string())
|
|
||||||
})?;
|
|
||||||
let authority = match (ip, parsed.port()) {
|
|
||||||
(std::net::IpAddr::V6(_), Some(_)) => format!("[{host}]:{port}"),
|
|
||||||
(std::net::IpAddr::V6(_), None) => format!("[{host}]"),
|
|
||||||
(std::net::IpAddr::V4(_), Some(_)) => format!("{host}:{port}"),
|
|
||||||
(std::net::IpAddr::V4(_), None) => host.clone(),
|
|
||||||
};
|
|
||||||
Ok(WebRuntimeDecoy::HttpUpstream {
|
|
||||||
addr: SocketAddr::new(ip, port),
|
|
||||||
authority,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||||
let site = load_static_site(directory, index, limits, static_files, static_bytes)?;
|
let site = load_static_site(directory, index, limits, static_files, static_bytes)?;
|
||||||
|
|||||||
@@ -264,6 +264,7 @@ const WEB_CONFIG_KEYS: &[&str] = &[
|
|||||||
"enabled",
|
"enabled",
|
||||||
"carrier",
|
"carrier",
|
||||||
"carrier_method",
|
"carrier_method",
|
||||||
|
"conveyor",
|
||||||
"carriers",
|
"carriers",
|
||||||
"carrier_learning",
|
"carrier_learning",
|
||||||
"carrier_negotiation_aggressiveness",
|
"carrier_negotiation_aggressiveness",
|
||||||
@@ -364,10 +365,11 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"http_overload_timeout_ms",
|
"http_overload_timeout_ms",
|
||||||
"shutdown_secs",
|
"shutdown_secs",
|
||||||
"decoy_header_secs",
|
"decoy_header_secs",
|
||||||
|
"decoy_resolve_secs",
|
||||||
];
|
];
|
||||||
|
|
||||||
const WEB_VHOST_CONFIG_KEYS: &[&str] = &["host", "base_path", "public_addr", "decoy", "profiles"];
|
const WEB_VHOST_CONFIG_KEYS: &[&str] = &["host", "base_path", "public_addr", "decoy", "profiles"];
|
||||||
const WEB_DECOY_CONFIG_KEYS: &[&str] = &["mode", "upstream", "directory", "index"];
|
const WEB_DECOY_CONFIG_KEYS: &[&str] = &["mode", "upstream", "directory", "index", "resolve"];
|
||||||
const WEB_PROFILE_CONFIG_KEYS: &[&str] = &[
|
const WEB_PROFILE_CONFIG_KEYS: &[&str] = &[
|
||||||
"user",
|
"user",
|
||||||
"secret_mode",
|
"secret_mode",
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ const MAX_WEB_MEMORY_ENVELOPE_BYTES: usize = 4 * 1024 * 1024 * 1024;
|
|||||||
|
|
||||||
/// Validates WEB policy and resource bounds before building runtime state.
|
/// Validates WEB policy and resource bounds before building runtime state.
|
||||||
pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
||||||
|
validate_source(config)?;
|
||||||
|
validate_decoy_listener_separation(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validates source policy without requiring DNS evidence for opted-in origins.
|
||||||
|
pub(super) fn validate_source(config: &mut ProxyConfig) -> Result<()> {
|
||||||
let web_listener_count = config
|
let web_listener_count = config
|
||||||
.server
|
.server
|
||||||
.listeners
|
.listeners
|
||||||
@@ -83,12 +89,19 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
timeouts::validate(&config.web.timeouts)?;
|
timeouts::validate(&config.web.timeouts)?;
|
||||||
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
|
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
|
||||||
validate_vhosts(config)?;
|
validate_vhosts(config)?;
|
||||||
validate_decoy_listener_separation(config)?;
|
validate_decoy_listener_separation_inner(config, false)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rejects a direct decoy recursion into an effective WEB listener.
|
/// Rejects a direct decoy recursion into an effective WEB listener.
|
||||||
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
|
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
|
||||||
|
validate_decoy_listener_separation_inner(config, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_decoy_listener_separation_inner(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
require_resolved: bool,
|
||||||
|
) -> Result<()> {
|
||||||
let web_listeners = config
|
let web_listeners = config
|
||||||
.server
|
.server
|
||||||
.listeners
|
.listeners
|
||||||
@@ -101,30 +114,19 @@ pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result
|
|||||||
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
|
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
|
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
|
||||||
let WebDecoyConfig::HttpUpstream { upstream } = &vhost.decoy else {
|
let WebDecoyConfig::HttpUpstream { upstream, resolve } = &vhost.decoy else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
let parsed = decoy_dns::parse_origin(vhost_idx, upstream, *resolve)?;
|
||||||
ProxyError::Config(format!(
|
for upstream_addr in decoy_dns::addresses(config, vhost_idx, &parsed, require_resolved)? {
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
if web_listeners
|
||||||
))
|
.iter()
|
||||||
})?;
|
.any(|listener| listener_covers(*listener, upstream_addr))
|
||||||
let Some(port) = parsed.port_or_known_default() else {
|
{
|
||||||
continue;
|
return config_error(&format!(
|
||||||
};
|
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
|
||||||
let upstream_ip = match parsed.host() {
|
));
|
||||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
}
|
||||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
|
||||||
_ => continue,
|
|
||||||
};
|
|
||||||
let upstream_addr = SocketAddr::new(upstream_ip, port);
|
|
||||||
if web_listeners
|
|
||||||
.iter()
|
|
||||||
.any(|listener| listener_covers(*listener, upstream_addr))
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ const WEB_DEBUG_RENDERERS: usize = 2;
|
|||||||
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
||||||
const WEB_LANE_STATE_BYTES: usize = 512;
|
// Includes the lane plus bounded conveyor slots, Notify ownership, and both hash tables.
|
||||||
|
const WEB_LANE_STATE_BYTES: usize = 1024;
|
||||||
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
||||||
// Each profile capability is stored in its vhost and in the global containment table.
|
// Each profile capability is stored in its vhost and in the global containment table.
|
||||||
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 64;
|
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 64;
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
|||||||
("http_idle_secs", timeouts.http_idle_secs),
|
("http_idle_secs", timeouts.http_idle_secs),
|
||||||
("shutdown_secs", timeouts.shutdown_secs),
|
("shutdown_secs", timeouts.shutdown_secs),
|
||||||
("decoy_header_secs", timeouts.decoy_header_secs),
|
("decoy_header_secs", timeouts.decoy_header_secs),
|
||||||
|
("decoy_resolve_secs", timeouts.decoy_resolve_secs),
|
||||||
];
|
];
|
||||||
if let Some((field, _)) = values
|
if let Some((field, _)) = values
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
|||||||
@@ -150,45 +150,8 @@ pub(super) fn web_host_last_label_is_numeric(host: &str) -> bool {
|
|||||||
|
|
||||||
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||||
match decoy {
|
match decoy {
|
||||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
WebDecoyConfig::HttpUpstream { upstream, resolve } => {
|
||||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
decoy_dns::parse_origin(vhost_idx, upstream, *resolve)?;
|
||||||
ProxyError::Config(format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
if parsed.scheme() != "http"
|
|
||||||
|| parsed.host_str().is_none()
|
|
||||||
|| !parsed.username().is_empty()
|
|
||||||
|| parsed.password().is_some()
|
|
||||||
|| parsed.query().is_some()
|
|
||||||
|| parsed.fragment().is_some()
|
|
||||||
|| parsed.path() != "/"
|
|
||||||
|| parsed.port() == Some(0)
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let ip = match parsed.host() {
|
|
||||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
|
||||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
|
||||||
_ => {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let private = match ip {
|
|
||||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
|
||||||
IpAddr::V6(ip) => {
|
|
||||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if !private {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||||
if !directory.is_absolute() {
|
if !directory.is_absolute() {
|
||||||
|
|||||||
+1
-1
@@ -6,5 +6,5 @@ mod load;
|
|||||||
mod types;
|
mod types;
|
||||||
|
|
||||||
pub use load::ProxyConfig;
|
pub use load::ProxyConfig;
|
||||||
pub(crate) use load::{ConfigSourceGraph, LoadedConfig};
|
pub(crate) use load::{ConfigSourceGraph, LoadedConfig, ParsedConfigSource};
|
||||||
pub use types::*;
|
pub use types::*;
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ mod base_path_tests;
|
|||||||
#[path = "web_tests/carrier_method_tests.rs"]
|
#[path = "web_tests/carrier_method_tests.rs"]
|
||||||
mod carrier_method_tests;
|
mod carrier_method_tests;
|
||||||
|
|
||||||
|
#[path = "web_tests/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
#[path = "web_tests/decoy_dns_tests.rs"]
|
||||||
|
mod decoy_dns_tests;
|
||||||
|
|
||||||
const WEB_CONFIG: &str = r#"
|
const WEB_CONFIG: &str = r#"
|
||||||
[access.users]
|
[access.users]
|
||||||
alice = "000102030405060708090a0b0c0d0e0f"
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_defaults_on_and_roundtrips_without_changing_capabilities() {
|
||||||
|
let default = load_config_from_temp_toml(WEB_CONFIG);
|
||||||
|
assert!(default.web.conveyor);
|
||||||
|
assert!(ProxyConfig::default().web.conveyor);
|
||||||
|
let capabilities = default.web.runtime.as_ref().unwrap().capabilities.clone();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\nconveyor = {enabled}"),
|
||||||
|
);
|
||||||
|
let config =
|
||||||
|
load_config_from_temp_toml(&format!("[general]\nconfig_strict = true\n{configured}",));
|
||||||
|
assert_eq!(config.web.conveyor, enabled);
|
||||||
|
assert_eq!(
|
||||||
|
config.web.runtime.as_ref().unwrap().capabilities,
|
||||||
|
capabilities
|
||||||
|
);
|
||||||
|
let json = serde_json::to_value(&config.web).unwrap();
|
||||||
|
assert_eq!(json["conveyor"], enabled);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WebConfig>(json).unwrap().conveyor,
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
let toml = toml::to_string(&config.web).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
toml::from_str::<WebConfig>(&toml).unwrap().conveyor,
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_rejects_non_boolean_values() {
|
||||||
|
for value in ["\"true\"", "1", "0", "[]", "{}"] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\nconveyor = {value}"),
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&configured).contains("conveyor"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_resolution_defaults_to_never() {
|
||||||
|
let decoy: WebDecoyConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"mode": "http_upstream",
|
||||||
|
"upstream": "http://127.0.0.1:18081"
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
let serialized = serde_json::to_value(decoy).unwrap();
|
||||||
|
assert_eq!(serialized["resolve"], "never");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_timeout_defaults_to_five_seconds() {
|
||||||
|
let serialized = serde_json::to_value(WebTimeoutsConfig::default()).unwrap();
|
||||||
|
assert_eq!(serialized["decoy_resolve_secs"], 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_resolution_is_opt_in_and_mode_specific() {
|
||||||
|
let hostname = WEB_CONFIG.replace("127.0.0.1:18081", "example.com:18081");
|
||||||
|
assert!(load_config_error_from_temp_toml(&hostname).contains("IP literal"));
|
||||||
|
let never = hostname.replace(
|
||||||
|
"mode = \"http_upstream\"",
|
||||||
|
"mode = \"http_upstream\"\nresolve = \"never\"",
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&never).contains("IP literal"));
|
||||||
|
let unknown = WEB_CONFIG.replace(
|
||||||
|
"mode = \"http_upstream\"",
|
||||||
|
"mode = \"http_upstream\"\nresolve = \"always\"",
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&unknown).contains("unknown variant"));
|
||||||
|
let wrong_mode = WEB_CONFIG.replace(
|
||||||
|
"mode = \"http_upstream\"",
|
||||||
|
"mode = \"static_directory\"\nresolve = \"startup\"\ndirectory = \"/tmp\"",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&wrong_mode).contains("only valid for mode=http_upstream")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_timeout_bounds_are_independent_from_request_deadlines() {
|
||||||
|
for seconds in [0, 3601] {
|
||||||
|
let text = format!("{WEB_CONFIG}\n[web.timeouts]\ndecoy_resolve_secs = {seconds}\n");
|
||||||
|
assert!(load_config_error_from_temp_toml(&text).contains("decoy_resolve_secs"));
|
||||||
|
}
|
||||||
|
let text = format!("{WEB_CONFIG}\n[web.timeouts]\ndecoy_resolve_secs = 3600\n");
|
||||||
|
assert_eq!(
|
||||||
|
load_config_from_temp_toml(&text)
|
||||||
|
.web
|
||||||
|
.timeouts
|
||||||
|
.decoy_resolve_secs,
|
||||||
|
3600
|
||||||
|
);
|
||||||
|
}
|
||||||
+4
-4
@@ -54,12 +54,12 @@ pub use server::{
|
|||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub use web::{
|
pub use web::{
|
||||||
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebDecoyFastTrackMode,
|
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebDecoyFastTrackMode,
|
||||||
WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
WebDecoyResolve, WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig,
|
||||||
WebTimeoutsConfig, WebVhostConfig,
|
WebSecretMode, WebTimeoutsConfig, WebVhostConfig,
|
||||||
};
|
};
|
||||||
pub(crate) use web::{
|
pub(crate) use web::{
|
||||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
WebDecoyDnsSnapshot, WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost,
|
||||||
WebStaticSite,
|
WebStaticAsset, WebStaticSite,
|
||||||
};
|
};
|
||||||
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
|
|||||||
@@ -18,6 +18,10 @@ pub use fasttrack::WebDecoyFastTrackMode;
|
|||||||
// Accepted-socket overload policy remains separate from the bulky WEB data model.
|
// Accepted-socket overload policy remains separate from the bulky WEB data model.
|
||||||
mod overload;
|
mod overload;
|
||||||
pub use overload::WebHttpConnectionCapacityAction;
|
pub use overload::WebHttpConnectionCapacityAction;
|
||||||
|
// DNS evidence belongs to configuration generations, never request-time routing.
|
||||||
|
mod decoy_dns;
|
||||||
|
pub(crate) use decoy_dns::WebDecoyDnsSnapshot;
|
||||||
|
pub use decoy_dns::WebDecoyResolve;
|
||||||
|
|
||||||
/// Client-facing secret representation used to derive a WEB capability.
|
/// Client-facing secret representation used to derive a WEB capability.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
@@ -55,6 +59,9 @@ pub enum WebDecoyConfig {
|
|||||||
HttpUpstream {
|
HttpUpstream {
|
||||||
/// Origin URL without a query or fragment.
|
/// Origin URL without a query or fragment.
|
||||||
upstream: String,
|
upstream: String,
|
||||||
|
/// Opt-in configuration-time resolution; literal addresses never require DNS.
|
||||||
|
#[serde(default)]
|
||||||
|
resolve: WebDecoyResolve,
|
||||||
},
|
},
|
||||||
/// Serve an immutable, bounded snapshot of a local directory.
|
/// Serve an immutable, bounded snapshot of a local directory.
|
||||||
StaticDirectory {
|
StaticDirectory {
|
||||||
@@ -358,6 +365,9 @@ pub struct WebTimeoutsConfig {
|
|||||||
/// Deadline for connecting to and receiving headers from an HTTP decoy.
|
/// Deadline for connecting to and receiving headers from an HTTP decoy.
|
||||||
#[serde(default = "default_web_decoy_header_timeout_secs")]
|
#[serde(default = "default_web_decoy_header_timeout_secs")]
|
||||||
pub decoy_header_secs: u64,
|
pub decoy_header_secs: u64,
|
||||||
|
/// Maximum wait for each unique HTTP decoy hostname during preparation.
|
||||||
|
#[serde(default = "default_web_decoy_resolve_secs")]
|
||||||
|
pub decoy_resolve_secs: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for WebTimeoutsConfig {
|
impl Default for WebTimeoutsConfig {
|
||||||
@@ -387,6 +397,7 @@ impl Default for WebTimeoutsConfig {
|
|||||||
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
|
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
|
||||||
shutdown_secs: default_web_shutdown_secs(),
|
shutdown_secs: default_web_shutdown_secs(),
|
||||||
decoy_header_secs: default_web_decoy_header_timeout_secs(),
|
decoy_header_secs: default_web_decoy_header_timeout_secs(),
|
||||||
|
decoy_resolve_secs: default_web_decoy_resolve_secs(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -416,6 +427,9 @@ pub struct WebConfig {
|
|||||||
/// Page-owned HTTP method for HTTPS uplink and downlink, including retries.
|
/// Page-owned HTTP method for HTTPS uplink and downlink, including retries.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub carrier_method: WebCarrierMethod,
|
pub carrier_method: WebCarrierMethod,
|
||||||
|
/// Enables negotiated bounded HTTP uplink pipelining for new WEB sessions.
|
||||||
|
#[serde(default = "default_web_conveyor")]
|
||||||
|
pub conveyor: bool,
|
||||||
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub carriers: WebCarriers,
|
pub carriers: WebCarriers,
|
||||||
@@ -446,6 +460,9 @@ pub struct WebConfig {
|
|||||||
/// Validated immutable runtime snapshot built during configuration loading.
|
/// Validated immutable runtime snapshot built during configuration loading.
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub(crate) runtime: Option<Arc<WebRuntimeConfig>>,
|
pub(crate) runtime: Option<Arc<WebRuntimeConfig>>,
|
||||||
|
/// Immutable DNS evidence omitted from serialized configuration and revisions.
|
||||||
|
#[serde(skip)]
|
||||||
|
pub(crate) decoy_dns: Arc<WebDecoyDnsSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WebConfig {
|
impl WebConfig {
|
||||||
@@ -475,6 +492,7 @@ impl Default for WebConfig {
|
|||||||
enabled: false,
|
enabled: false,
|
||||||
carrier: WebCarrier::default(),
|
carrier: WebCarrier::default(),
|
||||||
carrier_method: WebCarrierMethod::default(),
|
carrier_method: WebCarrierMethod::default(),
|
||||||
|
conveyor: default_web_conveyor(),
|
||||||
carriers: WebCarriers::default(),
|
carriers: WebCarriers::default(),
|
||||||
carrier_learning: default_web_carrier_learning(),
|
carrier_learning: default_web_carrier_learning(),
|
||||||
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||||
@@ -485,6 +503,7 @@ impl Default for WebConfig {
|
|||||||
timeouts: WebTimeoutsConfig::default(),
|
timeouts: WebTimeoutsConfig::default(),
|
||||||
vhosts: Vec::new(),
|
vhosts: Vec::new(),
|
||||||
runtime: None,
|
runtime: None,
|
||||||
|
decoy_dns: Arc::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Configuration-time DNS policy for an HTTP decoy origin.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebDecoyResolve {
|
||||||
|
/// Accept IP literals only, without consulting DNS.
|
||||||
|
#[default]
|
||||||
|
Never,
|
||||||
|
/// Pin validated DNS answers for the lifetime of the prepared configuration.
|
||||||
|
Startup,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Complete resolver evidence, retained for validation against effective listeners.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub(crate) struct WebDecoyDnsSnapshot {
|
||||||
|
/// Resolver-order answers keyed by canonical URL hostname and effective port.
|
||||||
|
pub(crate) origins: BTreeMap<(String, u16), Vec<SocketAddr>>,
|
||||||
|
}
|
||||||
@@ -2,6 +2,11 @@ pub(super) fn default_web_static_index() -> String {
|
|||||||
"index.html".to_string()
|
"index.html".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Enables explicitly negotiated HTTP pipelining unless the operator disables it.
|
||||||
|
pub(super) fn default_web_conveyor() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
macro_rules! usize_default {
|
macro_rules! usize_default {
|
||||||
($name:ident, $value:expr) => {
|
($name:ident, $value:expr) => {
|
||||||
pub(super) fn $name() -> usize {
|
pub(super) fn $name() -> usize {
|
||||||
@@ -110,3 +115,4 @@ u64_default!(default_web_http_idle_secs, 75);
|
|||||||
u64_default!(default_web_http_overload_timeout_ms, 250);
|
u64_default!(default_web_http_overload_timeout_ms, 250);
|
||||||
u64_default!(default_web_shutdown_secs, 15);
|
u64_default!(default_web_shutdown_secs, 15);
|
||||||
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
||||||
|
u64_default!(default_web_decoy_resolve_secs, 5);
|
||||||
|
|||||||
+30
-2
@@ -8,13 +8,17 @@ use crate::config::ProxyConfig;
|
|||||||
|
|
||||||
const HEALTHCHECK_RESPONSE_MAX_BYTES: u64 = 64 * 1024;
|
const HEALTHCHECK_RESPONSE_MAX_BYTES: u64 = 64 * 1024;
|
||||||
|
|
||||||
|
/// API health contract selected by the standalone healthcheck command.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub(crate) enum HealthcheckMode {
|
pub(crate) enum HealthcheckMode {
|
||||||
|
/// Require a responsive API process.
|
||||||
Liveness,
|
Liveness,
|
||||||
|
/// Require the active runtime to report readiness.
|
||||||
Ready,
|
Ready,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HealthcheckMode {
|
impl HealthcheckMode {
|
||||||
|
/// Parses the supported command-line healthcheck modes.
|
||||||
pub(crate) fn from_cli_arg(value: &str) -> Option<Self> {
|
pub(crate) fn from_cli_arg(value: &str) -> Option<Self> {
|
||||||
match value {
|
match value {
|
||||||
"liveness" => Some(Self::Liveness),
|
"liveness" => Some(Self::Liveness),
|
||||||
@@ -31,6 +35,7 @@ impl HealthcheckMode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Runs the bounded API healthcheck without preparing a new proxy generation.
|
||||||
pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
|
pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
|
||||||
match run_inner(config_path, mode) {
|
match run_inner(config_path, mode) {
|
||||||
Ok(()) => 0,
|
Ok(()) => 0,
|
||||||
@@ -42,8 +47,9 @@ pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
||||||
let config =
|
let config = ProxyConfig::parse_source(config_path)
|
||||||
ProxyConfig::load(config_path).map_err(|error| format!("config load failed: {error}"))?;
|
.map_err(|error| format!("config load failed: {error}"))?
|
||||||
|
.config;
|
||||||
let api_cfg = &config.server.api;
|
let api_cfg = &config.server.api;
|
||||||
if !api_cfg.enabled {
|
if !api_cfg.enabled {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -229,4 +235,26 @@ mod tests {
|
|||||||
|
|
||||||
assert!(read_response_bounded(&mut payload.as_slice()).is_err());
|
assert!(read_response_bounded(&mut payload.as_slice()).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_dns_healthcheck_reads_source_without_resolution() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
r#"
|
||||||
|
[server.api]
|
||||||
|
enabled = false
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://unresolvable.example.invalid:8080"
|
||||||
|
resolve = "startup"
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
super::run_inner(path.to_str().unwrap(), HealthcheckMode::Liveness).unwrap();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -95,8 +95,8 @@ pub(super) async fn bootstrap(
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut config = match ProxyConfig::load(&config_path) {
|
let mut config = match ProxyConfig::load_prepared(config_path.clone()).await {
|
||||||
Ok(c) => c,
|
Ok(loaded) => loaded.config,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if config_path.exists() {
|
if config_path.exists() {
|
||||||
eprintln!("[telemt] Error: {}", e);
|
eprintln!("[telemt] Error: {}", e);
|
||||||
|
|||||||
@@ -512,6 +512,7 @@ pub(crate) fn deferred_process_fields(
|
|||||||
|
|
||||||
fn configs_equal(old: &ProxyConfig, new: &ProxyConfig) -> bool {
|
fn configs_equal(old: &ProxyConfig, new: &ProxyConfig) -> bool {
|
||||||
serde_json::to_value(old).ok() == serde_json::to_value(new).ok()
|
serde_json::to_value(old).ok() == serde_json::to_value(new).ok()
|
||||||
|
&& old.web_decoy_endpoints_equal(new)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -265,7 +265,6 @@ impl MePool {
|
|||||||
let transport_peer_addr = stream.peer_addr().map_err(ProxyError::Io)?;
|
let transport_peer_addr = stream.peer_addr().map_err(ProxyError::Io)?;
|
||||||
let peer_addr = addr;
|
let peer_addr = addr;
|
||||||
|
|
||||||
let _ = self.maybe_detect_nat_ip(local_addr.ip()).await;
|
|
||||||
let family = if local_addr.ip().is_ipv4() {
|
let family = if local_addr.ip().is_ipv4() {
|
||||||
IpFamily::V4
|
IpFamily::V4
|
||||||
} else {
|
} else {
|
||||||
@@ -324,9 +323,34 @@ impl MePool {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let local_addr_nat = self.translate_our_addr_with_reflection(local_addr, reflected);
|
// Authoritative sources do not wait behind unrelated HTTP discovery. Capture once
|
||||||
let peer_addr_nat =
|
// before deriving either side of the tuple so a refresh cannot mix observations.
|
||||||
SocketAddr::new(self.translate_ip_for_nat(peer_addr.ip()), peer_addr.port());
|
let detected = if socks_bound_kdf_addr.is_some()
|
||||||
|
|| self.nat_runtime.nat_ip_cfg.is_some()
|
||||||
|
|| reflected.is_some()
|
||||||
|
{
|
||||||
|
self.cached_http_nat_ip()
|
||||||
|
} else {
|
||||||
|
self.maybe_detect_nat_ip(local_addr.ip()).await
|
||||||
|
};
|
||||||
|
if self.nat_runtime.nat_probe
|
||||||
|
&& local_addr.is_ipv4()
|
||||||
|
&& is_bogon(local_addr.ip())
|
||||||
|
&& socks_bound_kdf_addr.is_none()
|
||||||
|
&& self.nat_runtime.nat_ip_cfg.is_none()
|
||||||
|
&& reflected.is_none()
|
||||||
|
&& detected.is_none()
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::WouldBlock,
|
||||||
|
"ME NAT source observation is unavailable or expired",
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let local_addr_nat = self.translate_our_addr_with_nat(local_addr, reflected, detected);
|
||||||
|
let peer_addr_nat = SocketAddr::new(
|
||||||
|
Self::translate_ip_with_nat(peer_addr.ip(), self.nat_runtime.nat_ip_cfg.or(detected)),
|
||||||
|
peer_addr.port(),
|
||||||
|
);
|
||||||
let client_addr_for_kdf = socks_bound_kdf_addr.unwrap_or(local_addr_nat);
|
let client_addr_for_kdf = socks_bound_kdf_addr.unwrap_or(local_addr_nat);
|
||||||
if let Some(upstream_info) = upstream_egress {
|
if let Some(upstream_info) = upstream_egress {
|
||||||
record_upstream_bnd_status(
|
record_upstream_bnd_status(
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ use super::pool::{
|
|||||||
MeFamilyRuntimeState, RefillTargetKey, WriterContour, WriterOpenIntent, WriterRole,
|
MeFamilyRuntimeState, RefillTargetKey, WriterContour, WriterOpenIntent, WriterRole,
|
||||||
};
|
};
|
||||||
use super::pool_writer::WriterReplacementPurpose;
|
use super::pool_writer::WriterReplacementPurpose;
|
||||||
|
// Bounded cross-family coverage transfers preserve the donor's current base floor.
|
||||||
|
mod coverage;
|
||||||
|
|
||||||
// Bounds reconnect jitter to at most half of the current backoff.
|
// Bounds reconnect jitter to at most half of the current backoff.
|
||||||
const JITTER_FRAC_NUM: u64 = 2;
|
const JITTER_FRAC_NUM: u64 = 2;
|
||||||
@@ -110,6 +112,8 @@ impl Drop for ScheduledReconnects<'_> {
|
|||||||
|
|
||||||
// Periodic family health monitor orchestration.
|
// Periodic family health monitor orchestration.
|
||||||
mod monitor;
|
mod monitor;
|
||||||
|
// Shared-family fair scheduling and independently tracked writer-operation quanta.
|
||||||
|
mod scheduler;
|
||||||
// Draining-writer deadline enforcement.
|
// Draining-writer deadline enforcement.
|
||||||
mod drain;
|
mod drain;
|
||||||
// Per-family reconnect scheduling and health state.
|
// Per-family reconnect scheduling and health state.
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
|
/// Selects globally because the Active cap is shared across DCs and address families.
|
||||||
|
pub(super) async fn transfer_coverage(
|
||||||
|
pool: &Arc<MePool>,
|
||||||
|
rng: &Arc<SecureRandom>,
|
||||||
|
dc: i32,
|
||||||
|
endpoint: SocketAddr,
|
||||||
|
required: usize,
|
||||||
|
) -> Option<bool> {
|
||||||
|
let capacity = pool
|
||||||
|
.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
dc,
|
||||||
|
endpoint,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
drop(capacity);
|
||||||
|
let authority = pool.floor_authority();
|
||||||
|
let endpoints = pool.endpoint_snapshot.load_full();
|
||||||
|
let generation = pool.current_generation();
|
||||||
|
let writers = pool.writers.read().await;
|
||||||
|
let activity = pool.registry.writer_activity_snapshot().await;
|
||||||
|
let mut counts = HashMap::new();
|
||||||
|
for writer in writers.iter() {
|
||||||
|
if !writer.draining.load(Ordering::Acquire)
|
||||||
|
&& writer.generation == generation
|
||||||
|
&& WriterContour::from_u8(writer.contour.load(Ordering::Acquire))
|
||||||
|
== WriterContour::Active
|
||||||
|
&& endpoints.contains_dc_endpoint(writer.writer_dc, writer.addr)
|
||||||
|
{
|
||||||
|
*counts
|
||||||
|
.entry((writer.writer_dc, writer.addr.is_ipv4()))
|
||||||
|
.or_insert(0usize) += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut candidates = Vec::new();
|
||||||
|
for writer in writers.iter() {
|
||||||
|
if writer.draining.load(Ordering::Acquire)
|
||||||
|
|| WriterContour::from_u8(writer.contour.load(Ordering::Acquire))
|
||||||
|
!= WriterContour::Active
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let role = WriterRole::from_writer(writer);
|
||||||
|
let obsolete = writer.generation != generation
|
||||||
|
|| !endpoints.contains_dc_endpoint(writer.writer_dc, writer.addr);
|
||||||
|
let protected = pool.required_writers_for_dc_with_floor_mode(
|
||||||
|
endpoints
|
||||||
|
.endpoints_for_dc_family(writer.writer_dc, role.family)
|
||||||
|
.len(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
if !obsolete
|
||||||
|
&& counts
|
||||||
|
.get(&(writer.writer_dc, writer.addr.is_ipv4()))
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0)
|
||||||
|
<= protected
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let busy = activity
|
||||||
|
.bound_clients_by_writer
|
||||||
|
.get(&writer.id)
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0)
|
||||||
|
> 0;
|
||||||
|
candidates.push((busy, !obsolete, writer.created_at, writer.id, role));
|
||||||
|
}
|
||||||
|
drop(writers);
|
||||||
|
candidates
|
||||||
|
.sort_unstable_by_key(|candidate| (candidate.0, candidate.1, candidate.2, candidate.3));
|
||||||
|
for (busy, _, _, id, role) in candidates {
|
||||||
|
let reservation = if busy {
|
||||||
|
pool.registry
|
||||||
|
.try_reserve_writer_replacement_preserving_clients(id)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
pool.registry.try_reserve_writer_replacement(id).await
|
||||||
|
};
|
||||||
|
let Some(mut reservation) = reservation else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
return Some(matches!(
|
||||||
|
tokio::time::timeout(
|
||||||
|
pool.reconnect_runtime.me_one_timeout,
|
||||||
|
pool.replace_writer_with_generation_contour_for_dc(
|
||||||
|
endpoint,
|
||||||
|
rng.as_ref(),
|
||||||
|
generation,
|
||||||
|
WriterContour::Active,
|
||||||
|
dc,
|
||||||
|
role,
|
||||||
|
WriterReplacementPurpose::CoverageTransfer {
|
||||||
|
authority,
|
||||||
|
receiver_floor: required
|
||||||
|
},
|
||||||
|
&mut reservation,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Ok(Ok(()))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
@@ -1,449 +1,67 @@
|
|||||||
|
use super::scheduler::HealthObservation;
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
pub(super) async fn check_family(
|
/// Takes a control-plane observation without awaiting any network operation.
|
||||||
family: IpFamily,
|
pub(super) async fn check_family(family: IpFamily, pool: &Arc<MePool>) -> HealthObservation {
|
||||||
pool: &Arc<MePool>,
|
let authority = pool.floor_authority();
|
||||||
rng: &Arc<SecureRandom>,
|
let generation = pool.current_generation();
|
||||||
backoff: &mut HashMap<(i32, IpFamily), u64>,
|
let snapshot = pool.endpoint_snapshot.load_full();
|
||||||
next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
let enabled = if family == IpFamily::V4 {
|
||||||
inflight: &mut HashMap<(i32, IpFamily), usize>,
|
pool.decision.ipv4_me
|
||||||
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
|
} else {
|
||||||
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
pool.decision.ipv6_me
|
||||||
single_endpoint_outage: &mut HashSet<(i32, IpFamily)>,
|
|
||||||
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
) -> bool {
|
|
||||||
let enabled = match family {
|
|
||||||
IpFamily::V4 => pool.decision.ipv4_me,
|
|
||||||
IpFamily::V6 => pool.decision.ipv6_me,
|
|
||||||
};
|
};
|
||||||
if !enabled {
|
let source = if family == IpFamily::V4 {
|
||||||
return false;
|
&snapshot.map_v4
|
||||||
}
|
} else {
|
||||||
|
&snapshot.map_v6
|
||||||
let mut family_degraded = false;
|
|
||||||
|
|
||||||
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
|
|
||||||
let endpoint_snapshot = pool.endpoint_snapshot.load();
|
|
||||||
let endpoint_revision = endpoint_snapshot.revision;
|
|
||||||
let map_guard = match family {
|
|
||||||
IpFamily::V4 => &endpoint_snapshot.map_v4,
|
|
||||||
IpFamily::V6 => &endpoint_snapshot.map_v6,
|
|
||||||
};
|
};
|
||||||
for (dc, addrs) in map_guard.iter() {
|
let mut endpoints = HashMap::new();
|
||||||
let entry = dc_endpoints.entry(*dc).or_default();
|
if enabled {
|
||||||
for (ip, port) in addrs.iter().copied() {
|
for (dc, addresses) in source {
|
||||||
entry.push(SocketAddr::new(ip, port));
|
let mut addresses: Vec<_> = addresses
|
||||||
|
.iter()
|
||||||
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
|
.collect();
|
||||||
|
addresses.sort_unstable();
|
||||||
|
addresses.dedup();
|
||||||
|
endpoints.insert(*dc, addresses);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(endpoint_snapshot);
|
let mut counts = HashMap::new();
|
||||||
for endpoints in dc_endpoints.values_mut() {
|
let mut ids = HashMap::<_, Vec<_>>::new();
|
||||||
endpoints.sort_unstable();
|
for writer in pool.writers.read().await.iter() {
|
||||||
endpoints.dedup();
|
if !writer.draining.load(std::sync::atomic::Ordering::Acquire)
|
||||||
}
|
&& writer.generation == generation
|
||||||
let reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len());
|
&& WriterContour::from_u8(writer.contour.load(std::sync::atomic::Ordering::Acquire))
|
||||||
let reconnect_sem = Arc::new(Semaphore::new(reconnect_budget));
|
== WriterContour::Active
|
||||||
|
&& endpoints
|
||||||
if pool.floor_mode() == MeFloorMode::Static {}
|
.get(&writer.writer_dc)
|
||||||
|
.is_some_and(|addresses| addresses.contains(&writer.addr))
|
||||||
let mut live_addr_counts = HashMap::<(i32, SocketAddr), usize>::new();
|
|
||||||
let mut live_writer_ids_by_addr = HashMap::<(i32, SocketAddr), Vec<u64>>::new();
|
|
||||||
let active_generation = pool.current_generation();
|
|
||||||
for writer in pool
|
|
||||||
.writers
|
|
||||||
.read()
|
|
||||||
.await
|
|
||||||
.iter()
|
|
||||||
.filter(|w| !w.draining.load(std::sync::atomic::Ordering::Relaxed))
|
|
||||||
{
|
|
||||||
if !matches!(
|
|
||||||
crate::transport::middle_proxy::pool::WriterContour::from_u8(
|
|
||||||
writer.contour.load(std::sync::atomic::Ordering::Relaxed),
|
|
||||||
),
|
|
||||||
crate::transport::middle_proxy::pool::WriterContour::Active
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if writer.generation != active_generation {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if !dc_endpoints
|
|
||||||
.get(&writer.writer_dc)
|
|
||||||
.is_some_and(|endpoints| endpoints.contains(&writer.addr))
|
|
||||||
{
|
{
|
||||||
continue;
|
*counts
|
||||||
|
.entry((writer.writer_dc, writer.addr))
|
||||||
|
.or_insert(0usize) += 1;
|
||||||
|
ids.entry((writer.writer_dc, writer.addr))
|
||||||
|
.or_default()
|
||||||
|
.push(writer.id);
|
||||||
}
|
}
|
||||||
let key = (writer.writer_dc, writer.addr);
|
|
||||||
*live_addr_counts.entry(key).or_insert(0) += 1;
|
|
||||||
live_writer_ids_by_addr
|
|
||||||
.entry(key)
|
|
||||||
.or_default()
|
|
||||||
.push(writer.id);
|
|
||||||
}
|
}
|
||||||
let writer_idle_since = pool.registry.writer_idle_since_snapshot().await;
|
let idle = pool.registry.writer_idle_since_snapshot().await;
|
||||||
let bound_clients_by_writer = pool
|
let bound = pool
|
||||||
.registry
|
.registry
|
||||||
.writer_activity_snapshot()
|
.writer_activity_snapshot()
|
||||||
.await
|
.await
|
||||||
.bound_clients_by_writer;
|
.bound_clients_by_writer;
|
||||||
let floor_plan = build_family_floor_plan(
|
let floor = build_family_floor_plan(pool, family, &endpoints, &counts, &ids, &bound).await;
|
||||||
pool,
|
HealthObservation {
|
||||||
family,
|
authority,
|
||||||
&dc_endpoints,
|
generation,
|
||||||
&live_addr_counts,
|
floor,
|
||||||
&live_writer_ids_by_addr,
|
ids,
|
||||||
&bound_clients_by_writer,
|
idle,
|
||||||
)
|
bound,
|
||||||
.await;
|
|
||||||
pool.set_adaptive_floor_runtime_caps(
|
|
||||||
floor_plan.active_cap_configured_total,
|
|
||||||
floor_plan.active_cap_effective_total,
|
|
||||||
floor_plan.warm_cap_configured_total,
|
|
||||||
floor_plan.warm_cap_effective_total,
|
|
||||||
floor_plan.target_writers_total,
|
|
||||||
floor_plan.active_writers_current,
|
|
||||||
floor_plan.warm_writers_current,
|
|
||||||
);
|
|
||||||
let floor_targets_by_dc = Arc::new(
|
|
||||||
floor_plan
|
|
||||||
.by_dc
|
|
||||||
.iter()
|
|
||||||
.map(|(dc, entry)| (*dc, entry.target_required))
|
|
||||||
.collect::<HashMap<_, _>>(),
|
|
||||||
);
|
|
||||||
let live_writer_ids_by_addr = Arc::new(live_writer_ids_by_addr);
|
|
||||||
let writer_idle_since = Arc::new(writer_idle_since);
|
|
||||||
let bound_clients_by_writer = Arc::new(bound_clients_by_writer);
|
|
||||||
let mut reconnect_set = JoinSet::<FamilyReconnectOutcome>::new();
|
|
||||||
let mut scheduled_reconnects = ScheduledReconnects {
|
|
||||||
inflight,
|
|
||||||
keys: Vec::new(),
|
|
||||||
};
|
|
||||||
|
|
||||||
for (dc, endpoints) in dc_endpoints {
|
|
||||||
if endpoints.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let key = (dc, family);
|
|
||||||
let required = floor_plan
|
|
||||||
.by_dc
|
|
||||||
.get(&dc)
|
|
||||||
.map(|entry| entry.target_required)
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
pool.required_writers_for_dc_with_floor_mode(endpoints.len(), false)
|
|
||||||
});
|
|
||||||
let alive = endpoints
|
|
||||||
.iter()
|
|
||||||
.map(|addr| *live_addr_counts.get(&(dc, *addr)).unwrap_or(&0))
|
|
||||||
.sum::<usize>();
|
|
||||||
|
|
||||||
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
|
|
||||||
family_degraded = true;
|
|
||||||
if single_endpoint_outage.insert(key) {
|
|
||||||
pool.stats.increment_me_single_endpoint_outage_enter_total();
|
|
||||||
warn!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
required,
|
|
||||||
endpoint_count = endpoints.len(),
|
|
||||||
"Single-endpoint DC outage detected"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
recover_single_endpoint_outage(
|
|
||||||
pool,
|
|
||||||
rng,
|
|
||||||
key,
|
|
||||||
endpoints[0],
|
|
||||||
required,
|
|
||||||
outage_backoff,
|
|
||||||
outage_next_attempt,
|
|
||||||
&reconnect_sem,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if single_endpoint_outage.remove(&key) {
|
|
||||||
pool.stats.increment_me_single_endpoint_outage_exit_total();
|
|
||||||
outage_backoff.remove(&key);
|
|
||||||
outage_next_attempt.remove(&key);
|
|
||||||
shadow_rotate_deadline.remove(&key);
|
|
||||||
idle_refresh_next_attempt.remove(&key);
|
|
||||||
info!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
endpoint_count = endpoints.len(),
|
|
||||||
"Single-endpoint DC outage recovered"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
maybe_refresh_idle_writer_for_dc(
|
|
||||||
pool,
|
|
||||||
rng,
|
|
||||||
key,
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
&endpoints,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
live_writer_ids_by_addr.as_ref(),
|
|
||||||
writer_idle_since.as_ref(),
|
|
||||||
bound_clients_by_writer.as_ref(),
|
|
||||||
idle_refresh_next_attempt,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if alive >= required {
|
|
||||||
maybe_rotate_single_endpoint_shadow(
|
|
||||||
pool,
|
|
||||||
rng,
|
|
||||||
key,
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
&endpoints,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
live_writer_ids_by_addr.as_ref(),
|
|
||||||
bound_clients_by_writer.as_ref(),
|
|
||||||
shadow_rotate_deadline,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let missing = required - alive;
|
|
||||||
family_degraded = true;
|
|
||||||
|
|
||||||
let now = Instant::now();
|
|
||||||
if reconnect_sem.available_permits() == 0 {
|
|
||||||
let base_ms = pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64;
|
|
||||||
let next_ms = (*backoff.get(&key).unwrap_or(&base_ms)).max(base_ms);
|
|
||||||
let jitter = next_ms / JITTER_FRAC_NUM;
|
|
||||||
let wait = Duration::from_millis(next_ms)
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
|
||||||
next_attempt.insert(key, now + wait);
|
|
||||||
debug!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
endpoint_count = endpoints.len(),
|
|
||||||
reconnect_budget,
|
|
||||||
"Skipping reconnect due to per-tick health reconnect budget"
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if let Some(ts) = next_attempt.get(&key)
|
|
||||||
&& now < *ts
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let max_concurrent = pool
|
|
||||||
.reconnect_runtime
|
|
||||||
.me_reconnect_max_concurrent_per_dc
|
|
||||||
.max(1) as usize;
|
|
||||||
if scheduled_reconnects.current(&key) >= max_concurrent {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if pool
|
|
||||||
.has_refill_inflight_for_target(RefillTargetKey {
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
generation: pool.current_generation(),
|
|
||||||
endpoint_revision,
|
|
||||||
contour: WriterContour::Active,
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
debug!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
endpoint_count = endpoints.len(),
|
|
||||||
"Skipping health reconnect: immediate refill is already in flight for this DC group"
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
scheduled_reconnects.reserve(key);
|
|
||||||
let pool_for_reconnect = pool.clone();
|
|
||||||
let rng_for_reconnect = rng.clone();
|
|
||||||
let reconnect_sem_for_dc = reconnect_sem.clone();
|
|
||||||
let endpoints_for_dc = endpoints.clone();
|
|
||||||
let live_writer_ids_by_addr_for_dc = live_writer_ids_by_addr.clone();
|
|
||||||
let writer_idle_since_for_dc = writer_idle_since.clone();
|
|
||||||
let bound_clients_by_writer_for_dc = bound_clients_by_writer.clone();
|
|
||||||
let floor_targets_by_dc_for_reconnect = floor_targets_by_dc.clone();
|
|
||||||
let active_cap_effective_total = floor_plan.active_cap_effective_total;
|
|
||||||
reconnect_set.spawn(async move {
|
|
||||||
let mut restored = 0usize;
|
|
||||||
for _ in 0..missing {
|
|
||||||
let Ok(reconnect_permit) = reconnect_sem_for_dc.clone().try_acquire_owned() else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
let base_req = pool_for_reconnect
|
|
||||||
.required_writers_for_dc_with_floor_mode(endpoints_for_dc.len(), false);
|
|
||||||
if pool_for_reconnect.active_contour_writer_count_total().await
|
|
||||||
>= active_cap_effective_total
|
|
||||||
{
|
|
||||||
let swapped = maybe_swap_idle_writer_for_cap(
|
|
||||||
&pool_for_reconnect,
|
|
||||||
&rng_for_reconnect,
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
&endpoints_for_dc,
|
|
||||||
required,
|
|
||||||
floor_targets_by_dc_for_reconnect.as_ref(),
|
|
||||||
live_writer_ids_by_addr_for_dc.as_ref(),
|
|
||||||
writer_idle_since_for_dc.as_ref(),
|
|
||||||
bound_clients_by_writer_for_dc.as_ref(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
if swapped {
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_swap_idle_total();
|
|
||||||
restored += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if alive + restored >= base_req {
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_cap_block_total();
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_swap_idle_failed_total();
|
|
||||||
debug!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
active_cap_effective_total,
|
|
||||||
"Adaptive floor cap reached, reconnect attempt blocked"
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
|
||||||
let intent = if alive + restored < base_req {
|
|
||||||
WriterOpenIntent::Coverage
|
|
||||||
} else {
|
|
||||||
WriterOpenIntent::Normal
|
|
||||||
};
|
|
||||||
let res = tokio::time::timeout(
|
|
||||||
pool_for_reconnect.reconnect_runtime.me_one_timeout,
|
|
||||||
pool_for_reconnect.connect_endpoints_round_robin_with_generation_contour(
|
|
||||||
dc,
|
|
||||||
&endpoints_for_dc,
|
|
||||||
rng_for_reconnect.as_ref(),
|
|
||||||
pool_for_reconnect.current_generation(),
|
|
||||||
WriterContour::Active,
|
|
||||||
intent,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
match res {
|
|
||||||
Ok(true) => {
|
|
||||||
restored += 1;
|
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_success();
|
|
||||||
}
|
|
||||||
Ok(false) => {
|
|
||||||
debug!(dc = %dc, ?family, "ME round-robin reconnect failed")
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
debug!(dc = %dc, ?family, "ME reconnect timed out");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
drop(reconnect_permit);
|
|
||||||
}
|
|
||||||
|
|
||||||
FamilyReconnectOutcome {
|
|
||||||
key,
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
required,
|
|
||||||
endpoint_count: endpoints_for_dc.len(),
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
while let Some(joined) = reconnect_set.join_next().await {
|
|
||||||
let outcome = match joined {
|
|
||||||
Ok(outcome) => outcome,
|
|
||||||
Err(join_error) => {
|
|
||||||
debug!(error = %join_error, "Health reconnect task failed");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let now = Instant::now();
|
|
||||||
let now_alive = live_active_writers_for_dc_family(pool, outcome.dc, outcome.family).await;
|
|
||||||
if now_alive >= outcome.required {
|
|
||||||
info!(
|
|
||||||
dc = %outcome.dc,
|
|
||||||
family = ?outcome.family,
|
|
||||||
alive = now_alive,
|
|
||||||
required = outcome.required,
|
|
||||||
endpoint_count = outcome.endpoint_count,
|
|
||||||
"ME writer floor restored for DC"
|
|
||||||
);
|
|
||||||
backoff.insert(
|
|
||||||
outcome.key,
|
|
||||||
pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64,
|
|
||||||
);
|
|
||||||
let jitter = pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64
|
|
||||||
/ JITTER_FRAC_NUM;
|
|
||||||
let wait = pool.reconnect_runtime.me_reconnect_backoff_base
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
|
||||||
next_attempt.insert(outcome.key, now + wait);
|
|
||||||
} else {
|
|
||||||
let curr = *backoff
|
|
||||||
.get(&outcome.key)
|
|
||||||
.unwrap_or(&(pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64));
|
|
||||||
let next_ms = (curr.saturating_mul(2))
|
|
||||||
.min(pool.reconnect_runtime.me_reconnect_backoff_cap.as_millis() as u64);
|
|
||||||
backoff.insert(outcome.key, next_ms);
|
|
||||||
let jitter = next_ms / JITTER_FRAC_NUM;
|
|
||||||
let wait = Duration::from_millis(next_ms)
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
|
||||||
next_attempt.insert(outcome.key, now + wait);
|
|
||||||
if pool.is_runtime_ready() {
|
|
||||||
let warn_cooldown = pool.warn_rate_limit_duration();
|
|
||||||
if should_emit_rate_limited_warn(
|
|
||||||
floor_warn_next_allowed,
|
|
||||||
outcome.key,
|
|
||||||
now,
|
|
||||||
warn_cooldown,
|
|
||||||
) {
|
|
||||||
warn!(
|
|
||||||
dc = %outcome.dc,
|
|
||||||
family = ?outcome.family,
|
|
||||||
alive = now_alive,
|
|
||||||
required = outcome.required,
|
|
||||||
endpoint_count = outcome.endpoint_count,
|
|
||||||
backoff_ms = next_ms,
|
|
||||||
"DC writer floor is below required level, scheduled reconnect"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
info!(
|
|
||||||
dc = %outcome.dc,
|
|
||||||
family = ?outcome.family,
|
|
||||||
alive = now_alive,
|
|
||||||
required = outcome.required,
|
|
||||||
endpoint_count = outcome.endpoint_count,
|
|
||||||
backoff_ms = next_ms,
|
|
||||||
"DC writer floor is below required level during startup, scheduled reconnect"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
family_degraded
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
pub(super) fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
||||||
|
|||||||
@@ -1,59 +1,293 @@
|
|||||||
|
use super::scheduler::{
|
||||||
|
HealthJob, HealthObservation, HealthScheduler, JobKind, JobOutcome, idle_refresh_ready, run_job,
|
||||||
|
};
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Observes family health independently of bounded, lifecycle-owned network operations.
|
||||||
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
||||||
let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
let mut scheduler = HealthScheduler::default();
|
||||||
let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut tasks = JoinSet::new();
|
||||||
let mut inflight: HashMap<(i32, IpFamily), usize> = HashMap::new();
|
let mut observations = HashMap::<IpFamily, Arc<HealthObservation>>::new();
|
||||||
let mut outage_backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
let mut drain_warn_next_allowed = HashMap::new();
|
||||||
let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut observe = true;
|
||||||
let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new();
|
let mut health_tick = true;
|
||||||
let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut next_tick = tokio::time::Instant::now();
|
||||||
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
|
||||||
let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
|
||||||
let mut drain_warn_next_allowed: HashMap<u64, Instant> = HashMap::new();
|
|
||||||
let mut degraded_interval = true;
|
|
||||||
loop {
|
loop {
|
||||||
let interval = if degraded_interval {
|
if observe {
|
||||||
pool.health_interval_unhealthy()
|
if health_tick {
|
||||||
} else {
|
pool.prune_closed_writers().await;
|
||||||
pool.health_interval_healthy()
|
pool.sweep_endpoint_quarantine().await;
|
||||||
};
|
reap_draining_writers(&pool, &mut drain_warn_next_allowed).await;
|
||||||
tokio::time::sleep(interval).await;
|
}
|
||||||
pool.prune_closed_writers().await;
|
let mut target_total = 0usize;
|
||||||
pool.sweep_endpoint_quarantine().await;
|
let mut effective_cap = 0usize;
|
||||||
reap_draining_writers(&pool, &mut drain_warn_next_allowed).await;
|
let mut cap_observation = None;
|
||||||
let v4_degraded = check_family(
|
let mut degraded = false;
|
||||||
IpFamily::V4,
|
for family in [IpFamily::V4, IpFamily::V6] {
|
||||||
&pool,
|
let observation = Arc::new(check_family(family, &pool).await);
|
||||||
&rng,
|
let family_degraded = observation
|
||||||
&mut backoff,
|
.floor
|
||||||
&mut next_attempt,
|
.by_dc
|
||||||
&mut inflight,
|
.values()
|
||||||
&mut outage_backoff,
|
.any(|entry| entry.alive < entry.target_required);
|
||||||
&mut outage_next_attempt,
|
if health_tick {
|
||||||
&mut single_endpoint_outage,
|
update_family_runtime_state(&pool, family, family_degraded);
|
||||||
&mut shadow_rotate_deadline,
|
}
|
||||||
&mut idle_refresh_next_attempt,
|
degraded |= family_degraded;
|
||||||
&mut floor_warn_next_allowed,
|
target_total += observation
|
||||||
)
|
.floor
|
||||||
.await;
|
.by_dc
|
||||||
let v6_degraded = check_family(
|
.values()
|
||||||
IpFamily::V6,
|
.map(|entry| entry.target_required)
|
||||||
&pool,
|
.sum::<usize>();
|
||||||
&rng,
|
effective_cap = effective_cap.max(observation.floor.active_cap_effective_total);
|
||||||
&mut backoff,
|
if cap_observation.is_none() {
|
||||||
&mut next_attempt,
|
cap_observation = Some(observation.clone());
|
||||||
&mut inflight,
|
}
|
||||||
&mut outage_backoff,
|
observations.insert(family, observation);
|
||||||
&mut outage_next_attempt,
|
}
|
||||||
&mut single_endpoint_outage,
|
if let Some(observation) = cap_observation
|
||||||
&mut shadow_rotate_deadline,
|
&& observations
|
||||||
&mut idle_refresh_next_attempt,
|
.values()
|
||||||
&mut floor_warn_next_allowed,
|
.all(|other| other.authority == observation.authority)
|
||||||
)
|
{
|
||||||
.await;
|
let floor = &observation.floor;
|
||||||
update_family_runtime_state(&pool, IpFamily::V4, v4_degraded);
|
pool.set_adaptive_floor_runtime_caps(
|
||||||
update_family_runtime_state(&pool, IpFamily::V6, v6_degraded);
|
observation.authority,
|
||||||
degraded_interval = v4_degraded || v6_degraded;
|
floor.active_cap_configured_total,
|
||||||
|
effective_cap.max(target_total),
|
||||||
|
floor.warm_cap_configured_total,
|
||||||
|
floor.warm_cap_effective_total,
|
||||||
|
target_total,
|
||||||
|
floor.active_writers_current,
|
||||||
|
floor.warm_writers_current,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
scheduler.observe(&observations);
|
||||||
|
for (key, state) in &mut scheduler.groups {
|
||||||
|
if health_tick {
|
||||||
|
state.deferred = false;
|
||||||
|
}
|
||||||
|
let Some(entry) = observations
|
||||||
|
.get(&key.1)
|
||||||
|
.and_then(|o| o.floor.by_dc.get(&key.0))
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let outage = entry.endpoints.len() == 1
|
||||||
|
&& entry.alive == 0
|
||||||
|
&& pool.single_endpoint_outage_mode_enabled();
|
||||||
|
if outage != state.outage {
|
||||||
|
if outage {
|
||||||
|
pool.stats.increment_me_single_endpoint_outage_enter_total();
|
||||||
|
warn!(dc = key.0, family = ?key.1, required = entry.target_required,
|
||||||
|
endpoint_count = entry.endpoints.len(), "Single-endpoint DC outage detected");
|
||||||
|
} else {
|
||||||
|
pool.stats.increment_me_single_endpoint_outage_exit_total();
|
||||||
|
info!(dc = key.0, family = ?key.1, alive = entry.alive, required = entry.target_required,
|
||||||
|
endpoint_count = entry.endpoints.len(), "Single-endpoint DC outage recovered");
|
||||||
|
}
|
||||||
|
state.outage = outage;
|
||||||
|
// Outage retries and ordinary missing-floor rounds use different bounds.
|
||||||
|
// A successful first writer must not inherit an outage failure delay.
|
||||||
|
state.backoff_ms = 0;
|
||||||
|
state.due = None;
|
||||||
|
state.round_left = 0;
|
||||||
|
state.round_active = false;
|
||||||
|
}
|
||||||
|
if state.round_active
|
||||||
|
&& (state.round_left == 0 || entry.alive >= entry.target_required)
|
||||||
|
{
|
||||||
|
let (base, cap) = if state.outage {
|
||||||
|
pool.single_endpoint_outage_backoff_bounds_ms()
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64,
|
||||||
|
pool.reconnect_runtime.me_reconnect_backoff_cap.as_millis() as u64,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let restored = entry.alive >= entry.target_required;
|
||||||
|
state.backoff_ms = if restored {
|
||||||
|
base
|
||||||
|
} else {
|
||||||
|
state.backoff_ms.max(base).saturating_mul(2).min(cap)
|
||||||
|
};
|
||||||
|
let wait = Duration::from_millis(state.backoff_ms)
|
||||||
|
+ Duration::from_millis(
|
||||||
|
rand::rng()
|
||||||
|
.random_range(0..=(state.backoff_ms / JITTER_FRAC_NUM).max(1)),
|
||||||
|
);
|
||||||
|
state.due = Some(Instant::now() + wait);
|
||||||
|
state.round_active = false;
|
||||||
|
if restored {
|
||||||
|
info!(dc = key.0, family = ?key.1, alive = entry.alive, required = entry.target_required,
|
||||||
|
endpoint_count = entry.endpoints.len(), "ME writer floor restored for DC");
|
||||||
|
}
|
||||||
|
if !restored && state.warn_due.is_none_or(|due| due <= Instant::now()) {
|
||||||
|
state.warn_due = Some(Instant::now() + pool.warn_rate_limit_duration());
|
||||||
|
if state.outage {
|
||||||
|
warn!(dc = key.0, family = ?key.1, endpoint = %entry.endpoints[0], required = entry.target_required,
|
||||||
|
backoff_ms = state.backoff_ms, "Single-endpoint outage reconnect scheduled");
|
||||||
|
} else if pool.is_runtime_ready() {
|
||||||
|
warn!(dc = key.0, family = ?key.1, alive = entry.alive, required = entry.target_required,
|
||||||
|
endpoint_count = entry.endpoints.len(), backoff_ms = state.backoff_ms,
|
||||||
|
"DC writer floor is below required level, scheduled reconnect");
|
||||||
|
} else {
|
||||||
|
info!(dc = key.0, family = ?key.1, alive = entry.alive, required = entry.target_required,
|
||||||
|
endpoint_count = entry.endpoints.len(), backoff_ms = state.backoff_ms,
|
||||||
|
"DC writer floor is below required level during startup, scheduled reconnect");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if health_tick {
|
||||||
|
next_tick = tokio::time::Instant::now()
|
||||||
|
+ if degraded {
|
||||||
|
pool.health_interval_unhealthy()
|
||||||
|
} else {
|
||||||
|
pool.health_interval_healthy()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
observe = false;
|
||||||
|
health_tick = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let budget = family::health_reconnect_budget(&pool, scheduler.groups.len());
|
||||||
|
for _ in 0..scheduler.queue.len() {
|
||||||
|
if tasks.len() >= budget {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let Some(key) = scheduler.queue.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
scheduler.queue.push_back(key);
|
||||||
|
if scheduler.busy(key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(observation) = observations.get(&key.1).cloned() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if pool.floor_authority() != observation.authority
|
||||||
|
|| pool.current_generation() != observation.generation
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(entry) = observation.floor.by_dc.get(&key.0).cloned() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if entry.endpoints.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(state) = scheduler.groups.get_mut(&key) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if state.deferred {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let now = Instant::now();
|
||||||
|
let recover =
|
||||||
|
entry.alive < entry.target_required && state.due.is_none_or(|due| due <= now);
|
||||||
|
let refresh = state.refresh_due.is_none_or(|due| due <= now)
|
||||||
|
&& idle_refresh_ready(&observation, &entry);
|
||||||
|
let mut kind = if refresh && (!recover || state.prefer_refresh) {
|
||||||
|
JobKind::Refresh
|
||||||
|
} else if recover {
|
||||||
|
JobKind::Recover
|
||||||
|
} else if entry.alive >= entry.target_required
|
||||||
|
&& entry.endpoints.len() == 1
|
||||||
|
&& pool.single_endpoint_shadow_rotate_interval().is_some()
|
||||||
|
&& state.shadow_due.is_none_or(|due| due <= now)
|
||||||
|
{
|
||||||
|
JobKind::Shadow
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if kind == JobKind::Recover {
|
||||||
|
if pool
|
||||||
|
.has_refill_inflight_for_target(RefillTargetKey {
|
||||||
|
dc: key.0,
|
||||||
|
family: key.1,
|
||||||
|
generation: observation.generation,
|
||||||
|
endpoint_revision: observation.authority.0,
|
||||||
|
contour: WriterContour::Active,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Preflight distinguishes capacity pressure from unreachable endpoints without
|
||||||
|
// retaining an opening token while a group waits in the fair queue.
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
key.0,
|
||||||
|
entry.endpoints[0],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if reservation.is_none() {
|
||||||
|
kind = JobKind::Transfer;
|
||||||
|
}
|
||||||
|
drop(reservation);
|
||||||
|
if !state.round_active {
|
||||||
|
state.round = state.round.wrapping_add(1);
|
||||||
|
state.round_left = if state.outage {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
entry.target_required.saturating_sub(entry.alive)
|
||||||
|
};
|
||||||
|
state.round_active = true;
|
||||||
|
}
|
||||||
|
state.prefer_refresh = true;
|
||||||
|
} else if kind == JobKind::Refresh {
|
||||||
|
state.prefer_refresh = false;
|
||||||
|
}
|
||||||
|
let job = HealthJob {
|
||||||
|
key,
|
||||||
|
kind,
|
||||||
|
observation,
|
||||||
|
entry,
|
||||||
|
round: state.round,
|
||||||
|
};
|
||||||
|
let future = run_job(pool.clone(), rng.clone(), job.clone());
|
||||||
|
let Ok(tracked) = pool.lifecycle.track_producer(future) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let id = tasks.spawn(tracked).id();
|
||||||
|
scheduler.owners.insert(id, job);
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = Instant::now();
|
||||||
|
let deadline = scheduler
|
||||||
|
.groups
|
||||||
|
.values()
|
||||||
|
.flat_map(|state| [state.due, state.refresh_due, state.shadow_due])
|
||||||
|
.flatten()
|
||||||
|
.filter(|due| *due > now)
|
||||||
|
.min()
|
||||||
|
.map(tokio::time::Instant::from_std)
|
||||||
|
.map_or(next_tick, |due| due.min(next_tick));
|
||||||
|
tokio::select! {
|
||||||
|
joined = tasks.join_next_with_id(), if !tasks.is_empty() => {
|
||||||
|
let Some(joined) = joined else { continue; };
|
||||||
|
let (id, output) = match joined {
|
||||||
|
Ok((id, output)) => (id, output),
|
||||||
|
Err(error) => {
|
||||||
|
debug!(error = %error, "Health writer operation failed");
|
||||||
|
(error.id(), Some(JobOutcome::Completed(None)))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(job) = scheduler.owners.remove(&id)
|
||||||
|
&& let Some(deadline) = output {
|
||||||
|
scheduler.completed(&pool, job, deadline);
|
||||||
|
}
|
||||||
|
observe = true;
|
||||||
|
}
|
||||||
|
_ = tokio::time::sleep_until(deadline) => {
|
||||||
|
observe = true;
|
||||||
|
health_tick = tokio::time::Instant::now() >= next_tick;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,311 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use tokio::task::Id;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
/// Exact fairness and single-owner domain; DC sign remains part of the identity.
|
||||||
|
pub(super) type Group = (i32, IpFamily);
|
||||||
|
|
||||||
|
/// Immutable observation is fenced again before dispatch and when accepting completion.
|
||||||
|
pub(super) struct HealthObservation {
|
||||||
|
/// Endpoint and floor-policy revisions used to derive this plan.
|
||||||
|
pub(super) authority: (u64, u64),
|
||||||
|
/// Active generation whose writers contribute coverage.
|
||||||
|
pub(super) generation: u64,
|
||||||
|
/// Family-local targets and global capacity projection.
|
||||||
|
pub(super) floor: FamilyFloorPlan,
|
||||||
|
/// Current non-draining writer IDs grouped by DC and endpoint.
|
||||||
|
pub(super) ids: HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
|
/// Registry idle timestamps used only for refresh selection.
|
||||||
|
pub(super) idle: HashMap<u64, u64>,
|
||||||
|
/// Captured bindings; commit revalidates victim idleness when required.
|
||||||
|
pub(super) bound: HashMap<u64, usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mutually exclusive network work selected for one group quantum.
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(super) enum JobKind {
|
||||||
|
/// Restores one missing writer through ordinary bounded admission.
|
||||||
|
Recover,
|
||||||
|
/// Restores coverage by replacing an eligible donor.
|
||||||
|
Transfer,
|
||||||
|
/// Replaces an idle writer before upstream expiry.
|
||||||
|
Refresh,
|
||||||
|
/// Applies the configured single-endpoint rotation cadence.
|
||||||
|
Shadow,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resource deferral consumes no network retry budget and preserves the eligibility deadline.
|
||||||
|
pub(super) enum JobOutcome {
|
||||||
|
/// Work consumed its quantum and may return a refresh or shadow deadline.
|
||||||
|
Completed(Option<Instant>),
|
||||||
|
/// Capacity or authority prevented an operation; do not charge network backoff.
|
||||||
|
Deferred,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captured authority accompanies both successful results and JoinError ownership.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(super) struct HealthJob {
|
||||||
|
/// Group holding the unique inflight slot.
|
||||||
|
pub(super) key: Group,
|
||||||
|
/// Determines which deadline or round may change on completion.
|
||||||
|
pub(super) kind: JobKind,
|
||||||
|
/// Immutable selection evidence shared by jobs from one observation.
|
||||||
|
pub(super) observation: Arc<HealthObservation>,
|
||||||
|
/// The selected group's target and endpoint candidates.
|
||||||
|
pub(super) entry: DcFloorPlanEntry,
|
||||||
|
/// Recovery round identity; refresh and shadow operations do not consume it.
|
||||||
|
pub(super) round: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retry and cooldown state owned exclusively by the monitor.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(super) struct GroupState {
|
||||||
|
/// Endpoint, policy and generation fence for every mutable field below.
|
||||||
|
pub(super) authority: Option<((u64, u64), u64)>,
|
||||||
|
/// Earliest recovery eligibility; resource deferral leaves it unchanged.
|
||||||
|
pub(super) due: Option<Instant>,
|
||||||
|
/// Idle-refresh cooldown independent of missing-floor recovery.
|
||||||
|
pub(super) refresh_due: Option<Instant>,
|
||||||
|
/// Single-endpoint rotation cooldown.
|
||||||
|
pub(super) shadow_due: Option<Instant>,
|
||||||
|
/// Backoff advances once per completed recovery round.
|
||||||
|
pub(super) backoff_ms: u64,
|
||||||
|
/// Remaining writer operations in the current round.
|
||||||
|
pub(super) round_left: usize,
|
||||||
|
/// Distinguishes a finalized round from one that has no operations left yet.
|
||||||
|
pub(super) round_active: bool,
|
||||||
|
/// Monotonic identity prevents late outcomes from changing a newer round.
|
||||||
|
pub(super) round: u64,
|
||||||
|
/// Alternates independently eligible refresh and recovery work.
|
||||||
|
pub(super) prefer_refresh: bool,
|
||||||
|
/// Selects the single-endpoint outage retry namespace.
|
||||||
|
pub(super) outage: bool,
|
||||||
|
/// Rate limit for missing-floor warnings.
|
||||||
|
pub(super) warn_due: Option<Instant>,
|
||||||
|
/// Resource-blocked groups wait until the next health tick.
|
||||||
|
pub(super) deferred: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One queue and one owner table bound work across both address families.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(super) struct HealthScheduler {
|
||||||
|
/// Persistent cursor order survives observations and worker completions.
|
||||||
|
pub(super) queue: VecDeque<Group>,
|
||||||
|
/// Coalesced work and deadlines for configured groups only.
|
||||||
|
pub(super) groups: HashMap<Group, GroupState>,
|
||||||
|
/// Task IDs release group ownership even after panic or abort.
|
||||||
|
pub(super) owners: HashMap<Id, HealthJob>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealthScheduler {
|
||||||
|
/// Refreshes desired groups without resetting the round-robin cursor.
|
||||||
|
pub(super) fn observe(&mut self, observations: &HashMap<IpFamily, Arc<HealthObservation>>) {
|
||||||
|
let mut current = HashSet::new();
|
||||||
|
for (family, observation) in observations {
|
||||||
|
for dc in observation.floor.by_dc.keys() {
|
||||||
|
let key = (*dc, *family);
|
||||||
|
current.insert(key);
|
||||||
|
let state = self.groups.entry(key).or_default();
|
||||||
|
let authority = (observation.authority, observation.generation);
|
||||||
|
if state.authority != Some(authority) {
|
||||||
|
*state = GroupState {
|
||||||
|
authority: Some(authority),
|
||||||
|
..GroupState::default()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if !self.queue.contains(&key) {
|
||||||
|
self.queue.push_back(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.queue.retain(|key| current.contains(key));
|
||||||
|
self.groups.retain(|key, _| current.contains(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Includes obsolete tasks until their actual completion releases ownership.
|
||||||
|
pub(super) fn busy(&self, key: Group) -> bool {
|
||||||
|
self.owners.values().any(|job| job.key == key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accepts only an authoritative outcome from a still-active recovery round.
|
||||||
|
pub(super) fn completed(&mut self, pool: &Arc<MePool>, job: HealthJob, outcome: JobOutcome) {
|
||||||
|
let Some(state) = self.groups.get_mut(&job.key) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if state.authority != Some((job.observation.authority, job.observation.generation))
|
||||||
|
|| pool.floor_authority() != job.observation.authority
|
||||||
|
|| pool.current_generation() != job.observation.generation
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if matches!(job.kind, JobKind::Recover | JobKind::Transfer)
|
||||||
|
&& (!state.round_active || state.round != job.round)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let JobOutcome::Completed(deadline) = outcome else {
|
||||||
|
state.deferred = true;
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match job.kind {
|
||||||
|
JobKind::Refresh => {
|
||||||
|
state.refresh_due = Some(deadline.unwrap_or_else(|| {
|
||||||
|
Instant::now() + Duration::from_secs(IDLE_REFRESH_RETRY_SECS)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
JobKind::Shadow => {
|
||||||
|
state.shadow_due = Some(deadline.unwrap_or_else(|| {
|
||||||
|
Instant::now() + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
JobKind::Recover | JobKind::Transfer => {
|
||||||
|
state.round_left = state.round_left.saturating_sub(1);
|
||||||
|
// Round finalization uses the next fresh observation, never the job's old count.
|
||||||
|
state.due = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds refresh work without waiting for another group's network operation.
|
||||||
|
pub(super) fn idle_refresh_ready(
|
||||||
|
observation: &HealthObservation,
|
||||||
|
entry: &DcFloorPlanEntry,
|
||||||
|
) -> bool {
|
||||||
|
let now = MePool::now_epoch_secs();
|
||||||
|
entry.endpoints.iter().any(|endpoint| {
|
||||||
|
observation
|
||||||
|
.ids
|
||||||
|
.get(&(entry.dc, *endpoint))
|
||||||
|
.is_some_and(|ids| {
|
||||||
|
ids.iter().any(|id| {
|
||||||
|
observation.bound.get(id).copied().unwrap_or(0) == 0
|
||||||
|
&& observation.idle.get(id).is_some_and(|since| {
|
||||||
|
now.saturating_sub(*since)
|
||||||
|
>= IDLE_REFRESH_TRIGGER_BASE_SECS
|
||||||
|
+ id % (IDLE_REFRESH_TRIGGER_JITTER_SECS + 1)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Executes at most one bounded writer operation, never a whole missing-writer batch.
|
||||||
|
pub(super) async fn run_job(
|
||||||
|
pool: Arc<MePool>,
|
||||||
|
rng: Arc<SecureRandom>,
|
||||||
|
job: HealthJob,
|
||||||
|
) -> JobOutcome {
|
||||||
|
if pool.floor_authority() != job.observation.authority
|
||||||
|
|| pool.current_generation() != job.observation.generation
|
||||||
|
{
|
||||||
|
return JobOutcome::Deferred;
|
||||||
|
}
|
||||||
|
let key = job.key;
|
||||||
|
let entry = &job.entry;
|
||||||
|
let observation = &job.observation;
|
||||||
|
let mut deadlines = HashMap::new();
|
||||||
|
match job.kind {
|
||||||
|
JobKind::Refresh => {
|
||||||
|
maybe_refresh_idle_writer_for_dc(
|
||||||
|
&pool,
|
||||||
|
&rng,
|
||||||
|
key,
|
||||||
|
key.0,
|
||||||
|
key.1,
|
||||||
|
&entry.endpoints,
|
||||||
|
entry.alive,
|
||||||
|
entry.target_required,
|
||||||
|
&observation.ids,
|
||||||
|
&observation.idle,
|
||||||
|
&observation.bound,
|
||||||
|
&mut deadlines,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
JobKind::Shadow => {
|
||||||
|
maybe_rotate_single_endpoint_shadow(
|
||||||
|
&pool,
|
||||||
|
&rng,
|
||||||
|
key,
|
||||||
|
key.0,
|
||||||
|
key.1,
|
||||||
|
&entry.endpoints,
|
||||||
|
entry.alive,
|
||||||
|
entry.target_required,
|
||||||
|
&observation.ids,
|
||||||
|
&observation.bound,
|
||||||
|
&mut deadlines,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
JobKind::Transfer => {
|
||||||
|
if coverage::transfer_coverage(
|
||||||
|
&pool,
|
||||||
|
&rng,
|
||||||
|
key.0,
|
||||||
|
entry.endpoints[0],
|
||||||
|
entry.target_required,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
return JobOutcome::Deferred;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
JobKind::Recover => {
|
||||||
|
let outage = entry.endpoints.len() == 1
|
||||||
|
&& entry.alive == 0
|
||||||
|
&& pool.single_endpoint_outage_mode_enabled();
|
||||||
|
pool.stats.increment_me_reconnect_attempt();
|
||||||
|
if outage {
|
||||||
|
pool.stats
|
||||||
|
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||||
|
}
|
||||||
|
let base = pool.required_writers_for_dc_with_floor_mode(entry.endpoints.len(), false);
|
||||||
|
let intent = if entry.alive < base {
|
||||||
|
WriterOpenIntent::Coverage
|
||||||
|
} else {
|
||||||
|
WriterOpenIntent::Normal
|
||||||
|
};
|
||||||
|
let connected = tokio::time::timeout(pool.reconnect_runtime.me_one_timeout, async {
|
||||||
|
if outage && pool.single_endpoint_outage_disable_quarantine() {
|
||||||
|
pool.stats
|
||||||
|
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||||
|
pool.connect_one_with_generation_contour_for_dc_with_intent(
|
||||||
|
entry.endpoints[0],
|
||||||
|
rng.as_ref(),
|
||||||
|
observation.generation,
|
||||||
|
WriterContour::Active,
|
||||||
|
key.0,
|
||||||
|
intent,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
} else {
|
||||||
|
pool.connect_endpoints_round_robin_with_generation_contour(
|
||||||
|
key.0,
|
||||||
|
&entry.endpoints,
|
||||||
|
rng.as_ref(),
|
||||||
|
observation.generation,
|
||||||
|
WriterContour::Active,
|
||||||
|
intent,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or(false);
|
||||||
|
if connected {
|
||||||
|
pool.stats.increment_me_reconnect_success();
|
||||||
|
if outage {
|
||||||
|
pool.stats
|
||||||
|
.increment_me_single_endpoint_outage_reconnect_success_total();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
JobOutcome::Completed(deadlines.remove(&key))
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn deferred_and_stale_completions_preserve_round_budget_and_deadlines() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
let observation = Arc::new(check_family(IpFamily::V4, &pool).await);
|
||||||
|
let mut scheduler = HealthScheduler::default();
|
||||||
|
scheduler.observe(&HashMap::from([(IpFamily::V4, observation.clone())]));
|
||||||
|
let key = (2, IpFamily::V4);
|
||||||
|
let entry = observation.floor.by_dc[&2].clone();
|
||||||
|
let job = HealthJob {
|
||||||
|
key,
|
||||||
|
kind: JobKind::Transfer,
|
||||||
|
observation,
|
||||||
|
entry,
|
||||||
|
round: 4,
|
||||||
|
};
|
||||||
|
let due = Instant::now() - Duration::from_secs(1);
|
||||||
|
let state = scheduler.groups.get_mut(&key).unwrap();
|
||||||
|
state.round = 4;
|
||||||
|
state.round_left = 3;
|
||||||
|
state.round_active = true;
|
||||||
|
state.due = Some(due);
|
||||||
|
scheduler.completed(&pool, job.clone(), JobOutcome::Deferred);
|
||||||
|
let state = &scheduler.groups[&key];
|
||||||
|
assert!(state.deferred);
|
||||||
|
assert_eq!(state.round_left, 3);
|
||||||
|
assert_eq!(state.due, Some(due));
|
||||||
|
let state = scheduler.groups.get_mut(&key).unwrap();
|
||||||
|
state.deferred = false;
|
||||||
|
state.round_active = false;
|
||||||
|
scheduler.completed(&pool, job.clone(), JobOutcome::Completed(None));
|
||||||
|
scheduler.completed(&pool, job, JobOutcome::Deferred);
|
||||||
|
let state = &scheduler.groups[&key];
|
||||||
|
assert!(!state.deferred);
|
||||||
|
assert_eq!(state.round_left, 3);
|
||||||
|
assert_eq!(state.due, Some(due));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn queue_survives_repeated_observations_without_starving_later_groups() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
(1..=12)
|
||||||
|
.map(|dc| (dc, vec![("127.0.0.1".parse().unwrap(), 443)]))
|
||||||
|
.collect(),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let observation = Arc::new(check_family(IpFamily::V4, &pool).await);
|
||||||
|
let observations = HashMap::from([(IpFamily::V4, observation)]);
|
||||||
|
let mut scheduler = HealthScheduler::default();
|
||||||
|
scheduler.observe(&observations);
|
||||||
|
let expected = scheduler.queue.iter().copied().collect::<HashSet<_>>();
|
||||||
|
let mut visited = HashSet::new();
|
||||||
|
for _ in 0..expected.len() {
|
||||||
|
let key = scheduler.queue.pop_front().unwrap();
|
||||||
|
scheduler.queue.push_back(key);
|
||||||
|
visited.insert(key);
|
||||||
|
scheduler.observe(&observations);
|
||||||
|
}
|
||||||
|
assert_eq!(visited, expected);
|
||||||
|
}
|
||||||
@@ -342,6 +342,8 @@ pub(super) struct ReinitCoordinatorState {
|
|||||||
/// Latest endpoint authority revision accepted by the coordinator.
|
/// Latest endpoint authority revision accepted by the coordinator.
|
||||||
pub(super) endpoint_revision: u64,
|
pub(super) endpoint_revision: u64,
|
||||||
pub(super) pending: Option<ReinitPendingState>,
|
pub(super) pending: Option<ReinitPendingState>,
|
||||||
|
/// Revision of the coherent floor and capacity policy.
|
||||||
|
pub(super) floor_policy_revision: u64,
|
||||||
pub(super) attempts: HashMap<u64, ReinitAttemptState>,
|
pub(super) attempts: HashMap<u64, ReinitAttemptState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,7 +411,7 @@ pub(super) struct BindingPolicyCore {
|
|||||||
pub(super) struct NatRuntimeCore {
|
pub(super) struct NatRuntimeCore {
|
||||||
pub(super) discovery: super::pool_nat::NatDiscovery,
|
pub(super) discovery: super::pool_nat::NatDiscovery,
|
||||||
pub(super) nat_ip_cfg: Option<IpAddr>,
|
pub(super) nat_ip_cfg: Option<IpAddr>,
|
||||||
pub(super) nat_ip_detected: Arc<RwLock<Option<IpAddr>>>,
|
pub(super) nat_ip_detected: Arc<RwLock<Option<super::pool_nat::HttpNatObservation>>>,
|
||||||
pub(super) nat_probe: bool,
|
pub(super) nat_probe: bool,
|
||||||
pub(super) nat_stun: Option<String>,
|
pub(super) nat_stun: Option<String>,
|
||||||
pub(super) nat_stun_servers: Vec<String>,
|
pub(super) nat_stun_servers: Vec<String>,
|
||||||
@@ -503,10 +505,10 @@ pub struct MePool {
|
|||||||
pub(super) proxy_secret: Arc<RwLock<SecretSnapshot>>,
|
pub(super) proxy_secret: Arc<RwLock<SecretSnapshot>>,
|
||||||
pub(super) default_dc: AtomicI32,
|
pub(super) default_dc: AtomicI32,
|
||||||
pub(super) next_writer_id: AtomicU64,
|
pub(super) next_writer_id: AtomicU64,
|
||||||
pub(super) writer_connect_active_reserved: AtomicUsize,
|
pub(super) writer_connect_active_reserved: Arc<AtomicUsize>,
|
||||||
pub(super) writer_connect_warm_reserved: AtomicUsize,
|
pub(super) writer_connect_warm_reserved: Arc<AtomicUsize>,
|
||||||
/// Replacement connections opened but not yet committed to writer visibility.
|
/// Physical replacement overlap: preparation plus victims awaiting transport destruction.
|
||||||
pub(super) writer_replacement_open_reserved: AtomicUsize,
|
pub(super) writer_replacement_open_reserved: Arc<AtomicUsize>,
|
||||||
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
||||||
/// Coalesced refill state keyed by exact generation and contour ownership.
|
/// Coalesced refill state keyed by exact generation and contour ownership.
|
||||||
pub(super) refill_states: Arc<ParkingMutex<HashMap<RefillTargetKey, RefillTargetState>>>,
|
pub(super) refill_states: Arc<ParkingMutex<HashMap<RefillTargetKey, RefillTargetState>>>,
|
||||||
@@ -545,6 +547,8 @@ mod transport_policy;
|
|||||||
mod selection_policy;
|
mod selection_policy;
|
||||||
// Bounded writer-open admission and coverage accounting.
|
// Bounded writer-open admission and coverage accounting.
|
||||||
mod writer_admission;
|
mod writer_admission;
|
||||||
|
// Coherent policy revisions fence derived health plans and replacement decisions.
|
||||||
|
mod floor_authority;
|
||||||
pub(super) use writer_admission::{WriterOpenIntent, WriterOpenReservation, WriterRole};
|
pub(super) use writer_admission::{WriterOpenIntent, WriterOpenReservation, WriterRole};
|
||||||
// Endpoint-to-DC routing and health timing policy.
|
// Endpoint-to-DC routing and health timing policy.
|
||||||
mod routing;
|
mod routing;
|
||||||
|
|||||||
@@ -165,6 +165,7 @@ impl MePool {
|
|||||||
active_generation: 1,
|
active_generation: 1,
|
||||||
desired_map_hash: 0,
|
desired_map_hash: 0,
|
||||||
endpoint_revision: 1,
|
endpoint_revision: 1,
|
||||||
|
floor_policy_revision: 1,
|
||||||
pending: None,
|
pending: None,
|
||||||
attempts: HashMap::new(),
|
attempts: HashMap::new(),
|
||||||
}),
|
}),
|
||||||
@@ -395,9 +396,9 @@ impl MePool {
|
|||||||
pool_size: 2,
|
pool_size: 2,
|
||||||
default_dc: AtomicI32::new(default_dc.unwrap_or(2)),
|
default_dc: AtomicI32::new(default_dc.unwrap_or(2)),
|
||||||
next_writer_id: AtomicU64::new(1),
|
next_writer_id: AtomicU64::new(1),
|
||||||
writer_connect_active_reserved: AtomicUsize::new(0),
|
writer_connect_active_reserved: Arc::new(AtomicUsize::new(0)),
|
||||||
writer_connect_warm_reserved: AtomicUsize::new(0),
|
writer_connect_warm_reserved: Arc::new(AtomicUsize::new(0)),
|
||||||
writer_replacement_open_reserved: AtomicUsize::new(0),
|
writer_replacement_open_reserved: Arc::new(AtomicUsize::new(0)),
|
||||||
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
||||||
refill_states: Arc::new(ParkingMutex::new(HashMap::new())),
|
refill_states: Arc::new(ParkingMutex::new(HashMap::new())),
|
||||||
refill_running: AtomicUsize::new(0),
|
refill_running: AtomicUsize::new(0),
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
use super::*;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
impl MePool {
|
||||||
|
/// Captures the two independent authorities used by a health plan.
|
||||||
|
pub(in crate::transport::middle_proxy) fn floor_authority(&self) -> (u64, u64) {
|
||||||
|
let coordinator = self.reinit.coordinator.lock();
|
||||||
|
(
|
||||||
|
coordinator.endpoint_revision,
|
||||||
|
coordinator.floor_policy_revision,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captures floor-affecting values under the coordinator without recursively acquiring it.
|
||||||
|
pub(super) fn floor_policy_values(&self) -> [u64; 14] {
|
||||||
|
let floor = &self.floor_runtime;
|
||||||
|
[
|
||||||
|
self.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_shadow_writers
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor.me_floor_mode.load(Ordering::Relaxed) as u64,
|
||||||
|
floor.me_adaptive_floor_idle_secs.load(Ordering::Relaxed),
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_min_writers_single_endpoint
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_min_writers_multi_endpoint
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_recover_grace_secs
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_writers_per_core_total
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_cpu_cores_override
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_extra_writers_single_per_core
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
floor
|
||||||
|
.me_adaptive_floor_max_warm_writers_global
|
||||||
|
.load(Ordering::Relaxed) as u64,
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::config::GeneralConfig;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||||
|
|
||||||
|
fn reload(pool: &MePool, cfg: &GeneralConfig) {
|
||||||
|
pool.update_runtime_reinit_policy(
|
||||||
|
cfg.hardswap,
|
||||||
|
cfg.me_pool_drain_ttl_secs,
|
||||||
|
cfg.me_instadrain,
|
||||||
|
cfg.me_pool_drain_threshold,
|
||||||
|
cfg.me_pool_drain_soft_evict_enabled,
|
||||||
|
cfg.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
cfg.me_pool_drain_soft_evict_per_writer,
|
||||||
|
cfg.me_pool_drain_soft_evict_budget_per_core,
|
||||||
|
cfg.me_pool_drain_soft_evict_cooldown_ms,
|
||||||
|
cfg.effective_me_pool_force_close_secs(),
|
||||||
|
cfg.me_pool_min_fresh_ratio,
|
||||||
|
cfg.me_hardswap_warmup_delay_min_ms,
|
||||||
|
cfg.me_hardswap_warmup_delay_max_ms,
|
||||||
|
cfg.me_hardswap_warmup_extra_passes,
|
||||||
|
cfg.me_hardswap_warmup_pass_backoff_base_ms,
|
||||||
|
cfg.me_bind_stale_mode,
|
||||||
|
cfg.me_bind_stale_ttl_secs,
|
||||||
|
cfg.me_secret_atomic_snapshot,
|
||||||
|
cfg.me_deterministic_writer_sort,
|
||||||
|
cfg.me_writer_pick_mode,
|
||||||
|
cfg.me_writer_pick_sample_size,
|
||||||
|
cfg.me_single_endpoint_shadow_writers,
|
||||||
|
cfg.me_single_endpoint_outage_mode_enabled,
|
||||||
|
cfg.me_single_endpoint_outage_disable_quarantine,
|
||||||
|
cfg.me_single_endpoint_outage_backoff_min_ms,
|
||||||
|
cfg.me_single_endpoint_outage_backoff_max_ms,
|
||||||
|
cfg.me_single_endpoint_shadow_rotate_every_secs,
|
||||||
|
cfg.me_floor_mode,
|
||||||
|
cfg.me_adaptive_floor_idle_secs,
|
||||||
|
cfg.me_adaptive_floor_min_writers_single_endpoint,
|
||||||
|
cfg.me_adaptive_floor_min_writers_multi_endpoint,
|
||||||
|
cfg.me_adaptive_floor_recover_grace_secs,
|
||||||
|
cfg.me_adaptive_floor_writers_per_core_total,
|
||||||
|
cfg.me_adaptive_floor_cpu_cores_override,
|
||||||
|
cfg.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
|
cfg.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
cfg.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
cfg.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
cfg.me_adaptive_floor_max_active_writers_global,
|
||||||
|
cfg.me_adaptive_floor_max_warm_writers_global,
|
||||||
|
cfg.me_health_interval_ms_unhealthy,
|
||||||
|
cfg.me_health_interval_ms_healthy,
|
||||||
|
cfg.me_warn_rate_limit_ms,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn policy_reload_invalidates_old_caps_and_rejects_stale_plan_publication() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let mut cfg = GeneralConfig::default();
|
||||||
|
reload(&pool, &cfg);
|
||||||
|
let authority = pool.floor_authority();
|
||||||
|
pool.set_adaptive_floor_runtime_caps(authority, 10, 12, 10, 12, 6, 6, 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
12
|
||||||
|
);
|
||||||
|
cfg.me_single_endpoint_shadow_writers += 1;
|
||||||
|
reload(&pool, &cfg);
|
||||||
|
assert_ne!(pool.floor_authority(), authority);
|
||||||
|
assert_eq!(
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
pool.set_adaptive_floor_runtime_caps(authority, 10, 128, 10, 128, 100, 6, 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
let new_authority = pool.floor_authority();
|
||||||
|
reload(&pool, &cfg);
|
||||||
|
assert_eq!(
|
||||||
|
pool.floor_authority(),
|
||||||
|
new_authority,
|
||||||
|
"unchanged reload must not supersede work"
|
||||||
|
);
|
||||||
|
pool.set_adaptive_floor_runtime_caps(new_authority, 10, 11, 10, 11, 6, 6, 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
11
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -305,6 +305,8 @@ impl MePool {
|
|||||||
if previous_writer_pick_mode != writer_pick_mode {
|
if previous_writer_pick_mode != writer_pick_mode {
|
||||||
self.stats.increment_me_writer_pick_mode_switch_total();
|
self.stats.increment_me_writer_pick_mode_switch_total();
|
||||||
}
|
}
|
||||||
|
let mut coordinator = self.reinit.coordinator.lock();
|
||||||
|
let previous_floor_policy = self.floor_policy_values();
|
||||||
self.single_endpoint_runtime
|
self.single_endpoint_runtime
|
||||||
.me_single_endpoint_shadow_writers
|
.me_single_endpoint_shadow_writers
|
||||||
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
|
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
|
||||||
@@ -381,6 +383,17 @@ impl MePool {
|
|||||||
self.floor_runtime
|
self.floor_runtime
|
||||||
.me_adaptive_floor_max_warm_writers_global
|
.me_adaptive_floor_max_warm_writers_global
|
||||||
.store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed);
|
.store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed);
|
||||||
|
if previous_floor_policy != self.floor_policy_values() {
|
||||||
|
coordinator.floor_policy_revision = coordinator.floor_policy_revision.wrapping_add(1);
|
||||||
|
// Cached caps belong to the old policy until a new health plan is accepted.
|
||||||
|
self.floor_runtime
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.store(0, Ordering::Release);
|
||||||
|
self.floor_runtime
|
||||||
|
.me_adaptive_floor_warm_cap_effective
|
||||||
|
.store(0, Ordering::Release);
|
||||||
|
}
|
||||||
|
drop(coordinator);
|
||||||
self.health_runtime
|
self.health_runtime
|
||||||
.me_health_interval_ms_unhealthy
|
.me_health_interval_ms_unhealthy
|
||||||
.store(me_health_interval_ms_unhealthy.max(1), Ordering::Relaxed);
|
.store(me_health_interval_ms_unhealthy.max(1), Ordering::Relaxed);
|
||||||
|
|||||||
@@ -230,8 +230,10 @@ impl MePool {
|
|||||||
configured
|
configured
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Publishes derived caps only while both endpoint and floor-policy authority remain current.
|
||||||
pub(in crate::transport::middle_proxy) fn set_adaptive_floor_runtime_caps(
|
pub(in crate::transport::middle_proxy) fn set_adaptive_floor_runtime_caps(
|
||||||
&self,
|
&self,
|
||||||
|
authority: (u64, u64),
|
||||||
active_cap_configured: usize,
|
active_cap_configured: usize,
|
||||||
active_cap_effective: usize,
|
active_cap_effective: usize,
|
||||||
warm_cap_configured: usize,
|
warm_cap_configured: usize,
|
||||||
@@ -240,6 +242,15 @@ impl MePool {
|
|||||||
active_writers_current: usize,
|
active_writers_current: usize,
|
||||||
warm_writers_current: usize,
|
warm_writers_current: usize,
|
||||||
) {
|
) {
|
||||||
|
let coordinator = self.reinit.coordinator.lock();
|
||||||
|
if authority
|
||||||
|
!= (
|
||||||
|
coordinator.endpoint_revision,
|
||||||
|
coordinator.floor_policy_revision,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
self.floor_runtime
|
self.floor_runtime
|
||||||
.me_adaptive_floor_global_cap_raw
|
.me_adaptive_floor_global_cap_raw
|
||||||
.store(active_cap_configured as u64, Ordering::Relaxed);
|
.store(active_cap_configured as u64, Ordering::Relaxed);
|
||||||
|
|||||||
@@ -48,13 +48,13 @@ pub(in crate::transport::middle_proxy) enum WriterOpenIntent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// RAII ownership of one bounded in-flight writer open.
|
/// RAII ownership of one bounded in-flight writer open.
|
||||||
pub(in crate::transport::middle_proxy) struct WriterOpenReservation<'a> {
|
pub(in crate::transport::middle_proxy) struct WriterOpenReservation {
|
||||||
counter: Option<&'a AtomicUsize>,
|
counter: Option<Arc<AtomicUsize>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for WriterOpenReservation<'_> {
|
impl Drop for WriterOpenReservation {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if let Some(counter) = self.counter {
|
if let Some(counter) = &self.counter {
|
||||||
counter.fetch_sub(1, Ordering::AcqRel);
|
counter.fetch_sub(1, Ordering::AcqRel);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -172,7 +172,7 @@ impl MePool {
|
|||||||
intent: WriterOpenIntent,
|
intent: WriterOpenIntent,
|
||||||
writer_dc: i32,
|
writer_dc: i32,
|
||||||
target_addr: SocketAddr,
|
target_addr: SocketAddr,
|
||||||
) -> Option<WriterOpenReservation<'_>> {
|
) -> Option<WriterOpenReservation> {
|
||||||
let counter = match contour {
|
let counter = match contour {
|
||||||
WriterContour::Active => &self.writer_connect_active_reserved,
|
WriterContour::Active => &self.writer_connect_active_reserved,
|
||||||
WriterContour::Warm => &self.writer_connect_warm_reserved,
|
WriterContour::Warm => &self.writer_connect_warm_reserved,
|
||||||
@@ -182,6 +182,8 @@ impl MePool {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if intent == WriterOpenIntent::Replacement {
|
if intent == WriterOpenIntent::Replacement {
|
||||||
|
// Policy reload and replacement admission share the same linearization point.
|
||||||
|
let _coordinator = self.reinit.coordinator.lock();
|
||||||
let configured_cap = match contour {
|
let configured_cap = match contour {
|
||||||
WriterContour::Active => self.adaptive_floor_active_cap_configured_total(),
|
WriterContour::Active => self.adaptive_floor_active_cap_configured_total(),
|
||||||
WriterContour::Warm => self.adaptive_floor_warm_cap_configured_total(),
|
WriterContour::Warm => self.adaptive_floor_warm_cap_configured_total(),
|
||||||
@@ -220,7 +222,7 @@ impl MePool {
|
|||||||
.is_ok()
|
.is_ok()
|
||||||
{
|
{
|
||||||
return Some(WriterOpenReservation {
|
return Some(WriterOpenReservation {
|
||||||
counter: Some(&self.writer_replacement_open_reserved),
|
counter: Some(self.writer_replacement_open_reserved.clone()),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -266,7 +268,7 @@ impl MePool {
|
|||||||
.is_ok()
|
.is_ok()
|
||||||
{
|
{
|
||||||
return Some(WriterOpenReservation {
|
return Some(WriterOpenReservation {
|
||||||
counter: Some(counter),
|
counter: Some(counter.clone()),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -111,6 +111,31 @@ impl MePoolLifecycle {
|
|||||||
self.admission.try_register()
|
self.admission.try_register()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Registers a cancellation-aware producer for a caller-owned JoinSet without a second spawn.
|
||||||
|
pub(super) fn track_producer<F>(
|
||||||
|
&self,
|
||||||
|
future: F,
|
||||||
|
) -> Result<impl Future<Output = Option<F::Output>> + Send + 'static + use<F>, F>
|
||||||
|
where
|
||||||
|
F: Future + Send + 'static,
|
||||||
|
F::Output: Send + 'static,
|
||||||
|
{
|
||||||
|
let Some(registration) = self.try_register() else {
|
||||||
|
return Err(future);
|
||||||
|
};
|
||||||
|
let cancel = self.producer_cancel.clone();
|
||||||
|
let tracked = self.producer_tasks.track_future(async move {
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = cancel.cancelled() => None,
|
||||||
|
output = future => Some(output),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// The tracker owns destruction before shutdown can observe no registrations.
|
||||||
|
drop(registration);
|
||||||
|
Ok(tracked)
|
||||||
|
}
|
||||||
|
|
||||||
/// Registers and spawns one cancellation-aware ME producer.
|
/// Registers and spawns one cancellation-aware ME producer.
|
||||||
pub(super) fn spawn_producer<F>(&self, future: F) -> Result<(), F>
|
pub(super) fn spawn_producer<F>(&self, future: F) -> Result<(), F>
|
||||||
where
|
where
|
||||||
|
|||||||
@@ -26,6 +26,9 @@ const HTTP_FAILURE_TTL: Duration = Duration::from_secs(60);
|
|||||||
// Discovery ownership and source-bound cache admission for this pool generation.
|
// Discovery ownership and source-bound cache admission for this pool generation.
|
||||||
mod state;
|
mod state;
|
||||||
pub(super) use state::NatDiscovery;
|
pub(super) use state::NatDiscovery;
|
||||||
|
// Successful HTTP observations expire independently of failure backoff.
|
||||||
|
mod http;
|
||||||
|
pub(super) use http::HttpNatObservation;
|
||||||
|
|
||||||
// Local network regressions exercise cancellation before discovery publication.
|
// Local network regressions exercise cancellation before discovery publication.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -167,14 +170,16 @@ impl MePool {
|
|||||||
|
|
||||||
/// Applies the configured or HTTP-detected NAT IP to non-public addresses.
|
/// Applies the configured or HTTP-detected NAT IP to non-public addresses.
|
||||||
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
||||||
let nat_ip = self.nat_runtime.nat_ip_cfg.or_else(|| {
|
Self::translate_ip_with_nat(
|
||||||
|
ip,
|
||||||
self.nat_runtime
|
self.nat_runtime
|
||||||
.nat_ip_detected
|
.nat_ip_cfg
|
||||||
.try_read()
|
.or_else(|| self.cached_http_nat_ip()),
|
||||||
.ok()
|
)
|
||||||
.and_then(|g| *g)
|
}
|
||||||
});
|
|
||||||
|
|
||||||
|
/// Applies a captured NAT observation without rereading mutable discovery state.
|
||||||
|
pub(super) fn translate_ip_with_nat(ip: IpAddr, nat_ip: Option<IpAddr>) -> IpAddr {
|
||||||
let Some(nat_ip) = nat_ip else {
|
let Some(nat_ip) = nat_ip else {
|
||||||
return ip;
|
return ip;
|
||||||
};
|
};
|
||||||
@@ -197,6 +202,16 @@ impl MePool {
|
|||||||
&self,
|
&self,
|
||||||
addr: std::net::SocketAddr,
|
addr: std::net::SocketAddr,
|
||||||
reflected: Option<std::net::SocketAddr>,
|
reflected: Option<std::net::SocketAddr>,
|
||||||
|
) -> std::net::SocketAddr {
|
||||||
|
self.translate_our_addr_with_nat(addr, reflected, self.cached_http_nat_ip())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retains the actual TCP port while using one handshake-local NAT snapshot.
|
||||||
|
pub(super) fn translate_our_addr_with_nat(
|
||||||
|
&self,
|
||||||
|
addr: std::net::SocketAddr,
|
||||||
|
reflected: Option<std::net::SocketAddr>,
|
||||||
|
detected: Option<IpAddr>,
|
||||||
) -> std::net::SocketAddr {
|
) -> std::net::SocketAddr {
|
||||||
let ip = if let Some(nat_ip) = self.nat_runtime.nat_ip_cfg {
|
let ip = if let Some(nat_ip) = self.nat_runtime.nat_ip_cfg {
|
||||||
match (addr.ip(), nat_ip) {
|
match (addr.ip(), nat_ip) {
|
||||||
@@ -209,63 +224,16 @@ impl MePool {
|
|||||||
if is_bogon(addr.ip()) || addr.ip().is_loopback() || addr.ip().is_unspecified() {
|
if is_bogon(addr.ip()) || addr.ip().is_loopback() || addr.ip().is_unspecified() {
|
||||||
r.ip()
|
r.ip()
|
||||||
} else {
|
} else {
|
||||||
self.translate_ip_for_nat(addr.ip())
|
Self::translate_ip_with_nat(addr.ip(), detected)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
self.translate_ip_for_nat(addr.ip())
|
Self::translate_ip_with_nat(addr.ip(), detected)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Keep the kernel-assigned TCP source port; STUN port can differ.
|
// Keep the kernel-assigned TCP source port; STUN port can differ.
|
||||||
std::net::SocketAddr::new(ip, addr.port())
|
std::net::SocketAddr::new(ip, addr.port())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Waits for pool-owned HTTP discovery without owning its cancellation.
|
|
||||||
pub(super) async fn maybe_detect_nat_ip(self: &Arc<Self>, local_ip: IpAddr) -> Option<IpAddr> {
|
|
||||||
if self.nat_runtime.nat_ip_cfg.is_some() {
|
|
||||||
return self.nat_runtime.nat_ip_cfg;
|
|
||||||
}
|
|
||||||
|
|
||||||
if !self.nat_runtime.nat_probe {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
if !(is_bogon(local_ip) || local_ip.is_loopback() || local_ip.is_unspecified()) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(ip) = *self.nat_runtime.nat_ip_detected.read().await {
|
|
||||||
return Some(ip);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut flight = self.nat_runtime.discovery.http.clone().lock_owned().await;
|
|
||||||
if let Some(ip) = *self.nat_runtime.nat_ip_detected.read().await {
|
|
||||||
return Some(ip);
|
|
||||||
}
|
|
||||||
if flight.is_some_and(|until| Instant::now() < until) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let pool = self.clone();
|
|
||||||
let (send, receive) = oneshot::channel();
|
|
||||||
// Transfer singleflight ownership before the producer can be cancelled or polled.
|
|
||||||
self.lifecycle
|
|
||||||
.spawn_producer(async move {
|
|
||||||
let detected = detect_public_ipv4_http(&pool.nat_runtime.http_ip_detect_urls)
|
|
||||||
.await
|
|
||||||
.map(IpAddr::V4);
|
|
||||||
if let Some(ip) = detected {
|
|
||||||
*pool.nat_runtime.nat_ip_detected.write().await = Some(ip);
|
|
||||||
*flight = None;
|
|
||||||
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
|
||||||
} else {
|
|
||||||
*flight = Some(Instant::now() + HTTP_FAILURE_TTL);
|
|
||||||
}
|
|
||||||
let _ = send.send(detected);
|
|
||||||
})
|
|
||||||
.ok()?;
|
|
||||||
receive.await.ok().flatten()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for a pool-owned STUN refresh in the exact source-address context.
|
/// Waits for a pool-owned STUN refresh in the exact source-address context.
|
||||||
pub(super) async fn maybe_reflect_public_addr(
|
pub(super) async fn maybe_reflect_public_addr(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const HTTP_FRESH_TTL: Duration = Duration::from_secs(600);
|
||||||
|
const HTTP_MAX_AGE: Duration = Duration::from_secs(1200);
|
||||||
|
|
||||||
|
/// A successful HTTP observation has an immutable monotonic timestamp.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub(in crate::transport::middle_proxy) struct HttpNatObservation {
|
||||||
|
/// Successful detector result, never updated by a failed refresh.
|
||||||
|
pub(in crate::transport::middle_proxy) ip: IpAddr,
|
||||||
|
/// Original observation time used for both freshness and bounded stale fallback.
|
||||||
|
pub(super) observed_at: tokio::time::Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpNatObservation {
|
||||||
|
fn usable(self) -> Option<IpAddr> {
|
||||||
|
(self.observed_at.elapsed() < HTTP_MAX_AGE).then_some(self.ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fresh(self) -> bool {
|
||||||
|
self.observed_at.elapsed() < HTTP_FRESH_TTL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MePool {
|
||||||
|
/// Reads a bounded cache snapshot without discovery or data-plane waiting.
|
||||||
|
pub(in crate::transport::middle_proxy) fn cached_http_nat_ip(&self) -> Option<IpAddr> {
|
||||||
|
self.nat_runtime
|
||||||
|
.nat_ip_detected
|
||||||
|
.try_read()
|
||||||
|
.ok()
|
||||||
|
.and_then(|cache| cache.and_then(HttpNatObservation::usable))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Refreshes once per pool, retaining stale success only within its original grace period.
|
||||||
|
pub(in crate::transport::middle_proxy) async fn maybe_detect_nat_ip(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
local_ip: IpAddr,
|
||||||
|
) -> Option<IpAddr> {
|
||||||
|
if self.nat_runtime.nat_ip_cfg.is_some() {
|
||||||
|
return self.nat_runtime.nat_ip_cfg;
|
||||||
|
}
|
||||||
|
if !self.nat_runtime.nat_probe
|
||||||
|
|| !local_ip.is_ipv4()
|
||||||
|
|| !(is_bogon(local_ip) || local_ip.is_loopback() || local_ip.is_unspecified())
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if let Some(cached) = *self.nat_runtime.nat_ip_detected.read().await
|
||||||
|
&& cached.fresh()
|
||||||
|
{
|
||||||
|
return Some(cached.ip);
|
||||||
|
}
|
||||||
|
let mut flight = self.nat_runtime.discovery.http.clone().lock_owned().await;
|
||||||
|
let cached = *self.nat_runtime.nat_ip_detected.read().await;
|
||||||
|
if let Some(cached) = cached
|
||||||
|
&& cached.fresh()
|
||||||
|
{
|
||||||
|
return Some(cached.ip);
|
||||||
|
}
|
||||||
|
if flight.is_some_and(|until| Instant::now() < until) {
|
||||||
|
return cached.and_then(HttpNatObservation::usable);
|
||||||
|
}
|
||||||
|
let pool = self.clone();
|
||||||
|
let (send, receive) = oneshot::channel();
|
||||||
|
// Discovery owns the flight; cancellation of a handshake cannot cancel publication.
|
||||||
|
self.lifecycle
|
||||||
|
.spawn_producer(async move {
|
||||||
|
let detected = detect_public_ipv4_http(&pool.nat_runtime.http_ip_detect_urls)
|
||||||
|
.await
|
||||||
|
.map(IpAddr::V4);
|
||||||
|
if let Some(ip) = detected {
|
||||||
|
*pool.nat_runtime.nat_ip_detected.write().await = Some(HttpNatObservation {
|
||||||
|
ip,
|
||||||
|
observed_at: tokio::time::Instant::now(),
|
||||||
|
});
|
||||||
|
*flight = None;
|
||||||
|
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
||||||
|
} else {
|
||||||
|
*flight = Some(Instant::now() + HTTP_FAILURE_TTL);
|
||||||
|
}
|
||||||
|
let _ = send.send(());
|
||||||
|
})
|
||||||
|
.ok()?;
|
||||||
|
let _ = receive.await;
|
||||||
|
self.cached_http_nat_ip()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn observation_age_is_not_renewed_by_reads_or_failure() {
|
||||||
|
let cached = HttpNatObservation {
|
||||||
|
ip: "91.108.56.1".parse().unwrap(),
|
||||||
|
observed_at: tokio::time::Instant::now(),
|
||||||
|
};
|
||||||
|
tokio::time::advance(HTTP_FRESH_TTL).await;
|
||||||
|
assert!(!cached.fresh());
|
||||||
|
for _ in 0..100 {
|
||||||
|
assert_eq!(cached.usable(), Some(cached.ip));
|
||||||
|
}
|
||||||
|
tokio::time::advance(HTTP_MAX_AGE - HTTP_FRESH_TTL).await;
|
||||||
|
assert_eq!(cached.usable(), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,7 @@ pub(super) const PUBLIC_IP: Ipv4Addr = Ipv4Addr::new(91, 108, 56, 1);
|
|||||||
pub(super) const TEST_DEADLINE: Duration = Duration::from_secs(2);
|
pub(super) const TEST_DEADLINE: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
// Concurrency and end-to-end regressions share gated local discovery servers.
|
// Concurrency and end-to-end regressions share gated local discovery servers.
|
||||||
|
mod freshness;
|
||||||
pub(super) mod lifecycle;
|
pub(super) mod lifecycle;
|
||||||
mod recovery;
|
mod recovery;
|
||||||
|
|
||||||
@@ -243,7 +244,11 @@ async fn http_discovery_survives_reconnect_timeout() {
|
|||||||
.await
|
.await
|
||||||
.expect("pool-owned HTTP discovery must publish after caller cancellation");
|
.expect("pool-owned HTTP discovery must publish after caller cancellation");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
*pool.nat_runtime.nat_ip_detected.read().await,
|
pool.nat_runtime
|
||||||
|
.nat_ip_detected
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.map(|observation| observation.ip),
|
||||||
Some(PUBLIC_IP.into())
|
Some(PUBLIC_IP.into())
|
||||||
);
|
);
|
||||||
assert_eq!(http.requests.load(Ordering::Relaxed), 1);
|
assert_eq!(http.requests.load(Ordering::Relaxed), 1);
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_refresh_retains_original_age_but_never_expired_material() {
|
||||||
|
let http = http_server("invalid address").await;
|
||||||
|
http.release.add_permits(8);
|
||||||
|
let pool = nat_pool(vec![], vec![format!("http://{}/", http.addr)]).await;
|
||||||
|
let observed_at = tokio::time::Instant::now() - Duration::from_secs(601);
|
||||||
|
*pool.nat_runtime.nat_ip_detected.write().await = Some(HttpNatObservation {
|
||||||
|
ip: PUBLIC_IP.into(),
|
||||||
|
observed_at,
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
Some(PUBLIC_IP.into())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_ip_detected
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.observed_at,
|
||||||
|
observed_at
|
||||||
|
);
|
||||||
|
assert_eq!(http.requests.load(Ordering::Acquire), 1);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
Some(PUBLIC_IP.into())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
http.requests.load(Ordering::Acquire),
|
||||||
|
1,
|
||||||
|
"failure backoff remains singleflight"
|
||||||
|
);
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_ip_detected
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.as_mut()
|
||||||
|
.unwrap()
|
||||||
|
.observed_at = tokio::time::Instant::now() - Duration::from_secs(1201);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.translate_ip_for_nat(Ipv4Addr::LOCALHOST.into()),
|
||||||
|
IpAddr::V4(Ipv4Addr::LOCALHOST)
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expired_http_observation_defers_handshake_before_sending_kdf_material() {
|
||||||
|
let pool = nat_pool(vec![], vec![]).await;
|
||||||
|
*pool.nat_runtime.nat_ip_detected.write().await = Some(HttpNatObservation {
|
||||||
|
ip: PUBLIC_IP.into(),
|
||||||
|
observed_at: tokio::time::Instant::now() - Duration::from_secs(1201),
|
||||||
|
});
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
||||||
|
let (mut peer, _) = listener.accept().await.unwrap();
|
||||||
|
let error = pool
|
||||||
|
.handshake_only(stream, addr, None, pool.rng.as_ref())
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
matches!(error, ProxyError::Io(ref error) if error.kind() == std::io::ErrorKind::WouldBlock)
|
||||||
|
);
|
||||||
|
assert_eq!(peer.read(&mut [0u8; 1]).await.unwrap(), 0);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
@@ -24,7 +24,11 @@ impl Drop for MiddlePeer {
|
|||||||
|
|
||||||
impl MiddlePeer {
|
impl MiddlePeer {
|
||||||
async fn new(server_ip: Ipv4Addr) -> Self {
|
async fn new(server_ip: Ipv4Addr) -> Self {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
Self::with_bind("127.0.0.1:0", server_ip.into(), PUBLIC_IP.into()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn with_bind(bind: &str, server_ip: IpAddr, client_ip: IpAddr) -> Self {
|
||||||
|
let listener = TcpListener::bind(bind).await.unwrap();
|
||||||
let addr = listener.local_addr().unwrap();
|
let addr = listener.local_addr().unwrap();
|
||||||
let handshakes = Arc::new(AtomicUsize::new(0));
|
let handshakes = Arc::new(AtomicUsize::new(0));
|
||||||
let cancelled = Arc::new(AtomicUsize::new(0));
|
let cancelled = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -37,7 +41,7 @@ impl MiddlePeer {
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
accepted = listener.accept() => {
|
accepted = listener.accept() => {
|
||||||
let (stream, _) = accepted.unwrap();
|
let (stream, _) = accepted.unwrap();
|
||||||
pending.spawn(accept_handshake(stream, server_ip));
|
pending.spawn(accept_handshake(stream, server_ip, client_ip));
|
||||||
}
|
}
|
||||||
completed = pending.join_next(), if !pending.is_empty() => {
|
completed = pending.join_next(), if !pending.is_empty() => {
|
||||||
match completed.unwrap().expect("fake ME handshake task must not panic") {
|
match completed.unwrap().expect("fake ME handshake task must not panic") {
|
||||||
@@ -60,7 +64,11 @@ impl MiddlePeer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn accept_handshake(mut stream: TcpStream, server_ip: Ipv4Addr) -> Option<TcpStream> {
|
async fn accept_handshake(
|
||||||
|
mut stream: TcpStream,
|
||||||
|
server_ip: IpAddr,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
) -> Option<TcpStream> {
|
||||||
let client_port = stream.peer_addr().unwrap().port();
|
let client_port = stream.peer_addr().unwrap().port();
|
||||||
let server_port = stream.local_addr().unwrap().port();
|
let server_port = stream.local_addr().unwrap().port();
|
||||||
let (seq, payload) = match read_rpc_frame_plaintext(&mut stream).await {
|
let (seq, payload) = match read_rpc_frame_plaintext(&mut stream).await {
|
||||||
@@ -79,35 +87,43 @@ async fn accept_handshake(mut stream: TcpStream, server_ip: Ipv4Addr) -> Option<
|
|||||||
.write_all(&build_rpc_frame(-2, &nonce, RpcChecksumMode::Crc32))
|
.write_all(&build_rpc_frame(-2, &nonce, RpcChecksumMode::Crc32))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut client_ip = PUBLIC_IP.octets();
|
let split = |ip: IpAddr| match ip {
|
||||||
client_ip.reverse();
|
IpAddr::V4(ip) => {
|
||||||
let mut server_ip = server_ip.octets();
|
let mut octets = ip.octets();
|
||||||
server_ip.reverse();
|
octets.reverse();
|
||||||
|
(Some(octets), None)
|
||||||
|
}
|
||||||
|
IpAddr::V6(ip) => (None, Some(ip.octets())),
|
||||||
|
};
|
||||||
|
let (client_v4, client_v6) = split(client_ip);
|
||||||
|
let (server_v4, server_v6) = split(server_ip);
|
||||||
|
let client_ip = client_v4.unwrap_or([0; 4]);
|
||||||
|
let server_ip = server_v4.unwrap_or([0; 4]);
|
||||||
let (read_key, read_iv) = derive_middleproxy_keys(
|
let (read_key, read_iv) = derive_middleproxy_keys(
|
||||||
&server_nonce,
|
&server_nonce,
|
||||||
&client_nonce,
|
&client_nonce,
|
||||||
×tamp.to_le_bytes(),
|
×tamp.to_le_bytes(),
|
||||||
Some(&server_ip),
|
server_v4.as_ref().map(|ip| ip.as_slice()),
|
||||||
&client_port.to_le_bytes(),
|
&client_port.to_le_bytes(),
|
||||||
b"CLIENT",
|
b"CLIENT",
|
||||||
Some(&client_ip),
|
client_v4.as_ref().map(|ip| ip.as_slice()),
|
||||||
&server_port.to_le_bytes(),
|
&server_port.to_le_bytes(),
|
||||||
&[1u8; 32],
|
&[1u8; 32],
|
||||||
None,
|
client_v6.as_ref(),
|
||||||
None,
|
server_v6.as_ref(),
|
||||||
);
|
);
|
||||||
let (write_key, write_iv) = derive_middleproxy_keys(
|
let (write_key, write_iv) = derive_middleproxy_keys(
|
||||||
&server_nonce,
|
&server_nonce,
|
||||||
&client_nonce,
|
&client_nonce,
|
||||||
×tamp.to_le_bytes(),
|
×tamp.to_le_bytes(),
|
||||||
Some(&server_ip),
|
server_v4.as_ref().map(|ip| ip.as_slice()),
|
||||||
&client_port.to_le_bytes(),
|
&client_port.to_le_bytes(),
|
||||||
b"SERVER",
|
b"SERVER",
|
||||||
Some(&client_ip),
|
client_v4.as_ref().map(|ip| ip.as_slice()),
|
||||||
&server_port.to_le_bytes(),
|
&server_port.to_le_bytes(),
|
||||||
&[1u8; 32],
|
&[1u8; 32],
|
||||||
None,
|
client_v6.as_ref(),
|
||||||
None,
|
server_v6.as_ref(),
|
||||||
);
|
);
|
||||||
let mut encrypted = [0u8; 48];
|
let mut encrypted = [0u8; 48];
|
||||||
stream.read_exact(&mut encrypted).await.unwrap();
|
stream.read_exact(&mut encrypted).await.unwrap();
|
||||||
@@ -151,12 +167,8 @@ async fn suppressed_pool_recovers_after_delayed_discovery(http_slow: bool) {
|
|||||||
let http = http_server_with_delay("91.108.56.1", delay).await;
|
let http = http_server_with_delay("91.108.56.1", delay).await;
|
||||||
stun.release.add_permits(64);
|
stun.release.add_permits(64);
|
||||||
http.release.add_permits(64);
|
http.release.add_permits(64);
|
||||||
let peer = MiddlePeer::new(if http_slow {
|
// A valid STUN source bypasses HTTP; the peer remains the configured loopback endpoint.
|
||||||
PUBLIC_IP
|
let peer = MiddlePeer::new(Ipv4Addr::LOCALHOST).await;
|
||||||
} else {
|
|
||||||
Ipv4Addr::LOCALHOST
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let urls = if http_slow {
|
let urls = if http_slow {
|
||||||
vec![format!("http://{}/", http.addr)]
|
vec![format!("http://{}/", http.addr)]
|
||||||
} else {
|
} else {
|
||||||
@@ -178,6 +190,9 @@ async fn suppressed_pool_recovers_after_delayed_discovery(http_slow: bool) {
|
|||||||
pool.single_endpoint_runtime
|
pool.single_endpoint_runtime
|
||||||
.me_single_endpoint_shadow_writers
|
.me_single_endpoint_shadow_writers
|
||||||
.store(0, Ordering::Relaxed);
|
.store(0, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_shadow_rotate_every_secs
|
||||||
|
.store(0, Ordering::Relaxed);
|
||||||
pool.update_proxy_maps(
|
pool.update_proxy_maps(
|
||||||
HashMap::from([(2, vec![(peer.addr.ip(), peer.addr.port())])]),
|
HashMap::from([(2, vec![(peer.addr.ip(), peer.addr.port())])]),
|
||||||
None,
|
None,
|
||||||
@@ -206,20 +221,18 @@ async fn suppressed_pool_recovers_after_delayed_discovery(http_slow: bool) {
|
|||||||
Arc::new(SecureRandom::new()),
|
Arc::new(SecureRandom::new()),
|
||||||
0,
|
0,
|
||||||
));
|
));
|
||||||
if http_slow {
|
if !http_slow {
|
||||||
http.received().await;
|
|
||||||
} else {
|
|
||||||
stun.received().await;
|
stun.received().await;
|
||||||
|
timeout(Duration::from_secs(5), async {
|
||||||
|
while peer.cancelled.load(Ordering::Acquire) == 0 {
|
||||||
|
assert!(!peer.task.is_finished());
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the actual 1200ms reconnect must cancel a pre-KDF TCP connection");
|
||||||
|
assert_eq!(peer.handshakes.load(Ordering::Acquire), 0);
|
||||||
}
|
}
|
||||||
timeout(Duration::from_secs(5), async {
|
|
||||||
while peer.cancelled.load(Ordering::Acquire) == 0 {
|
|
||||||
assert!(!peer.task.is_finished());
|
|
||||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("the actual 1200ms reconnect must cancel at least one pre-KDF TCP connection");
|
|
||||||
assert_eq!(peer.handshakes.load(Ordering::Acquire), 0);
|
|
||||||
let expected = 2 * pool.required_writers_for_dc(1);
|
let expected = 2 * pool.required_writers_for_dc(1);
|
||||||
timeout(Duration::from_secs(10), async {
|
timeout(Duration::from_secs(10), async {
|
||||||
while !pool.admission_ready_full_floor().await
|
while !pool.admission_ready_full_floor().await
|
||||||
@@ -244,10 +257,7 @@ async fn suppressed_pool_recovers_after_delayed_discovery(http_slow: bool) {
|
|||||||
);
|
);
|
||||||
assert_eq!(pool.writers.read().await.len(), expected);
|
assert_eq!(pool.writers.read().await.len(), expected);
|
||||||
assert!(peer.handshakes.load(Ordering::Acquire) >= expected);
|
assert!(peer.handshakes.load(Ordering::Acquire) >= expected);
|
||||||
assert_eq!(
|
assert_eq!(http.requests.load(Ordering::Relaxed), 0);
|
||||||
http.requests.load(Ordering::Relaxed),
|
|
||||||
usize::from(http_slow)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
stun.requests.load(Ordering::Relaxed),
|
stun.requests.load(Ordering::Relaxed),
|
||||||
usize::from(!http_slow)
|
usize::from(!http_slow)
|
||||||
@@ -264,3 +274,58 @@ async fn suppressed_pool_recovers_full_floor_after_expired_stun_refresh() {
|
|||||||
async fn suppressed_pool_recovers_full_floor_after_slow_http_with_fresh_stun() {
|
async fn suppressed_pool_recovers_full_floor_after_slow_http_with_fresh_stun() {
|
||||||
suppressed_pool_recovers_after_delayed_discovery(true).await;
|
suppressed_pool_recovers_after_delayed_discovery(true).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stalled_ipv4_does_not_delay_real_ipv6_handshakes() {
|
||||||
|
let stalled = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let stalled_addr = stalled.local_addr().unwrap();
|
||||||
|
let healthy =
|
||||||
|
MiddlePeer::with_bind("[::1]:0", "::1".parse().unwrap(), "::1".parse().unwrap()).await;
|
||||||
|
let mut pool = make_pool_with_decision(NetworkDecision {
|
||||||
|
ipv4_me: true,
|
||||||
|
ipv6_me: true,
|
||||||
|
..NetworkDecision::default()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
Arc::get_mut(&mut Arc::get_mut(&mut pool).unwrap().reconnect_runtime)
|
||||||
|
.unwrap()
|
||||||
|
.me_one_timeout = Duration::from_secs(5);
|
||||||
|
pool.health_runtime
|
||||||
|
.me_health_interval_ms_unhealthy
|
||||||
|
.store(10, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_shadow_rotate_every_secs
|
||||||
|
.store(0, Ordering::Relaxed);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([(2, vec![(stalled_addr.ip(), stalled_addr.port())])]),
|
||||||
|
Some(HashMap::from([(
|
||||||
|
2,
|
||||||
|
vec![(healthy.addr.ip(), healthy.addr.port())],
|
||||||
|
)])),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let monitor = tokio::spawn(me_health_monitor(pool.clone(), pool.rng.clone(), 0));
|
||||||
|
let (blocked_stream, _) = timeout(TEST_DEADLINE, stalled.accept())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
timeout(TEST_DEADLINE, async {
|
||||||
|
while healthy.handshakes.load(Ordering::Acquire) < pool.required_writers_for_dc(1) {
|
||||||
|
assert!(!healthy.task.is_finished());
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("IPv6 must restore coverage while IPv4 still owns a stalled handshake");
|
||||||
|
monitor.abort();
|
||||||
|
let _ = monitor.await;
|
||||||
|
assert!(
|
||||||
|
pool.shutdown_until(TEST_DEADLINE).await,
|
||||||
|
"tracked child transports must join on shutdown"
|
||||||
|
);
|
||||||
|
drop(blocked_stream);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -491,7 +491,11 @@ impl MePool {
|
|||||||
rand::rng().random_range(min_ms..=max_ms)
|
rand::rng().random_range(min_ms..=max_ms)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn hardswap_warmup_backoff_ms(&self, pass_idx: usize) -> u64 {
|
/// Returns capped exponential retry delay with bounded positive jitter.
|
||||||
|
pub(in crate::transport::middle_proxy) fn hardswap_warmup_backoff_ms(
|
||||||
|
&self,
|
||||||
|
pass_idx: usize,
|
||||||
|
) -> u64 {
|
||||||
let base_ms = self
|
let base_ms = self
|
||||||
.reinit
|
.reinit
|
||||||
.me_hardswap_warmup_pass_backoff_base_ms
|
.me_hardswap_warmup_pass_backoff_base_ms
|
||||||
|
|||||||
@@ -169,11 +169,21 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconciles one immutable endpoint snapshot without draining uncovered groups.
|
||||||
pub async fn zero_downtime_reinit_after_map_change(
|
pub async fn zero_downtime_reinit_after_map_change(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let endpoint_snapshot = self.endpoint_snapshot.load_full();
|
let endpoint_snapshot = self.endpoint_snapshot.load_full();
|
||||||
|
self.reinit_endpoint_snapshot(rng, endpoint_snapshot).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconciles exactly the revision owned by the scheduler's retry objective.
|
||||||
|
pub(in crate::transport::middle_proxy) async fn reinit_endpoint_snapshot(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
endpoint_snapshot: Arc<EndpointSnapshot>,
|
||||||
|
) -> bool {
|
||||||
let desired_by_dc = self.desired_dc_endpoints_from_snapshot(&endpoint_snapshot);
|
let desired_by_dc = self.desired_dc_endpoints_from_snapshot(&endpoint_snapshot);
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
|
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
|
||||||
@@ -407,6 +417,7 @@ impl MePool {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Applies periodic rotation using the same coverage and generation fences as map changes.
|
||||||
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) -> bool {
|
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) -> bool {
|
||||||
self.zero_downtime_reinit_after_map_change(rng).await
|
self.zero_downtime_reinit_after_map_change(rng).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -149,6 +149,7 @@ fn stale_concurrent_attempt_cannot_regress_active_generation() {
|
|||||||
active_generation: 1,
|
active_generation: 1,
|
||||||
desired_map_hash: 22,
|
desired_map_hash: 22,
|
||||||
endpoint_revision: 7,
|
endpoint_revision: 7,
|
||||||
|
floor_policy_revision: 1,
|
||||||
pending: Some(ReinitPendingState {
|
pending: Some(ReinitPendingState {
|
||||||
generation: 3,
|
generation: 3,
|
||||||
started_at_epoch_secs: 1,
|
started_at_epoch_secs: 1,
|
||||||
|
|||||||
@@ -32,6 +32,9 @@ use super::wire::build_proxy_req_payload;
|
|||||||
mod publication;
|
mod publication;
|
||||||
mod replacement;
|
mod replacement;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
|
// Physical replacement overlap survives logical writer removal until transport destruction.
|
||||||
|
pub(super) mod overlap;
|
||||||
|
use overlap::WriterTransport;
|
||||||
|
|
||||||
// Recovery regressions exercise the serialized production publication boundary.
|
// Recovery regressions exercise the serialized production publication boundary.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -42,9 +45,8 @@ struct PreparedWriter<'a> {
|
|||||||
tx: mpsc::Sender<WriterCommand>,
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
byte_budget: Arc<tokio::sync::Semaphore>,
|
byte_budget: Arc<tokio::sync::Semaphore>,
|
||||||
task_registration: MeTaskRegistration<'a>,
|
task_registration: MeTaskRegistration<'a>,
|
||||||
writer_task: Pin<Box<dyn Future<Output = ()> + Send + 'static>>,
|
writer_task: WriterTransport,
|
||||||
intent: WriterOpenIntent,
|
intent: WriterOpenIntent,
|
||||||
_open_reservation: WriterOpenReservation<'a>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
@@ -59,6 +61,13 @@ pub(super) enum WriterReplacementPurpose {
|
|||||||
/// Target count above which the receiver no longer needs the slot.
|
/// Target count above which the receiver no longer needs the slot.
|
||||||
receiver_floor: usize,
|
receiver_floor: usize,
|
||||||
},
|
},
|
||||||
|
/// Transfers one slot without taking a current donor below its protected base floor.
|
||||||
|
CoverageTransfer {
|
||||||
|
/// Endpoint and floor-policy authority captured before selecting the donor.
|
||||||
|
authority: (u64, u64),
|
||||||
|
/// Planned receiver target, revalidated against current policy at commit.
|
||||||
|
receiver_floor: usize,
|
||||||
|
},
|
||||||
/// Replaces a stale-generation writer with an active-generation writer.
|
/// Replaces a stale-generation writer with an active-generation writer.
|
||||||
GenerationConvergence,
|
GenerationConvergence,
|
||||||
/// Rotates a single-endpoint shadow while retaining coverage.
|
/// Rotates a single-endpoint shadow while retaining coverage.
|
||||||
@@ -72,6 +81,7 @@ impl WriterReplacementPurpose {
|
|||||||
match self {
|
match self {
|
||||||
Self::IdleRefresh => "idle_refresh",
|
Self::IdleRefresh => "idle_refresh",
|
||||||
Self::FloorRebalance { .. } => "floor_rebalance",
|
Self::FloorRebalance { .. } => "floor_rebalance",
|
||||||
|
Self::CoverageTransfer { .. } => "coverage_transfer",
|
||||||
Self::GenerationConvergence => "generation_convergence",
|
Self::GenerationConvergence => "generation_convergence",
|
||||||
Self::ShadowRotation => "shadow_rotation",
|
Self::ShadowRotation => "shadow_rotation",
|
||||||
Self::SecretRotation => "secret_rotation",
|
Self::SecretRotation => "secret_rotation",
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::task::{Context, Poll};
|
||||||
|
|
||||||
|
/// Writer-local physical lifetime, independent of registry visibility.
|
||||||
|
pub(in crate::transport::middle_proxy) struct ReplacementHandoff {
|
||||||
|
state: parking_lot::Mutex<HandoffState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum HandoffState {
|
||||||
|
Open(Option<WriterOpenReservation>),
|
||||||
|
Closed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReplacementHandoff {
|
||||||
|
/// Creates a handoff that may accept one physical-overlap token.
|
||||||
|
pub(in crate::transport::middle_proxy) fn new() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
state: parking_lot::Mutex::new(HandoffState::Open(None)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn close(&self) {
|
||||||
|
*self.state.lock() = HandoffState::Closed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Keeps attachment reversible until the final registry validation has succeeded.
|
||||||
|
pub(super) fn commit(
|
||||||
|
&self,
|
||||||
|
permit: &mut Option<WriterOpenReservation>,
|
||||||
|
validate: impl FnOnce() -> bool,
|
||||||
|
) -> bool {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
match &mut *state {
|
||||||
|
HandoffState::Open(slot) if slot.is_none() && permit.is_some() && validate() => {
|
||||||
|
*slot = permit.take();
|
||||||
|
true
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Destroys transport futures before releasing either opening or retiring capacity.
|
||||||
|
pub(super) struct WriterTransport {
|
||||||
|
future: Option<Pin<Box<dyn Future<Output = ()> + Send + 'static>>>,
|
||||||
|
/// Shared with the registry but closed only by transport destruction.
|
||||||
|
pub(super) lifetime: Arc<ReplacementHandoff>,
|
||||||
|
/// Moves to the victim on replacement or is released after normal publication.
|
||||||
|
pub(super) opening: Option<WriterOpenReservation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WriterTransport {
|
||||||
|
/// Owns prepared transports even if their task is never polled or published.
|
||||||
|
pub(super) fn new(
|
||||||
|
future: Pin<Box<dyn Future<Output = ()> + Send + 'static>>,
|
||||||
|
opening: WriterOpenReservation,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
future: Some(future),
|
||||||
|
lifetime: ReplacementHandoff::new(),
|
||||||
|
opening: Some(opening),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Future for WriterTransport {
|
||||||
|
type Output = ();
|
||||||
|
|
||||||
|
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
|
||||||
|
match self.future.as_mut() {
|
||||||
|
Some(future) => future.as_mut().poll(cx),
|
||||||
|
None => Poll::Ready(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for WriterTransport {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
// Explicit order also applies to unpolled futures, panic unwinding and abort.
|
||||||
|
drop(self.future.take());
|
||||||
|
self.lifetime.close();
|
||||||
|
drop(self.opening.take());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
|
struct DestructionProbe {
|
||||||
|
counter: Arc<AtomicUsize>,
|
||||||
|
destroyed: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for DestructionProbe {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
assert!(
|
||||||
|
self.counter.load(Ordering::Acquire) > 0,
|
||||||
|
"transport must be destroyed before its overlap token"
|
||||||
|
);
|
||||||
|
self.destroyed.store(true, Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn transport(pool: &Arc<MePool>, destroyed: Arc<AtomicBool>) -> WriterTransport {
|
||||||
|
let permit = pool
|
||||||
|
.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
"127.0.0.1:443".parse().unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let probe = DestructionProbe {
|
||||||
|
counter: pool.writer_replacement_open_reserved.clone(),
|
||||||
|
destroyed,
|
||||||
|
};
|
||||||
|
WriterTransport::new(
|
||||||
|
Box::pin(async move {
|
||||||
|
let _probe = probe;
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
}),
|
||||||
|
permit,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unpolled_transport_and_aborted_task_release_after_destruction() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
for abort in [false, true] {
|
||||||
|
let destroyed = Arc::new(AtomicBool::new(false));
|
||||||
|
let task = transport(&pool, destroyed.clone()).await;
|
||||||
|
if abort {
|
||||||
|
let task = tokio::spawn(task);
|
||||||
|
task.abort();
|
||||||
|
let _ = task.await;
|
||||||
|
} else {
|
||||||
|
drop(task);
|
||||||
|
}
|
||||||
|
assert!(destroyed.load(Ordering::Acquire));
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retiring_transport_owns_capacity_after_logical_removal() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let destroyed = Arc::new(AtomicBool::new(false));
|
||||||
|
let mut victim = transport(&pool, destroyed.clone()).await;
|
||||||
|
drop(victim.opening.take());
|
||||||
|
let lifetime = victim.lifetime.clone();
|
||||||
|
let mut successor = transport(&pool, Arc::new(AtomicBool::new(false))).await;
|
||||||
|
successor.future = Some(Box::pin(std::future::pending()));
|
||||||
|
assert!(lifetime.commit(&mut successor.opening, || true));
|
||||||
|
assert!(successor.opening.is_none());
|
||||||
|
// Registry removal only drops its handle; the real transport retains the token.
|
||||||
|
drop(lifetime);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
drop(victim);
|
||||||
|
assert!(destroyed.load(Ordering::Acquire));
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
// Successor is no longer overlap after its victim has physically gone.
|
||||||
|
drop(successor);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_or_closed_handoff_keeps_the_prepared_token() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let mut prepared = transport(&pool, Arc::new(AtomicBool::new(false))).await;
|
||||||
|
let lifetime = ReplacementHandoff::new();
|
||||||
|
assert!(!lifetime.commit(&mut prepared.opening, || false));
|
||||||
|
assert!(prepared.opening.is_some());
|
||||||
|
lifetime.close();
|
||||||
|
assert!(!lifetime.commit(&mut prepared.opening, || true));
|
||||||
|
assert!(prepared.opening.is_some());
|
||||||
|
drop(prepared);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn overlap_limit_survives_commit_and_cap_shrink_until_teardown() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_cpu_cores_override
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.set_adaptive_floor_runtime_caps(pool.floor_authority(), 1, 2, 1, 1, 1, 0, 0);
|
||||||
|
let mut victim = transport(&pool, Arc::new(AtomicBool::new(false))).await;
|
||||||
|
drop(victim.opening.take());
|
||||||
|
let mut successor = transport(&pool, Arc::new(AtomicBool::new(false))).await;
|
||||||
|
successor.future = Some(Box::pin(std::future::pending()));
|
||||||
|
assert!(victim.lifetime.commit(&mut successor.opening, || true));
|
||||||
|
let second = transport(&pool, Arc::new(AtomicBool::new(false))).await;
|
||||||
|
pool.set_adaptive_floor_runtime_caps(pool.floor_authority(), 1, 1, 1, 1, 1, 1, 0);
|
||||||
|
let addr = "127.0.0.1:443".parse().unwrap();
|
||||||
|
assert!(
|
||||||
|
pool.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
addr
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
drop(second);
|
||||||
|
assert!(
|
||||||
|
pool.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
addr
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
drop(victim);
|
||||||
|
assert!(
|
||||||
|
pool.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
addr
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
drop(successor);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn panicking_transport_releases_capacity_after_unwinding_its_resources() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let permit = pool
|
||||||
|
.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
"127.0.0.1:443".parse().unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let destroyed = Arc::new(AtomicBool::new(false));
|
||||||
|
let probe = DestructionProbe {
|
||||||
|
counter: pool.writer_replacement_open_reserved.clone(),
|
||||||
|
destroyed: destroyed.clone(),
|
||||||
|
};
|
||||||
|
let task = WriterTransport::new(
|
||||||
|
Box::pin(async move {
|
||||||
|
let _probe = probe;
|
||||||
|
panic!("injected transport panic");
|
||||||
|
}),
|
||||||
|
permit,
|
||||||
|
);
|
||||||
|
assert!(tokio::spawn(task).await.unwrap_err().is_panic());
|
||||||
|
assert!(destroyed.load(Ordering::Acquire));
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn racing_commit_and_close_release_each_token_exactly_once() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
for _ in 0..128 {
|
||||||
|
let permit = pool
|
||||||
|
.reserve_writer_open(
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Replacement,
|
||||||
|
2,
|
||||||
|
"127.0.0.1:443".parse().unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let lifetime = ReplacementHandoff::new();
|
||||||
|
let closing = lifetime.clone();
|
||||||
|
let close = tokio::spawn(async move {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
closing.close();
|
||||||
|
});
|
||||||
|
let commit = tokio::spawn(async move {
|
||||||
|
let mut permit = Some(permit);
|
||||||
|
lifetime.commit(&mut permit, || true);
|
||||||
|
drop(permit);
|
||||||
|
});
|
||||||
|
close.await.unwrap();
|
||||||
|
commit.await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,9 +11,8 @@ impl MePool {
|
|||||||
tx,
|
tx,
|
||||||
byte_budget,
|
byte_budget,
|
||||||
task_registration,
|
task_registration,
|
||||||
writer_task,
|
mut writer_task,
|
||||||
intent,
|
intent,
|
||||||
_open_reservation,
|
|
||||||
} = prepared;
|
} = prepared;
|
||||||
let writer_id = writer.id;
|
let writer_id = writer.id;
|
||||||
let mut writers = self.writers.write().await;
|
let mut writers = self.writers.write().await;
|
||||||
@@ -22,9 +21,15 @@ impl MePool {
|
|||||||
let contour = self.authorize_writer_publication(&writer, &coordinator)?;
|
let contour = self.authorize_writer_publication(&writer, &coordinator)?;
|
||||||
self.authorize_writer_publication_capacity(&writer, contour, intent, writers.as_slice())?;
|
self.authorize_writer_publication_capacity(&writer, contour, intent, writers.as_slice())?;
|
||||||
writer.contour.store(contour.as_u8(), Ordering::Release);
|
writer.contour.store(contour.as_u8(), Ordering::Release);
|
||||||
registry_registration.install(writer_id, tx, byte_budget);
|
registry_registration.install_with_lifetime(
|
||||||
|
writer_id,
|
||||||
|
tx,
|
||||||
|
byte_budget,
|
||||||
|
writer_task.lifetime.clone(),
|
||||||
|
);
|
||||||
writers.push(writer);
|
writers.push(writer);
|
||||||
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
||||||
|
drop(writer_task.opening.take());
|
||||||
self.lifecycle
|
self.lifecycle
|
||||||
.spawn_registered_writer(task_registration, writer_task);
|
.spawn_registered_writer(task_registration, writer_task);
|
||||||
drop(coordinator);
|
drop(coordinator);
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ fn prepared<'a>(
|
|||||||
pool: &'a Arc<MePool>,
|
pool: &'a Arc<MePool>,
|
||||||
writer: MeWriter,
|
writer: MeWriter,
|
||||||
intent: WriterOpenIntent,
|
intent: WriterOpenIntent,
|
||||||
reservation: WriterOpenReservation<'a>,
|
reservation: WriterOpenReservation,
|
||||||
task_started: Arc<AtomicBool>,
|
task_started: Arc<AtomicBool>,
|
||||||
) -> PreparedWriter<'a> {
|
) -> PreparedWriter<'a> {
|
||||||
let cancel = writer.cancel.clone();
|
let cancel = writer.cancel.clone();
|
||||||
@@ -45,12 +45,14 @@ fn prepared<'a>(
|
|||||||
byte_budget: writer.byte_budget.clone(),
|
byte_budget: writer.byte_budget.clone(),
|
||||||
writer,
|
writer,
|
||||||
task_registration: pool.lifecycle.try_register().unwrap(),
|
task_registration: pool.lifecycle.try_register().unwrap(),
|
||||||
writer_task: Box::pin(async move {
|
writer_task: WriterTransport::new(
|
||||||
task_started.store(true, Ordering::Release);
|
Box::pin(async move {
|
||||||
cancel.cancelled().await;
|
task_started.store(true, Ordering::Release);
|
||||||
}),
|
cancel.cancelled().await;
|
||||||
|
}),
|
||||||
|
reservation,
|
||||||
|
),
|
||||||
intent,
|
intent,
|
||||||
_open_reservation: reservation,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,9 +44,8 @@ impl MePool {
|
|||||||
tx,
|
tx,
|
||||||
byte_budget,
|
byte_budget,
|
||||||
task_registration,
|
task_registration,
|
||||||
writer_task,
|
mut writer_task,
|
||||||
intent,
|
intent: _,
|
||||||
_open_reservation,
|
|
||||||
} = prepared;
|
} = prepared;
|
||||||
let replacement_writer_id = writer.id;
|
let replacement_writer_id = writer.id;
|
||||||
let victim_writer_id = reservation.writer_id();
|
let victim_writer_id = reservation.writer_id();
|
||||||
@@ -57,7 +56,25 @@ impl MePool {
|
|||||||
let mut registry_registration = self.registry.prepare_writer_registration().await;
|
let mut registry_registration = self.registry.prepare_writer_registration().await;
|
||||||
let coordinator = self.reinit.coordinator.lock();
|
let coordinator = self.reinit.coordinator.lock();
|
||||||
let contour = self.authorize_writer_publication(&writer, &coordinator)?;
|
let contour = self.authorize_writer_publication(&writer, &coordinator)?;
|
||||||
self.authorize_writer_publication_capacity(&writer, contour, intent, writers.as_slice())?;
|
// Only this victim-backed transaction may replace Active coverage during suppression.
|
||||||
|
let family = if writer.addr.is_ipv4() {
|
||||||
|
crate::network::IpFamily::V4
|
||||||
|
} else {
|
||||||
|
crate::network::IpFamily::V6
|
||||||
|
};
|
||||||
|
let configured = if writer.addr.is_ipv4() {
|
||||||
|
self.decision.ipv4_me
|
||||||
|
} else {
|
||||||
|
self.decision.ipv6_me
|
||||||
|
};
|
||||||
|
if !configured
|
||||||
|
|| (contour != WriterContour::Active
|
||||||
|
&& self.is_family_temporarily_suppressed(family, Self::now_epoch_secs()))
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME replacement family lost publication authority".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
writer.contour.store(contour.as_u8(), Ordering::Release);
|
writer.contour.store(contour.as_u8(), Ordering::Release);
|
||||||
let Some(victim_pos) = writers
|
let Some(victim_pos) = writers
|
||||||
.iter()
|
.iter()
|
||||||
@@ -73,6 +90,17 @@ impl MePool {
|
|||||||
"ME replacement victim changed role before commit".into(),
|
"ME replacement victim changed role before commit".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if !matches!(
|
||||||
|
purpose,
|
||||||
|
WriterReplacementPurpose::FloorRebalance { .. }
|
||||||
|
| WriterReplacementPurpose::CoverageTransfer { .. }
|
||||||
|
) && (expected_victim_role.dc != writer.writer_dc
|
||||||
|
|| expected_victim_role.family != family)
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME replacement purpose cannot move coverage between groups".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
if let WriterReplacementPurpose::FloorRebalance {
|
if let WriterReplacementPurpose::FloorRebalance {
|
||||||
donor_floor,
|
donor_floor,
|
||||||
receiver_floor,
|
receiver_floor,
|
||||||
@@ -122,19 +150,103 @@ impl MePool {
|
|||||||
}) == receiver_family
|
}) == receiver_family
|
||||||
})
|
})
|
||||||
.count();
|
.count();
|
||||||
if donor_count <= donor_floor || receiver_count >= receiver_floor {
|
let protected_floor = self.required_writers_for_dc_with_floor_mode(
|
||||||
|
endpoint_snapshot
|
||||||
|
.endpoints_for_dc_family(expected_victim_role.dc, expected_victim_role.family)
|
||||||
|
.len(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
let current_receiver_floor = self.required_writers_for_dc_with_floor_mode(
|
||||||
|
endpoint_snapshot
|
||||||
|
.endpoints_for_dc_family(writer.writer_dc, receiver_family)
|
||||||
|
.len(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
if donor_count <= donor_floor.max(protected_floor)
|
||||||
|
|| receiver_count >= receiver_floor.min(current_receiver_floor)
|
||||||
|
{
|
||||||
return Err(ProxyError::Proxy(
|
return Err(ProxyError::Proxy(
|
||||||
"ME floor rebalance became unnecessary before commit".into(),
|
"ME floor rebalance became unnecessary before commit".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !registry_registration.prepare_replacement_commit(reservation) {
|
if let WriterReplacementPurpose::CoverageTransfer {
|
||||||
|
authority,
|
||||||
|
receiver_floor,
|
||||||
|
} = purpose
|
||||||
|
{
|
||||||
|
if authority
|
||||||
|
!= (
|
||||||
|
coordinator.endpoint_revision,
|
||||||
|
coordinator.floor_policy_revision,
|
||||||
|
)
|
||||||
|
|| contour != WriterContour::Active
|
||||||
|
|| expected_victim_role.contour != WriterContour::Active
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME coverage transfer lost policy authority".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let endpoints = self.endpoint_snapshot.load();
|
||||||
|
let count = |dc, family| {
|
||||||
|
writers
|
||||||
|
.iter()
|
||||||
|
.filter(|candidate| {
|
||||||
|
!candidate.draining.load(Ordering::Acquire)
|
||||||
|
&& candidate.generation == coordinator.active_generation
|
||||||
|
&& candidate.writer_dc == dc
|
||||||
|
&& WriterContour::from_u8(candidate.contour.load(Ordering::Acquire))
|
||||||
|
== WriterContour::Active
|
||||||
|
&& candidate.addr.is_ipv4()
|
||||||
|
== matches!(family, crate::network::IpFamily::V4)
|
||||||
|
&& endpoints.contains_dc_endpoint(dc, candidate.addr)
|
||||||
|
})
|
||||||
|
.count()
|
||||||
|
};
|
||||||
|
let donor_is_current = victim.generation == coordinator.active_generation
|
||||||
|
&& endpoints.contains_dc_endpoint(victim.writer_dc, victim.addr);
|
||||||
|
let donor_floor = self.required_writers_for_dc_with_floor_mode(
|
||||||
|
endpoints
|
||||||
|
.endpoints_for_dc_family(victim.writer_dc, expected_victim_role.family)
|
||||||
|
.len(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
if (donor_is_current
|
||||||
|
&& count(victim.writer_dc, expected_victim_role.family) <= donor_floor)
|
||||||
|
|| count(writer.writer_dc, family)
|
||||||
|
>= receiver_floor.min(
|
||||||
|
self.required_writers_for_dc_with_floor_mode(
|
||||||
|
endpoints
|
||||||
|
.endpoints_for_dc_family(writer.writer_dc, family)
|
||||||
|
.len(),
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME coverage transfer no longer preserves floors".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let lifetime = registry_registration
|
||||||
|
.writer_lifetime(victim_writer_id)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Proxy("ME replacement victim transport disappeared".into())
|
||||||
|
})?;
|
||||||
|
if !lifetime.commit(&mut writer_task.opening, || {
|
||||||
|
registry_registration.prepare_replacement_commit(reservation)
|
||||||
|
}) {
|
||||||
return Err(ProxyError::Proxy(
|
return Err(ProxyError::Proxy(
|
||||||
"ME replacement victim became active before commit".into(),
|
"ME replacement victim became active before commit".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
registry_registration.install(replacement_writer_id, tx, byte_budget);
|
registry_registration.install_with_lifetime(
|
||||||
|
replacement_writer_id,
|
||||||
|
tx,
|
||||||
|
byte_budget,
|
||||||
|
writer_task.lifetime.clone(),
|
||||||
|
);
|
||||||
writers.push(writer);
|
writers.push(writer);
|
||||||
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
||||||
writers.publish_current();
|
writers.publish_current();
|
||||||
@@ -158,6 +270,7 @@ impl MePool {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
mod convergence;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::{IpAddr, Ipv4Addr};
|
use std::net::{IpAddr, Ipv4Addr};
|
||||||
|
|
||||||
@@ -248,14 +361,54 @@ mod tests {
|
|||||||
tx,
|
tx,
|
||||||
byte_budget,
|
byte_budget,
|
||||||
task_registration,
|
task_registration,
|
||||||
writer_task: Box::pin(async move {
|
writer_task: WriterTransport::new(
|
||||||
drop(rx);
|
Box::pin(async move {
|
||||||
}),
|
drop(rx);
|
||||||
|
}),
|
||||||
|
open_reservation,
|
||||||
|
),
|
||||||
intent: WriterOpenIntent::Replacement,
|
intent: WriterOpenIntent::Replacement,
|
||||||
_open_reservation: open_reservation,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_victim_backed_replacement_keeps_coverage() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = endpoint(8);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
let victim = install_writer(&pool, 8001, 2, addr).await;
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
crate::network::IpFamily::V4,
|
||||||
|
crate::transport::middle_proxy::pool::MeFamilyRuntimeState::Suppressed,
|
||||||
|
MePool::now_epoch_secs(),
|
||||||
|
MePool::now_epoch_secs() + 3600,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let mut reservation = pool
|
||||||
|
.registry
|
||||||
|
.try_reserve_writer_replacement(victim.id)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let prepared = prepared_writer(&pool, 8002, 2, addr).await;
|
||||||
|
pool.publish_prepared_replacement_writer(
|
||||||
|
prepared,
|
||||||
|
WriterRole::from_writer(&victim),
|
||||||
|
WriterReplacementPurpose::IdleRefresh,
|
||||||
|
&mut reservation,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("a reserved active victim must remain replaceable during suppression");
|
||||||
|
assert!(victim.draining.load(Ordering::Acquire));
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
1,
|
||||||
|
"overlap is owned until the victim transport is destroyed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn replacement_commit_publishes_successor_before_draining_victim() {
|
async fn replacement_commit_publishes_successor_before_draining_victim() {
|
||||||
let pool = make_pool().await;
|
let pool = make_pool().await;
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::registry::ConnMeta;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn busy_obsolete_donor_preserves_clients_and_rechecks_policy() {
|
||||||
|
for stale in [false, true] {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let obsolete = endpoint(21);
|
||||||
|
let receiver = endpoint(22);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([(2, vec![(receiver.ip(), receiver.port())])]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let victim = install_writer(&pool, 9001, 1, obsolete).await;
|
||||||
|
let (client, _messages) = pool.registry.register().await;
|
||||||
|
assert!(
|
||||||
|
pool.registry
|
||||||
|
.bind_writer(
|
||||||
|
client,
|
||||||
|
victim.id,
|
||||||
|
ConnMeta {
|
||||||
|
target_dc: 1,
|
||||||
|
client_addr: obsolete,
|
||||||
|
our_addr: obsolete,
|
||||||
|
proto_flags: 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
let authority = pool.floor_authority();
|
||||||
|
let mut reservation = pool
|
||||||
|
.registry
|
||||||
|
.try_reserve_writer_replacement_preserving_clients(victim.id)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let prepared = prepared_writer(&pool, 9002, 2, receiver).await;
|
||||||
|
if stale {
|
||||||
|
pool.reinit.coordinator.lock().floor_policy_revision += 1;
|
||||||
|
}
|
||||||
|
let result = pool
|
||||||
|
.publish_prepared_replacement_writer(
|
||||||
|
prepared,
|
||||||
|
WriterRole::from_writer(&victim),
|
||||||
|
WriterReplacementPurpose::CoverageTransfer {
|
||||||
|
authority,
|
||||||
|
receiver_floor: pool.required_writers_for_dc(1),
|
||||||
|
},
|
||||||
|
&mut reservation,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(result.is_ok(), !stale);
|
||||||
|
assert_eq!(victim.draining.load(Ordering::Acquire), !stale);
|
||||||
|
assert_eq!(
|
||||||
|
pool.registry.get_writer(client).await.unwrap().writer_id,
|
||||||
|
victim.id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
usize::from(!stale)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn transfer_commit_matches_independent_donor_floor_model() {
|
||||||
|
// The model counts only current endpoints; obsolete sockets contribute no coverage.
|
||||||
|
for obsolete in [false, true] {
|
||||||
|
for donor_delta in [0usize, 1] {
|
||||||
|
for receiver_full in [false, true] {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let donor = endpoint(31);
|
||||||
|
let receiver = endpoint(32);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([
|
||||||
|
(1, vec![(donor.ip(), donor.port())]),
|
||||||
|
(2, vec![(receiver.ip(), receiver.port())]),
|
||||||
|
]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let floor = pool.required_writers_for_dc(1);
|
||||||
|
let mut victim = None;
|
||||||
|
for id in 0..(floor + donor_delta) {
|
||||||
|
let writer = install_writer(&pool, 10000 + id as u64, 1, donor).await;
|
||||||
|
if id == 0 {
|
||||||
|
victim = Some(writer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut victim = victim.unwrap();
|
||||||
|
if obsolete {
|
||||||
|
let mut writers = pool.writers.write().await;
|
||||||
|
writers
|
||||||
|
.iter_mut()
|
||||||
|
.find(|writer| writer.id == victim.id)
|
||||||
|
.unwrap()
|
||||||
|
.addr = endpoint(33);
|
||||||
|
victim.addr = endpoint(33);
|
||||||
|
}
|
||||||
|
if receiver_full {
|
||||||
|
for id in 0..floor {
|
||||||
|
install_writer(&pool, 11000 + id as u64, 2, receiver).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut reservation = pool
|
||||||
|
.registry
|
||||||
|
.try_reserve_writer_replacement(victim.id)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let prepared = prepared_writer(&pool, 12000, 2, receiver).await;
|
||||||
|
let result = pool
|
||||||
|
.publish_prepared_replacement_writer(
|
||||||
|
prepared,
|
||||||
|
WriterRole::from_writer(&victim),
|
||||||
|
WriterReplacementPurpose::CoverageTransfer {
|
||||||
|
authority: pool.floor_authority(),
|
||||||
|
receiver_floor: floor,
|
||||||
|
},
|
||||||
|
&mut reservation,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
result.is_ok(),
|
||||||
|
!receiver_full && (obsolete || donor_delta > 0),
|
||||||
|
"obsolete={obsolete}, surplus={donor_delta}, receiver_full={receiver_full}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn concurrent_transfers_cannot_overfill_receiver_or_steal_donor_floor() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let donor = endpoint(41);
|
||||||
|
let receiver = endpoint(42);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([
|
||||||
|
(1, vec![(donor.ip(), donor.port())]),
|
||||||
|
(2, vec![(receiver.ip(), receiver.port())]),
|
||||||
|
]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
pool.set_adaptive_floor_runtime_caps(pool.floor_authority(), 128, 128, 128, 128, 6, 0, 0);
|
||||||
|
let floor = pool.required_writers_for_dc(1);
|
||||||
|
let count = floor + 8;
|
||||||
|
let mut victims = Vec::new();
|
||||||
|
for id in 0..count {
|
||||||
|
victims.push(install_writer(&pool, 20000 + id as u64, 1, donor).await);
|
||||||
|
}
|
||||||
|
let mut tasks = tokio::task::JoinSet::new();
|
||||||
|
for (index, victim) in victims.into_iter().enumerate() {
|
||||||
|
let pool = pool.clone();
|
||||||
|
tasks.spawn(async move {
|
||||||
|
let authority = pool.floor_authority();
|
||||||
|
let mut reservation = pool
|
||||||
|
.registry
|
||||||
|
.try_reserve_writer_replacement(victim.id)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let prepared = prepared_writer(&pool, 21000 + index as u64, 2, receiver).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
pool.publish_prepared_replacement_writer(
|
||||||
|
prepared,
|
||||||
|
WriterRole::from_writer(&victim),
|
||||||
|
WriterReplacementPurpose::CoverageTransfer {
|
||||||
|
authority,
|
||||||
|
receiver_floor: floor,
|
||||||
|
},
|
||||||
|
&mut reservation,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mut committed = 0;
|
||||||
|
while let Some(result) = tasks.join_next().await {
|
||||||
|
committed += usize::from(result.unwrap());
|
||||||
|
}
|
||||||
|
assert_eq!(committed, floor);
|
||||||
|
let writers = pool.writers.read().await;
|
||||||
|
let live = |dc| {
|
||||||
|
writers
|
||||||
|
.iter()
|
||||||
|
.filter(|writer| writer.writer_dc == dc && !writer.draining.load(Ordering::Acquire))
|
||||||
|
.count()
|
||||||
|
};
|
||||||
|
assert_eq!(live(2), floor);
|
||||||
|
assert_eq!(live(1), count - floor);
|
||||||
|
assert!(live(1) >= floor);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_replacement_open_reserved
|
||||||
|
.load(Ordering::Acquire),
|
||||||
|
committed
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -291,9 +291,8 @@ impl MePool {
|
|||||||
tx,
|
tx,
|
||||||
byte_budget,
|
byte_budget,
|
||||||
task_registration,
|
task_registration,
|
||||||
writer_task,
|
writer_task: WriterTransport::new(writer_task, writer_open_reservation),
|
||||||
intent,
|
intent,
|
||||||
_open_reservation: writer_open_reservation,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ struct WriterRoute {
|
|||||||
tx: mpsc::Sender<WriterCommand>,
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
byte_budget: Arc<Semaphore>,
|
byte_budget: Arc<Semaphore>,
|
||||||
replacement_state: Arc<AtomicU8>,
|
replacement_state: Arc<AtomicU8>,
|
||||||
|
lifetime: Arc<super::pool_writer::overlap::ReplacementHandoff>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|||||||
@@ -99,6 +99,34 @@ impl WriterRegistrationGuard<'_> {
|
|||||||
writer_id: u64,
|
writer_id: u64,
|
||||||
tx: mpsc::Sender<WriterCommand>,
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
byte_budget: Arc<Semaphore>,
|
byte_budget: Arc<Semaphore>,
|
||||||
|
) {
|
||||||
|
self.install_with_lifetime(
|
||||||
|
writer_id,
|
||||||
|
tx,
|
||||||
|
byte_budget,
|
||||||
|
super::super::pool_writer::overlap::ReplacementHandoff::new(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retrieves the physical lifetime while binding ownership fences victim replacement.
|
||||||
|
pub(in crate::transport::middle_proxy) fn writer_lifetime(
|
||||||
|
&self,
|
||||||
|
writer_id: u64,
|
||||||
|
) -> Option<Arc<super::super::pool_writer::overlap::ReplacementHandoff>> {
|
||||||
|
self.registry
|
||||||
|
.writers
|
||||||
|
.map
|
||||||
|
.get(&writer_id)
|
||||||
|
.map(|route| route.lifetime.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publishes a route associated with its transport destruction authority.
|
||||||
|
pub(in crate::transport::middle_proxy) fn install_with_lifetime(
|
||||||
|
&mut self,
|
||||||
|
writer_id: u64,
|
||||||
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
|
byte_budget: Arc<Semaphore>,
|
||||||
|
lifetime: Arc<super::super::pool_writer::overlap::ReplacementHandoff>,
|
||||||
) {
|
) {
|
||||||
self.binding.conns_for_writer.entry(writer_id).or_default();
|
self.binding.conns_for_writer.entry(writer_id).or_default();
|
||||||
self.registry
|
self.registry
|
||||||
@@ -117,6 +145,7 @@ impl WriterRegistrationGuard<'_> {
|
|||||||
tx,
|
tx,
|
||||||
byte_budget,
|
byte_budget,
|
||||||
replacement_state: Arc::new(AtomicU8::new(WriterReplacementState::Open as u8)),
|
replacement_state: Arc::new(AtomicU8::new(WriterReplacementState::Open as u8)),
|
||||||
|
lifetime,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -415,15 +415,10 @@ impl ConnRegistry {
|
|||||||
.map
|
.map
|
||||||
.get(&conn_id)
|
.get(&conn_id)
|
||||||
.map(|entry| entry.writer_id)?;
|
.map(|entry| entry.writer_id)?;
|
||||||
let writer = self
|
self.writers.map.get(&writer_id).map(|entry| ConnWriter {
|
||||||
.writers
|
|
||||||
.map
|
|
||||||
.get(&writer_id)
|
|
||||||
.map(|entry| entry.value().clone())?;
|
|
||||||
Some(ConnWriter {
|
|
||||||
writer_id,
|
writer_id,
|
||||||
tx: writer.tx,
|
tx: entry.tx.clone(),
|
||||||
byte_budget: writer.byte_budget,
|
byte_budget: entry.byte_budget.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,19 +431,12 @@ impl ConnRegistry {
|
|||||||
let hot = self.hot_binding.map.get(&conn_id)?;
|
let hot = self.hot_binding.map.get(&conn_id)?;
|
||||||
let writer_id = hot.writer_id;
|
let writer_id = hot.writer_id;
|
||||||
let meta = hot.meta.clone();
|
let meta = hot.meta.clone();
|
||||||
let writer = self
|
let writer = self.writers.map.get(&writer_id).map(|entry| ConnWriter {
|
||||||
.writers
|
writer_id,
|
||||||
.map
|
tx: entry.tx.clone(),
|
||||||
.get(&writer_id)
|
byte_budget: entry.byte_budget.clone(),
|
||||||
.map(|entry| entry.value().clone())?;
|
})?;
|
||||||
Some((
|
Some((writer, meta))
|
||||||
ConnWriter {
|
|
||||||
writer_id,
|
|
||||||
tx: writer.tx,
|
|
||||||
byte_budget: writer.byte_budget,
|
|
||||||
},
|
|
||||||
meta,
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
||||||
|
|||||||
@@ -9,10 +9,17 @@ use crate::config::ProxyConfig;
|
|||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
|
|
||||||
use super::MePool;
|
use super::MePool;
|
||||||
|
// Durable scheduler intent is independent of pending generation creation.
|
||||||
|
mod objective;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod retry_tests;
|
||||||
|
|
||||||
|
/// External causes that request generation reconciliation.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum MeReinitTrigger {
|
pub enum MeReinitTrigger {
|
||||||
|
/// The configured rotation interval elapsed.
|
||||||
Periodic,
|
Periodic,
|
||||||
|
/// Endpoint authority changed.
|
||||||
MapChanged,
|
MapChanged,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,6 +32,7 @@ impl MeReinitTrigger {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Coalesces an external request; endpoint epochs independently preserve map-change intent.
|
||||||
pub fn enqueue_reinit_trigger(tx: &mpsc::Sender<MeReinitTrigger>, trigger: MeReinitTrigger) {
|
pub fn enqueue_reinit_trigger(tx: &mpsc::Sender<MeReinitTrigger>, trigger: MeReinitTrigger) {
|
||||||
match tx.try_send(trigger) {
|
match tx.try_send(trigger) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
@@ -82,6 +90,7 @@ fn trigger_reason(pending: u8) -> &'static str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retries durable reinitialization objectives until their exact endpoint revision converges.
|
||||||
pub async fn me_reinit_scheduler(
|
pub async fn me_reinit_scheduler(
|
||||||
pool: Arc<MePool>,
|
pool: Arc<MePool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
@@ -90,9 +99,14 @@ pub async fn me_reinit_scheduler(
|
|||||||
me_ready_tx: watch::Sender<u64>,
|
me_ready_tx: watch::Sender<u64>,
|
||||||
) {
|
) {
|
||||||
info!("ME reinit scheduler started");
|
info!("ME reinit scheduler started");
|
||||||
let mut tasks = JoinSet::<bool>::new();
|
let mut tasks = JoinSet::<Option<bool>>::new();
|
||||||
let mut pending = 0u8;
|
let mut owners = std::collections::HashMap::new();
|
||||||
let mut pending_deadline = None;
|
let mut objective: Option<objective::RetryObjective> = None;
|
||||||
|
let mut next_id = 1u64;
|
||||||
|
let mut epochs = pool.writer_epoch.subscribe();
|
||||||
|
let mut epoch_open = true;
|
||||||
|
let mut observed_revision = pool.endpoint_snapshot.load().revision;
|
||||||
|
let mut completed_revision = None;
|
||||||
let mut trigger_channel_open = true;
|
let mut trigger_channel_open = true;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
@@ -101,73 +115,152 @@ pub async fn me_reinit_scheduler(
|
|||||||
pool.reinit
|
pool.reinit
|
||||||
.max_concurrency_effective
|
.max_concurrency_effective
|
||||||
.store(max_concurrency, std::sync::atomic::Ordering::Release);
|
.store(max_concurrency, std::sync::atomic::Ordering::Release);
|
||||||
|
let snapshot = pool.endpoint_snapshot.load_full();
|
||||||
let pending_ready = pending != 0
|
if snapshot.revision != observed_revision {
|
||||||
&& pending_deadline.is_none_or(|deadline| deadline <= tokio::time::Instant::now());
|
observed_revision = snapshot.revision;
|
||||||
if pending_ready && tasks.len() < max_concurrency {
|
objective = Some(objective::RetryObjective::new(
|
||||||
let reason = trigger_reason(pending);
|
next_id,
|
||||||
pending = 0;
|
observed_revision,
|
||||||
pending_deadline = None;
|
REINIT_TRIGGER_MAP_CHANGED,
|
||||||
debug!(reason, max_concurrency, "ME reinit scheduled");
|
Duration::from_millis(cfg.general.me_reinit_coalesce_window_ms),
|
||||||
|
));
|
||||||
|
next_id = next_id.wrapping_add(1);
|
||||||
|
}
|
||||||
|
let owned = objective
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|current| owners.values().any(|identity| current.matches(*identity)));
|
||||||
|
if let Some(current) = objective.as_mut()
|
||||||
|
&& !owned
|
||||||
|
&& current.attempt > 0
|
||||||
|
{
|
||||||
|
let status = pool.reinit.status.load_full();
|
||||||
|
let authority = pool.floor_authority();
|
||||||
|
let readiness = pool.api_hardswap_snapshot().await;
|
||||||
|
// An API projection is usable only while its exact publication and policy survive
|
||||||
|
// the await. A changed snapshot is unknown, not a not-ready transition.
|
||||||
|
if Arc::ptr_eq(&status, &pool.reinit.status.load_full())
|
||||||
|
&& authority == pool.floor_authority()
|
||||||
|
{
|
||||||
|
if status.pending_hardswap_endpoint_revision == current.revision
|
||||||
|
&& readiness.pending_map_current == Some(true)
|
||||||
|
&& readiness.pending_writer_deficit == 0
|
||||||
|
{
|
||||||
|
current.ready((
|
||||||
|
status.pending_hardswap_generation,
|
||||||
|
status.pending_hardswap_map_hash,
|
||||||
|
status.pending_hardswap_endpoint_revision,
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
current.not_ready();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let can_launch = !owned
|
||||||
|
&& tasks.len() < max_concurrency
|
||||||
|
&& objective
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|current| current.attempt == 0 || tasks.is_empty());
|
||||||
|
if can_launch
|
||||||
|
&& objective.as_ref().is_some_and(|current| {
|
||||||
|
current
|
||||||
|
.deadline
|
||||||
|
.is_some_and(|deadline| deadline <= tokio::time::Instant::now())
|
||||||
|
})
|
||||||
|
{
|
||||||
|
if pool.endpoint_snapshot.load().revision != snapshot.revision {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(current) = objective.as_mut() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let identity = (current.id, current.revision);
|
||||||
|
current.deadline = None;
|
||||||
|
debug!(
|
||||||
|
reason = trigger_reason(current.triggers),
|
||||||
|
attempt = current.attempt,
|
||||||
|
endpoint_revision = current.revision,
|
||||||
|
"ME reinit scheduled"
|
||||||
|
);
|
||||||
let pool_clone = pool.clone();
|
let pool_clone = pool.clone();
|
||||||
let rng_clone = rng.clone();
|
let rng_clone = rng.clone();
|
||||||
pool.reinit
|
pool.reinit
|
||||||
.scheduler_inflight
|
.scheduler_inflight
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||||
let inflight = ReinitInflightGuard {
|
let inflight = ReinitInflightGuard {
|
||||||
pool: Arc::clone(&pool_clone),
|
pool: pool_clone.clone(),
|
||||||
};
|
};
|
||||||
tasks.spawn(async move {
|
let future = async move {
|
||||||
let _inflight = inflight;
|
let _inflight = inflight;
|
||||||
pool_clone
|
pool_clone
|
||||||
.zero_downtime_reinit_periodic(rng_clone.as_ref())
|
.reinit_endpoint_snapshot(rng_clone.as_ref(), snapshot)
|
||||||
.await
|
.await
|
||||||
});
|
};
|
||||||
|
// Pool shutdown joins destruction of attempts as well as their writer transports.
|
||||||
|
let Ok(tracked) = pool.lifecycle.track_producer(future) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let task = tasks.spawn(tracked);
|
||||||
|
owners.insert(task.id(), identity);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if !trigger_channel_open && objective.is_none() && tasks.is_empty() {
|
||||||
if !trigger_channel_open && pending == 0 && tasks.is_empty() {
|
|
||||||
warn!("ME reinit scheduler stopped: trigger channel closed");
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let deadline = objective.as_ref().and_then(|current| current.deadline);
|
||||||
let timer_enabled = pending != 0 && tasks.len() < max_concurrency;
|
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
trigger = trigger_rx.recv(), if trigger_channel_open => {
|
trigger = trigger_rx.recv(), if trigger_channel_open => {
|
||||||
match trigger {
|
match trigger {
|
||||||
Some(trigger) => {
|
Some(trigger) => {
|
||||||
if pending == 0 {
|
let revision = pool.endpoint_snapshot.load().revision;
|
||||||
pending_deadline = Some(
|
if trigger == MeReinitTrigger::MapChanged && completed_revision == Some(revision) {
|
||||||
tokio::time::Instant::now()
|
continue;
|
||||||
+ Duration::from_millis(
|
|
||||||
cfg.general.me_reinit_coalesce_window_ms,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
pending |= trigger_bit(trigger);
|
if let Some(current) = objective.as_mut()
|
||||||
|
&& current.revision == revision
|
||||||
|
&& (trigger == MeReinitTrigger::MapChanged
|
||||||
|
|| !owners.values().any(|identity| current.matches(*identity))) {
|
||||||
|
// Endpoint epochs and updater notifications describe the same objective.
|
||||||
|
current.triggers |= trigger_bit(trigger);
|
||||||
|
} else {
|
||||||
|
objective = Some(objective::RetryObjective::new(next_id, revision,
|
||||||
|
trigger_bit(trigger), Duration::from_millis(cfg.general.me_reinit_coalesce_window_ms)));
|
||||||
|
next_id = next_id.wrapping_add(1);
|
||||||
|
}
|
||||||
|
observed_revision = revision;
|
||||||
}
|
}
|
||||||
None => trigger_channel_open = false,
|
None => trigger_channel_open = false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
joined = tasks.join_next(), if !tasks.is_empty() => {
|
changed = epochs.changed(), if epoch_open => {
|
||||||
match joined {
|
if changed.is_err() { epoch_open = false; }
|
||||||
Some(Ok(true)) => {
|
}
|
||||||
me_ready_tx.send_modify(|version| {
|
joined = tasks.join_next_with_id(), if !tasks.is_empty() => {
|
||||||
*version = version.saturating_add(1);
|
let Some(joined) = joined else { continue; };
|
||||||
});
|
let (id, succeeded) = match joined {
|
||||||
}
|
Ok((id, Some(succeeded))) => (id, succeeded),
|
||||||
Some(Ok(false)) => {}
|
Ok((_, None)) => return,
|
||||||
Some(Err(error)) => {
|
Err(error) => {
|
||||||
warn!(error = %error, "ME reinit task failed");
|
warn!(error = %error, "ME reinit task failed");
|
||||||
|
(error.id(), false)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Some(identity) = owners.remove(&id) else { continue; };
|
||||||
|
if succeeded {
|
||||||
|
me_ready_tx.send_modify(|version| *version = version.saturating_add(1));
|
||||||
|
}
|
||||||
|
if let Some(current) = objective.as_mut()
|
||||||
|
&& current.matches(identity) {
|
||||||
|
if succeeded {
|
||||||
|
completed_revision = Some(current.revision);
|
||||||
|
objective = None;
|
||||||
|
} else {
|
||||||
|
current.failed(Duration::from_millis(pool.hardswap_warmup_backoff_ms(current.attempt)));
|
||||||
}
|
}
|
||||||
None => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = async {
|
_ = async {
|
||||||
if let Some(deadline) = pending_deadline {
|
if let Some(deadline) = deadline { tokio::time::sleep_until(deadline).await; }
|
||||||
tokio::time::sleep_until(deadline).await;
|
}, if can_launch && deadline.is_some() => {}
|
||||||
}
|
|
||||||
}, if timer_enabled => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
use std::time::Duration;
|
||||||
|
use tokio::time::Instant;
|
||||||
|
|
||||||
|
/// Durable intent exists even if a reinit returns before creating a pending generation.
|
||||||
|
pub(super) struct RetryObjective {
|
||||||
|
/// Distinguishes separate external requests for the same endpoint revision.
|
||||||
|
pub(super) id: u64,
|
||||||
|
/// Exact endpoint snapshot to reconcile.
|
||||||
|
pub(super) revision: u64,
|
||||||
|
/// Coalesced external causes used for diagnostic logging.
|
||||||
|
pub(super) triggers: u8,
|
||||||
|
/// Completed failures; the count saturates but never terminates recovery.
|
||||||
|
pub(super) attempt: usize,
|
||||||
|
/// A missing deadline means the objective currently owns a running task.
|
||||||
|
pub(super) deadline: Option<Instant>,
|
||||||
|
ready_wake: Option<(u64, u64, u64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RetryObjective {
|
||||||
|
/// Creates retryable intent before any asynchronous attempt can fail.
|
||||||
|
pub(super) fn new(id: u64, revision: u64, triggers: u8, delay: Duration) -> Self {
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
revision,
|
||||||
|
triggers,
|
||||||
|
attempt: 0,
|
||||||
|
deadline: Some(Instant::now() + delay),
|
||||||
|
ready_wake: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rejects completions from superseded objectives, including the same endpoint revision.
|
||||||
|
pub(super) fn matches(&self, identity: (u64, u64)) -> bool {
|
||||||
|
(self.id, self.revision) == identity
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Measures backoff from completion and never caps the number of retry attempts.
|
||||||
|
pub(super) fn failed(&mut self, delay: Duration) {
|
||||||
|
self.attempt = self.attempt.saturating_add(1);
|
||||||
|
self.deadline = Some(Instant::now() + delay);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accelerates once per observed ready tuple without allowing an epoch storm to spin.
|
||||||
|
pub(super) fn ready(&mut self, key: (u64, u64, u64)) {
|
||||||
|
if self.ready_wake != Some(key) {
|
||||||
|
self.ready_wake = Some(key);
|
||||||
|
self.deadline = Some(Instant::now());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-arms acceleration only after a coherent observation of lost readiness.
|
||||||
|
pub(super) fn not_ready(&mut self) {
|
||||||
|
self.ready_wake = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
use super::objective::RetryObjective;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn retry_objective_survives_unlimited_failures_and_ready_storms() {
|
||||||
|
let mut objective = RetryObjective::new(1, 2, REINIT_TRIGGER_PERIODIC, Duration::ZERO);
|
||||||
|
assert!(objective.deadline.is_some());
|
||||||
|
for _ in 0..100 {
|
||||||
|
objective.deadline = None;
|
||||||
|
tokio::time::advance(Duration::from_secs(70)).await;
|
||||||
|
let completed = tokio::time::Instant::now();
|
||||||
|
objective.failed(Duration::from_secs(30));
|
||||||
|
assert_eq!(
|
||||||
|
objective.deadline,
|
||||||
|
Some(completed + Duration::from_secs(30))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
objective.ready((3, 4, 2));
|
||||||
|
objective.failed(Duration::from_secs(30));
|
||||||
|
let deadline = objective.deadline;
|
||||||
|
for _ in 0..100 {
|
||||||
|
objective.ready((3, 4, 2));
|
||||||
|
}
|
||||||
|
assert_eq!(objective.deadline, deadline);
|
||||||
|
objective.not_ready();
|
||||||
|
objective.ready((3, 4, 2));
|
||||||
|
assert!(
|
||||||
|
objective.deadline < deadline,
|
||||||
|
"a genuine loss and recovery of the same floor re-arms wakeup"
|
||||||
|
);
|
||||||
|
assert!(!objective.matches((1, 1)));
|
||||||
|
assert!(!objective.matches((2, 2)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pre_pending_failure_remains_live_without_a_periodic_trigger() {
|
||||||
|
let pool = crate::transport::middle_proxy::pool_writer_security_tests::make_pool().await;
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
std::collections::HashMap::from([(2, vec![(addr.ip(), addr.port())])]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
crate::network::IpFamily::V4,
|
||||||
|
crate::transport::middle_proxy::pool::MeFamilyRuntimeState::Suppressed,
|
||||||
|
MePool::now_epoch_secs(),
|
||||||
|
MePool::now_epoch_secs() + 3600,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
pool.reinit
|
||||||
|
.me_hardswap_warmup_pass_backoff_base_ms
|
||||||
|
.store(10, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
pool.reinit
|
||||||
|
.me_hardswap_warmup_delay_min_ms
|
||||||
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
pool.reinit
|
||||||
|
.me_hardswap_warmup_delay_max_ms
|
||||||
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.general.me_reinit_coalesce_window_ms = 0;
|
||||||
|
let (_config_tx, config_rx) = watch::channel(Arc::new(config));
|
||||||
|
let (tx, rx) = mpsc::channel(1);
|
||||||
|
let (ready_tx, _) = watch::channel(0);
|
||||||
|
tx.send(MeReinitTrigger::Periodic).await.unwrap();
|
||||||
|
drop(tx);
|
||||||
|
let scheduler = tokio::spawn(me_reinit_scheduler(
|
||||||
|
pool.clone(),
|
||||||
|
pool.rng.clone(),
|
||||||
|
config_rx,
|
||||||
|
rx,
|
||||||
|
ready_tx,
|
||||||
|
));
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
assert!(
|
||||||
|
!scheduler.is_finished(),
|
||||||
|
"false before pending creation must not discard recovery intent"
|
||||||
|
);
|
||||||
|
assert!(pool.reinit.coordinator.lock().pending.is_none());
|
||||||
|
// Family policy changes do not emit endpoint or writer epochs.
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
crate::network::IpFamily::V4,
|
||||||
|
crate::transport::middle_proxy::pool::MeFamilyRuntimeState::Healthy,
|
||||||
|
MePool::now_epoch_secs(),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let (stream, _) = tokio::time::timeout(Duration::from_secs(2), listener.accept())
|
||||||
|
.await
|
||||||
|
.expect("autonomous retry must actually reach TCP without another trigger")
|
||||||
|
.unwrap();
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), scheduler)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
pool.reinit
|
||||||
|
.scheduler_inflight
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
drop(stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn panic_retains_the_task_identity_for_retry() {
|
||||||
|
let mut tasks = JoinSet::<bool>::new();
|
||||||
|
let id = tasks.spawn(async { panic!("injected reinit panic") }).id();
|
||||||
|
let error = tasks.join_next_with_id().await.unwrap().unwrap_err();
|
||||||
|
assert_eq!(error.id(), id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_inflight_trigger(trigger: MeReinitTrigger, starts_successor: bool) {
|
||||||
|
let pool = crate::transport::middle_proxy::pool_writer_security_tests::make_pool().await;
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
std::collections::HashMap::from([(2, vec![(addr.ip(), addr.port())])]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
pool.reinit
|
||||||
|
.me_hardswap_warmup_delay_min_ms
|
||||||
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
pool.reinit
|
||||||
|
.me_hardswap_warmup_delay_max_ms
|
||||||
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.general.me_reinit_coalesce_window_ms = 0;
|
||||||
|
config.general.me_reinit_singleflight = false;
|
||||||
|
config.general.me_reinit_max_concurrency = 2;
|
||||||
|
let (_config_tx, config_rx) = watch::channel(Arc::new(config));
|
||||||
|
let (tx, rx) = mpsc::channel(1);
|
||||||
|
let (ready_tx, _) = watch::channel(0);
|
||||||
|
tx.send(MeReinitTrigger::MapChanged).await.unwrap();
|
||||||
|
let scheduler = tokio::spawn(me_reinit_scheduler(
|
||||||
|
pool.clone(),
|
||||||
|
pool.rng.clone(),
|
||||||
|
config_rx,
|
||||||
|
rx,
|
||||||
|
ready_tx,
|
||||||
|
));
|
||||||
|
let (first, _) = tokio::time::timeout(Duration::from_secs(1), listener.accept())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
pool.reinit
|
||||||
|
.scheduler_inflight
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
// Keep the first attempt blocked before handshake completion while delivering another trigger.
|
||||||
|
tx.send(trigger).await.unwrap();
|
||||||
|
let successor = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await;
|
||||||
|
let inflight = pool
|
||||||
|
.reinit
|
||||||
|
.scheduler_inflight
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire);
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), scheduler)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(successor.is_ok(), starts_successor);
|
||||||
|
assert_eq!(inflight, if starts_successor { 2 } else { 1 });
|
||||||
|
assert_eq!(
|
||||||
|
pool.reinit
|
||||||
|
.scheduler_inflight
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
drop(first);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn duplicate_map_change_coalesces_with_its_inflight_revision() {
|
||||||
|
check_inflight_trigger(MeReinitTrigger::MapChanged, false).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn periodic_trigger_retains_successor_intent_while_map_change_runs() {
|
||||||
|
check_inflight_trigger(MeReinitTrigger::Periodic, true).await;
|
||||||
|
}
|
||||||
@@ -65,6 +65,8 @@ pub(crate) fn render(
|
|||||||
.replace("__REQUEST_RUNTIME__", REQUEST_RUNTIME)
|
.replace("__REQUEST_RUNTIME__", REQUEST_RUNTIME)
|
||||||
.replace("__BUFFER_RUNTIME__", BUFFER_RUNTIME)
|
.replace("__BUFFER_RUNTIME__", BUFFER_RUNTIME)
|
||||||
.replace("__RECOVERY_RUNTIME__", RECOVERY_RUNTIME)
|
.replace("__RECOVERY_RUNTIME__", RECOVERY_RUNTIME)
|
||||||
|
.replace("__DOWNLINK_RUNTIME__", DOWNLINK_RUNTIME)
|
||||||
|
.replace("__CONVEYOR_RUNTIME__", CONVEYOR_RUNTIME)
|
||||||
.replace("__RUNTIME__", RUNTIME)
|
.replace("__RUNTIME__", RUNTIME)
|
||||||
.replace(
|
.replace(
|
||||||
"__DIAGNOSTIC_BINDING__;\n",
|
"__DIAGNOSTIC_BINDING__;\n",
|
||||||
@@ -174,6 +176,8 @@ const RESPONSE_RUNTIME: &str = include_str!("bridge/response.js");
|
|||||||
const REQUEST_RUNTIME: &str = include_str!("bridge/request.js");
|
const REQUEST_RUNTIME: &str = include_str!("bridge/request.js");
|
||||||
const BUFFER_RUNTIME: &str = include_str!("bridge/buffers.js");
|
const BUFFER_RUNTIME: &str = include_str!("bridge/buffers.js");
|
||||||
const RECOVERY_RUNTIME: &str = include_str!("bridge/recovery.js");
|
const RECOVERY_RUNTIME: &str = include_str!("bridge/recovery.js");
|
||||||
|
const DOWNLINK_RUNTIME: &str = include_str!("bridge/downlink.js");
|
||||||
|
const CONVEYOR_RUNTIME: &str = include_str!("bridge/conveyor.js");
|
||||||
const RUNTIME: &str = include_str!("bridge/runtime.js");
|
const RUNTIME: &str = include_str!("bridge/runtime.js");
|
||||||
|
|
||||||
// Rendered wire-contract tests remain separate from the embedded document.
|
// Rendered wire-contract tests remain separate from the embedded document.
|
||||||
|
|||||||
@@ -0,0 +1,265 @@
|
|||||||
|
'use strict';
|
||||||
|
const assert=require('node:assert/strict');
|
||||||
|
const fs=require('node:fs');
|
||||||
|
const vm=require('node:vm');
|
||||||
|
const path=require('node:path');
|
||||||
|
|
||||||
|
function renderedPage(){
|
||||||
|
if(process.argv.includes('--stdin'))return fs.readFileSync(0,'utf8');
|
||||||
|
let page=fs.readFileSync(path.join(__dirname,'document.html'),'utf8');
|
||||||
|
const modules={RESPONSE:'response',REQUEST:'request',BUFFER:'buffers',RECOVERY:'recovery',CONVEYOR:'conveyor',DOWNLINK:'downlink'};
|
||||||
|
page=page.replace('__DIAGNOSTIC_RUNTIME__','');
|
||||||
|
for(const [key,file] of Object.entries(modules))if(page.includes('__'+key+'_RUNTIME__'))page=page.replace('__'+key+'_RUNTIME__',()=>fs.readFileSync(path.join(__dirname,file+'.js'),'utf8'));
|
||||||
|
page=page.replace('__RUNTIME__',()=>fs.readFileSync(path.join(__dirname,'runtime.js'),'utf8'));
|
||||||
|
const values={BOOTSTRAP:'A'.repeat(43),HOST:'proxy.example.com',BASE_PREFIX:'',CARRIER_METHOD:'POST',NEGOTIATION_ENABLED:'true',CANDIDATE_COUNT:4,CARRIER_DEADLINES:'3,5,8,12',LONG_POLL_SECS:25,BRIDGE_REQUEST_SECS:10,BRIDGE_RETRY_SECS:90,BRIDGE_RECOVERY_SECS:15,WEBSOCKET_OPEN_SECS:15,RECONNECT_GRACE_SECS:120,CARRIER_PROBE_COALESCE_MS:0,BATCH_LIMIT:2097152,QUEUE_LIMIT:33554432,QUEUE_ITEMS:16384,MAX_STREAMS:1024,STATUS_FUNCTION:"state=>{if(port&&!closed)port.postMessage({t:'status',state})}",HELLO_TIMEOUT_CALLBACK:"()=>fail('timeout')",PAGEHIDE_CALLBACK:'()=>close(true)'};
|
||||||
|
return page.replace(/__([A-Z_]+)__/g,(all,key)=>key.startsWith('DIAGNOSTIC_')?'':String(values[key]??''));
|
||||||
|
}
|
||||||
|
function frame(type,id=0,payload=[]){
|
||||||
|
const data=new Uint8Array(8+payload.length),view=new DataView(data.buffer);
|
||||||
|
data[0]=type;data[1]=id>>>16;data[2]=id>>>8;data[3]=id;view.setUint32(4,payload.length);data.set(payload,8);return data.buffer;
|
||||||
|
}
|
||||||
|
function join(...frames){const data=new Uint8Array(frames.reduce((n,f)=>n+f.byteLength,0));let offset=0;for(const f of frames){data.set(new Uint8Array(f),offset);offset+=f.byteLength}return data.buffer}
|
||||||
|
async function flush(){for(let i=0;i<40;i++)await Promise.resolve()}
|
||||||
|
function environment(page,android=false){
|
||||||
|
let now=1000,nextTimer=1;const timers=new Map(),events=new Map(),requests=[],received=[];
|
||||||
|
const port={onmessage:null,start(){},close(){},postMessage(value,transfer){received.push(structuredClone(value,{transfer:transfer||[]}))}};
|
||||||
|
const context={ArrayBuffer,Uint8Array,DataView,TextDecoder,TextEncoder,URL,Headers,AbortController,ReadableStream,structuredClone,console,
|
||||||
|
location:{hash:android?'#android='+'A'.repeat(43):'',pathname:'/',search:'?bridge=test'},history:{replaceState(){}},parent:{},
|
||||||
|
performance:{now:()=>now},Date:class extends Date{static now(){return now}},document:{visibilityState:'visible',addEventListener(){}},
|
||||||
|
addEventListener:(name,fn)=>events.set(name,fn),setTimeout:(fn,delay)=>{const id=nextTimer++;timers.set(id,{fn,at:now+delay});return id},clearTimeout:id=>timers.delete(id),
|
||||||
|
fetch:(url,options)=>new Promise((resolve,reject)=>{const request={url,options,resolve,reject};requests.push(request);options.signal?.addEventListener('abort',()=>reject(new Error('aborted')),{once:true})}),
|
||||||
|
WebSocket:class {static OPEN=1;static CLOSING=2;constructor(){throw new Error('unexpected websocket')}}
|
||||||
|
};
|
||||||
|
if(android)context.TelegramWebProxy={onmessage:null,postMessage(value){received.push(typeof value==='string'?JSON.parse(value):structuredClone(value))}};
|
||||||
|
vm.createContext(context);
|
||||||
|
for(const match of page.matchAll(/<script\b[^>]*>([\s\S]*?)<\/script>/g))vm.runInContext(match[1],context);
|
||||||
|
if(!android)events.get('message')({source:context.parent,origin:'http://127.0.0.1:12345',data:{t:'tproxy-init',v:1},ports:[port]});
|
||||||
|
const send=data=>android?context.TelegramWebProxy.onmessage({data}):port.onmessage({data});
|
||||||
|
const pending=suffix=>requests.filter(r=>r.url.endsWith(suffix)&&!r.answered);
|
||||||
|
const answer=(request,status,body=null,headers={})=>{request.answered=true;request.resolve({status,headers:new Headers(headers),body:body===null?null:new ReadableStream({start(c){c.enqueue(new Uint8Array(body));c.close()}})})};
|
||||||
|
return {context,send,pending,answer,received,requests,async tick(ms){const until=now+ms;for(;;){const entry=[...timers].filter(([,t])=>t.at<=until).sort((a,b)=>a[1].at-b[1].at)[0];if(!entry)break;now=entry[1].at;timers.delete(entry[0]);entry[1].fn();await flush()}now=until;await flush()},close(){events.get('pagehide')();}};
|
||||||
|
}
|
||||||
|
async function session(page,carrier='https',window=1,android=false){
|
||||||
|
const env=environment(page,android);env.send(frame(16,0,[1]));await flush();
|
||||||
|
const request=env.pending('/api/v1/session')[0];assert.ok(request,'session requested');
|
||||||
|
const headers={'X-Session-Token':'B'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':carrier,'X-Carrier-Attempt':'1','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional'};
|
||||||
|
if(window!==null)headers['X-Telemt-Up-Window']=String(window);
|
||||||
|
env.answer(request,200,frame(17),headers);await flush();return env;
|
||||||
|
}
|
||||||
|
const tests=[];
|
||||||
|
function test(name,run){tests.push({name,run})}
|
||||||
|
function ack(env,request){env.answer(request,204,null,{'X-Up-Ack':request.options.headers['X-Up-Seq']})}
|
||||||
|
function binary(env){return env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]!==17)}
|
||||||
|
function streaming(env,request,next='1',length=null){
|
||||||
|
let controller;const body=new ReadableStream({start(value){controller=value}}),headers={'X-Down-Cursor':next};if(length!==null)headers['Content-Length']=String(length);
|
||||||
|
request.answered=true;request.resolve({status:200,headers:new Headers(headers),body});return controller;
|
||||||
|
}
|
||||||
|
test('already queued OPEN and DATA share the zero-wait startup request',async page=>{
|
||||||
|
const env=await session(page);const body=join(frame(1,1),frame(2,1,[7,8]));env.send(body);await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up')[0].options.body.byteLength,body.byteLength);env.close();await flush();
|
||||||
|
});
|
||||||
|
test('negotiated conveyor sends DATA before the OPEN acknowledgement',async page=>{
|
||||||
|
for(const android of [false,true])for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4,android);env.send(frame(1,1));await flush();env.send(frame(2,1,[9]));await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up').length,2,carrier+' startup must not wait for ACK1');env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('unsupported server and disabled conveyor remain serialized',async page=>{
|
||||||
|
for(const window of [null,1]){
|
||||||
|
const env=await session(page,'https',window);env.send(frame(1,1));await flush();env.send(frame(2,1,[9]));await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up').length,1);env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('lost ACK1 cannot slide the window after ACK2 through ACK4',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();
|
||||||
|
for(let i=0;i<6;i++){env.send(frame(2,1,[i]));await flush()}
|
||||||
|
const up=env.pending('/api/v1/up');assert.equal(up.length,4);
|
||||||
|
for(const request of up.slice(1)){ack(env,request);await flush()}
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/up')).length,4);
|
||||||
|
ack(env,up[0]);await flush();
|
||||||
|
const next=env.pending('/api/v1/up')[0];assert.equal(next.options.headers['X-Up-Seq'],'5');assert.equal(next.options.headers['X-Telemt-Up-Confirmed'],'4');
|
||||||
|
env.close();await flush();
|
||||||
|
});
|
||||||
|
test('early downlink cannot deliver native data before a verified commit',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();
|
||||||
|
const down=env.pending('/api/v1/down')[0];assert.ok(down,'down starts before ACK');const data=frame(2,1,[4]);
|
||||||
|
const stream=streaming(env,down,'1',data.byteLength);stream.enqueue(new Uint8Array(data));await flush();assert.equal(binary(env).length,0);
|
||||||
|
ack(env,env.pending('/api/v1/up')[0]);await flush();assert.equal(binary(env).length,1);
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/down')).length,1,'cursor cannot advance before EOF');
|
||||||
|
stream.close();await flush();assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');env.close();await flush();
|
||||||
|
});
|
||||||
|
test('DATA and WINDOW prefixes are delivered once across a mid-body network failure',async page=>{
|
||||||
|
for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const first=frame(2,1,[1,2]),credit=frame(4,1,[0,0,0,2]),last=frame(3,1),body=join(first,credit,last);
|
||||||
|
const stream=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(join(first,credit)));await flush();assert.equal(binary(env).length,2);
|
||||||
|
stream.error(new Error('connection lost'));await flush();
|
||||||
|
const replay=env.pending('/api/v1/down')[0];assert.ok(replay);assert.equal(replay.options.headers['X-Down-Cursor'],'0');
|
||||||
|
env.answer(replay,200,body,{'X-Down-Cursor':'1','Content-Length':String(body.byteLength)});await flush();
|
||||||
|
assert.deepEqual(binary(env).map(data=>new Uint8Array(data)[0]),[2,4,3]);
|
||||||
|
assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/down')).length,3,'successful replay is consumed without another old-cursor request');env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('changed replay prefix fails closed without duplicate native frames',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[7]),tail=frame(2,1,[8]),body=join(data,tail),stream=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(data));await flush();stream.error(new Error('lost'));await flush();
|
||||||
|
const changed=join(frame(2,1,[9]),tail);env.answer(env.pending('/api/v1/down')[0],200,changed,{'X-Down-Cursor':'1','Content-Length':String(changed.byteLength)});await flush();
|
||||||
|
assert.equal(binary(env).length,1);assert.ok(env.received.some(value=>value&&value.t==='close'));env.close();await flush();
|
||||||
|
});
|
||||||
|
test('partial frame boundaries never escape to native and cancellation owns every lease',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4,true);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[1,2,3]),stream=streaming(env,env.pending('/api/v1/down')[0],'1',data.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(data,0,5));await flush();assert.equal(binary(env).length,0);
|
||||||
|
stream.enqueue(new Uint8Array(data,5));await flush();assert.equal(binary(env).length,1);
|
||||||
|
env.close();await flush();assert.equal(binary(env).length,1);
|
||||||
|
});
|
||||||
|
test('post-commit lane batching keeps OPEN and DATA from one native message together',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',1);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const body=join(frame(1,2),frame(2,2,[7]));env.send(body);await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up')[0].options.body.byteLength,body.byteLength);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('page-owned HTTP method and identity survive frozen retries',async page=>{
|
||||||
|
const expected=/const carrierMethod='(POST|PUT)'/.exec(page)[1],env=await session(page,'https',4);
|
||||||
|
assert.equal(env.requests.find(r=>r.url.endsWith('/api/v1/session')).options.method,'POST');
|
||||||
|
env.send(frame(1,1));await flush();const first=env.pending('/api/v1/up')[0];ack(env,first);await flush();
|
||||||
|
env.send(frame(2,1,[7]));await flush();const original=env.pending('/api/v1/up')[0];
|
||||||
|
original.answered=true;original.reject(new Error('lost'));await flush();const replay=env.pending('/api/v1/up')[0];
|
||||||
|
assert.equal(replay.options.body,original.options.body);assert.deepEqual(replay.options.headers,original.options.headers);
|
||||||
|
assert.equal(replay.options.method,expected);ack(env,replay);await flush();
|
||||||
|
for(const request of env.requests.filter(r=>/\/api\/v1\/(up|down)$/.test(r.url)))assert.equal(request.options.method,expected);
|
||||||
|
env.close();await flush();assert.equal(env.requests.find(r=>r.options.method==='DELETE').options.method,'DELETE');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('RTT scales time but adds no OPEN ACK barrier or coalescing timer',async page=>{
|
||||||
|
for(const rtt of [40,100,400]){
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();await env.tick(1);
|
||||||
|
env.send(frame(2,1,[5]));await flush();assert.equal(env.pending('/api/v1/up').length,2);
|
||||||
|
await env.tick(rtt-1);for(const request of env.pending('/api/v1/up'))ack(env,request);await flush();
|
||||||
|
assert.equal(env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]===17).length,1);
|
||||||
|
assert.ok(env.received.filter(value=>value&&value.t==='status').every(value=>Object.keys(value).sort().join(',')==='state,t'));
|
||||||
|
env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('simultaneous failed lane polls each consume their own exact replay',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
env.send(frame(1,2));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const streams=[];
|
||||||
|
for(const request of env.pending('/api/v1/down')){
|
||||||
|
const id=Number(request.options.headers['X-Lane-ID']),data=frame(2,id,[id]),body=join(data,frame(4,id,[0,0,0,1]));
|
||||||
|
const response=streaming(env,request,'1',body.byteLength);response.enqueue(new Uint8Array(data));streams.push({id,response,body});
|
||||||
|
}
|
||||||
|
await flush();assert.equal(binary(env).length,2);for(const item of streams)item.response.error(new Error('lost'));await flush();
|
||||||
|
for(let round=0;round<3;round++){
|
||||||
|
for(const request of env.pending('/api/v1/down').filter(r=>r.options.headers['X-Down-Cursor']==='0')){
|
||||||
|
const body=streams.find(item=>String(item.id)===request.options.headers['X-Lane-ID']).body;
|
||||||
|
env.answer(request,200,body,{'X-Down-Cursor':'1','Content-Length':String(body.byteLength)});
|
||||||
|
}await flush();
|
||||||
|
}
|
||||||
|
assert.equal(binary(env).length,4);assert.equal(env.pending('/api/v1/down').length,2);
|
||||||
|
assert.ok(env.pending('/api/v1/down').every(r=>r.options.headers['X-Down-Cursor']==='1'));env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('incremental decoder rejects changed cursor, invalid WINDOW and cross-lane frames',async page=>{
|
||||||
|
for(const invalid of ['cursor','credit','lane']){
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const body=invalid==='credit'?frame(4,1,[0,0,0,0]):frame(2,invalid==='lane'?2:1,[1]);
|
||||||
|
env.answer(env.pending('/api/v1/down')[0],200,body,{'X-Down-Cursor':invalid==='cursor'?'2':'1','Content-Length':String(body.byteLength)});await flush();
|
||||||
|
assert.equal(binary(env).length,0);assert.ok(env.received.some(value=>value&&value.t==='close'));env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a replay without Content-Length cannot truncate an already known batch',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const prefix=frame(2,1,[1]),tail=frame(2,1,[2]),body=join(prefix,tail),response=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
response.enqueue(new Uint8Array(prefix));await flush();response.error(new Error('lost'));await flush();
|
||||||
|
env.answer(env.pending('/api/v1/down')[0],200,prefix,{'X-Down-Cursor':'1'});await flush();
|
||||||
|
assert.equal(binary(env).length,1);assert.ok(env.received.some(value=>value&&value.t==='close'));
|
||||||
|
assert.ok(!env.requests.some(r=>r.url.endsWith('/api/v1/down')&&r.options.headers['X-Down-Cursor']==='1'));env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a pipelined startup flight adopts the recovery budget once another ACK commits',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const [head,tail]=env.pending('/api/v1/up');ack(env,head);await flush();tail.answered=true;tail.reject(new Error('lost'));await flush();
|
||||||
|
const replay=env.pending('/api/v1/up')[0];assert.ok(replay,'post-commit recovery replays immediately instead of staying in the startup backoff');
|
||||||
|
assert.equal(replay.options.body,tail.options.body);ack(env,replay);await flush();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('candidate rollback restores every unacknowledged OPEN and DATA without a second WELCOME',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const retired=env.requests.filter(r=>/\/api\/v1\/(up|down)$/.test(r.url));await env.tick(3001);
|
||||||
|
const next=env.pending('/api/v1/session')[0];assert.ok(next,'candidate fallback creates a new session');
|
||||||
|
env.answer(next,200,frame(17),{'X-Session-Token':'C'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':'https-lanes','X-Carrier-Attempt':'2','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional','X-Telemt-Up-Window':'4'});await flush();
|
||||||
|
const first=env.requests.find(r=>r.url.endsWith('/api/v1/up')&&r.options.headers.Authorization==='Bearer '+'C'.repeat(43));
|
||||||
|
assert.ok(first);assert.deepEqual(new Uint8Array(first.options.body),new Uint8Array(join(frame(1,1),frame(2,1,[7]))));
|
||||||
|
assert.ok(retired.every(r=>r.options.signal.aborted));assert.equal(env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]===17).length,1);
|
||||||
|
ack(env,first);await flush();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('downlink prefix reservations are bounded, cancellable and released on retirement',async page=>{
|
||||||
|
const env=environment(page),budget=env.context.TelemtBridgeDownlink.budget(()=>16),first=await budget.acquire(16);
|
||||||
|
const controller=new AbortController();let acquired=false;
|
||||||
|
const parked=budget.acquire(8,controller.signal).then(release=>{acquired=true;release()},()=>{});await flush();assert.equal(acquired,false);
|
||||||
|
controller.abort();await parked;first();assert.equal(budget.used(),0);
|
||||||
|
const release=await budget.acquire(16);release();release();assert.equal(budget.used(),0);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('two interrupted and differently chunked replays never duplicate DATA WINDOW or CLOSE',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[1]),credit=frame(4,1,[0,0,0,1]),close=frame(3,1),body=join(data,credit,close);
|
||||||
|
const first=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
first.enqueue(new Uint8Array(data));await flush();first.error(new Error('lost once'));await flush();
|
||||||
|
const second=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
const prefix=new Uint8Array(join(data,credit));second.enqueue(prefix.subarray(0,3));second.enqueue(prefix.subarray(3));await flush();
|
||||||
|
assert.deepEqual(binary(env).map(value=>new Uint8Array(value)[0]),[2,4]);second.error(new Error('lost twice'));await flush();await env.tick(1000);
|
||||||
|
const last=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
for(const byte of new Uint8Array(body))last.enqueue(new Uint8Array([byte]));last.close();
|
||||||
|
for(let i=0;i<20&&(!env.pending('/api/v1/down').length);i++)await flush();
|
||||||
|
assert.deepEqual(binary(env).map(value=>new Uint8Array(value)[0]),[2,4,3]);assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');
|
||||||
|
env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('multiple precommit lanes survive fallback without losing stream ordering',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(join(frame(1,1),frame(1,2)));await flush();
|
||||||
|
env.send(join(frame(2,2,[2]),frame(2,1,[1])));await flush();await env.tick(3001);
|
||||||
|
const next=env.pending('/api/v1/session')[0];env.answer(next,200,frame(17),{'X-Session-Token':'C'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':'https','X-Carrier-Attempt':'2','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional','X-Telemt-Up-Window':'4'});await flush();
|
||||||
|
const requests=env.requests.filter(r=>r.url.endsWith('/api/v1/up')&&r.options.headers.Authorization==='Bearer '+'C'.repeat(43));
|
||||||
|
const frames=[];for(const request of requests){const bytes=new Uint8Array(request.options.body);for(let offset=0;offset<bytes.length;){const view=new DataView(bytes.buffer,offset);frames.push([bytes[offset],bytes[offset+3]]);offset+=8+view.getUint32(4);}}
|
||||||
|
for(const id of [1,2])assert.deepEqual(frames.filter(value=>value[1]===id),[[1,id],[2,id]]);
|
||||||
|
assert.equal(frames.length,4);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('receiver retirement releases shared prefix capacity without exposing a partial frame',async page=>{
|
||||||
|
const env=environment(page),module=env.context.TelemtBridgeDownlink,budget=module.budget(()=>18),received=[];
|
||||||
|
function receiver(id){
|
||||||
|
let cursor='0',stream;const controller=new AbortController(),response={status:200,headers:new Headers({'X-Down-Cursor':'1','Content-Length':'18'}),body:new ReadableStream({start(value){stream=value}})};
|
||||||
|
const owner=module.create({failure:(reason,message)=>Object.assign(new Error(message),{telemtReason:reason}),alive:()=>true,cursor:()=>cursor,lane:id,limit:()=>18,budget,cancel(){},ready:()=>Promise.resolve(),deliver:data=>received.push(new Uint8Array(data)),advance:value=>{cursor=value},finished(){}});
|
||||||
|
const result=owner.read(response,controller.signal).catch(()=>{});return {owner,result,stream,controller,cursor:()=>cursor};
|
||||||
|
}
|
||||||
|
const a=receiver(1);a.stream.enqueue(new Uint8Array(frame(2,1,[1])));await flush();
|
||||||
|
const b=receiver(2);b.stream.enqueue(new Uint8Array(join(frame(2,2,[2]),frame(2,2,[3]))));b.stream.close();await flush();
|
||||||
|
assert.equal(budget.used(),18);assert.equal(received.length,1);assert.equal(b.cursor(),'0');
|
||||||
|
a.owner.close();a.controller.abort();a.stream.error(new Error('retired'));await a.result;await b.result;
|
||||||
|
assert.equal(received.length,3);assert.equal(b.cursor(),'1');assert.equal(budget.used(),0);
|
||||||
|
b.owner.close();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('native close cancels every HTTP request and pagehide cannot repeat cleanup',async page=>{
|
||||||
|
for(const android of [false,true])for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4,android);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const up=env.pending('/api/v1/up'),down=env.pending('/api/v1/down'),requests=up.concat(down);
|
||||||
|
assert.equal(up.length,2);assert.equal(down.length,1);assert.ok(requests.every(request=>!request.options.signal.aborted));
|
||||||
|
env.send({t:'close'});await flush();
|
||||||
|
assert.ok(requests.every(request=>request.options.signal.aborted));
|
||||||
|
assert.equal(env.requests.filter(request=>request.options.method==='DELETE').length,1);
|
||||||
|
const count=env.requests.length;env.close();await env.tick(120000);
|
||||||
|
assert.equal(env.requests.filter(request=>request.options.method==='DELETE').length,1);
|
||||||
|
assert.equal(env.requests.length,count);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(async()=>{const page=renderedPage();let failed=0;for(const {name,run} of tests){let timer;try{await Promise.race([run(page),new Promise((_,reject)=>{timer=setTimeout(()=>reject(new Error('test made no bounded progress')),5000)})]);console.log('ok - '+name)}catch(error){failed++;console.error('not ok - '+name+'\n'+error.stack)}finally{clearTimeout(timer)}}if(failed)process.exitCode=1})();
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
use std::io::Write;
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
|
/// Exercises the actual generated page with deterministic network and native-boundary doubles.
|
||||||
|
pub(super) fn run(page: &super::BridgePage) {
|
||||||
|
let mut child = Command::new("node")
|
||||||
|
.args(["-e", include_str!("behavior_tests.js"), "--", "--stdin"])
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.expect("Node.js 22+ is required for executable WEB bridge regressions");
|
||||||
|
child
|
||||||
|
.stdin
|
||||||
|
.take()
|
||||||
|
.unwrap()
|
||||||
|
.write_all(page.body.as_bytes())
|
||||||
|
.unwrap();
|
||||||
|
let output = child.wait_with_output().unwrap();
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"bridge regressions failed:\n{}\n{}",
|
||||||
|
String::from_utf8_lossy(&output.stdout),
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
(()=>{'use strict';
|
||||||
|
// HTTP request ownership is separate from the native WINDOW credit protocol.
|
||||||
|
function create(settings){
|
||||||
|
const channels=new Map(),window=settings.window,token=settings.token,downBudget=globalThis.TelemtBridgeDownlink.budget(settings.queueLimit);
|
||||||
|
let closed=false,order=0,ready=settings.committed(),resolveReady;
|
||||||
|
const readyPromise=new Promise(resolve=>{resolveReady=resolve});if(ready)resolveReady();
|
||||||
|
const alive=channel=>!closed&&!channel.closed&&settings.alive(channel.lane);
|
||||||
|
const markReady=()=>{if(!ready){ready=true;resolveReady();settings.commit()}};
|
||||||
|
function waitReady(signal){
|
||||||
|
if(ready)return Promise.resolve();
|
||||||
|
return new Promise((resolve,reject)=>{const abort=()=>reject(new Error('carrier retired'));if(signal&&signal.aborted){abort();return}if(signal)signal.addEventListener('abort',abort,{once:true});readyPromise.then(()=>{if(signal)signal.removeEventListener('abort',abort);resolve()})});
|
||||||
|
}
|
||||||
|
function get(id,type){
|
||||||
|
let channel=channels.get(id);if(channel)return channel;
|
||||||
|
const lane=id===null?null:settings.lane(id,type);if(id!==null&&!lane)return null;
|
||||||
|
channel={id,lane,pending:[],flights:new Map(),acks:new Set(),next:1,confirmed:0,cursor:'0',controller:null,polling:false,closed:false};
|
||||||
|
channel.receiver=globalThis.TelemtBridgeDownlink.create({
|
||||||
|
failure:settings.failure,alive:()=>alive(channel),cursor:()=>channel.cursor,lane:id,limit:settings.batchLimit,budget:downBudget,
|
||||||
|
cancel:settings.cancel,ready:waitReady,deliver:settings.deliver,advance:next=>{channel.cursor=next},finished:()=>{if(lane)settings.finish(lane,false)}
|
||||||
|
});channels.set(id,channel);return channel;
|
||||||
|
}
|
||||||
|
function enqueue(data,defer){
|
||||||
|
if(closed)return;
|
||||||
|
const entries=settings.lanes?settings.buffers.splitFrames(data):[{id:null,type:0,data}],touched=new Set();
|
||||||
|
for(const entry of entries){
|
||||||
|
const channel=get(entry.id,entry.type);if(!channel)continue;
|
||||||
|
if(!settings.buffers.reserve(entry.data,channel.lane))throw settings.failure('capacity','carrier queue full');
|
||||||
|
channel.pending.push(entry.data);touched.add(channel);
|
||||||
|
}
|
||||||
|
if(!defer)for(const channel of touched)pump(channel);
|
||||||
|
}
|
||||||
|
function checkAck(response,sequence){
|
||||||
|
if(response.status!==204)throw settings.failure('http','uplink rejected');
|
||||||
|
if(response.headers.get('X-Up-Ack')!==String(sequence))throw settings.failure('protocol','uplink acknowledgement rejected');
|
||||||
|
}
|
||||||
|
function acknowledge(channel,sequence,lease){
|
||||||
|
if(!alive(channel)||channel.flights.get(sequence)!==lease)return;
|
||||||
|
settings.buffers.settleBatch(lease);channel.flights.delete(sequence);channel.acks.add(sequence);
|
||||||
|
while(channel.acks.delete(channel.confirmed+1))channel.confirmed++;
|
||||||
|
markReady();settings.traffic(lease.total,0);pump(channel);
|
||||||
|
}
|
||||||
|
async function send(channel,sequence,lease){
|
||||||
|
const headers={'X-Up-Seq':String(sequence)};if(channel.id!==null)headers['X-Lane-ID']=String(channel.id);
|
||||||
|
if(window>1)headers['X-Telemt-Up-Confirmed']=String(channel.confirmed);
|
||||||
|
const frozen=settings.options(settings.method,token,lease.body,headers,lease.controller.signal);
|
||||||
|
try{
|
||||||
|
while(alive(channel)&&!lease.cancelled){
|
||||||
|
try{const response=await settings.request('/api/v1/up',frozen,null,()=>ready?1:9);if(!alive(channel))return;checkAck(response,sequence);acknowledge(channel,sequence,lease);return}
|
||||||
|
catch(error){
|
||||||
|
if(!alive(channel)||lease.cancelled)return;
|
||||||
|
if(!ready){settings.failed(error);return}
|
||||||
|
let replayed=false;
|
||||||
|
const recovered=await settings.recover(error,async(signal,remaining)=>{
|
||||||
|
const response=await settings.request('/api/v1/up',Object.assign({},frozen,{signal}),remaining,2);
|
||||||
|
if(!alive(channel)||lease.cancelled)return;checkAck(response,sequence);replayed=true;acknowledge(channel,sequence,lease);
|
||||||
|
});
|
||||||
|
if(!recovered||!alive(channel)||replayed)return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(error){if(alive(channel)&&!lease.cancelled)settings.failed(error)}
|
||||||
|
}
|
||||||
|
function pump(channel){
|
||||||
|
while(alive(channel)&&channel.pending.length&&channel.next<=channel.confirmed+window){
|
||||||
|
const lease=settings.buffers.takeBatch(channel.pending,channel.lane),sequence=channel.next++;
|
||||||
|
lease.controller=new AbortController();lease.order=order++;channel.flights.set(sequence,lease);send(channel,sequence,lease);
|
||||||
|
if(!channel.polling)poll(channel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function poll(channel){
|
||||||
|
if(channel.polling||!alive(channel))return;channel.polling=true;
|
||||||
|
try{
|
||||||
|
while(alive(channel)){
|
||||||
|
const cursor=channel.cursor,headers={'X-Down-Cursor':cursor};if(channel.id!==null)headers['X-Lane-ID']=String(channel.id);
|
||||||
|
channel.controller=new AbortController();const frozen=settings.options(settings.method,token,null,headers,channel.controller.signal);
|
||||||
|
try{
|
||||||
|
const response=await settings.request('/api/v1/down',frozen,null,1,channel.receiver.read);
|
||||||
|
if(!alive(channel))return;if(response.status!==200&&response.status!==204)throw settings.failure('http','downlink rejected');
|
||||||
|
if(ready)settings.connected();
|
||||||
|
}catch(error){
|
||||||
|
if(!alive(channel))return;
|
||||||
|
if(!ready){settings.failed(error);return}
|
||||||
|
const recovered=await settings.recover(error,async(signal,remaining)=>{
|
||||||
|
const response=await settings.request('/api/v1/down',Object.assign({},frozen,{signal}),remaining,2,channel.receiver.read);
|
||||||
|
if(response.status!==200&&response.status!==204)throw settings.failure('http','downlink replay rejected');
|
||||||
|
});
|
||||||
|
if(!recovered||!alive(channel))return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(error){if(alive(channel))settings.failed(error)}
|
||||||
|
finally{channel.polling=false;channel.controller=null}
|
||||||
|
}
|
||||||
|
function retire(channel,restore){
|
||||||
|
channel.closed=true;if(channel.controller)channel.controller.abort();channel.receiver.close();
|
||||||
|
const saved=[];
|
||||||
|
for(const lease of channel.flights.values()){if(restore)saved.push({order:lease.order,data:lease.body});settings.buffers.cancelBatch(lease);}
|
||||||
|
channel.flights.clear();channel.acks.clear();
|
||||||
|
for(const data of channel.pending)if(restore)saved.push({order:order++,data});
|
||||||
|
settings.buffers.releasePending(channel.pending,channel.lane);return saved;
|
||||||
|
}
|
||||||
|
return {enqueue,flush(){for(const channel of channels.values())pump(channel)},finish(lane){const channel=channels.get(lane.id);if(channel){channels.delete(lane.id);retire(channel,false)}},close(restore){
|
||||||
|
if(closed)return [];closed=true;const saved=[];for(const channel of channels.values())saved.push(...retire(channel,restore));channels.clear();
|
||||||
|
return saved.sort((a,b)=>a.order-b.order).map(entry=>entry.data);
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
globalThis.TelemtBridgeConveyor=Object.freeze({create});
|
||||||
|
})();
|
||||||
@@ -20,6 +20,12 @@ __BUFFER_RUNTIME__
|
|||||||
__RECOVERY_RUNTIME__
|
__RECOVERY_RUNTIME__
|
||||||
</script>
|
</script>
|
||||||
<script nonce="__NONCE__">
|
<script nonce="__NONCE__">
|
||||||
|
__DOWNLINK_RUNTIME__
|
||||||
|
</script>
|
||||||
|
<script nonce="__NONCE__">
|
||||||
|
__CONVEYOR_RUNTIME__
|
||||||
|
</script>
|
||||||
|
<script nonce="__NONCE__">
|
||||||
__RUNTIME__
|
__RUNTIME__
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
(()=>{'use strict';
|
||||||
|
// Prefix ownership survives HTTP retries, but never a session or lane replacement.
|
||||||
|
function budget(limit){
|
||||||
|
let used=0;const waiters=[];
|
||||||
|
function wake(){for(const next of waiters.slice())next()}
|
||||||
|
function acquire(size,signal){
|
||||||
|
if(size>limit())return Promise.reject(new Error('downlink budget exceeded'));
|
||||||
|
return new Promise((resolve,reject)=>{
|
||||||
|
const remove=()=>{const at=waiters.indexOf(tryAcquire);if(at>=0)waiters.splice(at,1);if(signal)signal.removeEventListener('abort',abort)};
|
||||||
|
const abort=()=>{remove();reject(new Error('downlink aborted'))};
|
||||||
|
const tryAcquire=()=>{if(signal&&signal.aborted){abort();return}if(size<=limit()-used){remove();used+=size;let released=false;resolve(()=>{if(!released){released=true;used-=size;wake()}})}};
|
||||||
|
waiters.push(tryAcquire);if(signal)signal.addEventListener('abort',abort,{once:true});tryAcquire();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {acquire,used:()=>used};
|
||||||
|
}
|
||||||
|
function create(settings){
|
||||||
|
let transaction=null,closed=false;
|
||||||
|
const protocol=message=>settings.failure('protocol',message);
|
||||||
|
const current=signal=>!closed&&settings.alive()&&!(signal&&signal.aborted);
|
||||||
|
function dispose(){if(transaction){transaction.release();transaction=null}}
|
||||||
|
function shape(type,id,size){
|
||||||
|
if(type===2)return id!==0&&size>0&&size<=1048576;
|
||||||
|
if(type===3)return id!==0&&size===0;
|
||||||
|
if(type===4)return id!==0&&size===4;
|
||||||
|
if(type===5)return id===0&&size<=64;
|
||||||
|
return type===31&&id===0&&size===0;
|
||||||
|
}
|
||||||
|
async function read(response,signal){
|
||||||
|
if(!current(signal))throw new Error('downlink retired');
|
||||||
|
if(response.status===204){
|
||||||
|
settings.cancel(response);if(transaction)throw protocol('missing replay batch');
|
||||||
|
const cursor=response.headers.get('X-Down-Cursor');
|
||||||
|
if(cursor!==null&&cursor!==settings.cursor())throw protocol('changed empty cursor');
|
||||||
|
if(response.headers.get('X-Lane-Closed')==='1')settings.finished();
|
||||||
|
return new ArrayBuffer(0);
|
||||||
|
}
|
||||||
|
const next=response.headers.get('X-Down-Cursor')||'',base=settings.cursor();
|
||||||
|
if(!/^[1-9]\d*$/.test(next)||BigInt(next)!==BigInt(base)+1n||BigInt(next)>18446744073709551615n)throw protocol('invalid downlink cursor');
|
||||||
|
const length=response.headers.get('Content-Length');let declared=null;
|
||||||
|
if(length!==null){if(!/^(0|[1-9]\d*)$/.test(length))throw protocol('invalid response length');declared=Number(length);if(!Number.isSafeInteger(declared)||declared===0||declared>settings.limit())throw protocol('response body overflow');}
|
||||||
|
if(transaction&&(transaction.base!==base||transaction.next!==next||(transaction.declared!==null&&declared!==null&&transaction.declared!==declared)))throw protocol('changed replay metadata');
|
||||||
|
if(!transaction){
|
||||||
|
const capacity=declared===null?settings.limit():declared,release=await settings.budget.acquire(capacity,signal);
|
||||||
|
if(!current(signal)){release();throw new Error('downlink retired');}
|
||||||
|
transaction={base,next,declared,bytes:new Uint8Array(capacity),retained:0,delivered:0,release};
|
||||||
|
}
|
||||||
|
const owner=transaction;
|
||||||
|
if(owner.declared===null&&declared!==null)owner.declared=declared;
|
||||||
|
await settings.ready(signal);
|
||||||
|
if(!current(signal)||transaction!==owner)throw new Error('downlink retired');
|
||||||
|
if(!response.body)throw protocol('missing response body');
|
||||||
|
const reader=response.body.getReader();let position=0,offset=0,frames=0,chunks=0,complete=false;
|
||||||
|
try{
|
||||||
|
for(;;){
|
||||||
|
let part;try{part=await reader.read();}catch(error){throw settings.failure('network','response stream interrupted');}
|
||||||
|
if(!current(signal)||transaction!==owner)throw new Error('downlink retired');
|
||||||
|
if(part.done)break;
|
||||||
|
const data=part.value;if(!(data instanceof Uint8Array)||++chunks>65536||data.byteLength>owner.bytes.byteLength-position)throw protocol('response body overflow');
|
||||||
|
const compared=Math.min(data.byteLength,Math.max(0,owner.retained-position));
|
||||||
|
for(let i=0;i<compared;i++)if(data[i]!==owner.bytes[position+i])throw protocol('changed replay prefix');
|
||||||
|
owner.bytes.set(data.subarray(compared),position+compared);position+=data.byteLength;owner.retained=Math.max(owner.retained,position);
|
||||||
|
while(position-offset>=8){
|
||||||
|
const view=new DataView(owner.bytes.buffer,offset),type=view.getUint8(0),id=(view.getUint8(1)<<16)|(view.getUint8(2)<<8)|view.getUint8(3),size=view.getUint32(4),end=offset+8+size;
|
||||||
|
if(!shape(type,id,size)||(settings.lane!==null&&id!==settings.lane)||end>owner.bytes.byteLength)throw protocol('invalid downlink frame');
|
||||||
|
if(end>position)break;if(++frames>4096)throw protocol('too many downlink frames');
|
||||||
|
if(type===4&&view.getUint32(8)===0)throw protocol('invalid WINDOW credit');
|
||||||
|
if(end>owner.delivered){
|
||||||
|
if(offset!==owner.delivered)throw protocol('invalid replay boundary');
|
||||||
|
const frame=owner.bytes.slice(offset,end).buffer;
|
||||||
|
settings.deliver(frame);owner.delivered=end;
|
||||||
|
}
|
||||||
|
offset=end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!position||offset!==position||position<owner.retained||(owner.declared!==null&&position!==owner.declared))throw protocol('incomplete downlink batch');
|
||||||
|
settings.advance(next);complete=true;dispose();return new ArrayBuffer(0);
|
||||||
|
}finally{
|
||||||
|
if(!complete)try{const cancelled=reader.cancel();if(cancelled)cancelled.catch(()=>{});}catch(error){}
|
||||||
|
reader.releaseLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {read,close(){closed=true;dispose()}};
|
||||||
|
}
|
||||||
|
globalThis.TelemtBridgeDownlink=Object.freeze({create,budget});
|
||||||
|
})();
|
||||||
@@ -26,12 +26,13 @@ function create(settings){
|
|||||||
if(status===204&&(path==='/api/v1/up'||path==='/api/v1/down'))return {limit:0,exact:true,reason:'protocol'};
|
if(status===204&&(path==='/api/v1/up'||path==='/api/v1/down'))return {limit:0,exact:true,reason:'protocol'};
|
||||||
return {limit:0,exact:true,reason:'http'};
|
return {limit:0,exact:true,reason:'http'};
|
||||||
}
|
}
|
||||||
async function send(path,frozenOptions,remainingBudget,maxAttempts){
|
async function send(path,frozenOptions,remainingBudget,maxAttempts,receiver){
|
||||||
let delay=250,attempt=0,lastReason='network';maxAttempts=maxAttempts||9;
|
let delay=250,attempt=0,lastReason='network';maxAttempts=maxAttempts||9;
|
||||||
|
const attempts=()=>typeof maxAttempts==='function'?maxAttempts():maxAttempts;
|
||||||
const initialBudget=remainingBudget?Math.min(settings.retryMs(),remainingBudget()):settings.retryMs();
|
const initialBudget=remainingBudget?Math.min(settings.retryMs(),remainingBudget()):settings.retryMs();
|
||||||
const deadline=Date.now()+Math.max(0,initialBudget),external=frozenOptions.signal;
|
const deadline=Date.now()+Math.max(0,initialBudget),external=frozenOptions.signal;
|
||||||
const attemptLimit=path==='/api/v1/down'?settings.longPollMs()+settings.requestMs():settings.requestMs();
|
const attemptLimit=path==='/api/v1/down'?settings.longPollMs()+settings.requestMs():settings.requestMs();
|
||||||
while(attempt<maxAttempts){
|
while(attempt<attempts()){
|
||||||
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
||||||
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
||||||
const controller=new AbortController(),abort=()=>controller.abort();let timedOut=false;
|
const controller=new AbortController(),abort=()=>controller.abort();let timedOut=false;
|
||||||
@@ -45,12 +46,12 @@ function create(settings){
|
|||||||
lastReason='http';wait=retryAfterMs(fetched);settings.cancel(fetched);
|
lastReason='http';wait=retryAfterMs(fetched);settings.cancel(fetched);
|
||||||
}else{
|
}else{
|
||||||
const policy=responsePolicy(path,fetched.status);let body;
|
const policy=responsePolicy(path,fetched.status);let body;
|
||||||
try{body=await settings.read(fetched,policy.limit,policy.exact,controller.signal)}
|
try{body=receiver&&(fetched.status===200||fetched.status===204)?await receiver(fetched,controller.signal):await settings.read(fetched,policy.limit,policy.exact,controller.signal);}
|
||||||
catch(error){
|
catch(error){
|
||||||
controller.abort();
|
controller.abort();
|
||||||
if(external&&external.aborted)throw error;
|
if(external&&external.aborted)throw error;
|
||||||
if(timedOut)throw settings.failure('timeout','response deadline exceeded');
|
if(timedOut)throw settings.failure('timeout','response deadline exceeded');
|
||||||
throw settings.failure(policy.reason,error&&error.message);
|
throw settings.failure(settings.reason(error,policy.reason),error&&error.message);
|
||||||
}
|
}
|
||||||
response={status:fetched.status,headers:fetched.headers,body};return response;
|
response={status:fetched.status,headers:fetched.headers,body};return response;
|
||||||
}
|
}
|
||||||
@@ -60,7 +61,7 @@ function create(settings){
|
|||||||
if(timedOut)lastReason='timeout';
|
if(timedOut)lastReason='timeout';
|
||||||
if(settings.reason(error,'')==='protocol')throw error;
|
if(settings.reason(error,'')==='protocol')throw error;
|
||||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||||
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=maxAttempts||after<=0)break;
|
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=attempts()||after<=0)break;
|
||||||
settings.retrying();
|
settings.retrying();
|
||||||
const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
|
const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
|
||||||
await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,2000);
|
await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,2000);
|
||||||
|
|||||||
+43
-143
@@ -19,7 +19,7 @@ let laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItem
|
|||||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'',recoveryPath=location.pathname+location.search;
|
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'',recoveryPath=location.pathname+location.search;
|
||||||
history.replaceState(null,'',location.pathname);
|
history.replaceState(null,'',location.pathname);
|
||||||
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
||||||
let upSequence=1,downCursor='0',upRunning=false,upLease=null,pollController=null;
|
let upSequence=1,downCursor='0',upRunning=false,upLease=null,httpCarrier=null,upWindow=1;
|
||||||
let helloFrame=null,helloTimer=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false,terminalFailure='';
|
let helloFrame=null,helloTimer=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false,terminalFailure='';
|
||||||
let negotiationStartedAt=0,carrierTimer=null,probeTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
let negotiationStartedAt=0,carrierTimer=null,probeTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
||||||
let recoveryController=null,recoveryCommit=null,recoveryReplaced=false,lastSchedulerWall=Date.now(),lastSchedulerMonotonic=performance.now(),schedulerGapPending=0,schedulerTimer=null;
|
let recoveryController=null,recoveryCommit=null,recoveryReplaced=false,lastSchedulerWall=Date.now(),lastSchedulerMonotonic=performance.now(),schedulerGapPending=0,schedulerTimer=null;
|
||||||
@@ -44,7 +44,7 @@ const {reserve,release,releasePending,frameBound,splitFrames,acceptNativeFrames,
|
|||||||
function detachLease(lease){if(lease.lane){if(lease.lane.upLease===lease)lease.lane.upLease=null}else if(upLease===lease)upLease=null}
|
function detachLease(lease){if(lease.lane){if(lease.lane.upLease===lease)lease.lane.upLease=null}else if(upLease===lease)upLease=null}
|
||||||
function settleBatch(lease){if(!buffers.settleBatch(lease))return false;detachLease(lease);return true}
|
function settleBatch(lease){if(!buffers.settleBatch(lease))return false;detachLease(lease);return true}
|
||||||
function cancelBatch(lease){if(!lease||lease.settled)return;buffers.cancelBatch(lease);detachLease(lease)}
|
function cancelBatch(lease){if(!lease||lease.settled)return;buffers.cancelBatch(lease);detachLease(lease)}
|
||||||
const attemptHeaders=(attempt,failure)=>negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{};
|
const attemptHeaders=(attempt,failure)=>Object.assign({'X-Telemt-Up-Window':'4'},negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{});
|
||||||
function finishOldRecovery(){
|
function finishOldRecovery(){
|
||||||
recoveryReplaced=false;resetScheduler();status('connected');
|
recoveryReplaced=false;resetScheduler();status('connected');
|
||||||
while(recoveryPending.length&&!closed){const data=recoveryPending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
while(recoveryPending.length&&!closed){const data=recoveryPending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
||||||
@@ -58,8 +58,9 @@ function resolveRecoveryCommit(){
|
|||||||
commit.signal.removeEventListener('abort',commit.abort);commit.resolve();
|
commit.signal.removeEventListener('abort',commit.abort);commit.resolve();
|
||||||
}
|
}
|
||||||
function retireCarrier(policy){
|
function retireCarrier(policy){
|
||||||
|
stopHttp(false);upWindow=1;
|
||||||
recoveryReplaced=true;attemptEpoch++;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;clearProbeTimer();
|
recoveryReplaced=true;attemptEpoch++;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;clearProbeTimer();
|
||||||
if(attemptController)attemptController.abort();attemptController=null;if(pollController)pollController.abort();pollController=null;
|
if(attemptController)attemptController.abort();attemptController=null;
|
||||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;cancelBatch(upLease);releasePending(upPending,null);
|
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||||
@@ -118,9 +119,10 @@ function fail(reason){
|
|||||||
}
|
}
|
||||||
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
||||||
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
||||||
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'';
|
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'',windowHeader=response.headers.get('X-Telemt-Up-Window');
|
||||||
|
if(windowHeader!==null&&!/^[1-4]$/.test(windowHeader))throw new Error('invalid conveyor window');const upWindow=windowHeader===null?1:Number(windowHeader);
|
||||||
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
||||||
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:''}}
|
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:'',upWindow}}
|
||||||
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
||||||
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
||||||
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
||||||
@@ -129,7 +131,7 @@ function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
|||||||
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
||||||
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
||||||
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
||||||
return {selected,state};
|
return {selected,state,upWindow};
|
||||||
}
|
}
|
||||||
function armCarrierDeadline(epoch){
|
function armCarrierDeadline(epoch){
|
||||||
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
||||||
@@ -140,8 +142,8 @@ function armCarrierDeadline(epoch){
|
|||||||
}
|
}
|
||||||
function clearProbeTimer(){if(probeTimer){clearTimeout(probeTimer.timer);probeTimer=null}}
|
function clearProbeTimer(){if(probeTimer){clearTimeout(probeTimer.timer);probeTimer=null}}
|
||||||
function resetCandidate(){
|
function resetCandidate(){
|
||||||
|
stopHttp(true);upWindow=1;
|
||||||
clearProbeTimer();
|
clearProbeTimer();
|
||||||
if(pollController)pollController.abort();pollController=null;
|
|
||||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
||||||
cancelBatch(upLease);releasePending(upPending,null);
|
cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
@@ -198,8 +200,8 @@ function startCandidate(probe,epoch){
|
|||||||
else advanceCarrier('protocol',epoch);
|
else advanceCarrier('protocol',epoch);
|
||||||
}
|
}
|
||||||
function maybeStartCandidate(){
|
function maybeStartCandidate(){
|
||||||
if(closed||carrierCommitted||!sessionToken||candidateRunning)return;const epoch=attemptEpoch;
|
if(closed||carrierCommitted||!sessionToken)return;if(candidateRunning){try{flushHttpPending()}catch(error){fail(failureReason(error,'protocol'))}return}const epoch=attemptEpoch;
|
||||||
let probe;try{probe=findProbe(probeCoalesceMs>0)}catch(error){fail('protocol');return}if(!probe)return;
|
let probe;try{probe=findProbe(true)}catch(error){fail('protocol');return}if(!probe)return;
|
||||||
if(!probeCoalesceMs||probe.hasData){startCandidate(probe,epoch);return}
|
if(!probeCoalesceMs||probe.hasData){startCandidate(probe,epoch);return}
|
||||||
if(probeTimer)return;const owner={epoch,timer:null};
|
if(probeTimer)return;const owner={epoch,timer:null};
|
||||||
owner.timer=setTimeout(()=>{if(probeTimer!==owner||closed||owner.epoch!==attemptEpoch)return;probeTimer=null;let current;try{current=findProbe(false)}catch(error){fail('protocol');return}startCandidate(current,owner.epoch)},probeCoalesceMs);
|
owner.timer=setTimeout(()=>{if(probeTimer!==owner||closed||owner.epoch!==attemptEpoch)return;probeTimer=null;let current;try{current=findProbe(false)}catch(error){fail('protocol');return}startCandidate(current,owner.epoch)},probeCoalesceMs);
|
||||||
@@ -219,74 +221,58 @@ async function createSession(epoch){
|
|||||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||||
if(!token||cursor!=='0'){advanceCarrier('protocol',epoch);return}
|
if(!token||cursor!=='0'){advanceCarrier('protocol',epoch);return}
|
||||||
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
||||||
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;
|
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;upWindow=echo.upWindow;
|
||||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome]);status('connecting')}
|
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome]);status('connecting')}
|
||||||
if(carrier==='websocket')openCandidateSocket(null,null,epoch);
|
if(carrier==='websocket')openCandidateSocket(null,null,epoch);
|
||||||
maybeStartCandidate();
|
maybeStartCandidate();
|
||||||
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier(failureReason(error,'network'),epoch)}
|
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier(failureReason(error,'network'),epoch)}
|
||||||
}
|
}
|
||||||
async function probeHttp(probe,laneID,epoch){
|
function stopHttp(restore){
|
||||||
try{
|
if(!httpCarrier)return;const saved=httpCarrier.close(restore);httpCarrier=null;
|
||||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
for(const data of saved){if(!reserve(data,null)){fail('capacity');return}pending.push(data)}
|
||||||
const response=await request('/api/v1/up',options(carrierMethod,token,body,headers,controller.signal));
|
}
|
||||||
if(closed||epoch!==attemptEpoch)return
|
function flushHttpPending(){
|
||||||
if(response.status!==204){advanceCarrier('http',epoch);return}
|
while(httpCarrier&&pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);httpCarrier.enqueue(data,true)}
|
||||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
if(httpCarrier&&!closed)httpCarrier.flush();
|
||||||
if(laneID===null)upSequence=2;else ensureLane(laneID).sequence=2;
|
}
|
||||||
commitCarrier(probe,epoch);
|
function probeHttp(probe,laneID,epoch){
|
||||||
}catch(error){if(!closed&&epoch===attemptEpoch)advanceCarrier(failureReason(error,'network'),epoch)}
|
const token=sessionToken;
|
||||||
|
httpCarrier=globalThis.TelemtBridgeConveyor.create({
|
||||||
|
token,window:upWindow,method:carrierMethod,lanes:carrier==='https-lanes',buffers,options,request,
|
||||||
|
queueLimit:()=>queueLimit,batchLimit:()=>batchLimit,committed:()=>carrierCommitted,failure,cancel:responseBody.cancel,
|
||||||
|
alive:lane=>!closed&&epoch===attemptEpoch&&token===sessionToken&&(!lane||lanes.get(lane.id)===lane),
|
||||||
|
commit:()=>commitCarrier(null,epoch),recover:recoverTransport,connected:()=>status('connected'),
|
||||||
|
traffic:(up,down)=>port.postMessage({t:'traffic',up,down}),
|
||||||
|
deliver:data=>{observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data])},
|
||||||
|
lane:(id,type)=>{
|
||||||
|
let lane=lanes.get(id);if(!lane&&(type===2||type===3||type===4))return null;
|
||||||
|
if(!lane&&closedLanes.has(id))throw failure('protocol','closed lane reused');
|
||||||
|
if(!lane&&id!==0&&type!==1)throw failure('protocol','lane did not begin with OPEN');
|
||||||
|
return lane||ensureLane(id);
|
||||||
|
},
|
||||||
|
finish:finishLane,failed:error=>{if(closed||epoch!==attemptEpoch)return;if(carrierCommitted)fail(failureReason(error,'network'));else advanceCarrier(failureReason(error,'network'),epoch)}
|
||||||
|
});
|
||||||
|
try{flushHttpPending()}catch(error){fail(failureReason(error,'protocol'))}
|
||||||
}
|
}
|
||||||
function commitCarrier(probe,epoch){
|
function commitCarrier(probe,epoch){
|
||||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||||
if(switching){fail('protocol');return}
|
if(switching){fail('protocol');return}
|
||||||
clearProbeTimer();try{consumeProbe(probe)}catch(error){fail('protocol');return}
|
clearProbeTimer();try{if(probe)consumeProbe(probe)}catch(error){fail('protocol');return}
|
||||||
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||||
resetScheduler();
|
resetScheduler();
|
||||||
attemptController=null;currentAttempt=null;
|
attemptController=null;currentAttempt=null;
|
||||||
status('connected');
|
status('connected');
|
||||||
if(carrier==='https')poll();
|
|
||||||
else if(carrier==='https-lanes'){const lane=lanes.get(probe.id);if(lane&&!lane.polling)pollLane(lane)}
|
|
||||||
while(pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
while(pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
||||||
resolveRecoveryCommit();
|
resolveRecoveryCommit();
|
||||||
}
|
}
|
||||||
function queueCarrier(data){
|
function queueCarrier(data){
|
||||||
if(closed)return;
|
if(closed)return;
|
||||||
try{
|
try{
|
||||||
if(carrier==='https')queueUp(data);
|
if(carrier==='https'||carrier==='https-lanes')httpCarrier.enqueue(data);
|
||||||
else if(carrier==='websocket')queueSocket(data);
|
else if(carrier==='websocket')queueSocket(data);
|
||||||
else for(const value of splitFrames(data)){if(closed)break;queueLane(value)}
|
else for(const value of splitFrames(data)){if(closed)break;queueLane(value)}
|
||||||
}catch(error){fail('protocol')}
|
}catch(error){fail('protocol')}
|
||||||
}
|
}
|
||||||
function queueUp(data){if(!reserve(data,null)){fail('capacity');return}upPending.push(data);runUp()}
|
|
||||||
async function runUp(){
|
|
||||||
if(upRunning)return;upRunning=true;const ownerEpoch=attemptEpoch;let lease=null;
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&upPending.length){
|
|
||||||
lease=takeBatch(upPending,null);upLease=lease;lease.controller=new AbortController();const sequence=String(upSequence),token=sessionToken;
|
|
||||||
for(;;){
|
|
||||||
try{
|
|
||||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence},lease.controller.signal),null,1);
|
|
||||||
if(response.status!==204)throw failure('http','uplink rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink acknowledgement rejected');
|
|
||||||
break;
|
|
||||||
}catch(error){
|
|
||||||
let replayed=false;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence},signal),remaining,2);
|
|
||||||
if(response.status!==204)throw failure('http','uplink replay rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink replay acknowledgement rejected');
|
|
||||||
replayed=true;
|
|
||||||
});
|
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token)return;
|
|
||||||
if(!replayed)continue;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});upSequence++;lease=null;
|
|
||||||
}
|
|
||||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))fail(failureReason(error,'network'))}
|
|
||||||
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}}
|
|
||||||
}
|
|
||||||
function sendCandidateSocket(next){
|
function sendCandidateSocket(next){
|
||||||
const state=next.telemt;if(!state||state.sent||next.readyState!==WebSocket.OPEN||!state.probe)return;
|
const state=next.telemt;if(!state||state.sent||next.readyState!==WebSocket.OPEN||!state.probe)return;
|
||||||
let probe=state.probe;
|
let probe=state.probe;
|
||||||
@@ -340,30 +326,6 @@ async function runSocketUp(){
|
|||||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))recoverTransport(error,null)}
|
}catch(error){if(!closed&&!(lease&&lease.cancelled))recoverTransport(error,null)}
|
||||||
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}}
|
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}}
|
||||||
}
|
}
|
||||||
async function poll(){
|
|
||||||
while(!closed&&sessionToken){
|
|
||||||
const token=sessionToken,cursor=downCursor;
|
|
||||||
try{
|
|
||||||
pollController=new AbortController();
|
|
||||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor},pollController.signal),null,1);
|
|
||||||
if(closed||sessionToken!==token)return;
|
|
||||||
if(response.status===204){status('connected');continue}
|
|
||||||
if(response.status!==200)throw failure('http','downlink rejected');
|
|
||||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
|
||||||
if(!next||!data.byteLength)throw failure('protocol','invalid downlink response');
|
|
||||||
if(closed)return;
|
|
||||||
observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);downCursor=next;status('connected');
|
|
||||||
}catch(error){
|
|
||||||
if(closed)return;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor},signal),remaining,2);
|
|
||||||
if(response.status===204)return;
|
|
||||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','downlink replay rejected');
|
|
||||||
});
|
|
||||||
if(!recovered||closed||sessionToken!==token)return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
function ensureLane(id){
|
function ensureLane(id){
|
||||||
let lane=lanes.get(id);
|
let lane=lanes.get(id);
|
||||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,upLease:null,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,upLease:null,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
||||||
@@ -375,7 +337,7 @@ function rememberLaneClosed(id){
|
|||||||
closedLanes.add(id);closedLaneOrder.push(id);
|
closedLanes.add(id);closedLaneOrder.push(id);
|
||||||
}
|
}
|
||||||
function finishLane(lane,notifyClient){
|
function finishLane(lane,notifyClient){
|
||||||
if(lanes.get(lane.id)!==lane)return;
|
if(lanes.get(lane.id)!==lane)return;if(httpCarrier)httpCarrier.finish(lane);
|
||||||
if(lane.controller)lane.controller.abort();lane.controller=null;cancelBatch(lane.upLease);
|
if(lane.controller)lane.controller.abort();lane.controller=null;cancelBatch(lane.upLease);
|
||||||
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
||||||
releasePending(lane.pending,lane);lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
releasePending(lane.pending,lane);lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
||||||
@@ -389,7 +351,7 @@ function queueLane(value){
|
|||||||
lane=lane||ensureLane(value.id);
|
lane=lane||ensureLane(value.id);
|
||||||
if(!reserve(value.data,lane)){fail('capacity');return}
|
if(!reserve(value.data,lane)){fail('capacity');return}
|
||||||
lane.pending.push(value.data);
|
lane.pending.push(value.data);
|
||||||
if(carrier==='websocket-lanes'){openLaneSocket(lane);runLaneSocketUp(lane)}else runLaneUp(lane);
|
openLaneSocket(lane);runLaneSocketUp(lane);
|
||||||
}
|
}
|
||||||
function openLaneSocket(lane){
|
function openLaneSocket(lane){
|
||||||
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
||||||
@@ -416,74 +378,12 @@ async function runLaneSocketUp(lane){
|
|||||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))finishLane(lane,true)}
|
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))finishLane(lane,true)}
|
||||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
||||||
}
|
}
|
||||||
async function runLaneUp(lane){
|
|
||||||
if(lane.running)return;lane.running=true;let lease=null;
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&lane.pending.length){
|
|
||||||
lease=takeBatch(lane.pending,lane);lane.upLease=lease;lease.controller=new AbortController();
|
|
||||||
const sequence=String(lane.sequence),laneID=String(lane.id),token=sessionToken;
|
|
||||||
for(;;){
|
|
||||||
try{
|
|
||||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},lease.controller.signal),null,1);
|
|
||||||
if(response.status!==204)throw failure('http','lane uplink rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink acknowledgement rejected');
|
|
||||||
break;
|
|
||||||
}catch(error){
|
|
||||||
let replayed=false;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/up',options(carrierMethod,token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},signal),remaining,2);
|
|
||||||
if(response.status!==204)throw failure('http','lane uplink replay rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink replay acknowledgement rejected');
|
|
||||||
replayed=true;
|
|
||||||
});
|
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
|
||||||
if(!replayed)continue;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lane.sequence++;lease=null;
|
|
||||||
if(!lane.polling)pollLane(lane);
|
|
||||||
}
|
|
||||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))fail(failureReason(error,'network'))}
|
|
||||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&sessionToken&&lane.pending.length)runLaneUp(lane)}
|
|
||||||
}
|
|
||||||
async function pollLane(lane){
|
|
||||||
if(!lane||lane.polling)return;lane.polling=true;let restart=false,failedToken='',failedCursor='',failedLaneID='';
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
|
||||||
const controller=new AbortController(),laneID=String(lane.id),token=sessionToken,cursor=lane.cursor;lane.controller=controller;
|
|
||||||
failedToken=token;failedCursor=cursor;failedLaneID=laneID;
|
|
||||||
const response=await request('/api/v1/down',options(carrierMethod,token,null,{'X-Down-Cursor':cursor,'X-Lane-ID':laneID},controller.signal),null,1);
|
|
||||||
if(closed||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
|
||||||
if(response.status===204){
|
|
||||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
|
||||||
status('connected');continue;
|
|
||||||
}
|
|
||||||
if(response.status!==200)throw failure('http','lane downlink rejected');
|
|
||||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
|
||||||
if(!next||!data.byteLength)throw failure('protocol','invalid lane downlink response');
|
|
||||||
for(const value of splitFrames(data))if(value.id!==lane.id)throw new Error('cross-lane frame');
|
|
||||||
if(closed)return;
|
|
||||||
observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);lane.cursor=next;status('connected');
|
|
||||||
}
|
|
||||||
}catch(error){
|
|
||||||
if(!closed&&lanes.get(lane.id)===lane){
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/down',options(carrierMethod,failedToken,null,{'X-Down-Cursor':failedCursor,'X-Lane-ID':failedLaneID},signal),remaining,2);
|
|
||||||
if(response.status===204)return;
|
|
||||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','lane downlink replay rejected');
|
|
||||||
});
|
|
||||||
restart=recovered&&!closed&&sessionToken===failedToken&&lanes.get(lane.id)===lane;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally{lane.polling=false;lane.controller=null;if(restart)pollLane(lane)}
|
|
||||||
}
|
|
||||||
function deleteSession(){
|
function deleteSession(){
|
||||||
const token=cleanupToken||sessionToken,headers=canonicalFailures.includes(terminalFailure)?{'X-Carrier-Failure':terminalFailure}:null;
|
const token=cleanupToken||sessionToken,headers=canonicalFailures.includes(terminalFailure)?{'X-Carrier-Failure':terminalFailure}:null;
|
||||||
if(token)fetch(relayBase+'/api/v1/session',options('DELETE',token,null,headers,undefined,true)).catch(()=>{});
|
if(token)fetch(relayBase+'/api/v1/session',options('DELETE',token,null,headers,undefined,true)).catch(()=>{});
|
||||||
}
|
}
|
||||||
function close(notifyServer){
|
function close(notifyServer){
|
||||||
if(closed)return;closed=true;if(recoveryController)recoveryController.cancel();rejectRecoveryCommit(failure('network','bridge closed'));if(helloTimer)clearTimeout(helloTimer);helloTimer=null;if(carrierTimer)clearTimeout(carrierTimer);clearProbeTimer();if(schedulerTimer)clearTimeout(schedulerTimer);schedulerTimer=null;if(attemptController)attemptController.abort();if(pollController)pollController.abort();
|
if(closed)return;closed=true;stopHttp(false);if(recoveryController)recoveryController.cancel();rejectRecoveryCommit(failure('network','bridge closed'));if(helloTimer)clearTimeout(helloTimer);helloTimer=null;if(carrierTimer)clearTimeout(carrierTimer);clearProbeTimer();if(schedulerTimer)clearTimeout(schedulerTimer);schedulerTimer=null;if(attemptController)attemptController.abort();
|
||||||
if(socket)socket.close();cancelBatch(upLease);releasePending(upPending,null);
|
if(socket)socket.close();cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||||
|
|||||||
+8
-16
@@ -31,20 +31,8 @@ fn carrier_method_is_page_owned_and_used_by_every_https_request() {
|
|||||||
.contains(&format!("const carrierMethod='{}';", method.as_str()))
|
.contains(&format!("const carrierMethod='{}';", method.as_str()))
|
||||||
);
|
);
|
||||||
assert_eq!(page.body.matches("carrierMethod=").count(), 1);
|
assert_eq!(page.body.matches("carrierMethod=").count(), 1);
|
||||||
assert_eq!(page.body.matches("options(carrierMethod,").count(), 9);
|
// Execute the rendered POST/PUT page, including retries, instead of counting call sites.
|
||||||
let requests: Vec<_> = page
|
behavior_tests::run(&page);
|
||||||
.body
|
|
||||||
.lines()
|
|
||||||
.filter(|line| {
|
|
||||||
line.contains("request('/api/v1/up'") || line.contains("request('/api/v1/down'")
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
assert_eq!(requests.len(), 9);
|
|
||||||
assert!(
|
|
||||||
requests
|
|
||||||
.iter()
|
|
||||||
.all(|line| line.contains("options(carrierMethod,"))
|
|
||||||
);
|
|
||||||
assert_eq!(page.body.matches("options('POST',bootstrap,").count(), 2);
|
assert_eq!(page.body.matches("options('POST',bootstrap,").count(), 2);
|
||||||
assert!(
|
assert!(
|
||||||
page.body
|
page.body
|
||||||
@@ -68,6 +56,9 @@ fn carrier_method_is_page_owned_and_used_by_every_https_request() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[path = "behavior_tests.rs"]
|
||||||
|
mod behavior_tests;
|
||||||
|
|
||||||
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||||
render(
|
render(
|
||||||
"proxy.example.com",
|
"proxy.example.com",
|
||||||
@@ -289,8 +280,9 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
|||||||
fn retry_and_attempt_state_are_frozen_before_fetch() {
|
fn retry_and_attempt_state_are_frozen_before_fetch() {
|
||||||
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
||||||
assert!(
|
assert!(
|
||||||
page.body
|
page.body.contains(
|
||||||
.contains("async function send(path,frozenOptions,remainingBudget,maxAttempts)")
|
"async function send(path,frozenOptions,remainingBudget,maxAttempts,receiver)"
|
||||||
|
)
|
||||||
);
|
);
|
||||||
assert!(!page.body.contains("makeOptions"));
|
assert!(!page.body.contains("makeOptions"));
|
||||||
assert!(page.body.contains(
|
assert!(page.body.contains(
|
||||||
|
|||||||
+3
-70
@@ -36,6 +36,8 @@ mod decoy;
|
|||||||
mod diagnostic;
|
mod diagnostic;
|
||||||
// Downlink long-poll handling remains isolated from request routing.
|
// Downlink long-poll handling remains isolated from request routing.
|
||||||
mod down;
|
mod down;
|
||||||
|
// Uplink admission and conveyor waits own their bounded request bodies.
|
||||||
|
mod up;
|
||||||
// Canonical request parsing rejects ambiguous credentials before routing.
|
// Canonical request parsing rejects ambiguous credentials before routing.
|
||||||
mod request;
|
mod request;
|
||||||
// Positive-only recovery representation stays separate from ordinary bridge rendering.
|
// Positive-only recovery representation stays separate from ordinary bridge rendering.
|
||||||
@@ -68,6 +70,7 @@ use response::{
|
|||||||
insert_header, service_unavailable,
|
insert_header, service_unavailable,
|
||||||
};
|
};
|
||||||
use session::handle_session;
|
use session::handle_session;
|
||||||
|
use up::handle_up;
|
||||||
|
|
||||||
type BoxError = Box<dyn Error + Send + Sync>;
|
type BoxError = Box<dyn Error + Send + Sync>;
|
||||||
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
||||||
@@ -457,76 +460,6 @@ async fn handle_api(
|
|||||||
_ => serve_decoy(request, vhost, true, &runtime).await,
|
_ => serve_decoy(request, vhost, true, &runtime).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async fn handle_up(
|
|
||||||
request: Request<RequestBody>,
|
|
||||||
runtime: Arc<WebProcessRuntime>,
|
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
|
||||||
token_hash: crate::web::manager::TokenHash,
|
|
||||||
) -> HttpResponse {
|
|
||||||
if !matches!(*request.method(), Method::POST | Method::PUT) || !binary_content_type(&request) {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
let Some(sequence) = canonical_u64_header(&request, "x-up-seq").filter(|value| *value != 0)
|
|
||||||
else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
if session.carrier().uses_websocket() {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
if let Some(trace) = request_trace(&request) {
|
|
||||||
trace.set_route(TraceRoute::Uplink);
|
|
||||||
trace.bind_identity(session.trace_identity());
|
|
||||||
}
|
|
||||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let limit = session.limits().max_body_bytes;
|
|
||||||
let CollectedBody {
|
|
||||||
request,
|
|
||||||
body,
|
|
||||||
_body_budget,
|
|
||||||
} = match collect_body(
|
|
||||||
request,
|
|
||||||
&runtime,
|
|
||||||
Duration::from_secs(session.timeouts().body_secs),
|
|
||||||
limit,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
|
||||||
Err(CollectBodyError::Invalid(request)) => {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if let Some(trace) = request_trace(&request) {
|
|
||||||
trace.record_frames(TraceDirection::Request, &body, session.limits());
|
|
||||||
}
|
|
||||||
let result = match lane_id {
|
|
||||||
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
|
||||||
None => session.process_up(sequence, &body),
|
|
||||||
};
|
|
||||||
match result {
|
|
||||||
Ok(ack) => {
|
|
||||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
|
||||||
insert_header(
|
|
||||||
&mut response,
|
|
||||||
HeaderName::from_static("x-up-ack"),
|
|
||||||
&ack.to_string(),
|
|
||||||
);
|
|
||||||
response
|
|
||||||
}
|
|
||||||
Err(ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit) => {
|
|
||||||
service_unavailable()
|
|
||||||
}
|
|
||||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn strip_query<B>(request: &mut Request<B>) {
|
fn strip_query<B>(request: &mut Request<B>) {
|
||||||
if request.uri().query().is_some()
|
if request.uri().query().is_some()
|
||||||
&& let Ok(uri) = request.uri().path().parse()
|
&& let Ok(uri) = request.uri().path().parse()
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ use super::super::session_policy_tests::{open_keepalive, read_http_response};
|
|||||||
use super::super::*;
|
use super::super::*;
|
||||||
use crate::config::WebCarrierMethod;
|
use crate::config::WebCarrierMethod;
|
||||||
|
|
||||||
|
#[path = "conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|
||||||
fn carrier_request(method: &str, path: &str, token: &str, headers: &str, body: &[u8]) -> Vec<u8> {
|
fn carrier_request(method: &str, path: &str, token: &str, headers: &str, body: &[u8]) -> Vec<u8> {
|
||||||
let mut bytes = format!(
|
let mut bytes = format!(
|
||||||
"{method} {path} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nAuthorization: Bearer {token}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
"{method} {path} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nAuthorization: Bearer {token}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
|||||||
@@ -0,0 +1,358 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn create_with_headers(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
bootstrap: &str,
|
||||||
|
headers: &str,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
request(
|
||||||
|
listener,
|
||||||
|
runtime,
|
||||||
|
carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/session",
|
||||||
|
bootstrap,
|
||||||
|
&format!("Content-Type: application/octet-stream\r\n{headers}"),
|
||||||
|
&frame::encode(FrameType::Hello, 0, &[1]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn up(token: &str, seq: u64, confirmed: &str, lane: &str) -> Vec<u8> {
|
||||||
|
carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/up",
|
||||||
|
token,
|
||||||
|
&format!("Content-Type: application/octet-stream\r\nX-Up-Seq: {seq}\r\n{confirmed}{lane}"),
|
||||||
|
&frame::encode(FrameType::Pong, 0, &[]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn conveyor_negotiation_is_opt_in_bounded_and_invisible_to_native_welcome() {
|
||||||
|
for enabled in [false, true] {
|
||||||
|
for carrier in [
|
||||||
|
WebCarrier::Https,
|
||||||
|
WebCarrier::HttpsLanes,
|
||||||
|
WebCarrier::Websocket,
|
||||||
|
] {
|
||||||
|
let capability = [71; 32];
|
||||||
|
let mut config = runtime_config(capability, carrier);
|
||||||
|
config.web.conveyor = enabled;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime =
|
||||||
|
WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
for offered in [None, Some(1), Some(2), Some(4)] {
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
let offer = offered.map_or(String::new(), |value| {
|
||||||
|
format!("X-Telemt-Up-Window: {value}\r\n")
|
||||||
|
});
|
||||||
|
let response = create_with_headers(&listener, &runtime, bootstrap, &offer).await;
|
||||||
|
let (headers, body) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(body, frame::encode(FrameType::Welcome, 0, &[]));
|
||||||
|
if let Some(offered) = offered {
|
||||||
|
let window = if enabled && !carrier.uses_websocket() {
|
||||||
|
offered
|
||||||
|
} else {
|
||||||
|
1
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "x-telemt-up-window"),
|
||||||
|
window.to_string()
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert!(
|
||||||
|
!String::from_utf8_lossy(headers)
|
||||||
|
.to_lowercase()
|
||||||
|
.contains("x-telemt-up-window")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let replay = create_with_headers(&listener, &runtime, bootstrap, &offer).await;
|
||||||
|
let (replayed, _) = split_response(&replay);
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "x-session-token"),
|
||||||
|
response_header(replayed, "x-session-token")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn malformed_window_does_not_consume_bootstrap_or_enable_a_conveyor() {
|
||||||
|
let capability = [72; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
for offer in ["0", "5", "04", "-1", "4, 4", "4\r\nX-Telemt-Up-Window: 4"] {
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap,
|
||||||
|
&format!("X-Telemt-Up-Window: {offer}\r\n"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
response.starts_with(b"HTTP/1.1 404"),
|
||||||
|
"{offer}: {:?}",
|
||||||
|
response
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let response =
|
||||||
|
create_with_headers(&listener, &runtime, bootstrap, "X-Telemt-Up-Window: 4\r\n").await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 200"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reload_changes_only_new_sessions_and_exact_creation_replay_keeps_its_window() {
|
||||||
|
let capability = [73; 32];
|
||||||
|
let config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
let generation = test_runtime_generation(1, config.clone());
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
let offer = "X-Telemt-Up-Window: 4\r\n";
|
||||||
|
let original = create_with_headers(&listener, &runtime, bootstrap, offer).await;
|
||||||
|
let (original_headers, _) = split_response(&original);
|
||||||
|
let token = response_header(original_headers, "x-session-token");
|
||||||
|
let mut disabled = config;
|
||||||
|
disabled.web.conveyor = false;
|
||||||
|
let replacement = test_runtime_generation(2, disabled);
|
||||||
|
runtime.activate_generation(Arc::clone(&replacement));
|
||||||
|
let replay = create_with_headers(&listener, &runtime, bootstrap, offer).await;
|
||||||
|
let (headers, _) = split_response(&replay);
|
||||||
|
assert_eq!(response_header(headers, "x-telemt-up-window"), "4");
|
||||||
|
assert_eq!(response_header(headers, "x-session-token"), token);
|
||||||
|
let old_up = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(old_up.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let created = create_with_headers(&listener, &runtime, bootstrap_from(&page), offer).await;
|
||||||
|
let (headers, _) = split_response(&created);
|
||||||
|
assert_eq!(response_header(headers, "x-telemt-up-window"), "1");
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let mode_error = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(mode_error.starts_with(b"HTTP/1.1 409"));
|
||||||
|
let legacy = request(&listener, &runtime, up(token, 1, "", "")).await;
|
||||||
|
assert!(legacy.starts_with(b"HTTP/1.1 204"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
for generation in [generation, replacement] {
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn future_http_request_is_not_acked_until_head_applies_for_both_carriers() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [74; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, carrier));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let created = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&created);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let (mut tail, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
tail.write_all(&up(token, 2, "X-Telemt-Up-Confirmed: 0\r\n", lane))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_millis(20), tail.read_u8())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", lane),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let response = read_http_response(&mut tail).await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert_eq!(response_header(headers, "x-up-ack"), "2");
|
||||||
|
cancellation.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn parked_uplink_keeps_its_session_deadline_after_reload() {
|
||||||
|
let capability = [75; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.timeouts.body_secs = 4;
|
||||||
|
let generation = test_runtime_generation(1, config.clone());
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
config.web.timeouts.body_secs = 1;
|
||||||
|
config.web.timeouts.http_idle_secs = 2;
|
||||||
|
let replacement = test_runtime_generation(2, config);
|
||||||
|
runtime.activate_generation(Arc::clone(&replacement));
|
||||||
|
let (mut tail, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
tail.write_all(&up(token, 2, "X-Telemt-Up-Confirmed: 0\r\n", ""))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(2200)).await;
|
||||||
|
assert!(
|
||||||
|
!task.is_finished(),
|
||||||
|
"connection idle timeout must not replace the frozen parked-UP deadline"
|
||||||
|
);
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert!(
|
||||||
|
read_http_response(&mut tail)
|
||||||
|
.await
|
||||||
|
.starts_with(b"HTTP/1.1 204")
|
||||||
|
);
|
||||||
|
cancellation.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
for generation in [generation, replacement] {
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_long_poll_cannot_steal_the_reserved_head_body_capacity() {
|
||||||
|
let capability = [76; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.limits.max_body_readers = 3;
|
||||||
|
config.web.limits.max_body_bytes = 32;
|
||||||
|
config.web.limits.max_body_bytes_global = 96;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let (mut down, down_cancel, down_task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
down.write_all(&carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/down",
|
||||||
|
token,
|
||||||
|
"X-Down-Cursor: 0\r\n",
|
||||||
|
&[],
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let used = |resource| {
|
||||||
|
runtime
|
||||||
|
.capacity_snapshot()
|
||||||
|
.resources
|
||||||
|
.into_iter()
|
||||||
|
.find(|value| value.resource == resource)
|
||||||
|
.unwrap()
|
||||||
|
.used
|
||||||
|
};
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||||
|
while used("lane_polls") != 1 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
used("body_bytes"),
|
||||||
|
0,
|
||||||
|
"an empty long poll owns no buffered request body"
|
||||||
|
);
|
||||||
|
let mut tails = Vec::new();
|
||||||
|
for sequence in [2, 3] {
|
||||||
|
let (mut client, cancel, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
client
|
||||||
|
.write_all(&up(token, sequence, "X-Telemt-Up-Confirmed: 0\r\n", ""))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tails.push((client, cancel, task));
|
||||||
|
}
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||||
|
while used("body_bytes") != 64 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
for (mut client, cancel, task) in tails {
|
||||||
|
assert!(
|
||||||
|
read_http_response(&mut client)
|
||||||
|
.await
|
||||||
|
.starts_with(b"HTTP/1.1 204")
|
||||||
|
);
|
||||||
|
cancel.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
}
|
||||||
|
down_cancel.cancel();
|
||||||
|
down_task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
@@ -345,6 +345,10 @@ fn remove_hop_by_hop(headers: &mut hyper::HeaderMap) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "decoy_dns_tests.rs"]
|
||||||
|
mod decoy_dns_tests;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
use crate::config::WebCarrier;
|
||||||
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::http::tests::{request, runtime_config};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn decoy_dns_pinned_socket_keeps_configured_hostname_in_actual_http_request() {
|
||||||
|
let origin = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let mut config = runtime_config([17; 32], WebCarrier::Https);
|
||||||
|
let snapshot = Arc::get_mut(config.web.runtime.as_mut().unwrap()).unwrap();
|
||||||
|
let vhost = Arc::get_mut(snapshot.vhosts.get_mut("proxy.example.com").unwrap()).unwrap();
|
||||||
|
// A public hostname deliberately paired with a loopback socket proves no request-time lookup.
|
||||||
|
vhost.decoy = WebRuntimeDecoy::HttpUpstream {
|
||||||
|
addr: origin.local_addr().unwrap(),
|
||||||
|
authority: "bsi.bund.de:8080".to_string(),
|
||||||
|
};
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(generation.clone())));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let receive = async {
|
||||||
|
let (mut socket, _) = origin.accept().await.unwrap();
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let mut buf = [0; 1024];
|
||||||
|
while !bytes.windows(4).any(|part| part == b"\r\n\r\n") {
|
||||||
|
let count = socket.read(&mut buf).await.unwrap();
|
||||||
|
assert_ne!(count, 0);
|
||||||
|
bytes.extend_from_slice(&buf[..count]);
|
||||||
|
}
|
||||||
|
socket
|
||||||
|
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
String::from_utf8(bytes).unwrap()
|
||||||
|
};
|
||||||
|
let (response, upstream) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||||
|
tokio::join!(request(&listener, &runtime, b"GET /site?q=1 HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: close\r\n\r\n".to_vec()), receive)
|
||||||
|
}).await.unwrap();
|
||||||
|
assert!(upstream.starts_with("GET /site?q=1 HTTP/1.1\r\n"));
|
||||||
|
assert!(
|
||||||
|
upstream
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.contains("\r\nhost: bsi.bund.de:8080\r\n")
|
||||||
|
);
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert!(response.ends_with(b"OK"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
@@ -65,6 +65,8 @@ pub(super) async fn handle_down(
|
|||||||
if !body.is_empty() {
|
if !body.is_empty() {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
|
// Empty polls must not retain even one byte from a conveyor head's body reserve.
|
||||||
|
drop(_body_budget);
|
||||||
let Some(_down_poll) = runtime.try_lane_poll(false) else {
|
let Some(_down_poll) = runtime.try_lane_poll(false) else {
|
||||||
return service_unavailable();
|
return service_unavailable();
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -76,6 +76,17 @@ pub(super) async fn handle_session(
|
|||||||
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
|
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
|
let up_window = if request.headers().contains_key("x-telemt-up-window") {
|
||||||
|
let Some(window) = super::request::canonical_u64_header(&request, "x-telemt-up-window")
|
||||||
|
.filter(|window| (1..=4).contains(window))
|
||||||
|
else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
Some(window as u8)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let carrier_request = carrier_request.with_up_window(up_window);
|
||||||
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
||||||
let Some((trace_session_id, profile, body_timeout)) =
|
let Some((trace_session_id, profile, body_timeout)) =
|
||||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||||
@@ -124,6 +135,13 @@ pub(super) async fn handle_session(
|
|||||||
}
|
}
|
||||||
let mut response = full_response(StatusCode::OK, welcome);
|
let mut response = full_response(StatusCode::OK, welcome);
|
||||||
carrier_headers(&mut response);
|
carrier_headers(&mut response);
|
||||||
|
if let Some(window) = result.up_window {
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-telemt-up-window"),
|
||||||
|
&window.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
insert_header(
|
insert_header(
|
||||||
&mut response,
|
&mut response,
|
||||||
HeaderName::from_static("x-session-token"),
|
HeaderName::from_static("x-session-token"),
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::web::session::ConveyorError;
|
||||||
|
|
||||||
|
/// Handles one bounded legacy or explicitly negotiated conveyor uplink.
|
||||||
|
pub(super) async fn handle_up(
|
||||||
|
request: Request<RequestBody>,
|
||||||
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
|
token_hash: crate::web::manager::TokenHash,
|
||||||
|
) -> HttpResponse {
|
||||||
|
if !matches!(*request.method(), Method::POST | Method::PUT) || !binary_content_type(&request) {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
let Some(sequence) = canonical_u64_header(&request, "x-up-seq").filter(|value| *value != 0)
|
||||||
|
else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
if session.carrier().uses_websocket() {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Uplink);
|
||||||
|
trace.bind_identity(session.trace_identity());
|
||||||
|
}
|
||||||
|
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
|
||||||
|
let confirmed = if request.headers().contains_key("x-telemt-up-confirmed") {
|
||||||
|
let Some(value) = canonical_u64_header(&request, "x-telemt-up-confirmed") else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
Some(value)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let claim = match session.claim_conveyor(lane_id, sequence, confirmed) {
|
||||||
|
Ok(claim) => claim,
|
||||||
|
Err(ConveyorError::Stale | ConveyorError::Mode) => {
|
||||||
|
return carrier_empty(StatusCode::CONFLICT);
|
||||||
|
}
|
||||||
|
Err(ConveyorError::Manager(
|
||||||
|
ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit,
|
||||||
|
)) => {
|
||||||
|
return service_unavailable();
|
||||||
|
}
|
||||||
|
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||||
|
};
|
||||||
|
let limit = session.limits().max_body_bytes;
|
||||||
|
let CollectedBody {
|
||||||
|
request,
|
||||||
|
body,
|
||||||
|
_body_budget,
|
||||||
|
} = match collect_body(
|
||||||
|
request,
|
||||||
|
&runtime,
|
||||||
|
Duration::from_secs(session.timeouts().body_secs),
|
||||||
|
limit,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||||
|
Err(CollectBodyError::Invalid(request)) => {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.record_frames(TraceDirection::Request, &body, session.limits());
|
||||||
|
}
|
||||||
|
let result = if let Some(claim) = claim {
|
||||||
|
let _deadline_lease = match super::request_deadline(&request) {
|
||||||
|
Some(deadline) => {
|
||||||
|
match deadline.lease_for(Duration::from_secs(session.timeouts().body_secs)) {
|
||||||
|
Some(lease) => Some(lease),
|
||||||
|
None => return service_unavailable(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
claim.process(&body).await
|
||||||
|
} else {
|
||||||
|
match lane_id {
|
||||||
|
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
||||||
|
None => session.process_up(sequence, &body),
|
||||||
|
}
|
||||||
|
.map_err(ConveyorError::Manager)
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(ack) => {
|
||||||
|
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-up-ack"),
|
||||||
|
&ack.to_string(),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
Err(ConveyorError::Stale | ConveyorError::Mode) => carrier_empty(StatusCode::CONFLICT),
|
||||||
|
Err(ConveyorError::Manager(
|
||||||
|
ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit,
|
||||||
|
)) => service_unavailable(),
|
||||||
|
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -177,6 +177,8 @@ impl ManagerError {
|
|||||||
|
|
||||||
/// Successful idempotent session creation result.
|
/// Successful idempotent session creation result.
|
||||||
pub(crate) struct CreateResult {
|
pub(crate) struct CreateResult {
|
||||||
|
/// HTTP window echoed only to bridge pages advertising the extension.
|
||||||
|
pub(crate) up_window: Option<u8>,
|
||||||
/// Opaque bearer token for the created or replayed session.
|
/// Opaque bearer token for the created or replayed session.
|
||||||
pub(crate) token: String,
|
pub(crate) token: String,
|
||||||
/// Carrier frozen into the created or replayed session.
|
/// Carrier frozen into the created or replayed session.
|
||||||
@@ -230,6 +232,7 @@ pub(crate) struct WebProcessRuntime {
|
|||||||
lane_aux_polls: Arc<Semaphore>,
|
lane_aux_polls: Arc<Semaphore>,
|
||||||
body_readers: Arc<Semaphore>,
|
body_readers: Arc<Semaphore>,
|
||||||
body_bytes: Arc<Semaphore>,
|
body_bytes: Arc<Semaphore>,
|
||||||
|
conveyor_waiters: Arc<Semaphore>,
|
||||||
stream_handshakes: Arc<Semaphore>,
|
stream_handshakes: Arc<Semaphore>,
|
||||||
websocket_connections: Arc<Semaphore>,
|
websocket_connections: Arc<Semaphore>,
|
||||||
websockets: Mutex<websocket::WebSocketRegistry>,
|
websockets: Mutex<websocket::WebSocketRegistry>,
|
||||||
@@ -299,6 +302,9 @@ impl WebProcessRuntime {
|
|||||||
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
|
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
|
||||||
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
||||||
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
||||||
|
conveyor_waiters: Arc::new(Semaphore::new(crate::web::session::conveyor_waiter_limit(
|
||||||
|
&limits,
|
||||||
|
))),
|
||||||
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
||||||
websocket_connections: Arc::new(Semaphore::new(websocket_connections)),
|
websocket_connections: Arc::new(Semaphore::new(websocket_connections)),
|
||||||
websockets: Mutex::new(websocket::WebSocketRegistry::default()),
|
websockets: Mutex::new(websocket::WebSocketRegistry::default()),
|
||||||
|
|||||||
@@ -337,6 +337,11 @@ impl Drop for WebSocketBudgetLease {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WebProcessRuntime {
|
impl WebProcessRuntime {
|
||||||
|
/// Reserves a parked future uplink without consuming the head request's capacity.
|
||||||
|
pub(crate) fn try_conveyor_waiter(&self) -> Option<OwnedSemaphorePermit> {
|
||||||
|
Arc::clone(&self.conveyor_waiters).try_acquire_owned().ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Reserves one body reader and its declared bounded body allocation.
|
/// Reserves one body reader and its declared bounded body allocation.
|
||||||
pub(crate) fn try_body_budget(
|
pub(crate) fn try_body_budget(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ impl WebProcessRuntime {
|
|||||||
self.lane_aux_polls.close();
|
self.lane_aux_polls.close();
|
||||||
self.body_readers.close();
|
self.body_readers.close();
|
||||||
self.body_bytes.close();
|
self.body_bytes.close();
|
||||||
|
self.conveyor_waiters.close();
|
||||||
self.stream_handshakes.close();
|
self.stream_handshakes.close();
|
||||||
self.websocket_connections.close();
|
self.websocket_connections.close();
|
||||||
let sessions = {
|
let sessions = {
|
||||||
|
|||||||
@@ -130,6 +130,7 @@ pub(crate) struct CarrierRequest {
|
|||||||
attempt: Option<u8>,
|
attempt: Option<u8>,
|
||||||
failure: Option<CarrierFailure>,
|
failure: Option<CarrierFailure>,
|
||||||
user_agent_hash: [u8; 32],
|
user_agent_hash: [u8; 32],
|
||||||
|
up_window: Option<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CarrierRequest {
|
impl CarrierRequest {
|
||||||
@@ -141,6 +142,7 @@ impl CarrierRequest {
|
|||||||
attempt: None,
|
attempt: None,
|
||||||
failure: None,
|
failure: None,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,6 +154,7 @@ impl CarrierRequest {
|
|||||||
attempt: None,
|
attempt: None,
|
||||||
failure: None,
|
failure: None,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,9 +172,21 @@ impl CarrierRequest {
|
|||||||
attempt: Some(attempt),
|
attempt: Some(attempt),
|
||||||
failure,
|
failure,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attaches the separately validated HTTP conveyor capability.
|
||||||
|
pub(crate) const fn with_up_window(mut self, window: Option<u8>) -> Self {
|
||||||
|
self.up_window = window;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the separately advertised HTTP conveyor capability.
|
||||||
|
pub(crate) const fn up_window(self) -> Option<u8> {
|
||||||
|
self.up_window
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns whether this request participates in server-side negotiation.
|
/// Returns whether this request participates in server-side negotiation.
|
||||||
pub(crate) const fn is_automatic(self) -> bool {
|
pub(crate) const fn is_automatic(self) -> bool {
|
||||||
self.capabilities.is_some()
|
self.capabilities.is_some()
|
||||||
@@ -215,6 +230,7 @@ impl CarrierRequest {
|
|||||||
self.class == other.class
|
self.class == other.class
|
||||||
&& self.capabilities_bits() == other.capabilities_bits()
|
&& self.capabilities_bits() == other.capabilities_bits()
|
||||||
&& self.user_agent_hash == other.user_agent_hash
|
&& self.user_agent_hash == other.user_agent_hash
|
||||||
|
&& self.up_window == other.up_window
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks the complete idempotent identity of one exact attempt request.
|
/// Checks the complete idempotent identity of one exact attempt request.
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ impl WebProcessRuntime {
|
|||||||
entry.carrier_phase.as_str()
|
entry.carrier_phase.as_str()
|
||||||
};
|
};
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: carrier_request.up_window().map(|_| session.up_window()),
|
||||||
token: entry.session_token.as_str().to_owned(),
|
token: entry.session_token.as_str().to_owned(),
|
||||||
carrier: session.carrier(),
|
carrier: session.carrier(),
|
||||||
attempt: carrier_request.attempt(),
|
attempt: carrier_request.attempt(),
|
||||||
@@ -357,6 +358,11 @@ impl WebProcessRuntime {
|
|||||||
issued_timeouts.clone(),
|
issued_timeouts.clone(),
|
||||||
Some(user_registration),
|
Some(user_registration),
|
||||||
);
|
);
|
||||||
|
session.configure_conveyor(if config.web.conveyor {
|
||||||
|
carrier_request.up_window().unwrap_or(1)
|
||||||
|
} else {
|
||||||
|
1
|
||||||
|
});
|
||||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||||
@@ -399,6 +405,7 @@ impl WebProcessRuntime {
|
|||||||
}
|
}
|
||||||
let identity = session.trace_identity();
|
let identity = session.trace_identity();
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: carrier_request.up_window().map(|_| session.up_window()),
|
||||||
token: session_token,
|
token: session_token,
|
||||||
carrier,
|
carrier,
|
||||||
attempt: carrier_request.attempt(),
|
attempt: carrier_request.attempt(),
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ impl WebProcessRuntime {
|
|||||||
replacement.old_session.timeouts().clone(),
|
replacement.old_session.timeouts().clone(),
|
||||||
Some(user_registration),
|
Some(user_registration),
|
||||||
);
|
);
|
||||||
|
session.configure_conveyor(replacement.old_session.conveyor_offer());
|
||||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||||
drop(user_publication);
|
drop(user_publication);
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -149,6 +150,7 @@ impl WebProcessRuntime {
|
|||||||
crate::web::session::SessionCloseReason::CarrierSuperseded,
|
crate::web::session::SessionCloseReason::CarrierSuperseded,
|
||||||
);
|
);
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: replacement.request.up_window().map(|_| session.up_window()),
|
||||||
token: session_token,
|
token: session_token,
|
||||||
carrier: replacement.carrier,
|
carrier: replacement.carrier,
|
||||||
attempt: Some(replacement.attempt),
|
attempt: Some(replacement.attempt),
|
||||||
|
|||||||
@@ -54,6 +54,9 @@ use lifecycle::SessionNegotiationPhase;
|
|||||||
pub(crate) use lifecycle::{SessionCloseOutcome, SessionCloseReason};
|
pub(crate) use lifecycle::{SessionCloseOutcome, SessionCloseReason};
|
||||||
// Uplink batches own exactly-once sequencing and client-frame validation.
|
// Uplink batches own exactly-once sequencing and client-frame validation.
|
||||||
mod uplink;
|
mod uplink;
|
||||||
|
// HTTP conveyor ownership bounds reordered requests without changing native frames.
|
||||||
|
mod conveyor;
|
||||||
|
pub(crate) use conveyor::{ConveyorError, conveyor_waiter_limit};
|
||||||
// Logical stream polling owns cancellation-safe waker registration.
|
// Logical stream polling owns cancellation-safe waker registration.
|
||||||
mod stream_io;
|
mod stream_io;
|
||||||
|
|
||||||
@@ -160,6 +163,7 @@ impl CarrierLane {
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct SessionState {
|
struct SessionState {
|
||||||
|
conveyor: conveyor::ConveyorState,
|
||||||
streams: HashMap<u32, StreamState>,
|
streams: HashMap<u32, StreamState>,
|
||||||
closing_streams: HashMap<u32, u64>,
|
closing_streams: HashMap<u32, u64>,
|
||||||
next_stream_instance: u64,
|
next_stream_instance: u64,
|
||||||
@@ -295,6 +299,7 @@ impl WebSession {
|
|||||||
timeouts,
|
timeouts,
|
||||||
_user_registration: user_registration,
|
_user_registration: user_registration,
|
||||||
state: Mutex::new(SessionState {
|
state: Mutex::new(SessionState {
|
||||||
|
conveyor: conveyor::ConveyorState::default(),
|
||||||
streams: HashMap::new(),
|
streams: HashMap::new(),
|
||||||
closing_streams: HashMap::new(),
|
closing_streams: HashMap::new(),
|
||||||
next_stream_instance: 1,
|
next_stream_instance: 1,
|
||||||
|
|||||||
@@ -0,0 +1,421 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use subtle::ConstantTimeEq;
|
||||||
|
use tokio::sync::{Notify, OwnedSemaphorePermit};
|
||||||
|
|
||||||
|
use super::{SessionCloseReason, SessionNegotiationPhase, WebSession};
|
||||||
|
use crate::config::WebLimitsConfig;
|
||||||
|
use crate::web::manager::{ManagerError, TokenHash};
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
|
const MAX_WINDOW: usize = 4;
|
||||||
|
|
||||||
|
/// Preserves one head body and handler capacity beyond the bounded down-poll plane.
|
||||||
|
pub(crate) fn conveyor_waiter_limit(limits: &WebLimitsConfig) -> usize {
|
||||||
|
(limits.max_http_handlers / 4)
|
||||||
|
.min(limits.max_body_readers.saturating_sub(1))
|
||||||
|
.min((limits.max_body_bytes_global / limits.max_body_bytes.max(1)).saturating_sub(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// HTTP-only failures distinguish expired exchanges from terminal protocol faults.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum ConveyorError {
|
||||||
|
/// The bridge already confirmed this sequence, so its digest was retired.
|
||||||
|
Stale,
|
||||||
|
/// A negotiated mode cannot change within the same bearer session.
|
||||||
|
Mode,
|
||||||
|
/// Existing carrier failure semantics remain authoritative.
|
||||||
|
Manager(ManagerError),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ManagerError> for ConveyorError {
|
||||||
|
fn from(error: ManagerError) -> Self {
|
||||||
|
Self::Manager(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct Slot {
|
||||||
|
sequence: u64,
|
||||||
|
owner: u64,
|
||||||
|
digest: Option<TokenHash>,
|
||||||
|
committed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Channel {
|
||||||
|
confirmed: u64,
|
||||||
|
committed: u64,
|
||||||
|
slots: [Option<Slot>; MAX_WINDOW],
|
||||||
|
notify: Arc<Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Channel {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
confirmed: 0,
|
||||||
|
committed: 0,
|
||||||
|
slots: [None; MAX_WINDOW],
|
||||||
|
notify: Arc::new(Notify::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Session-owned replay metadata; bodies stay in their bounded HTTP handlers.
|
||||||
|
pub(super) struct ConveyorState {
|
||||||
|
offered: u8,
|
||||||
|
mode: Option<bool>,
|
||||||
|
next_owner: u64,
|
||||||
|
channels: HashMap<Option<u32>, Channel>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ConveyorState {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
offered: 1,
|
||||||
|
mode: None,
|
||||||
|
next_owner: 1,
|
||||||
|
channels: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConveyorState {
|
||||||
|
/// Publishes commit history under the same lock as frame application.
|
||||||
|
pub(super) fn commit(&mut self, lane: Option<u32>, sequence: u64, digest: TokenHash) {
|
||||||
|
if self.mode != Some(true) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(channel) = self.channels.get_mut(&lane) {
|
||||||
|
let index = sequence as usize % MAX_WINDOW;
|
||||||
|
channel.slots[index] = Some(Slot {
|
||||||
|
sequence,
|
||||||
|
digest: Some(digest),
|
||||||
|
committed: true,
|
||||||
|
owner: 0,
|
||||||
|
});
|
||||||
|
channel.committed = sequence;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detaches retired-lane ordering state for notification after unlocking.
|
||||||
|
pub(super) fn remove(&mut self, lane: Option<u32>) -> Option<Arc<Notify>> {
|
||||||
|
self.channels.remove(&lane).map(|channel| channel.notify)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detaches all waiter notifications for the session's deferred close effects.
|
||||||
|
pub(super) fn clear(&mut self) -> Vec<Arc<Notify>> {
|
||||||
|
self.channels
|
||||||
|
.drain()
|
||||||
|
.map(|(_, channel)| channel.notify)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An exact request owner; its body and ordinary body permit stay in the HTTP handler.
|
||||||
|
pub(crate) struct ConveyorClaim {
|
||||||
|
session: Arc<WebSession>,
|
||||||
|
lane: Option<u32>,
|
||||||
|
sequence: u64,
|
||||||
|
owner: u64,
|
||||||
|
notify: Arc<Notify>,
|
||||||
|
_waiter: Option<OwnedSemaphorePermit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebSession {
|
||||||
|
/// Freezes the offer before the newly created session is published by its manager.
|
||||||
|
pub(crate) fn configure_conveyor(&self, requested: u8) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
if state.conveyor.mode.is_none() {
|
||||||
|
state.conveyor.offered = if conveyor_waiter_limit(&self.limits) == 0 {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
requested.clamp(1, MAX_WINDOW as u8)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the immutable chain ceiling even across a WebSocket candidate.
|
||||||
|
pub(crate) fn conveyor_offer(&self) -> u8 {
|
||||||
|
self.state.lock().conveyor.offered
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the HTTP window without changing carrier capability negotiation.
|
||||||
|
pub(crate) fn up_window(&self) -> u8 {
|
||||||
|
if self.carrier().uses_websocket() {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
self.conveyor_offer()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claims an HTTP sequence before allocating a body, or locks in legacy serialization.
|
||||||
|
pub(crate) fn claim_conveyor(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
lane: Option<u32>,
|
||||||
|
sequence: u64,
|
||||||
|
confirmed: Option<u64>,
|
||||||
|
) -> Result<Option<ConveyorClaim>, ConveyorError> {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
|
if state.closed {
|
||||||
|
return Err(ManagerError::Closed.into());
|
||||||
|
}
|
||||||
|
let enabled = confirmed.is_some();
|
||||||
|
if (enabled && state.conveyor.offered <= 1)
|
||||||
|
|| state.conveyor.mode.is_some_and(|mode| mode != enabled)
|
||||||
|
{
|
||||||
|
return Err(ConveyorError::Mode);
|
||||||
|
}
|
||||||
|
if !enabled {
|
||||||
|
state.conveyor.mode = Some(false);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let confirmed = confirmed.unwrap_or(0);
|
||||||
|
let window = u64::from(state.conveyor.offered);
|
||||||
|
let existing = state.conveyor.channels.get(&lane);
|
||||||
|
// Lane eviction retires its bounded digest history, not the parent session.
|
||||||
|
if existing.is_none() && lane.is_some() && confirmed > 0 {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let committed = existing.map_or(0, |channel| channel.committed);
|
||||||
|
let floor = existing
|
||||||
|
.map_or(0, |channel| channel.confirmed)
|
||||||
|
.max(confirmed);
|
||||||
|
if confirmed > committed || sequence == 0 || sequence > floor.saturating_add(window) {
|
||||||
|
drop(state);
|
||||||
|
self.close(SessionCloseReason::Protocol);
|
||||||
|
return Err(ManagerError::Protocol.into());
|
||||||
|
}
|
||||||
|
if sequence <= floor {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let index = sequence as usize % MAX_WINDOW;
|
||||||
|
if existing
|
||||||
|
.and_then(|channel| channel.slots[index])
|
||||||
|
.is_some_and(|slot| slot.sequence == sequence && !slot.committed)
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Concurrent.into());
|
||||||
|
}
|
||||||
|
if existing.is_none() {
|
||||||
|
let limit = self
|
||||||
|
.profile
|
||||||
|
.max_streams_per_session
|
||||||
|
.saturating_add(self.limits.max_tombstones_per_session)
|
||||||
|
.saturating_add(1);
|
||||||
|
let provisional = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.keys()
|
||||||
|
.filter(|id| id.is_some_and(|id| !state.carrier_lanes.contains_key(&id)))
|
||||||
|
.count();
|
||||||
|
if state.conveyor.channels.len() >= limit
|
||||||
|
|| (lane.is_some_and(|id| !state.carrier_lanes.contains_key(&id))
|
||||||
|
&& state.carrier_lanes.len().saturating_add(provisional) >= limit)
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Limit.into());
|
||||||
|
}
|
||||||
|
if lane.is_some_and(|id| {
|
||||||
|
state.closed_streams.contains(&id) && !state.carrier_lanes.contains_key(&id)
|
||||||
|
}) {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let owner = state.conveyor.next_owner;
|
||||||
|
let next_owner = owner.checked_add(1).ok_or(ManagerError::Limit)?;
|
||||||
|
// All fallible validation precedes the permit so no early return drops it under state.
|
||||||
|
let waiter = if sequence > committed.saturating_add(1) {
|
||||||
|
Some(
|
||||||
|
self.manager
|
||||||
|
.upgrade()
|
||||||
|
.ok_or(ManagerError::Closed)?
|
||||||
|
.try_conveyor_waiter()
|
||||||
|
.ok_or(ManagerError::Backpressure)?,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
state.conveyor.mode = Some(true);
|
||||||
|
state.conveyor.next_owner = next_owner;
|
||||||
|
let channel = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.entry(lane)
|
||||||
|
.or_insert_with(Channel::new);
|
||||||
|
channel.confirmed = floor;
|
||||||
|
for slot in &mut channel.slots {
|
||||||
|
if slot.is_some_and(|slot| slot.sequence <= floor) {
|
||||||
|
*slot = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let replay = sequence <= channel.committed;
|
||||||
|
if !replay {
|
||||||
|
channel.slots[index] = Some(Slot {
|
||||||
|
sequence,
|
||||||
|
owner,
|
||||||
|
digest: None,
|
||||||
|
committed: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(Some(ConveyorClaim {
|
||||||
|
session: Arc::clone(self),
|
||||||
|
lane,
|
||||||
|
sequence,
|
||||||
|
owner: if replay { 0 } else { owner },
|
||||||
|
notify: Arc::clone(&channel.notify),
|
||||||
|
_waiter: waiter,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConveyorClaim {
|
||||||
|
/// Revalidates exact ownership under the frame-application transaction's lock.
|
||||||
|
pub(super) fn validate_locked(
|
||||||
|
&self,
|
||||||
|
state: &super::SessionState,
|
||||||
|
digest: &TokenHash,
|
||||||
|
) -> Result<(), ManagerError> {
|
||||||
|
let channel = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
.ok_or(ManagerError::Closed)?;
|
||||||
|
let slot = channel.slots[self.sequence as usize % MAX_WINDOW]
|
||||||
|
.filter(|slot| {
|
||||||
|
slot.sequence == self.sequence
|
||||||
|
&& slot.owner == self.owner
|
||||||
|
&& !slot.committed
|
||||||
|
&& slot
|
||||||
|
.digest
|
||||||
|
.is_some_and(|value| bool::from(value.ct_eq(digest)))
|
||||||
|
})
|
||||||
|
.ok_or(ManagerError::Closed)?;
|
||||||
|
if channel.committed.checked_add(1) != Some(slot.sequence) {
|
||||||
|
return Err(ManagerError::Concurrent);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies only the next sequence; duplicates verify the retained digest without replaying frames.
|
||||||
|
pub(crate) async fn process(&self, body: &[u8]) -> Result<u64, ConveyorError> {
|
||||||
|
let digest: TokenHash = Sha256::digest(body).into();
|
||||||
|
let operation = async {
|
||||||
|
loop {
|
||||||
|
let notified = self.notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
let ready = {
|
||||||
|
let mut state = self.session.state.lock();
|
||||||
|
self.session.ensure_carrier_active_locked(&state)?;
|
||||||
|
if state.closed {
|
||||||
|
return Err(ManagerError::Closed.into());
|
||||||
|
}
|
||||||
|
let Some(channel) = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get_mut(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
else {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
};
|
||||||
|
if self.sequence <= channel.confirmed {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let Some(slot) = &mut channel.slots[self.sequence as usize % MAX_WINDOW] else {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
};
|
||||||
|
if slot.sequence != self.sequence
|
||||||
|
|| (!slot.committed && slot.owner != self.owner)
|
||||||
|
{
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
if slot
|
||||||
|
.digest
|
||||||
|
.is_some_and(|previous| !bool::from(previous.ct_eq(&digest)))
|
||||||
|
{
|
||||||
|
drop(state);
|
||||||
|
self.session.close(SessionCloseReason::Protocol);
|
||||||
|
return Err(ManagerError::Protocol.into());
|
||||||
|
}
|
||||||
|
slot.digest = Some(digest);
|
||||||
|
if slot.committed {
|
||||||
|
self.session.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
|
if self.session.automatic_carrier
|
||||||
|
&& state.negotiation_phase != SessionNegotiationPhase::Committed
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Backpressure.into());
|
||||||
|
}
|
||||||
|
return Ok(self.sequence);
|
||||||
|
}
|
||||||
|
channel.committed.checked_add(1) == Some(self.sequence)
|
||||||
|
&& (self.lane.is_some()
|
||||||
|
|| !self
|
||||||
|
.session
|
||||||
|
.up_active
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire))
|
||||||
|
};
|
||||||
|
if ready {
|
||||||
|
let result = match self.lane {
|
||||||
|
Some(lane) => self.session.process_up_lane_inner(
|
||||||
|
lane,
|
||||||
|
self.sequence,
|
||||||
|
body,
|
||||||
|
Some((self, digest)),
|
||||||
|
),
|
||||||
|
None => self
|
||||||
|
.session
|
||||||
|
.process_claimed_up(self.sequence, body, self, digest),
|
||||||
|
};
|
||||||
|
self.notify.notify_waiters();
|
||||||
|
return result.map_err(ConveyorError::from);
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
_ = self.session.cancel.cancelled() => return Err(ManagerError::Closed.into()),
|
||||||
|
_ = notified => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_secs(self.session.timeouts.body_secs),
|
||||||
|
operation,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ConveyorError::Manager(ManagerError::Backpressure))?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ConveyorClaim {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.owner == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut state = self.session.state.lock();
|
||||||
|
if let Some(channel) = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get_mut(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
{
|
||||||
|
let slot = &mut channel.slots[self.sequence as usize % MAX_WINDOW];
|
||||||
|
if slot.is_some_and(|slot| !slot.committed && slot.owner == self.owner) {
|
||||||
|
*slot = None;
|
||||||
|
}
|
||||||
|
if channel.committed == 0 && channel.slots.iter().all(Option::is_none) {
|
||||||
|
state.conveyor.channels.remove(&self.lane);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(state);
|
||||||
|
self.notify.notify_waiters();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "conveyor/tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,524 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::task::{Context, Poll, Wake, Waker};
|
||||||
|
|
||||||
|
use crate::config::{ProxyConfig, WebCarrier, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig};
|
||||||
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::frame::{self, FrameType};
|
||||||
|
use crate::web::manager::{CarrierClientClass, WebProcessRuntime};
|
||||||
|
use crate::web::session::{CarrierLane, StreamState};
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
|
||||||
|
fn fixture(
|
||||||
|
carrier: WebCarrier,
|
||||||
|
limits: WebLimitsConfig,
|
||||||
|
automatic: bool,
|
||||||
|
) -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.web.limits = limits.clone();
|
||||||
|
let manager =
|
||||||
|
WebProcessRuntime::start(Arc::new(ArcSwap::from(test_runtime_generation(1, config))));
|
||||||
|
let profile = Arc::new(WebRuntimeProfile {
|
||||||
|
host: "proxy.example.com".into(),
|
||||||
|
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||||
|
user: "default".into(),
|
||||||
|
secret_mode: WebSecretMode::Plain,
|
||||||
|
carrier,
|
||||||
|
carrier_negotiation_enabled: automatic,
|
||||||
|
carrier_learning: false,
|
||||||
|
carriers: Arc::from([carrier]),
|
||||||
|
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||||
|
capability: [0; 32],
|
||||||
|
credential_id: [0; 16],
|
||||||
|
key_fingerprint: "0000000000000000".into(),
|
||||||
|
max_sessions: 2,
|
||||||
|
max_streams: 2,
|
||||||
|
max_streams_per_session: 2,
|
||||||
|
});
|
||||||
|
let session = WebSession::new(
|
||||||
|
Arc::downgrade(&manager),
|
||||||
|
[1; 32],
|
||||||
|
"192.0.2.1".parse().unwrap(),
|
||||||
|
1,
|
||||||
|
profile,
|
||||||
|
[2; 32],
|
||||||
|
carrier,
|
||||||
|
1,
|
||||||
|
[3; 32],
|
||||||
|
None,
|
||||||
|
if automatic {
|
||||||
|
CarrierClientClass::Bridge
|
||||||
|
} else {
|
||||||
|
CarrierClientClass::Legacy
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
automatic,
|
||||||
|
false,
|
||||||
|
limits,
|
||||||
|
WebTimeoutsConfig::default(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
session.configure_conveyor(4);
|
||||||
|
(session, manager)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn claim(session: &Arc<WebSession>, lane: Option<u32>, seq: u64, ack: u64) -> ConveyorClaim {
|
||||||
|
session
|
||||||
|
.claim_conveyor(lane, seq, Some(ack))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stream(session: &WebSession, carrier: WebCarrier) -> Option<u32> {
|
||||||
|
let mut state = session.state.lock();
|
||||||
|
state.streams.insert(
|
||||||
|
7,
|
||||||
|
StreamState {
|
||||||
|
instance: 1,
|
||||||
|
inbound: VecDeque::new(),
|
||||||
|
receive_window: frame::INITIAL_STREAM_WINDOW,
|
||||||
|
send_credit: 0,
|
||||||
|
read_waker: None,
|
||||||
|
write_waker: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if carrier.uses_lanes() {
|
||||||
|
state.carrier_lanes.insert(7, CarrierLane::new(7));
|
||||||
|
Some(7)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(session: Arc<WebSession>, manager: Arc<WebProcessRuntime>) {
|
||||||
|
session.close(SessionCloseReason::ApiClose);
|
||||||
|
session.wait().await;
|
||||||
|
manager.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reordered_data_and_window_apply_once_in_sequence_for_both_carriers() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let (session, manager) = fixture(carrier, WebLimitsConfig::default(), false);
|
||||||
|
let lane = stream(&session, carrier);
|
||||||
|
let first = frame::encode(FrameType::Data, 7, b"first");
|
||||||
|
let second = frame::encode(FrameType::Data, 7, b"second");
|
||||||
|
let credit = frame::encode(FrameType::Window, 7, &5u32.to_be_bytes());
|
||||||
|
let second_claim = claim(&session, lane, 2, 0);
|
||||||
|
let credit_claim = claim(&session, lane, 3, 0);
|
||||||
|
let mut second_wait = Box::pin(second_claim.process(&second));
|
||||||
|
let mut credit_wait = Box::pin(credit_claim.process(&credit));
|
||||||
|
assert!(futures::poll!(&mut second_wait).is_pending());
|
||||||
|
assert!(futures::poll!(&mut credit_wait).is_pending());
|
||||||
|
assert!(session.state.lock().streams[&7].inbound.is_empty());
|
||||||
|
assert_eq!(claim(&session, lane, 1, 0).process(&first).await, Ok(1));
|
||||||
|
assert!(futures::poll!(&mut credit_wait).is_pending());
|
||||||
|
assert_eq!(second_wait.await, Ok(2));
|
||||||
|
assert_eq!(credit_wait.await, Ok(3));
|
||||||
|
for (seq, body) in [(1, &first), (2, &second), (3, &credit)] {
|
||||||
|
assert_eq!(claim(&session, lane, seq, 0).process(body).await, Ok(seq));
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let state = session.state.lock();
|
||||||
|
let stream = &state.streams[&7];
|
||||||
|
assert_eq!(stream.inbound.len(), 2);
|
||||||
|
assert_eq!(stream.inbound[0].bytes.as_ref(), b"first");
|
||||||
|
assert_eq!(stream.inbound[1].bytes.as_ref(), b"second");
|
||||||
|
assert_eq!(stream.send_credit, 5);
|
||||||
|
assert_eq!(state.conveyor.channels[&lane].committed, 3);
|
||||||
|
}
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn confirmed_floor_retires_only_acknowledged_history_and_never_replays_it() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, b"same");
|
||||||
|
for seq in 1..=4 {
|
||||||
|
assert_eq!(claim(&session, None, seq, 0).process(&body).await, Ok(seq));
|
||||||
|
}
|
||||||
|
// ACK2 through ACK4 do not constitute confirmation of a lost ACK1.
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
assert_eq!(claim(&session, None, 5, 4).process(&body).await, Ok(5));
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 1, Some(0)),
|
||||||
|
Err(ConveyorError::Stale)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
assert_eq!(session.state.lock().last_up_sequence, 5);
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().conveyor.channels[&None]
|
||||||
|
.slots
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn digest_mismatch_and_window_violations_are_terminal() {
|
||||||
|
for invalid in [0, 1, 2] {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, b"first");
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
if invalid == 0 {
|
||||||
|
let changed = frame::encode(FrameType::Pong, 0, b"changed");
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, None, 1, 0).process(&changed).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
let (sequence, confirmed) = if invalid == 1 { (5, 0) } else { (3, 2) };
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, sequence, Some(confirmed)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert!(session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn duplicate_pending_claims_and_cancelled_owners_do_not_consume_or_erase_new_slots() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let first = claim(&session, None, 2, 0);
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 2, Some(0)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Concurrent))
|
||||||
|
));
|
||||||
|
drop(first);
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
let cancelled = claim(&session, None, 2, 0);
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
effects.notify(state.conveyor.remove(None).unwrap());
|
||||||
|
});
|
||||||
|
let replacement = claim(&session, None, 2, 0);
|
||||||
|
drop(cancelled);
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().conveyor.channels[&None].slots[2]
|
||||||
|
.unwrap()
|
||||||
|
.owner,
|
||||||
|
replacement.owner
|
||||||
|
);
|
||||||
|
drop(replacement);
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn missing_head_times_out_without_committing_and_returns_waiter_capacity() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let tail = claim(&session, None, 2, 0);
|
||||||
|
let mut waiting = Box::pin(tail.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
tokio::time::advance(Duration::from_secs(session.timeouts.body_secs)).await;
|
||||||
|
assert_eq!(
|
||||||
|
waiting.await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
);
|
||||||
|
assert_eq!(session.state.lock().last_up_sequence, 0);
|
||||||
|
drop(tail);
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
while let Some(permit) = manager.try_conveyor_waiter() {
|
||||||
|
permits.push(permit);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
permits.len(),
|
||||||
|
conveyor_waiter_limit(&WebLimitsConfig::default())
|
||||||
|
);
|
||||||
|
drop(permits);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn waiter_ceiling_always_preserves_head_handler_reader_and_bytes() {
|
||||||
|
let defaults = WebLimitsConfig::default();
|
||||||
|
assert_eq!(conveyor_waiter_limit(&defaults), 31);
|
||||||
|
for (handlers, readers, bodies, expected) in [
|
||||||
|
(8, 4, 8, 2),
|
||||||
|
(100, 2, 8, 1),
|
||||||
|
(100, 8, 2, 1),
|
||||||
|
(3, 8, 8, 0),
|
||||||
|
(8, 1, 8, 0),
|
||||||
|
(8, 8, 1, 0),
|
||||||
|
] {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_http_handlers: handlers,
|
||||||
|
max_body_readers: readers,
|
||||||
|
max_body_bytes: 32,
|
||||||
|
max_body_bytes_global: bodies * 32,
|
||||||
|
..defaults.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(conveyor_waiter_limit(&limits), expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn future_request_saturation_keeps_head_admissible_before_any_body_is_read() {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_http_handlers: 8,
|
||||||
|
max_body_readers: 3,
|
||||||
|
max_body_bytes: 32,
|
||||||
|
max_body_bytes_global: 96,
|
||||||
|
..WebLimitsConfig::default()
|
||||||
|
};
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, limits, false);
|
||||||
|
let a = claim(&session, Some(7), 2, 0);
|
||||||
|
let b = claim(&session, Some(8), 2, 0);
|
||||||
|
let a_body = manager.try_body_budget(32).unwrap();
|
||||||
|
let b_body = manager.try_body_budget(32).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(Some(9), 2, Some(0)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
!session
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.contains_key(&Some(9))
|
||||||
|
);
|
||||||
|
let head = claim(&session, Some(7), 1, 0);
|
||||||
|
let head_body = manager
|
||||||
|
.try_body_budget(32)
|
||||||
|
.expect("head reader and byte reserve");
|
||||||
|
assert!(!session.state.lock().carrier_lanes.contains_key(&7));
|
||||||
|
drop((a, b, head, a_body, b_body, head_body));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mode_is_frozen_and_control_only_ack_cannot_commit_an_automatic_carrier() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let (session, manager) = fixture(carrier, WebLimitsConfig::default(), true);
|
||||||
|
let lane = carrier.uses_lanes().then_some(0);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
for _ in 0..2 {
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, lane, 1, 0).process(&body).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!session.is_carrier_committed());
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(lane, 2, None),
|
||||||
|
Err(ConveyorError::Mode)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
assert!(session.claim_conveyor(None, 1, None).unwrap().is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 1, Some(0)),
|
||||||
|
Err(ConveyorError::Mode)
|
||||||
|
));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn first_data_on_unknown_lane_is_not_silently_acknowledged() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"not opened");
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, Some(7), 1, 0).process(&body).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
);
|
||||||
|
assert!(session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LockProbe {
|
||||||
|
session: Arc<WebSession>,
|
||||||
|
notified: std::sync::atomic::AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Wake for LockProbe {
|
||||||
|
fn wake(self: Arc<Self>) {
|
||||||
|
self.wake_by_ref();
|
||||||
|
}
|
||||||
|
fn wake_by_ref(self: &Arc<Self>) {
|
||||||
|
assert!(
|
||||||
|
self.session.state.try_lock().is_some(),
|
||||||
|
"wake under session lock"
|
||||||
|
);
|
||||||
|
self.notified
|
||||||
|
.store(true, std::sync::atomic::Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn close_and_lane_retirement_wake_parked_requests_only_after_unlock() {
|
||||||
|
for close in [false, true] {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"future");
|
||||||
|
let tail = claim(&session, Some(7), 2, 0);
|
||||||
|
let mut waiting = Box::pin(tail.process(&body));
|
||||||
|
let probe = Arc::new(LockProbe {
|
||||||
|
session: Arc::clone(&session),
|
||||||
|
notified: std::sync::atomic::AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
let waker = Waker::from(Arc::clone(&probe));
|
||||||
|
assert!(matches!(
|
||||||
|
waiting.as_mut().poll(&mut Context::from_waker(&waker)),
|
||||||
|
Poll::Pending
|
||||||
|
));
|
||||||
|
if close {
|
||||||
|
session.close(SessionCloseReason::ApiClose);
|
||||||
|
} else {
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
session.release_lane_locked(state, effects, 7);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert!(probe.notified.load(std::sync::atomic::Ordering::Acquire));
|
||||||
|
assert!(waiting.await.is_err());
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
drop(tail);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delayed_retry_after_lane_history_retirement_is_request_local() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let lane = stream(&session, WebCarrier::HttpsLanes);
|
||||||
|
let body = frame::encode(FrameType::Window, 7, &1u32.to_be_bytes());
|
||||||
|
for seq in 1..=2 {
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, lane, seq, seq - 1).process(&body).await,
|
||||||
|
Ok(seq)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
state.streams.remove(&7);
|
||||||
|
session.release_lane_locked(state, effects, 7);
|
||||||
|
});
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(lane, 2, Some(1)),
|
||||||
|
Err(ConveyorError::Stale)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn verified_replay_renews_peer_activity_without_reapplying_credit() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
stream(&session, WebCarrier::Https);
|
||||||
|
let body = frame::encode(FrameType::Window, 7, &1u32.to_be_bytes());
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
let now = std::time::Instant::now();
|
||||||
|
session.state.lock().activity =
|
||||||
|
crate::web::session::activity::SessionActivity::new(now - Duration::from_secs(60));
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
assert!(session.state.lock().activity.peer_idle(now) < Duration::from_secs(1));
|
||||||
|
assert_eq!(session.state.lock().streams[&7].send_credit, 1);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn full_lane_registry_still_admits_first_control_channel_request() {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_tombstones_per_session: 0,
|
||||||
|
..WebLimitsConfig::default()
|
||||||
|
};
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, limits, false);
|
||||||
|
stream(&session, WebCarrier::HttpsLanes);
|
||||||
|
session
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.carrier_lanes
|
||||||
|
.insert(8, CarrierLane::new(8));
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
assert_eq!(claim(&session, Some(0), 1, 0).process(&body).await, Ok(1));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn per_channel_memory_estimate_covers_window_notify_and_hash_table_slack() {
|
||||||
|
// The memory-envelope validator reserves 512 additional bytes per bounded lane slot.
|
||||||
|
let key_and_value = std::mem::size_of::<(Option<u32>, Channel)>();
|
||||||
|
let table_slack = key_and_value.div_ceil(7);
|
||||||
|
let notify_allocation = std::mem::size_of::<Notify>() + 2 * std::mem::size_of::<usize>();
|
||||||
|
assert!(key_and_value + table_slack + notify_allocation + 32 <= 512);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn concurrent_windows_preserve_contiguous_application_under_scheduler_pressure() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
for window in 0..256u64 {
|
||||||
|
let confirmed = window * 4;
|
||||||
|
let mut tasks = Vec::new();
|
||||||
|
for offset in (1..=4).rev() {
|
||||||
|
let claim = claim(&session, None, confirmed + offset, confirmed);
|
||||||
|
let body = body.clone();
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
claim.process(&body).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
for (task, offset) in tasks.into_iter().zip((1..=4).rev()) {
|
||||||
|
assert_eq!(task.await.unwrap(), Ok(confirmed + offset));
|
||||||
|
}
|
||||||
|
let state = session.state.lock();
|
||||||
|
assert_eq!(state.last_up_sequence, confirmed + 4);
|
||||||
|
assert_eq!(state.conveyor.channels[&None].committed, confirmed + 4);
|
||||||
|
assert_eq!(
|
||||||
|
state.conveyor.channels[&None]
|
||||||
|
.slots
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.count(),
|
||||||
|
4
|
||||||
|
);
|
||||||
|
}
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn next_head_waits_for_previous_synchronous_uplink_guard_to_drop() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let head = claim(&session, None, 1, 0);
|
||||||
|
session
|
||||||
|
.up_active
|
||||||
|
.store(true, std::sync::atomic::Ordering::Release);
|
||||||
|
let mut waiting = Box::pin(head.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
session
|
||||||
|
.up_active
|
||||||
|
.store(false, std::sync::atomic::Ordering::Release);
|
||||||
|
head.notify.notify_waiters();
|
||||||
|
assert_eq!(waiting.await, Ok(1));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn new_lane_future_data_waits_for_open_without_creating_a_backend_early() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), true);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"first inner bytes");
|
||||||
|
let future = claim(&session, Some(7), 2, 0);
|
||||||
|
let mut waiting = Box::pin(future.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
assert!(session.state.lock().streams.is_empty());
|
||||||
|
assert!(!session.state.lock().carrier_lanes.contains_key(&7));
|
||||||
|
assert_eq!(session.tasks_live(), 0);
|
||||||
|
assert!(!session.is_carrier_committed());
|
||||||
|
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||||
|
assert_eq!(claim(&session, Some(7), 1, 0).process(&open).await, Ok(1));
|
||||||
|
assert!(session.is_carrier_committed());
|
||||||
|
assert_eq!(waiting.await, Ok(2));
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().streams[&7].inbound[0].bytes,
|
||||||
|
b"first inner bytes"[..]
|
||||||
|
);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
@@ -20,6 +20,17 @@ impl WebSession {
|
|||||||
lane_id: u32,
|
lane_id: u32,
|
||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
|
) -> Result<u64, ManagerError> {
|
||||||
|
self.process_up_lane_inner(lane_id, sequence, body, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies one lane batch only while its optional conveyor claim still owns the turn.
|
||||||
|
pub(super) fn process_up_lane_inner(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
lane_id: u32,
|
||||||
|
sequence: u64,
|
||||||
|
body: &[u8],
|
||||||
|
claim: Option<(&super::conveyor::ConveyorClaim, TokenHash)>,
|
||||||
) -> Result<u64, ManagerError> {
|
) -> Result<u64, ManagerError> {
|
||||||
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
@@ -42,7 +53,8 @@ impl WebSession {
|
|||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
let digest: TokenHash = Sha256::digest(body).into();
|
let digest: TokenHash =
|
||||||
|
claim.map_or_else(|| Sha256::digest(body).into(), |(_, digest)| digest);
|
||||||
let mut opened = Vec::new();
|
let mut opened = Vec::new();
|
||||||
let mut committed = false;
|
let mut committed = false;
|
||||||
let mut healthy = None;
|
let mut healthy = None;
|
||||||
@@ -53,9 +65,13 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
self.ensure_carrier_active_locked(&state)?;
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
|
if let Some((claim, _)) = claim {
|
||||||
|
claim.validate_locked(&state, &digest)?;
|
||||||
|
}
|
||||||
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
||||||
if new_lane {
|
if new_lane {
|
||||||
if lane_id != 0
|
if claim.is_none()
|
||||||
|
&& lane_id != 0
|
||||||
&& frames
|
&& frames
|
||||||
.first()
|
.first()
|
||||||
.is_some_and(|value| value.frame_type != FrameType::Open)
|
.is_some_and(|value| value.frame_type != FrameType::Open)
|
||||||
@@ -190,6 +206,7 @@ impl WebSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if applied {
|
if applied {
|
||||||
|
state.conveyor.commit(Some(lane_id), sequence, digest);
|
||||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||||
}
|
}
|
||||||
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user