mirror of
https://github.com/telemt/telemt.git
synced 2026-11-05 15:21:57 +03:00
Split oversized runtime modules + Async tests hardened
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
use super::*;
|
||||
|
||||
// Read-only fixed API endpoints.
|
||||
mod read_routes;
|
||||
// Fixed configuration and lifecycle mutations.
|
||||
mod fixed_routes;
|
||||
// Dynamic reload and user-resource routes.
|
||||
mod user_routes;
|
||||
|
||||
pub(super) async fn handle(
|
||||
req: Request<Incoming>,
|
||||
peer: SocketAddr,
|
||||
shared: Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||
let runtime = shared.active_runtime.load_full();
|
||||
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||
if previous_cache_generation != runtime.id {
|
||||
*shared.minimal_cache.lock().await = None;
|
||||
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||
}
|
||||
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||
let config_rx = runtime.config_rx.clone();
|
||||
shared
|
||||
.runtime_state
|
||||
.admission_open
|
||||
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||
let request_id = shared.next_request_id();
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let api_cfg = &cfg.server.api;
|
||||
|
||||
if !api_cfg.enabled {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"api_disabled",
|
||||
"API is disabled",
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||
{
|
||||
return match api_cfg.gray_action {
|
||||
ApiGrayAction::Api => Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"forbidden",
|
||||
"Source IP is not allowed",
|
||||
),
|
||||
)),
|
||||
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/html; charset=utf-8")
|
||||
.body(Full::new(Bytes::new()))
|
||||
.unwrap()),
|
||||
ApiGrayAction::Drop => Err(IoError::new(
|
||||
ErrorKind::ConnectionAborted,
|
||||
"api request dropped by gray_action=drop",
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
if !api_cfg.auth_header.is_empty() {
|
||||
let auth_ok = req
|
||||
.headers()
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||
.unwrap_or(false);
|
||||
if !auth_ok {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"unauthorized",
|
||||
"Missing or invalid Authorization header",
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let normalized_path = if path.len() > 1 {
|
||||
path.trim_end_matches('/')
|
||||
} else {
|
||||
path.as_str()
|
||||
};
|
||||
let query = req.uri().query().map(str::to_string);
|
||||
let body_limit = api_cfg.request_body_limit_bytes;
|
||||
|
||||
let result = dispatch(
|
||||
req,
|
||||
method,
|
||||
&path,
|
||||
normalized_path,
|
||||
query.as_deref(),
|
||||
body_limit,
|
||||
&shared,
|
||||
cfg.as_ref(),
|
||||
&config_rx,
|
||||
request_id,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(resp) => Ok(resp),
|
||||
Err(error) => Ok(error_response(request_id, error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(
|
||||
req: Request<Incoming>,
|
||||
method: Method,
|
||||
path: &str,
|
||||
normalized_path: &str,
|
||||
query: Option<&str>,
|
||||
body_limit: usize,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
if web_runtime::is_route(normalized_path) {
|
||||
let web_mutation = method == Method::POST;
|
||||
let result = web_runtime::handle(
|
||||
method,
|
||||
normalized_path,
|
||||
query,
|
||||
req,
|
||||
shared.as_ref(),
|
||||
cfg,
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await;
|
||||
if web_mutation && let Err(error) = &result {
|
||||
shared.runtime_events.record(
|
||||
"api.web.control.failed",
|
||||
format!("path={} code={}", normalized_path, error.code),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if let Some(response) = read_routes::handle(
|
||||
&method,
|
||||
normalized_path,
|
||||
query,
|
||||
shared.as_ref(),
|
||||
cfg,
|
||||
config_rx,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
match (method.as_str(), normalized_path) {
|
||||
("POST", "/v1/users") => {
|
||||
fixed_routes::create_user_route(req, shared, cfg, config_rx, request_id, body_limit)
|
||||
.await
|
||||
}
|
||||
("GET", "/v1/config") => fixed_routes::get_config_route(shared).await,
|
||||
("POST", "/v1/system/reload") => {
|
||||
fixed_routes::reload_route(req, shared, cfg, request_id, body_limit).await
|
||||
}
|
||||
("PATCH", "/v1/config") => {
|
||||
fixed_routes::patch_config_route(req, shared, cfg, query, request_id, body_limit).await
|
||||
}
|
||||
_ => {
|
||||
user_routes::handle(
|
||||
req,
|
||||
&method,
|
||||
path,
|
||||
normalized_path,
|
||||
shared,
|
||||
cfg,
|
||||
config_rx,
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn create_user_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||
let requested_enabled = body.enabled;
|
||||
let result = create_user(body, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.create.failed", error.code);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
if let Some(enabled) = requested_enabled {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.user.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared
|
||||
.proxy_shared
|
||||
.cancel_user_sessions(&data.user.username);
|
||||
if cancelled > 0 {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.user.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.runtime_events.record(
|
||||
"api.user.create.ok",
|
||||
format!("username={}", data.user.username),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::CREATED
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
Ok(success_response(status, data, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn get_config_route(
|
||||
shared: &Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let (value, revision) = config_edit::read_managed_config(&shared.config_path).await?;
|
||||
Ok(success_response(StatusCode::OK, value, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn reload_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
request.validate().map_err(ApiFailure::bad_request)?;
|
||||
|
||||
let (accepted, revision) = submit_reload_from_disk(
|
||||
&shared.config_path,
|
||||
shared.mutation_lock.as_ref(),
|
||||
&shared.reload_control,
|
||||
expected_revision.as_deref(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn patch_config_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
query: Option<&str>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let reload_request = ReloadRequest::from_query(query).map_err(ApiFailure::bad_request)?;
|
||||
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||
match config_edit::patch_config(body, expected_revision, reload_request, shared).await {
|
||||
Ok(resp) => {
|
||||
let revision = resp.revision.clone();
|
||||
let status = if resp.reload.is_some() {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(success_response(status, resp, revision))
|
||||
}
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.failed", error.code);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn handle(
|
||||
method: &Method,
|
||||
normalized_path: &str,
|
||||
query: Option<&str>,
|
||||
shared: &ApiShared,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => Ok(web_status::render(query, &shared.web_trace).await),
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = HealthData {
|
||||
status: "ok",
|
||||
read_only: api_cfg.read_only,
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/health/ready") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||
let reason = if ready {
|
||||
None
|
||||
} else if !admission_open {
|
||||
Some("admission_closed")
|
||||
} else {
|
||||
Some("no_healthy_upstreams")
|
||||
};
|
||||
let data = HealthReadyData {
|
||||
ready,
|
||||
status: if ready { "ready" } else { "not_ready" },
|
||||
reason,
|
||||
admission_open,
|
||||
healthy_upstreams: upstream_health.healthy_total,
|
||||
total_upstreams: upstream_health.configured_total,
|
||||
};
|
||||
let status_code = if ready {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
Ok(success_response(status_code, data, revision))
|
||||
}
|
||||
("GET", "/v1/system/info") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_system_info_data(shared, cfg, &revision);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/gates") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_gates_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/initialization") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_initialization_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/limits/effective") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_limits_effective_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/posture") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_posture_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/whitelist") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_whitelist_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let connections_bad_by_class = shared
|
||||
.stats
|
||||
.get_connects_bad_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let handshake_failures_by_class = shared
|
||||
.stats
|
||||
.get_handshake_failure_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let data = SummaryData {
|
||||
uptime_seconds: shared.stats.uptime_secs(),
|
||||
connections_total: shared.stats.get_connects_all(),
|
||||
connections_bad_total: shared.stats.get_connects_bad(),
|
||||
connections_bad_by_class,
|
||||
handshake_failures_by_class,
|
||||
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
|
||||
configured_users: cfg.access.users.len(),
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/zero/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/upstreams") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_upstreams_data(shared, api_cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/minimal/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_minimal_all_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/me-writers") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_me_writers_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/dcs") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_dcs_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_pool_state_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_quality_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_upstream_quality_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_nat_stun_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-selftest") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_selftest_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/connections/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_connections_summary_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/events/recent") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_events_recent_data(shared, cfg, query);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_tls_fingerprints_data(shared, cfg, query);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/active-ips") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
||||
let mut data: Vec<UserActiveIps> = active_ips_map
|
||||
.into_iter()
|
||||
.filter(|(_, ips)| !ips.is_empty())
|
||||
.map(|(username, active_ips)| UserActiveIps {
|
||||
username,
|
||||
active_ips,
|
||||
})
|
||||
.collect();
|
||||
data.sort_by(|a, b| a.username.cmp(&b.username));
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(success_response(StatusCode::OK, users, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/quota") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
|
||||
_ => return Ok(None),
|
||||
}
|
||||
.map(Some)
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn handle(
|
||||
req: Request<Incoming>,
|
||||
method: &Method,
|
||||
path: &str,
|
||||
normalized_path: &str,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if method == Method::GET
|
||||
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||
{
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let status = shared
|
||||
.reload_control
|
||||
.status(reload_id)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"reload_not_found",
|
||||
format!("Reload {} was not found", reload_id),
|
||||
)
|
||||
})?;
|
||||
return Ok(success_response(StatusCode::OK, status, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/enable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = set_user_enabled(base_user, true, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.enable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/disable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = set_user_enabled(base_user, false, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.ok",
|
||||
format!(
|
||||
"username={} newly_disabled={} cancelled_sessions={}",
|
||||
base_user, newly_disabled, cancelled
|
||||
),
|
||||
);
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let _mutation_guard = shared.mutation_lock.lock().await;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
|
||||
if !disk_cfg.access.users.contains_key(user) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
let configured_users = disk_cfg
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
let snapshot = match shared.quota_state.reset_user(&configured_users, user).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.reset_quota.failed",
|
||||
format!("username={} error={}", user, error),
|
||||
);
|
||||
return Err(ApiFailure::internal(format!(
|
||||
"Failed to reset user quota: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
return Ok(success_response(
|
||||
StatusCode::OK,
|
||||
ResetUserQuotaResponse {
|
||||
username: user.to_string(),
|
||||
used_bytes: snapshot.used_bytes,
|
||||
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||
},
|
||||
revision,
|
||||
));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit).await?;
|
||||
let result = rotate_secret(
|
||||
base_user,
|
||||
body.unwrap_or_default(),
|
||||
expected_revision,
|
||||
shared,
|
||||
)
|
||||
.await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.ok",
|
||||
format!("username={}", base_user),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if method == Method::GET {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) {
|
||||
return Ok(success_response(StatusCode::OK, user_info, revision));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
if method == Method::PATCH {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||
let enabled_update = match &body.enabled {
|
||||
Patch::Unchanged => None,
|
||||
Patch::Remove => Some(true),
|
||||
Patch::Set(enabled) => Some(*enabled),
|
||||
};
|
||||
let result = patch_user(user, body, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.patch.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
if let Some(enabled) = enabled_update {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::DELETE {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = delete_user(user, expected_revision, shared).await;
|
||||
let (deleted_user, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.ok",
|
||||
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||
);
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||
let response = DeleteUserResponse {
|
||||
username: deleted_user,
|
||||
in_runtime,
|
||||
};
|
||||
let status = if response.in_runtime {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
return Ok(success_response(status, response, revision));
|
||||
}
|
||||
if method == Method::POST {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
if let Some(allow) = allowed_methods_for_path(normalized_path) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(allow),
|
||||
));
|
||||
}
|
||||
debug!(
|
||||
method = method.as_str(),
|
||||
path = %path,
|
||||
normalized_path = %normalized_path,
|
||||
"API route not found"
|
||||
);
|
||||
Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||
))
|
||||
}
|
||||
+4
-833
@@ -21,7 +21,7 @@ use tokio::sync::{Mutex, RwLock, Semaphore, watch};
|
||||
use tokio::time::timeout;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ApiGrayAction;
|
||||
use crate::config::{ApiGrayAction, ProxyConfig};
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::maestro::control_plane::ProcessControlPlane;
|
||||
use crate::maestro::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
@@ -40,6 +40,8 @@ mod config_edit;
|
||||
pub(crate) mod config_store;
|
||||
mod events;
|
||||
mod http_utils;
|
||||
// Authenticated request admission and route dispatch.
|
||||
mod handler;
|
||||
mod model;
|
||||
mod patch;
|
||||
#[cfg(test)]
|
||||
@@ -61,6 +63,7 @@ use config_store::{
|
||||
parse_if_match,
|
||||
};
|
||||
use events::ApiEventStore;
|
||||
use handler::handle;
|
||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||
use model::{
|
||||
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
|
||||
@@ -424,835 +427,3 @@ pub(crate) async fn serve(
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
req: Request<Incoming>,
|
||||
peer: SocketAddr,
|
||||
shared: Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||
let runtime = shared.active_runtime.load_full();
|
||||
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||
if previous_cache_generation != runtime.id {
|
||||
*shared.minimal_cache.lock().await = None;
|
||||
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||
}
|
||||
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||
let config_rx = runtime.config_rx.clone();
|
||||
shared
|
||||
.runtime_state
|
||||
.admission_open
|
||||
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||
let request_id = shared.next_request_id();
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let api_cfg = &cfg.server.api;
|
||||
|
||||
if !api_cfg.enabled {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"api_disabled",
|
||||
"API is disabled",
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||
{
|
||||
return match api_cfg.gray_action {
|
||||
ApiGrayAction::Api => Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"forbidden",
|
||||
"Source IP is not allowed",
|
||||
),
|
||||
)),
|
||||
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/html; charset=utf-8")
|
||||
.body(Full::new(Bytes::new()))
|
||||
.unwrap()),
|
||||
ApiGrayAction::Drop => Err(IoError::new(
|
||||
ErrorKind::ConnectionAborted,
|
||||
"api request dropped by gray_action=drop",
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
if !api_cfg.auth_header.is_empty() {
|
||||
let auth_ok = req
|
||||
.headers()
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||
.unwrap_or(false);
|
||||
if !auth_ok {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"unauthorized",
|
||||
"Missing or invalid Authorization header",
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let normalized_path = if path.len() > 1 {
|
||||
path.trim_end_matches('/')
|
||||
} else {
|
||||
path.as_str()
|
||||
};
|
||||
let query = req.uri().query().map(str::to_string);
|
||||
let body_limit = api_cfg.request_body_limit_bytes;
|
||||
|
||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||
if web_runtime::is_route(normalized_path) {
|
||||
let web_mutation = method == Method::POST;
|
||||
let result = web_runtime::handle(
|
||||
method,
|
||||
normalized_path,
|
||||
query.as_deref(),
|
||||
req,
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await;
|
||||
if web_mutation && let Err(error) = &result {
|
||||
shared.runtime_events.record(
|
||||
"api.web.control.failed",
|
||||
format!("path={} code={}", normalized_path, error.code),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => {
|
||||
Ok(web_status::render(query.as_deref(), &shared.web_trace).await)
|
||||
}
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = HealthData {
|
||||
status: "ok",
|
||||
read_only: api_cfg.read_only,
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/health/ready") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||
let reason = if ready {
|
||||
None
|
||||
} else if !admission_open {
|
||||
Some("admission_closed")
|
||||
} else {
|
||||
Some("no_healthy_upstreams")
|
||||
};
|
||||
let data = HealthReadyData {
|
||||
ready,
|
||||
status: if ready { "ready" } else { "not_ready" },
|
||||
reason,
|
||||
admission_open,
|
||||
healthy_upstreams: upstream_health.healthy_total,
|
||||
total_upstreams: upstream_health.configured_total,
|
||||
};
|
||||
let status_code = if ready {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
Ok(success_response(status_code, data, revision))
|
||||
}
|
||||
("GET", "/v1/system/info") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/gates") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/initialization") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_initialization_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/limits/effective") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_limits_effective_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/posture") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_posture_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/whitelist") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_whitelist_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let connections_bad_by_class = shared
|
||||
.stats
|
||||
.get_connects_bad_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let handshake_failures_by_class = shared
|
||||
.stats
|
||||
.get_handshake_failure_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let data = SummaryData {
|
||||
uptime_seconds: shared.stats.uptime_secs(),
|
||||
connections_total: shared.stats.get_connects_all(),
|
||||
connections_bad_total: shared.stats.get_connects_bad(),
|
||||
connections_bad_by_class,
|
||||
handshake_failures_by_class,
|
||||
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
|
||||
configured_users: cfg.access.users.len(),
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/zero/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/upstreams") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_upstreams_data(shared.as_ref(), api_cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/minimal/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_minimal_all_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/me-writers") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_me_writers_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/dcs") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_dcs_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_pool_state_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_quality_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_upstream_quality_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_nat_stun_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-selftest") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_selftest_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/connections/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data =
|
||||
build_runtime_connections_summary_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/events/recent") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_events_recent_data(
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
query.as_deref(),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_tls_fingerprints_data(
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
query.as_deref(),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/active-ips") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
||||
let mut data: Vec<UserActiveIps> = active_ips_map
|
||||
.into_iter()
|
||||
.filter(|(_, ips)| !ips.is_empty())
|
||||
.map(|(username, active_ips)| UserActiveIps {
|
||||
username,
|
||||
active_ips,
|
||||
})
|
||||
.collect();
|
||||
data.sort_by(|a, b| a.username.cmp(&b.username));
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(success_response(StatusCode::OK, users, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/quota") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("POST", "/v1/users") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||
let requested_enabled = body.enabled;
|
||||
let result = create_user(body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.create.failed", error.code);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
if let Some(enabled) = requested_enabled {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.user.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared
|
||||
.proxy_shared
|
||||
.cancel_user_sessions(&data.user.username);
|
||||
if cancelled > 0 {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.user.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.runtime_events.record(
|
||||
"api.user.create.ok",
|
||||
format!("username={}", data.user.username),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::CREATED
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
Ok(success_response(status, data, revision))
|
||||
}
|
||||
("GET", "/v1/config") => {
|
||||
let (value, revision) =
|
||||
config_edit::read_managed_config(&shared.config_path).await?;
|
||||
Ok(success_response(StatusCode::OK, value, revision))
|
||||
}
|
||||
("POST", "/v1/system/reload") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
request.validate().map_err(ApiFailure::bad_request)?;
|
||||
|
||||
let (accepted, revision) = submit_reload_from_disk(
|
||||
&shared.config_path,
|
||||
shared.mutation_lock.as_ref(),
|
||||
&shared.reload_control,
|
||||
expected_revision.as_deref(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||
}
|
||||
("PATCH", "/v1/config") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let reload_request =
|
||||
ReloadRequest::from_query(query.as_deref()).map_err(ApiFailure::bad_request)?;
|
||||
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||
match config_edit::patch_config(body, expected_revision, reload_request, &shared)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => {
|
||||
let revision = resp.revision.clone();
|
||||
let status = if resp.reload.is_some() {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(success_response(status, resp, revision))
|
||||
}
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.failed", error.code);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if method == Method::GET
|
||||
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||
{
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let status =
|
||||
shared
|
||||
.reload_control
|
||||
.status(reload_id)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"reload_not_found",
|
||||
format!("Reload {} was not found", reload_id),
|
||||
)
|
||||
})?;
|
||||
return Ok(success_response(StatusCode::OK, status, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/enable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, true, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.enable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/disable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, false, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.ok",
|
||||
format!(
|
||||
"username={} newly_disabled={} cancelled_sessions={}",
|
||||
base_user, newly_disabled, cancelled
|
||||
),
|
||||
);
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let _mutation_guard = shared.mutation_lock.lock().await;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
ensure_expected_revision(&shared.config_path, expected_revision.as_deref())
|
||||
.await?;
|
||||
if !disk_cfg.access.users.contains_key(user) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
let configured_users = disk_cfg
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
let snapshot = match shared
|
||||
.quota_state
|
||||
.reset_user(&configured_users, user)
|
||||
.await
|
||||
{
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.reset_quota.failed",
|
||||
format!("username={} error={}", user, error),
|
||||
);
|
||||
return Err(ApiFailure::internal(format!(
|
||||
"Failed to reset user quota: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
return Ok(success_response(
|
||||
StatusCode::OK,
|
||||
ResetUserQuotaResponse {
|
||||
username: user.to_string(),
|
||||
used_bytes: snapshot.used_bytes,
|
||||
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||
},
|
||||
revision,
|
||||
));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body =
|
||||
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
||||
.await?;
|
||||
let result = rotate_secret(
|
||||
base_user,
|
||||
body.unwrap_or_default(),
|
||||
expected_revision,
|
||||
&shared,
|
||||
)
|
||||
.await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime =
|
||||
runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.ok",
|
||||
format!("username={}", base_user),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if method == Method::GET {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
if let Some(user_info) =
|
||||
users.into_iter().find(|entry| entry.username == user)
|
||||
{
|
||||
return Ok(success_response(StatusCode::OK, user_info, revision));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
if method == Method::PATCH {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body =
|
||||
read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||
let enabled_update = match &body.enabled {
|
||||
Patch::Unchanged => None,
|
||||
Patch::Remove => Some(true),
|
||||
Patch::Set(enabled) => Some(*enabled),
|
||||
};
|
||||
let result = patch_user(user, body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.patch.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
if let Some(enabled) = enabled_update {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled =
|
||||
shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::DELETE {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = delete_user(user, expected_revision, &shared).await;
|
||||
let (deleted_user, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.ok",
|
||||
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||
);
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||
let response = DeleteUserResponse {
|
||||
username: deleted_user,
|
||||
in_runtime,
|
||||
};
|
||||
let status = if response.in_runtime {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
return Ok(success_response(status, response, revision));
|
||||
}
|
||||
if method == Method::POST {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
if let Some(allow) = allowed_methods_for_path(normalized_path) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(allow),
|
||||
));
|
||||
}
|
||||
debug!(
|
||||
method = method.as_str(),
|
||||
path = %path,
|
||||
normalized_path = %normalized_path,
|
||||
"API route not found"
|
||||
);
|
||||
Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(resp) => Ok(resp),
|
||||
Err(error) => Ok(error_response(request_id, error)),
|
||||
}
|
||||
}
|
||||
|
||||
+3
-161
@@ -466,164 +466,6 @@ pub(super) struct MinimalAllData {
|
||||
pub(super) data: Option<MinimalAllPayload>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserLinks {
|
||||
pub(super) classic: Vec<String>,
|
||||
pub(super) secure: Vec<String>,
|
||||
pub(super) tls: Vec<String>,
|
||||
pub(super) tls_domains: Vec<TlsDomainLink>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct TlsDomainLink {
|
||||
pub(super) domain: String,
|
||||
pub(super) link: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserInfo {
|
||||
pub(super) username: String,
|
||||
pub(super) enabled: bool,
|
||||
pub(super) in_runtime: bool,
|
||||
pub(super) user_ad_tag: Option<String>,
|
||||
pub(super) max_tcp_conns: Option<usize>,
|
||||
pub(super) expiration_rfc3339: Option<String>,
|
||||
pub(super) data_quota_bytes: Option<u64>,
|
||||
pub(super) rate_limit_up_bps: Option<u64>,
|
||||
pub(super) rate_limit_down_bps: Option<u64>,
|
||||
pub(super) max_unique_ips: Option<usize>,
|
||||
pub(super) current_connections: u64,
|
||||
pub(super) active_unique_ips: usize,
|
||||
pub(super) active_unique_ips_list: Vec<IpAddr>,
|
||||
pub(super) recent_unique_ips: usize,
|
||||
pub(super) recent_unique_ips_list: Vec<IpAddr>,
|
||||
pub(super) total_octets: u64,
|
||||
pub(super) links: UserLinks,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserActiveIps {
|
||||
pub(super) username: String,
|
||||
pub(super) active_ips: Vec<IpAddr>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct CreateUserResponse {
|
||||
pub(super) user: UserInfo,
|
||||
pub(super) secret: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct DeleteUserResponse {
|
||||
pub(super) username: String,
|
||||
pub(super) in_runtime: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct ResetUserQuotaResponse {
|
||||
pub(super) username: String,
|
||||
pub(super) used_bytes: u64,
|
||||
pub(super) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserQuotaListData {
|
||||
pub(super) users: Vec<UserQuotaEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserQuotaEntry {
|
||||
pub(super) username: String,
|
||||
pub(super) data_quota_bytes: u64,
|
||||
pub(super) used_bytes: u64,
|
||||
pub(super) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct CreateUserRequest {
|
||||
pub(super) username: String,
|
||||
pub(super) secret: Option<String>,
|
||||
pub(super) user_ad_tag: Option<String>,
|
||||
pub(super) max_tcp_conns: Option<usize>,
|
||||
pub(super) expiration_rfc3339: Option<String>,
|
||||
pub(super) data_quota_bytes: Option<u64>,
|
||||
pub(super) rate_limit_up_bps: Option<u64>,
|
||||
pub(super) rate_limit_down_bps: Option<u64>,
|
||||
pub(super) max_unique_ips: Option<usize>,
|
||||
pub(super) enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct PatchUserRequest {
|
||||
pub(super) secret: Option<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) user_ad_tag: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) max_tcp_conns: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) expiration_rfc3339: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) data_quota_bytes: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) rate_limit_up_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) rate_limit_down_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) max_unique_ips: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) enabled: Patch<bool>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
pub(super) struct RotateSecretRequest {
|
||||
pub(super) secret: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn parse_optional_expiration(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
||||
let Some(raw) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Some(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
|
||||
pub(super) fn parse_patch_expiration(
|
||||
value: &Patch<String>,
|
||||
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
||||
match value {
|
||||
Patch::Unchanged => Ok(Patch::Unchanged),
|
||||
Patch::Remove => Ok(Patch::Remove),
|
||||
Patch::Set(raw) => {
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_user_secret(secret: &str) -> bool {
|
||||
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_username(user: &str) -> bool {
|
||||
!user.is_empty()
|
||||
&& user.len() <= MAX_USERNAME_LEN
|
||||
&& user
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
||||
}
|
||||
|
||||
pub(super) fn random_user_secret() -> String {
|
||||
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
||||
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
||||
let mut bytes = [0u8; 16];
|
||||
rng.fill(&mut bytes);
|
||||
hex::encode(bytes)
|
||||
}
|
||||
// User-management request, response, and validation models.
|
||||
mod users;
|
||||
pub(super) use users::*;
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
use super::*;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserLinks {
|
||||
pub(in crate::api) classic: Vec<String>,
|
||||
pub(in crate::api) secure: Vec<String>,
|
||||
pub(in crate::api) tls: Vec<String>,
|
||||
pub(in crate::api) tls_domains: Vec<TlsDomainLink>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct TlsDomainLink {
|
||||
pub(in crate::api) domain: String,
|
||||
pub(in crate::api) link: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserInfo {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) enabled: bool,
|
||||
pub(in crate::api) in_runtime: bool,
|
||||
pub(in crate::api) user_ad_tag: Option<String>,
|
||||
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||
pub(in crate::api) current_connections: u64,
|
||||
pub(in crate::api) active_unique_ips: usize,
|
||||
pub(in crate::api) active_unique_ips_list: Vec<IpAddr>,
|
||||
pub(in crate::api) recent_unique_ips: usize,
|
||||
pub(in crate::api) recent_unique_ips_list: Vec<IpAddr>,
|
||||
pub(in crate::api) total_octets: u64,
|
||||
pub(in crate::api) links: UserLinks,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserActiveIps {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) active_ips: Vec<IpAddr>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct CreateUserResponse {
|
||||
pub(in crate::api) user: UserInfo,
|
||||
pub(in crate::api) secret: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct DeleteUserResponse {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) in_runtime: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct ResetUserQuotaResponse {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) used_bytes: u64,
|
||||
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserQuotaListData {
|
||||
pub(in crate::api) users: Vec<UserQuotaEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserQuotaEntry {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) data_quota_bytes: u64,
|
||||
pub(in crate::api) used_bytes: u64,
|
||||
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(in crate::api) struct CreateUserRequest {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
pub(in crate::api) user_ad_tag: Option<String>,
|
||||
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||
pub(in crate::api) enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(in crate::api) struct PatchUserRequest {
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) user_ad_tag: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) max_tcp_conns: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) expiration_rfc3339: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) data_quota_bytes: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) rate_limit_up_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) rate_limit_down_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) max_unique_ips: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) enabled: Patch<bool>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
pub(in crate::api) struct RotateSecretRequest {
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
}
|
||||
|
||||
pub(in crate::api) fn parse_optional_expiration(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
||||
let Some(raw) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Some(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
|
||||
pub(in crate::api) fn parse_patch_expiration(
|
||||
value: &Patch<String>,
|
||||
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
||||
match value {
|
||||
Patch::Unchanged => Ok(Patch::Unchanged),
|
||||
Patch::Remove => Ok(Patch::Remove),
|
||||
Patch::Set(raw) => {
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_user_secret(secret: &str) -> bool {
|
||||
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_username(user: &str) -> bool {
|
||||
!user.is_empty()
|
||||
&& user.len() <= MAX_USERNAME_LEN
|
||||
&& user
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
||||
}
|
||||
|
||||
pub(in crate::api) fn random_user_secret() -> String {
|
||||
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
||||
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
||||
let mut bytes = [0u8; 16];
|
||||
rng.fill(&mut bytes);
|
||||
hex::encode(bytes)
|
||||
}
|
||||
+4
-52
@@ -541,55 +541,7 @@ pub(super) async fn build_runtime_upstream_quality_data(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||
return RuntimeNatStunData {
|
||||
enabled: false,
|
||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: None,
|
||||
};
|
||||
};
|
||||
|
||||
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||
RuntimeNatStunData {
|
||||
enabled: true,
|
||||
reason: None,
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: Some(RuntimeNatStunPayload {
|
||||
flags: RuntimeNatStunFlagsData {
|
||||
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||
},
|
||||
servers: RuntimeNatStunServersData {
|
||||
configured: snapshot.configured_servers,
|
||||
live: snapshot.live_servers.clone(),
|
||||
live_total: snapshot.live_servers.len(),
|
||||
},
|
||||
reflection: RuntimeNatStunReflectionBlockData {
|
||||
v4: snapshot
|
||||
.reflection_v4
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
v6: snapshot
|
||||
.reflection_v6
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
},
|
||||
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
// NAT/STUN runtime projection and timestamping.
|
||||
mod nat;
|
||||
pub(super) use nat::build_runtime_nat_stun_data;
|
||||
use nat::now_epoch_secs;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use super::*;
|
||||
|
||||
pub(in crate::api) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||
return RuntimeNatStunData {
|
||||
enabled: false,
|
||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: None,
|
||||
};
|
||||
};
|
||||
|
||||
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||
RuntimeNatStunData {
|
||||
enabled: true,
|
||||
reason: None,
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: Some(RuntimeNatStunPayload {
|
||||
flags: RuntimeNatStunFlagsData {
|
||||
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||
},
|
||||
servers: RuntimeNatStunServersData {
|
||||
configured: snapshot.configured_servers,
|
||||
live: snapshot.live_servers.clone(),
|
||||
live_total: snapshot.live_servers.len(),
|
||||
},
|
||||
reflection: RuntimeNatStunReflectionBlockData {
|
||||
v4: snapshot
|
||||
.reflection_v4
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
v6: snapshot
|
||||
.reflection_v6
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
},
|
||||
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
@@ -84,8 +84,7 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
|
||||
reconnect_success_total: stats.get_me_reconnect_success(),
|
||||
handshake_reject_total: stats.get_me_handshake_reject_total(),
|
||||
handshake_error_codes,
|
||||
handshake_error_code_overflow_total: stats
|
||||
.get_me_handshake_error_code_overflow_total(),
|
||||
handshake_error_code_overflow_total: stats.get_me_handshake_error_code_overflow_total(),
|
||||
reader_eof_total: stats.get_me_reader_eof_total(),
|
||||
idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
|
||||
route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
|
||||
@@ -527,57 +526,6 @@ async fn get_minimal_payload_cached(
|
||||
Some((generated_at_epoch_secs, payload))
|
||||
}
|
||||
|
||||
fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
||||
MeWritersData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
summary: MeWritersSummary {
|
||||
configured_dc_groups: 0,
|
||||
configured_endpoints: 0,
|
||||
available_endpoints: 0,
|
||||
available_pct: 0.0,
|
||||
required_writers: 0,
|
||||
alive_writers: 0,
|
||||
coverage_pct: 0.0,
|
||||
fresh_alive_writers: 0,
|
||||
fresh_coverage_pct: 0.0,
|
||||
},
|
||||
writers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
||||
DcStatusData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
dcs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
||||
match value {
|
||||
UpstreamRouteKind::Direct => "direct",
|
||||
UpstreamRouteKind::Socks4 => "socks4",
|
||||
UpstreamRouteKind::Socks5 => "socks5",
|
||||
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
||||
}
|
||||
}
|
||||
|
||||
fn map_ip_preference(value: IpPreference) -> &'static str {
|
||||
match value {
|
||||
IpPreference::Unknown => "unknown",
|
||||
IpPreference::PreferV6 => "prefer_v6",
|
||||
IpPreference::PreferV4 => "prefer_v4",
|
||||
IpPreference::BothWork => "both_work",
|
||||
IpPreference::Unavailable => "unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
// Disabled-state builders and stable upstream enum mappings.
|
||||
mod helpers;
|
||||
use helpers::*;
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
||||
MeWritersData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
summary: MeWritersSummary {
|
||||
configured_dc_groups: 0,
|
||||
configured_endpoints: 0,
|
||||
available_endpoints: 0,
|
||||
available_pct: 0.0,
|
||||
required_writers: 0,
|
||||
alive_writers: 0,
|
||||
coverage_pct: 0.0,
|
||||
fresh_alive_writers: 0,
|
||||
fresh_coverage_pct: 0.0,
|
||||
},
|
||||
writers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
||||
DcStatusData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
dcs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
||||
match value {
|
||||
UpstreamRouteKind::Direct => "direct",
|
||||
UpstreamRouteKind::Socks4 => "socks4",
|
||||
UpstreamRouteKind::Socks5 => "socks5",
|
||||
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn map_ip_preference(value: IpPreference) -> &'static str {
|
||||
match value {
|
||||
IpPreference::Unknown => "unknown",
|
||||
IpPreference::PreferV6 => "prefer_v6",
|
||||
IpPreference::PreferV4 => "prefer_v4",
|
||||
IpPreference::BothWork => "both_work",
|
||||
IpPreference::Unavailable => "unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
@@ -4,8 +4,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::maestro::generation::RuntimeWatchState;
|
||||
use crate::maestro::control_plane::ProcessControlPlane;
|
||||
use crate::maestro::generation::RuntimeWatchState;
|
||||
|
||||
use super::ApiRuntimeState;
|
||||
use super::events::ApiEventStore;
|
||||
|
||||
+5
-450
@@ -1,6 +1,9 @@
|
||||
use ipnetwork::IpNetwork;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
|
||||
// Extended transport, masking, and ME default values.
|
||||
mod extended;
|
||||
|
||||
pub(crate) use extended::*;
|
||||
|
||||
// Helper defaults kept private to the config module.
|
||||
const DEFAULT_NETWORK_IPV6: Option<bool> = Some(false);
|
||||
@@ -520,451 +523,3 @@ pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize {
|
||||
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
|
||||
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||
DEFAULT_ME_ADMISSION_POLL_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
||||
256 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_max_client_frame() -> usize {
|
||||
16 * 1024 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_desync_all_full() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
||||
25
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
||||
120
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
||||
80
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_minutes() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
||||
15
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_file() -> String {
|
||||
"beobachten.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_serverhello_compact() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
||||
8
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
||||
24
|
||||
}
|
||||
|
||||
pub(crate) fn default_alpn_enforce() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
5 * 1024 * 1024
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
32 * 1024
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
60_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
5_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_stun_servers() -> Vec<String> {
|
||||
vec![
|
||||
"stun.l.google.com:5349".to_string(),
|
||||
"stun1.l.google.com:3478".to_string(),
|
||||
"stun.gmx.net:3478".to_string(),
|
||||
"stun.l.google.com:19302".to_string(),
|
||||
"stun.1und1.de:3478".to_string(),
|
||||
"stun1.l.google.com:19302".to_string(),
|
||||
"stun2.l.google.com:19302".to_string(),
|
||||
"stun3.l.google.com:19302".to_string(),
|
||||
"stun4.l.google.com:19302".to_string(),
|
||||
"stun.services.mozilla.com:3478".to_string(),
|
||||
"stun.stunprotocol.org:3478".to_string(),
|
||||
"stun.nextcloud.com:3478".to_string(),
|
||||
"stun.voip.eutelia.it:3478".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
||||
vec![
|
||||
"https://ifconfig.me/ip".to_string(),
|
||||
"https://api.ipify.org".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_cache_public_ip_path() -> String {
|
||||
"cache/public_ip.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every_secs() -> u64 {
|
||||
5 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every() -> Option<u64> {
|
||||
Some(default_update_every_secs())
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||
15 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_max_concurrency() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||
64
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||
1000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
||||
2000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
||||
500
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||
256
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_instadrain() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||
32
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||
default_me_pool_drain_ttl_secs()
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||
0.8
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_hardswap() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_check() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
||||
vec!["pool.ntp.org".to_string()]
|
||||
}
|
||||
|
||||
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6() -> String {
|
||||
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
||||
Some(default_listen_addr_ipv6())
|
||||
}
|
||||
|
||||
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
||||
HashMap::from([(
|
||||
DEFAULT_ACCESS_USER.to_string(),
|
||||
DEFAULT_ACCESS_SECRET.to_string(),
|
||||
)])
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
// Custom deserializer helpers
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum OneOrMany {
|
||||
One(String),
|
||||
Many(Vec<String>),
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
||||
where
|
||||
D: serde::de::Deserializer<'de>,
|
||||
{
|
||||
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
||||
let mut out = HashMap::new();
|
||||
for (dc, val) in raw {
|
||||
let mut addrs = match val {
|
||||
OneOrMany::One(s) => vec![s],
|
||||
OneOrMany::Many(v) => v,
|
||||
};
|
||||
addrs.retain(|s| !s.trim().is_empty());
|
||||
if !addrs.is_empty() {
|
||||
out.insert(dc, addrs);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||
DEFAULT_ME_ADMISSION_POLL_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
||||
256 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_max_client_frame() -> usize {
|
||||
16 * 1024 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_desync_all_full() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
||||
25
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
||||
120
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
||||
80
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_minutes() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
||||
15
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_file() -> String {
|
||||
"beobachten.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_serverhello_compact() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
||||
8
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
||||
24
|
||||
}
|
||||
|
||||
pub(crate) fn default_alpn_enforce() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
5 * 1024 * 1024
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
32 * 1024
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
60_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
5_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_stun_servers() -> Vec<String> {
|
||||
vec![
|
||||
"stun.l.google.com:5349".to_string(),
|
||||
"stun1.l.google.com:3478".to_string(),
|
||||
"stun.gmx.net:3478".to_string(),
|
||||
"stun.l.google.com:19302".to_string(),
|
||||
"stun.1und1.de:3478".to_string(),
|
||||
"stun1.l.google.com:19302".to_string(),
|
||||
"stun2.l.google.com:19302".to_string(),
|
||||
"stun3.l.google.com:19302".to_string(),
|
||||
"stun4.l.google.com:19302".to_string(),
|
||||
"stun.services.mozilla.com:3478".to_string(),
|
||||
"stun.stunprotocol.org:3478".to_string(),
|
||||
"stun.nextcloud.com:3478".to_string(),
|
||||
"stun.voip.eutelia.it:3478".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
||||
vec![
|
||||
"https://ifconfig.me/ip".to_string(),
|
||||
"https://api.ipify.org".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_cache_public_ip_path() -> String {
|
||||
"cache/public_ip.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every_secs() -> u64 {
|
||||
5 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every() -> Option<u64> {
|
||||
Some(default_update_every_secs())
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||
15 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_max_concurrency() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||
64
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||
1000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
||||
2000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
||||
500
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||
256
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_instadrain() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||
32
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||
default_me_pool_drain_ttl_secs()
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||
0.8
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_hardswap() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_check() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
||||
vec!["pool.ntp.org".to_string()]
|
||||
}
|
||||
|
||||
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6() -> String {
|
||||
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
||||
Some(default_listen_addr_ipv6())
|
||||
}
|
||||
|
||||
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
||||
HashMap::from([(
|
||||
DEFAULT_ACCESS_USER.to_string(),
|
||||
DEFAULT_ACCESS_SECRET.to_string(),
|
||||
)])
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
// Custom deserializer helpers
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum OneOrMany {
|
||||
One(String),
|
||||
Many(Vec<String>),
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
||||
where
|
||||
D: serde::de::Deserializer<'de>,
|
||||
{
|
||||
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
||||
let mut out = HashMap::new();
|
||||
for (dc, val) in raw {
|
||||
let mut addrs = match val {
|
||||
OneOrMany::One(s) => vec![s],
|
||||
OneOrMany::Many(v) => v,
|
||||
};
|
||||
addrs.retain(|s| !s.trim().is_empty());
|
||||
if !addrs.is_empty() {
|
||||
out.insert(dc, addrs);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -414,196 +414,9 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
let limits = &config.web.limits;
|
||||
if config.web.vhosts.len() > limits.max_vhosts {
|
||||
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
||||
}
|
||||
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
||||
let mut profile_count = 0usize;
|
||||
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
||||
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
||||
if !hosts.insert(vhost.host.clone()) {
|
||||
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
||||
}
|
||||
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
||||
));
|
||||
}
|
||||
if config.web.enabled && vhost.profiles.is_empty() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
||||
));
|
||||
}
|
||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||
));
|
||||
}
|
||||
if !config.access.users.contains_key(&profile.user) {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||
profile.user
|
||||
));
|
||||
}
|
||||
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
||||
return config_error(&format!(
|
||||
"duplicate WEB profile for user `{}` in vhost `{}`",
|
||||
profile.user, vhost.host
|
||||
));
|
||||
}
|
||||
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
||||
let max_streams_per_session = profile
|
||||
.max_streams_per_session
|
||||
.unwrap_or(limits.max_streams_per_session);
|
||||
if profile.max_sessions == Some(0)
|
||||
|| profile
|
||||
.max_sessions
|
||||
.is_some_and(|value| value > limits.max_sessions_global)
|
||||
|| profile.max_streams == Some(0)
|
||||
|| profile
|
||||
.max_streams
|
||||
.is_some_and(|value| value > limits.max_streams_global)
|
||||
|| profile.max_streams_per_session == Some(0)
|
||||
|| profile
|
||||
.max_streams_per_session
|
||||
.is_some_and(|value| value > limits.max_streams_per_session)
|
||||
|| max_streams_per_session > max_streams
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
||||
));
|
||||
}
|
||||
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
||||
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
if profile_count > limits.max_profiles {
|
||||
return config_error("WEB profiles exceed web.limits.max_profiles");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
||||
let input = value.trim();
|
||||
if input.is_empty()
|
||||
|| input.ends_with('.')
|
||||
|| input
|
||||
.chars()
|
||||
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
||||
));
|
||||
}
|
||||
let host = normalize_domain_to_ascii(input, field)?;
|
||||
if host.len() > 253
|
||||
|| !host.contains('.')
|
||||
|| host.parse::<IpAddr>().is_ok()
|
||||
|| web_host_last_label_is_numeric(&host)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
for label in host.split('.') {
|
||||
if label.is_empty()
|
||||
|| label.len() > 63
|
||||
|| label.starts_with('-')
|
||||
|| label.ends_with('-')
|
||||
|| !label
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} contains a hostname label rejected by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
fn web_host_last_label_is_numeric(host: &str) -> bool {
|
||||
let label = host.rsplit('.').next().unwrap_or_default();
|
||||
let digits = label
|
||||
.strip_prefix("0x")
|
||||
.or_else(|| label.strip_prefix("0X"));
|
||||
if let Some(digits) = digits {
|
||||
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
||||
}
|
||||
label.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||
match decoy {
|
||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||
))
|
||||
})?;
|
||||
if parsed.scheme() != "http"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.path() != "/"
|
||||
|| parsed.port() == Some(0)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||
));
|
||||
}
|
||||
let ip = match parsed.host() {
|
||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||
_ => {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
||||
));
|
||||
}
|
||||
};
|
||||
let private = match ip {
|
||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||
IpAddr::V6(ip) => {
|
||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
||||
}
|
||||
};
|
||||
if !private {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
||||
));
|
||||
}
|
||||
}
|
||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||
if !directory.is_absolute() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
||||
));
|
||||
}
|
||||
if index.is_empty()
|
||||
|| index.contains('\\')
|
||||
|| std::path::Path::new(index).components().count() != 1
|
||||
|| matches!(index.as_str(), "." | "..")
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn config_error<T>(message: &str) -> Result<T> {
|
||||
Err(ProxyError::Config(message.to_string()))
|
||||
}
|
||||
// Virtual-host, hostname, and decoy validation.
|
||||
mod vhosts;
|
||||
use vhosts::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
let limits = &config.web.limits;
|
||||
if config.web.vhosts.len() > limits.max_vhosts {
|
||||
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
||||
}
|
||||
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
||||
let mut profile_count = 0usize;
|
||||
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
||||
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
||||
if !hosts.insert(vhost.host.clone()) {
|
||||
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
||||
}
|
||||
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
||||
));
|
||||
}
|
||||
if config.web.enabled && vhost.profiles.is_empty() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
||||
));
|
||||
}
|
||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||
));
|
||||
}
|
||||
if !config.access.users.contains_key(&profile.user) {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||
profile.user
|
||||
));
|
||||
}
|
||||
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
||||
return config_error(&format!(
|
||||
"duplicate WEB profile for user `{}` in vhost `{}`",
|
||||
profile.user, vhost.host
|
||||
));
|
||||
}
|
||||
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
||||
let max_streams_per_session = profile
|
||||
.max_streams_per_session
|
||||
.unwrap_or(limits.max_streams_per_session);
|
||||
if profile.max_sessions == Some(0)
|
||||
|| profile
|
||||
.max_sessions
|
||||
.is_some_and(|value| value > limits.max_sessions_global)
|
||||
|| profile.max_streams == Some(0)
|
||||
|| profile
|
||||
.max_streams
|
||||
.is_some_and(|value| value > limits.max_streams_global)
|
||||
|| profile.max_streams_per_session == Some(0)
|
||||
|| profile
|
||||
.max_streams_per_session
|
||||
.is_some_and(|value| value > limits.max_streams_per_session)
|
||||
|| max_streams_per_session > max_streams
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
||||
));
|
||||
}
|
||||
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
||||
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
if profile_count > limits.max_profiles {
|
||||
return config_error("WEB profiles exceed web.limits.max_profiles");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
||||
let input = value.trim();
|
||||
if input.is_empty()
|
||||
|| input.ends_with('.')
|
||||
|| input
|
||||
.chars()
|
||||
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
||||
));
|
||||
}
|
||||
let host = normalize_domain_to_ascii(input, field)?;
|
||||
if host.len() > 253
|
||||
|| !host.contains('.')
|
||||
|| host.parse::<IpAddr>().is_ok()
|
||||
|| web_host_last_label_is_numeric(&host)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
for label in host.split('.') {
|
||||
if label.is_empty()
|
||||
|| label.len() > 63
|
||||
|| label.starts_with('-')
|
||||
|| label.ends_with('-')
|
||||
|| !label
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} contains a hostname label rejected by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
pub(super) fn web_host_last_label_is_numeric(host: &str) -> bool {
|
||||
let label = host.rsplit('.').next().unwrap_or_default();
|
||||
let digits = label
|
||||
.strip_prefix("0x")
|
||||
.or_else(|| label.strip_prefix("0X"));
|
||||
if let Some(digits) = digits {
|
||||
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
||||
}
|
||||
label.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||
match decoy {
|
||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||
))
|
||||
})?;
|
||||
if parsed.scheme() != "http"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.path() != "/"
|
||||
|| parsed.port() == Some(0)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||
));
|
||||
}
|
||||
let ip = match parsed.host() {
|
||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||
_ => {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
||||
));
|
||||
}
|
||||
};
|
||||
let private = match ip {
|
||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||
IpAddr::V6(ip) => {
|
||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
||||
}
|
||||
};
|
||||
if !private {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
||||
));
|
||||
}
|
||||
}
|
||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||
if !directory.is_absolute() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
||||
));
|
||||
}
|
||||
if index.is_empty()
|
||||
|| index.contains('\\')
|
||||
|| std::path::Path::new(index).components().count() != 1
|
||||
|| matches!(index.as_str(), "." | "..")
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn config_error<T>(message: &str) -> Result<T> {
|
||||
Err(ProxyError::Config(message.to_string()))
|
||||
}
|
||||
+6
-81
@@ -471,84 +471,9 @@ impl Default for WebConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeConfig {
|
||||
/// Canonical host lookup used by HTTP request routing.
|
||||
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
||||
/// Flat profile inventory used by startup link emission.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed immutable virtual-host data.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeVhost {
|
||||
/// Canonical lowercase ACE hostname.
|
||||
pub(crate) host: String,
|
||||
/// Immutable ordinary-site fallback snapshot.
|
||||
pub(crate) decoy: WebRuntimeDecoy,
|
||||
/// Upstream connect and response-head deadline.
|
||||
pub(crate) decoy_header_secs: u64,
|
||||
/// Exact capability profiles accepted by this host.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed exact-user capability entry.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeProfile {
|
||||
/// Canonical host that owns this profile.
|
||||
pub(crate) host: String,
|
||||
/// Stable public destination tuple supplied to relay routing.
|
||||
pub(crate) public_addr: SocketAddr,
|
||||
/// Exact access user authenticated by logical streams.
|
||||
pub(crate) user: String,
|
||||
/// Client secret representation and inner protocol policy.
|
||||
pub(crate) secret_mode: WebSecretMode,
|
||||
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||
pub(crate) carrier_negotiation_enabled: bool,
|
||||
/// Whether automatic outcomes consult and update process-local evidence.
|
||||
pub(crate) carrier_learning: bool,
|
||||
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
pub(crate) key_fingerprint: String,
|
||||
/// Per-profile live session ceiling.
|
||||
pub(crate) max_sessions: usize,
|
||||
/// Per-profile live logical-stream ceiling.
|
||||
pub(crate) max_streams: usize,
|
||||
/// Per-session live relay-task ceiling.
|
||||
pub(crate) max_streams_per_session: usize,
|
||||
}
|
||||
|
||||
/// Runtime-ready ordinary-site fallback.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum WebRuntimeDecoy {
|
||||
HttpUpstream { addr: SocketAddr, authority: String },
|
||||
StaticDirectory(Arc<WebStaticSite>),
|
||||
}
|
||||
|
||||
/// Immutable bounded static-site snapshot.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticSite {
|
||||
/// Canonical URL-path to immutable response asset mapping.
|
||||
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
||||
/// Configured root index file name.
|
||||
pub(crate) index: String,
|
||||
}
|
||||
|
||||
/// One immutable static response body and metadata.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticAsset {
|
||||
/// Immutable response body retained by the runtime snapshot.
|
||||
pub(crate) body: Bytes,
|
||||
/// Extension-derived static content type.
|
||||
pub(crate) content_type: &'static str,
|
||||
/// Strong SHA-256 entity tag.
|
||||
pub(crate) etag: String,
|
||||
}
|
||||
// Immutable runtime WEB configuration consumed by hot paths.
|
||||
mod runtime;
|
||||
pub(crate) use runtime::{
|
||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||
WebStaticSite,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
use super::*;
|
||||
|
||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeConfig {
|
||||
/// Canonical host lookup used by HTTP request routing.
|
||||
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
||||
/// Flat profile inventory used by startup link emission.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed immutable virtual-host data.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeVhost {
|
||||
/// Canonical lowercase ACE hostname.
|
||||
pub(crate) host: String,
|
||||
/// Immutable ordinary-site fallback snapshot.
|
||||
pub(crate) decoy: WebRuntimeDecoy,
|
||||
/// Upstream connect and response-head deadline.
|
||||
pub(crate) decoy_header_secs: u64,
|
||||
/// Exact capability profiles accepted by this host.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed exact-user capability entry.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeProfile {
|
||||
/// Canonical host that owns this profile.
|
||||
pub(crate) host: String,
|
||||
/// Stable public destination tuple supplied to relay routing.
|
||||
pub(crate) public_addr: SocketAddr,
|
||||
/// Exact access user authenticated by logical streams.
|
||||
pub(crate) user: String,
|
||||
/// Client secret representation and inner protocol policy.
|
||||
pub(crate) secret_mode: WebSecretMode,
|
||||
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||
pub(crate) carrier_negotiation_enabled: bool,
|
||||
/// Whether automatic outcomes consult and update process-local evidence.
|
||||
pub(crate) carrier_learning: bool,
|
||||
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
pub(crate) key_fingerprint: String,
|
||||
/// Per-profile live session ceiling.
|
||||
pub(crate) max_sessions: usize,
|
||||
/// Per-profile live logical-stream ceiling.
|
||||
pub(crate) max_streams: usize,
|
||||
/// Per-session live relay-task ceiling.
|
||||
pub(crate) max_streams_per_session: usize,
|
||||
}
|
||||
|
||||
/// Runtime-ready ordinary-site fallback.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum WebRuntimeDecoy {
|
||||
HttpUpstream { addr: SocketAddr, authority: String },
|
||||
StaticDirectory(Arc<WebStaticSite>),
|
||||
}
|
||||
|
||||
/// Immutable bounded static-site snapshot.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticSite {
|
||||
/// Canonical URL-path to immutable response asset mapping.
|
||||
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
||||
/// Configured root index file name.
|
||||
pub(crate) index: String,
|
||||
}
|
||||
|
||||
/// One immutable static response body and metadata.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticAsset {
|
||||
/// Immutable response body retained by the runtime snapshot.
|
||||
pub(crate) body: Bytes,
|
||||
/// Extension-derived static content type.
|
||||
pub(crate) content_type: &'static str,
|
||||
/// Strong SHA-256 entity tag.
|
||||
pub(crate) etag: String,
|
||||
}
|
||||
+2
-2
@@ -192,8 +192,8 @@ fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
HEALTHCHECK_RESPONSE_MAX_BYTES, HealthcheckMode, parse_status_code,
|
||||
read_response_bounded, split_response, validate_payload,
|
||||
HEALTHCHECK_RESPONSE_MAX_BYTES, HealthcheckMode, parse_status_code, read_response_bounded,
|
||||
split_response, validate_payload,
|
||||
};
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -129,10 +129,8 @@ impl ProcessControlPlane {
|
||||
if self.inner.shutdown_completed.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
let registrations_stopped = tokio::time::timeout_at(
|
||||
deadline,
|
||||
self.inner.admission.wait_for_registrations(),
|
||||
)
|
||||
let registrations_stopped =
|
||||
tokio::time::timeout_at(deadline, self.inner.admission.wait_for_registrations())
|
||||
.await
|
||||
.is_ok();
|
||||
let tasks_stopped = tokio::time::timeout_at(deadline, self.inner.tasks.wait())
|
||||
@@ -188,7 +186,8 @@ mod tests {
|
||||
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(1)).await });
|
||||
tokio::task::yield_now().await;
|
||||
let second_scope = scope.clone();
|
||||
let second = tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
|
||||
let second =
|
||||
tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!first.is_finished());
|
||||
@@ -204,7 +203,8 @@ mod tests {
|
||||
let scope = ProcessControlPlane::new();
|
||||
let registration = scope.inner.admission.try_register().unwrap();
|
||||
let first_scope = scope.clone();
|
||||
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
|
||||
let first =
|
||||
tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
first.abort();
|
||||
|
||||
@@ -213,9 +213,7 @@ async fn run_accept_loop(
|
||||
continue;
|
||||
}
|
||||
if web_runtime.is_shutdown() {
|
||||
web_runtime
|
||||
.telemetry()
|
||||
.record_rejection(
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
drop(stream);
|
||||
@@ -233,9 +231,8 @@ async fn run_accept_loop(
|
||||
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||
let config = web_runtime.active_generation().config();
|
||||
let action = config.web.http_connection_capacity_action;
|
||||
let phase_timeout = Duration::from_millis(
|
||||
config.web.timeouts.http_overload_timeout_ms,
|
||||
);
|
||||
let phase_timeout =
|
||||
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
|
||||
drop(config);
|
||||
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
@@ -247,8 +244,7 @@ async fn run_accept_loop(
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let overload_permit =
|
||||
match web_runtime.try_http_overload_connection() {
|
||||
let overload_permit = match web_runtime.try_http_overload_connection() {
|
||||
Ok(permit) => permit,
|
||||
Err(HttpConnectionAdmissionError::Closed) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
|
||||
+6
-115
@@ -22,57 +22,11 @@ use crate::transport::middle_proxy::MePool;
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::load_startup_proxy_config_snapshot;
|
||||
|
||||
async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
loop {
|
||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||
match result {
|
||||
Ok(()) => warn!(
|
||||
task = task_name,
|
||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||
),
|
||||
Err(error) => {
|
||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_me_supervisors(
|
||||
task_scope: RuntimeTaskScope,
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
min_connections: usize,
|
||||
) {
|
||||
let health_pool = pool.clone();
|
||||
let health_rng = rng;
|
||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||
let pool = health_pool.clone();
|
||||
let rng = health_rng.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||
}
|
||||
}));
|
||||
|
||||
let drain_pool = pool.clone();
|
||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||
let pool = drain_pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||
}
|
||||
}));
|
||||
|
||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||
let pool = pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Restarting supervisors for long-lived ME maintenance tasks.
|
||||
mod supervisor;
|
||||
use supervisor::spawn_me_supervisors;
|
||||
#[cfg(test)]
|
||||
use supervisor::supervise_me_task;
|
||||
|
||||
pub(crate) async fn initialize_me_pool(
|
||||
use_middle_proxy: bool,
|
||||
@@ -587,67 +541,4 @@ pub(crate) async fn initialize_me_pool(
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_supervisor_aborts_its_current_child() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let dropped_for_task = dropped.clone();
|
||||
scope.spawn(supervise_me_task("test", move || {
|
||||
let dropped = dropped_for_task.clone();
|
||||
async move {
|
||||
let _signal = DropSignal(dropped);
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
scope.stop().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let starts = Arc::new(AtomicUsize::new(0));
|
||||
let restarted = Arc::new(Notify::new());
|
||||
let starts_task = starts.clone();
|
||||
let restarted_task = restarted.clone();
|
||||
scope.spawn(supervise_me_task("restart_test", move || {
|
||||
let starts = starts_task.clone();
|
||||
let restarted = restarted_task.clone();
|
||||
async move {
|
||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||
restarted.notify_one();
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
scope.stop().await;
|
||||
let stopped_at = starts.load(Ordering::Acquire);
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(stopped_at >= 3);
|
||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||
}
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
loop {
|
||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||
match result {
|
||||
Ok(()) => warn!(
|
||||
task = task_name,
|
||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||
),
|
||||
Err(error) => {
|
||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn_me_supervisors(
|
||||
task_scope: RuntimeTaskScope,
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
min_connections: usize,
|
||||
) {
|
||||
let health_pool = pool.clone();
|
||||
let health_rng = rng;
|
||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||
let pool = health_pool.clone();
|
||||
let rng = health_rng.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||
}
|
||||
}));
|
||||
|
||||
let drain_pool = pool.clone();
|
||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||
let pool = drain_pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||
}
|
||||
}));
|
||||
|
||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||
let pool = pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_supervisor_aborts_its_current_child() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let dropped_for_task = dropped.clone();
|
||||
scope.spawn(supervise_me_task("test", move || {
|
||||
let dropped = dropped_for_task.clone();
|
||||
async move {
|
||||
let _signal = DropSignal(dropped);
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
scope.stop().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let starts = Arc::new(AtomicUsize::new(0));
|
||||
let restarted = Arc::new(Notify::new());
|
||||
let starts_task = starts.clone();
|
||||
let restarted_task = restarted.clone();
|
||||
scope.spawn(supervise_me_task("restart_test", move || {
|
||||
let starts = starts_task.clone();
|
||||
let restarted = restarted_task.clone();
|
||||
async move {
|
||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||
restarted.notify_one();
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
scope.stop().await;
|
||||
let stopped_at = starts.load(Ordering::Acquire);
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(stopped_at >= 3);
|
||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||
}
|
||||
@@ -52,16 +52,9 @@ pub(super) async fn run_telemt_core(
|
||||
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||
let quota_state_path = config.general.quota_state_path.clone();
|
||||
let quota_state = crate::quota_state::QuotaStateOwner::new(
|
||||
quota_state_path,
|
||||
quota_store.clone(),
|
||||
);
|
||||
let configured_quota_users = config
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
let quota_state =
|
||||
crate::quota_state::QuotaStateOwner::new(quota_state_path, quota_store.clone());
|
||||
let configured_quota_users = config.access.users.keys().cloned().collect::<BTreeSet<_>>();
|
||||
quota_state.load(&configured_quota_users).await;
|
||||
|
||||
let upstream_manager = Arc::new(
|
||||
|
||||
@@ -23,9 +23,9 @@ use super::control_plane::ProcessControlPlane;
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::helpers::{format_uptime, unit_label};
|
||||
use super::reload_supervisor::ReloadSupervisorHandle;
|
||||
use crate::quota_state::QuotaStateOwner;
|
||||
use crate::stats::Stats;
|
||||
use crate::synlimit_control;
|
||||
use crate::quota_state::QuotaStateOwner;
|
||||
|
||||
/// Signal that triggered shutdown.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -130,10 +130,7 @@ async fn perform_shutdown(
|
||||
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
||||
}
|
||||
|
||||
if !process_control_plane
|
||||
.shutdown(Duration::from_secs(5))
|
||||
.await
|
||||
{
|
||||
if !process_control_plane.shutdown(Duration::from_secs(5)).await {
|
||||
warn!("Process control-plane task shutdown deadline expired");
|
||||
}
|
||||
|
||||
|
||||
+4
-3938
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
use super::*;
|
||||
|
||||
// Process, buffer, and TLS cache metrics.
|
||||
mod process;
|
||||
// Connection, quota, and conntrack metrics.
|
||||
mod connections;
|
||||
// Rate limiter, upstream, and initial ME metrics.
|
||||
mod traffic;
|
||||
// ME lifecycle and relay event metrics.
|
||||
mod me_lifecycle;
|
||||
// ME batching and resident-memory metrics.
|
||||
mod me_buffers;
|
||||
// ME writer selection, KDF, and hardswap metrics.
|
||||
mod me_policy;
|
||||
// Adaptive-floor and writer-cap metrics.
|
||||
mod me_floor;
|
||||
// Desync, pool recovery, and refill metrics.
|
||||
mod me_recovery;
|
||||
// Bounded per-user and IP-tracker metrics.
|
||||
mod users;
|
||||
|
||||
pub(super) async fn render_metrics(
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
config: &ProxyConfig,
|
||||
ip_tracker: &UserIpTracker,
|
||||
tls_cache: Option<&TlsFrontCache>,
|
||||
tls_full_cert_budget: &TlsFullCertBudget,
|
||||
web_publication: &crate::web::control::WebRuntimePublication,
|
||||
) -> String {
|
||||
let mut out = String::with_capacity(4096);
|
||||
let telemetry = stats.telemetry_policy();
|
||||
let core_enabled = telemetry.core_enabled;
|
||||
let user_enabled = telemetry.user_enabled;
|
||||
let me_allows_normal = telemetry.me_level.allows_normal();
|
||||
let me_allows_debug = telemetry.me_level.allows_debug();
|
||||
|
||||
process::render(
|
||||
&mut out,
|
||||
stats,
|
||||
shared_state,
|
||||
telemetry,
|
||||
tls_full_cert_budget,
|
||||
);
|
||||
super::render_tls_front_profile_health(&mut out, config, tls_cache).await;
|
||||
connections::render(&mut out, stats, shared_state, core_enabled);
|
||||
traffic::render(
|
||||
&mut out,
|
||||
stats,
|
||||
shared_state,
|
||||
config,
|
||||
core_enabled,
|
||||
me_allows_normal,
|
||||
me_allows_debug,
|
||||
);
|
||||
me_lifecycle::render(&mut out, stats, me_allows_normal);
|
||||
me_buffers::render(
|
||||
&mut out,
|
||||
stats,
|
||||
core_enabled,
|
||||
me_allows_normal,
|
||||
me_allows_debug,
|
||||
);
|
||||
me_policy::render(&mut out, stats, me_allows_normal, me_allows_debug);
|
||||
me_floor::render(&mut out, stats, config, me_allows_normal);
|
||||
me_recovery::render(&mut out, stats, me_allows_normal, me_allows_debug);
|
||||
users::render(
|
||||
&mut out,
|
||||
stats,
|
||||
config,
|
||||
ip_tracker,
|
||||
core_enabled,
|
||||
user_enabled,
|
||||
)
|
||||
.await;
|
||||
super::web::render(&mut out, web_publication, config);
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
core_enabled: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_total Total accepted connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_total {}",
|
||||
if core_enabled {
|
||||
stats.get_connects_all()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_bad_total Bad/rejected connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_bad_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_bad_total {}",
|
||||
if core_enabled {
|
||||
stats.get_connects_bad()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter");
|
||||
if core_enabled {
|
||||
for (class, total) in stats.get_connects_bad_class_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_bad_by_class_total{{class=\"{}\"}} {}",
|
||||
class, total
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_handshake_timeouts_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_handshake_timeouts_total {}",
|
||||
if core_enabled {
|
||||
stats.get_handshake_timeouts()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_handshake_failures_by_class_total Handshake failures by class"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_handshake_failures_by_class_total counter"
|
||||
);
|
||||
if core_enabled {
|
||||
for (class, total) in stats.get_handshake_failure_class_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}",
|
||||
class, total
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_auth_expensive_checks_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_auth_expensive_checks_total {}",
|
||||
if core_enabled {
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_expensive_checks_total
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_auth_budget_exhausted_total Handshake validations that hit authentication candidate budget limits"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_auth_budget_exhausted_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_auth_budget_exhausted_total {}",
|
||||
if core_enabled {
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_budget_exhausted_total
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_accept_permit_timeout_total Accepted connections dropped due to permit wait timeout"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_accept_permit_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_accept_permit_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_accept_permit_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_route_cutover_parked_current Sessions currently parked in route cutover stagger delay"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_route_cutover_parked_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_current{{route=\"direct\"}} {}",
|
||||
stats.get_route_cutover_parked_direct_current()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_current{{route=\"middle\"}} {}",
|
||||
stats.get_route_cutover_parked_middle_current()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_route_cutover_parked_total Sessions parked in route cutover stagger delay"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_route_cutover_parked_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_total{{route=\"direct\"}} {}",
|
||||
stats.get_route_cutover_parked_direct_total()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_total{{route=\"middle\"}} {}",
|
||||
stats.get_route_cutover_parked_middle_total()
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_refund_bytes_total Reserved quota bytes returned before commit"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_refund_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_refund_bytes_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_refund_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_contention_total Quota reservation CAS contention events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_contention_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_contention_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_contention_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_contention_timeout_total Quota reservations that hit the bounded contention budget"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_contention_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_contention_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_contention_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_acquire_cancelled_total Quota acquisitions cancelled before reservation completed"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_acquire_cancelled_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_acquire_cancelled_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_acquire_cancelled_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_control_state gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"enabled\"}} {}",
|
||||
if stats.get_conntrack_control_enabled() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"available\"}} {}",
|
||||
if stats.get_conntrack_control_available() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"pressure_active\"}} {}",
|
||||
if stats.get_conntrack_pressure_active() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"rule_apply_ok\"}} {}",
|
||||
if stats.get_conntrack_rule_apply_ok() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_event_queue_depth Pending close events in conntrack control queue"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_event_queue_depth gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_event_queue_depth {}",
|
||||
stats.get_conntrack_event_queue_depth()
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_delete_total Conntrack delete attempts by outcome"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_delete_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"attempt\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"success\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"not_found\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_not_found_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"error\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_error_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_close_event_drop_total Dropped conntrack close events due to queue pressure or unavailable sender"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_conntrack_close_event_drop_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_close_event_drop_total {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_close_event_drop_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,489 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
core_enabled: bool,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"batch_bytes\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_batch_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"max_delay\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_max_delay_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"ack_immediate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_ack_immediate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"close\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_data_frames_total DC->Client data frames"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_data_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_data_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_data_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_ack_frames_total DC->Client quick-ack frames"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_ack_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_ack_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_ack_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_payload_bytes_total DC->Client payload bytes before transport framing"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_payload_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_payload_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_payload_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_write_mode_total DC->Client writer mode selection"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_write_mode_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_write_mode_total{{mode=\"coalesced\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_write_mode_coalesced_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_write_mode_total{{mode=\"split\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_write_mode_split_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_quota_reject_total DC->Client quota rejects"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_quota_reject_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_quota_reject_total{{stage=\"pre_write\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_quota_reject_pre_write_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_quota_reject_total{{stage=\"post_write\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_quota_reject_post_write_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_child_join_timeout_total Middle relay child tasks that did not join before cleanup deadline"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_child_join_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_child_join_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_me_child_join_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_child_abort_total Middle relay child tasks aborted after bounded cleanup timeout"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_child_abort_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_child_abort_total {}",
|
||||
if core_enabled {
|
||||
stats.get_me_child_abort_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_flow_wait_events_total Flow wait events by reason, direction, and outcome"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_flow_wait_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"waited\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"cancelled\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_cancelled_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_flow_wait_ms_total Flow wait time in milliseconds by reason and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_flow_wait_ms_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_ms_total{{reason=\"middle_rate_limit\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_ms_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_session_drop_fallback_total Session reservations cleaned by Drop instead of explicit async release"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_session_drop_fallback_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_session_drop_fallback_total {}",
|
||||
if core_enabled {
|
||||
stats.get_session_drop_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_frame_buf_shrink_total DC->Client reusable frame buffer shrink events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_frame_buf_shrink_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_frame_buf_shrink_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_frame_buf_shrink_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_frame_buf_shrink_bytes_total DC->Client reusable frame buffer bytes released"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_frame_buf_shrink_bytes_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_frame_buf_shrink_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_frame_buf_shrink_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_frames_bucket_total DC->Client batch frame count buckets"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_frames_bucket_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"1\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_1()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"2_4\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_2_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"5_8\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_5_8()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"9_16\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_9_16()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"17_32\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_17_32()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"gt_32\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_gt_32()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_bytes_bucket_total DC->Client batch byte size buckets"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_bytes_bucket_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"0_1k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_0_1k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"1k_4k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_1k_4k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"4k_16k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_4k_16k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"16k_64k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_16k_64k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"64k_128k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_64k_128k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"gt_128k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_gt_128k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_flush_duration_us_bucket_total DC->Client flush duration buckets"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"0_50\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_0_50()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"51_200\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_51_200()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"201_1000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_201_1000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"1001_5000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_1001_5000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"5001_20000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_5001_20000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"gt_20000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_gt_20000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_timeout_armed_total DC->Client max-delay timer armed events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_timeout_armed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_timeout_armed_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_timeout_armed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_timeout_fired_total DC->Client max-delay timer fired events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_timeout_fired_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_timeout_fired_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_timeout_fired_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_limit_bytes Configured resident-memory budget per ME writer"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_byte_budget_limit_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_limit_bytes {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_limit_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
me_allows_normal: bool,
|
||||
) {
|
||||
let floor_mode = config.general.me_floor_mode;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"static\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Static) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"adaptive\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Adaptive) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode_switch_all_total Runtime ME floor mode switches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode_switch_all_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_all_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"static\",to=\"adaptive\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_static_to_adaptive_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"adaptive\",to=\"static\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_adaptive_to_static_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_cpu_cores_detected Runtime detected logical CPU cores for adaptive floor"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_cpu_cores_detected gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_cpu_cores_detected {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cpu_cores_detected_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_cpu_cores_effective Runtime effective logical CPU cores for adaptive floor"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_cpu_cores_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_cpu_cores_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cpu_cores_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_global_cap_raw Runtime raw global adaptive floor cap"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_adaptive_floor_global_cap_raw gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_global_cap_raw {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_global_cap_raw_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_global_cap_effective Runtime effective global adaptive floor cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_global_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_global_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_global_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_target_writers_total Runtime adaptive floor target writers total"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_target_writers_total gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_target_writers_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_target_writers_total_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_active_cap_configured Runtime configured active writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_active_cap_configured gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_active_cap_configured {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_active_cap_configured_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_active_cap_effective Runtime effective active writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_active_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_active_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_active_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_warm_cap_configured Runtime configured warm writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_warm_cap_configured gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_warm_cap_configured {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_warm_cap_configured_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_warm_cap_effective Runtime effective warm writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_warm_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_warm_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_warm_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writers_active_current Current non-draining active ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writers_active_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writers_active_current {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writers_active_current_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writers_warm_current Current non-draining warm ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writers_warm_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writers_warm_current {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writers_warm_current_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_cap_block_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_cap_block_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cap_block_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_swap_idle_total Adaptive floor cap recovery via idle writer swap"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_swap_idle_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_swap_idle_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_swap_idle_failed_total Failed idle swap attempts under adaptive floor caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_swap_idle_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_swap_idle_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(out: &mut String, stats: &Stats, me_allows_normal: bool) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reconnect_attempts_total ME reconnect attempts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reconnect_attempts_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reconnect_attempts_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reconnect_attempts()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reconnect_success_total ME reconnect successes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reconnect_success_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reconnect_success_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reconnect_success()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_handshake_reject_total ME handshake rejects from upstream"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_reject_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_reject_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_handshake_reject_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_handshake_error_code_total ME handshake reject errors by code"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_error_code_total counter");
|
||||
if me_allows_normal {
|
||||
for (error_code, count) in stats.get_me_handshake_error_code_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_error_code_total{{error_code=\"{}\"}} {}",
|
||||
error_code, count
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_error_code_total{{error_code=\"overflow\"}} {}",
|
||||
stats.get_me_handshake_error_code_overflow_total()
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reader_eof_total ME reader EOF terminations"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reader_eof_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reader_eof_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reader_eof_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_idle_close_by_peer_total ME idle writers closed by peer"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_idle_close_by_peer_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_idle_close_by_peer_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_idle_close_by_peer_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_idle_soft_mark_total Middle-relay sessions marked as soft-idle candidates"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_idle_soft_mark_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_idle_soft_mark_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_idle_soft_mark_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_idle_hard_close_total Middle-relay sessions closed by hard-idle policy"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_idle_hard_close_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_idle_hard_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_idle_hard_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_pressure_evict_total Middle-relay sessions evicted under resource pressure"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_pressure_evict_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_pressure_evict_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_pressure_evict_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_protocol_desync_close_total Middle-relay sessions closed due to protocol desync"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_relay_protocol_desync_close_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_protocol_desync_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_protocol_desync_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_crc_mismatch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_crc_mismatch()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_seq_mismatch_total ME sequence mismatches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_seq_mismatch_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_seq_mismatch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_seq_mismatch()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_no_conn_total ME route drops: no conn"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_route_drop_no_conn_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_no_conn_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_no_conn()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_channel_closed_total ME route drops: channel closed"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_route_drop_channel_closed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_channel_closed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_channel_closed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_queue_full_total ME route drops: queue full"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_route_drop_queue_full_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_queue_full_profile_total ME route drops: queue full by adaptive profile"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_route_drop_queue_full_profile_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_profile_total{{profile=\"base\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full_base()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_profile_total{{profile=\"high\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full_high()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_pressure_state Worker-local fairness pressure state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_pressure_state gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_pressure_state {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_pressure_state_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_active_flows Fair-scheduler active flow count"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_active_flows gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_active_flows {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_active_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_queued_bytes Fair-scheduler queued bytes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_queued_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_queued_bytes {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_queued_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_flow_state_gauge Fair-scheduler flow health classes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_flow_state_gauge gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_flow_state_gauge{{class=\"standing\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_standing_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_flow_state_gauge{{class=\"backpressured\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_backpressured_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_events_total Fair-scheduler event counters"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"scheduler_round\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_scheduler_rounds_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"deficit_grant\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_deficit_grants_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"deficit_skip\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_deficit_skips_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"enqueue_reject\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_enqueue_rejects_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"shed_drop\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_shed_drops_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"penalty\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_penalties_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"downstream_stall\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_downstream_stalls_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_c2me_enqueue_events_total ME client->ME enqueue outcomes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_c2me_enqueue_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"high_water\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_high_water_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"timeout\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batches_total Total DC->Client flush batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batches_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batches_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batches_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_frames_total Total DC->Client frames flushed in batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batch_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_bytes_total Total DC->Client bytes flushed in batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batch_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_flush_reason_total DC->Client flush reasons"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_flush_reason_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"queue_drain\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_queue_drain_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"batch_frames\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_batch_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_reserved_bytes Aggregate ME writer memory reservations by lifecycle state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_reserved_bytes gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"queued\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_queued_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"inflight\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_inflight_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_events_total ME writer byte-budget outcomes"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_events_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"wait\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_wait_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"timeout\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"oversize\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_oversize_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_try\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_success_try_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_fallback\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_success_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"closed\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_closed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"no_candidate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_no_candidate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_try\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_success_try_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_fallback\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_success_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"closed\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_closed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"no_candidate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_no_candidate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_blocking_fallback_total ME writer-pick blocking fallback attempts"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_pick_blocking_fallback_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_blocking_fallback_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_blocking_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_mode_switch_total Writer-pick mode switches via runtime updates"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_pick_mode_switch_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_mode_switch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_mode_switch_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_socks_kdf_policy_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_socks_kdf_policy_total{{policy=\"strict\",outcome=\"reject\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_socks_kdf_strict_reject()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_socks_kdf_policy_total{{policy=\"compat\",outcome=\"fallback\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_socks_kdf_compat_fallback()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_total ME endpoint quarantines due to rapid flaps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_endpoint_quarantine_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_unexpected_total ME endpoint quarantines caused by unexpected writer removals"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_unexpected_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_unexpected_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_draining_suppressed_total Draining writer removals that skipped endpoint quarantine"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_draining_suppressed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_draining_suppressed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_kdf_drift_total ME KDF input drift detections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_drift_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_kdf_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_kdf_port_only_drift_total ME KDF client-port changes with stable non-port material"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_port_only_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_port_only_drift_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_kdf_port_only_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_reuse_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_reuse_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_hardswap_pending_reuse_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_ttl_expired_total Pending hardswap generations reset by TTL expiration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_hardswap_pending_ttl_expired_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_ttl_expired_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_hardswap_pending_ttl_expired_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_enter_total Single-endpoint DC outage transitions to active state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_enter_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_enter_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_enter_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_exit_total Single-endpoint DC outage recovery transitions"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_exit_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_exit_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_exit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_attempt_total Reconnect attempts performed during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_attempt_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_attempt_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_success_total Successful reconnect attempts during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_success_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_success_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_quarantine_bypass_total Outage reconnect attempts that bypassed quarantine"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_quarantine_bypass_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_quarantine_bypass_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_quarantine_bypass_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_total Successful periodic shadow rotations for single-endpoint DC groups"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total Shadow rotations skipped because endpoint is quarantined"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_skipped_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode Runtime ME writer floor policy mode"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode gauge");
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_secure_padding_invalid_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_secure_padding_invalid()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_total Total crypto-desync detections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_full_logged_total Full forensic desync logs emitted"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_full_logged_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_full_logged_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_full_logged()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_suppressed_total Suppressed desync forensic events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_suppressed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_suppressed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_suppressed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_frames_bucket_total Desync count by frames_ok bucket"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_frames_bucket_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"0\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_0()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"1_2\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_1_2()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"3_10\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_3_10()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"gt_10\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_gt_10()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_swap_total Successful ME pool swaps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_swap_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_swap_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_swap_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_drain_active Active draining ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_drain_active gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_drain_active {}",
|
||||
if me_allows_debug {
|
||||
stats.get_pool_drain_active()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_force_close_total Forced close events for draining writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_force_close_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_force_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_force_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_stale_pick_total Stale writer fallback picks for new binds"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_stale_pick_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_stale_pick_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_stale_pick_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_total Total ME writer removals"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_removed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_writer_removed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_unexpected_total Unexpected ME writer removals that triggered refill"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_removed_unexpected_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_unexpected_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_removed_unexpected_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_triggered_total Immediate ME refill runs started"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_refill_triggered_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_triggered_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_refill_triggered_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_skipped_inflight_total Immediate ME refill skips due to inflight dedup"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_refill_skipped_inflight_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_skipped_inflight_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_refill_skipped_inflight_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_failed_total Immediate ME refill failures"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_refill_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_refill_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_restored_same_endpoint_total Refilled ME writer restored on the same endpoint"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_restored_same_endpoint_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_restored_same_endpoint_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_restored_same_endpoint_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_restored_fallback_total Refilled ME writer restored via fallback endpoint"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_restored_fallback_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_restored_fallback_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_restored_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_no_writer_failfast_total ME route failfast errors due to missing writer in bounded wait window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_no_writer_failfast_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_no_writer_failfast_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_no_writer_failfast_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hybrid_timeout_total ME hybrid route timeouts after bounded retry window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hybrid_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hybrid_timeout_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_hybrid_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_async_recovery_trigger_total Async ME recovery trigger attempts from route path"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_async_recovery_trigger_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_async_recovery_trigger_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_async_recovery_trigger_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_inline_recovery_total Legacy inline ME recovery attempts from route path"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_inline_recovery_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_inline_recovery_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_inline_recovery_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let unresolved_writer_losses = if me_allows_normal {
|
||||
stats
|
||||
.get_me_writer_removed_unexpected_total()
|
||||
.saturating_sub(
|
||||
stats
|
||||
.get_me_writer_restored_same_endpoint_total()
|
||||
.saturating_add(stats.get_me_writer_restored_fallback_total()),
|
||||
)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_unexpected_minus_restored_total Unexpected writer removals not yet compensated by restore"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_unexpected_minus_restored_total {}",
|
||||
unresolved_writer_losses
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
telemetry: crate::stats::telemetry::TelemetryPolicy,
|
||||
tls_full_cert_budget: &TlsFullCertBudget,
|
||||
) {
|
||||
let core_enabled = telemetry.core_enabled;
|
||||
let user_enabled = telemetry.user_enabled;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_build_info Build information for the running telemt binary"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_build_info gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_uptime_seconds Proxy uptime");
|
||||
let _ = writeln!(out, "# TYPE telemt_uptime_seconds gauge");
|
||||
let _ = writeln!(out, "telemt_uptime_seconds {:.1}", stats.uptime_secs());
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_core_enabled Runtime core telemetry switch"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_core_enabled gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_core_enabled {}",
|
||||
if core_enabled { 1 } else { 0 }
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_enabled Runtime per-user telemetry switch"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_enabled gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_enabled {}",
|
||||
if user_enabled { 1 } else { 0 }
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_stats_user_entries Retained per-user stats entries"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_stats_user_entries gauge");
|
||||
let _ = writeln!(out, "telemt_stats_user_entries {}", stats.user_stats_len());
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_me_level Runtime ME telemetry level flag"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_me_level gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"silent\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Silent) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"normal\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Normal) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"debug\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Debug) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_buffer_pool_buffers_total Snapshot of pooled and allocated buffers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_buffer_pool_buffers_total gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"pooled\"}} {}",
|
||||
stats.get_buffer_pool_pooled_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"allocated\"}} {}",
|
||||
stats.get_buffer_pool_allocated_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"in_use\"}} {}",
|
||||
stats.get_buffer_pool_in_use_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_buffer_pool_events_total Buffer-pool allocation lifecycle events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_buffer_pool_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_events_total{{event=\"replaced_nonstandard\"}} {}",
|
||||
stats.get_buffer_pool_replaced_nonstandard_total()
|
||||
);
|
||||
|
||||
let direct_budget = shared_state.direct_buffer_budget.snapshot();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_bytes Direct relay copy-buffer budget and memory inputs"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_budget_bytes gauge");
|
||||
for (kind, value) in [
|
||||
("hard_limit", direct_budget.hard_limit_bytes),
|
||||
("target", direct_budget.target_bytes),
|
||||
("reserved", direct_budget.reserved_bytes),
|
||||
("memory_total", direct_budget.memory_total_bytes),
|
||||
("memory_available", direct_budget.memory_available_bytes),
|
||||
("process_rss", direct_budget.process_rss_bytes),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_bytes{{kind=\"{}\"}} {}",
|
||||
kind, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_events_total Direct relay buffer-budget lifecycle events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_direct_relay_buffer_budget_events_total counter"
|
||||
);
|
||||
for (result, value) in [
|
||||
("promotion", direct_budget.promotion_total),
|
||||
("promotion_denied", direct_budget.promotion_denied_total),
|
||||
("minimum_fallback", direct_budget.minimum_fallback_total),
|
||||
("admission_rejected", direct_budget.admission_rejected_total),
|
||||
("quiet_demotion", direct_budget.quiet_demotion_total),
|
||||
(
|
||||
"write_pressure_demotion",
|
||||
direct_budget.write_pressure_demotion_total,
|
||||
),
|
||||
(
|
||||
"global_pressure_demotion",
|
||||
direct_budget.global_pressure_demotion_total,
|
||||
),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_events_total{{result=\"{}\"}} {}",
|
||||
result, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_sessions Current Direct relay sessions by adaptive tier"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_sessions gauge");
|
||||
for (tier, value) in ["base", "tier1", "tier2", "tier3"]
|
||||
.into_iter()
|
||||
.zip(direct_budget.tier_sessions)
|
||||
{
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_sessions{{tier=\"{}\"}} {}",
|
||||
tier, value
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_fetch_profile_cache_entries Current adaptive TLS fetch profile-cache entries"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_tls_fetch_profile_cache_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_fetch_profile_cache_entries {}",
|
||||
fetcher::profile_cache_entries_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_fetch_profile_cache_cap_drops_total Profile-cache winner inserts skipped because the cache cap was reached"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_fetch_profile_cache_cap_drops_total {}",
|
||||
fetcher::profile_cache_cap_drops_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_full_cert_budget_entries Current domain and IP entries tracked by the process-owned TLS full-cert budget"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_full_cert_budget_entries gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_full_cert_budget_entries {}",
|
||||
tls_full_cert_budget.entries_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_full_cert_budget_cap_drops_total New domain and IP entries denied full-cert budget tracking because a bound was reached"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
|
||||
tls_full_cert_budget.cap_drops_for_metrics()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,516 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
config: &ProxyConfig,
|
||||
core_enabled: bool,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_burst_bound_bytes Configured upper bound for one direct relay rate-limit burst"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_burst_bound_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_burst_bound_bytes{{direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
config.general.direct_relay_copy_buf_c2s_bytes
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_burst_bound_bytes{{direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
config.general.direct_relay_copy_buf_s2c_bytes
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_throttle_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_throttle_up_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_throttle_down_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_throttle_up_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_throttle_down_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_wait_ms_total Traffic limiter accumulated wait time in milliseconds by scope and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_wait_ms_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_wait_up_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_wait_down_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_wait_up_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_wait_down_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_active_leases Active relay leases under rate limiting by scope"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_active_leases gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_active_leases{{scope=\"user\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_active_leases
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_active_leases{{scope=\"cidr\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_active_leases
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_policy_entries Active rate-limit policy entries by scope"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_policy_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_policy_entries{{scope=\"user\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_policy_entries
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_policy_entries{{scope=\"cidr\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_policy_entries
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_attempt_total Upstream connect attempts across all requests"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_attempt_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempt_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_success_total Successful upstream connect request cycles"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_success_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_success_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_fail_total Failed upstream connect request cycles"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_fail_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_fail_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_fail_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_failfast_hard_error_total Hard errors that triggered upstream connect failfast"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_failfast_hard_error_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_failfast_hard_error_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_failfast_hard_error_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_attempts_per_request Histogram-like buckets for attempts per upstream connect request cycle"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_attempts_per_request counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"1\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_1()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"2\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_2()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"3_4\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_3_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"gt_4\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_gt_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_duration_success_total Histogram-like buckets of successful upstream connect cycle duration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_duration_success_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"le_100ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_le_100ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"101_500ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_101_500ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"501_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_501_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"gt_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_gt_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_duration_fail_total Histogram-like buckets of failed upstream connect cycle duration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_duration_fail_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"le_100ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_le_100ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"101_500ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_101_500ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"501_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_501_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"gt_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_gt_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_sent_total ME keepalive frames sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_sent_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_sent_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_keepalive_sent()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_failed_total ME keepalive send failures"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_keepalive_failed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_pong_total ME keepalive pong replies"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_pong_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_pong_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_keepalive_pong()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_timeout_total ME keepalive ping timeouts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_timeout_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_keepalive_timeout()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_sent_total Service RPC_PROXY_REQ activity signals sent"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_sent_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_sent_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_failed_total Service RPC_PROXY_REQ activity signal failures"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_failed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_skipped_no_meta_total Service RPC_PROXY_REQ skipped due to missing writer metadata"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_skipped_no_meta_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_skipped_no_meta_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_skipped_no_meta_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_response_total Service RPC_PROXY_REQ responses observed"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_response_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_response_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_response_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_close_sent_total Service RPC_CLOSE_EXT sent after activity signals"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_close_sent_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_close_sent_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_close_sent_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) async fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
ip_tracker: &UserIpTracker,
|
||||
core_enabled: bool,
|
||||
user_enabled: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_connections_total Per-user total connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_connections_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_connections_current Per-user active connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_connections_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_octets_from_client_total Per-user total bytes received"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_octets_from_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_octets_to_client_total Per-user total bytes sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_octets_to_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_msgs_from_client_total Per-user total messages received"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_msgs_from_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_msgs_to_client_total Per-user total messages sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_msgs_to_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_reservation_rollback_total IP reservation rollbacks caused by later limit checks"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_reservation_rollback_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_reservation_rollback_total{{reason=\"tcp_limit\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_ip_reservation_rollback_tcp_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_reservation_rollback_total{{reason=\"quota_limit\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_ip_reservation_rollback_quota_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let ip_memory = ip_tracker.memory_stats().await;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_users Number of users tracked by IP limiter state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_users gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_users{{scope=\"active\"}} {}",
|
||||
ip_memory.active_users
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_users{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_users
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_entries Number of IP entries tracked by limiter state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_entries{{scope=\"active\"}} {}",
|
||||
ip_memory.active_entries
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_entries{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_entries
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cleanup_queue_len Deferred disconnect cleanup queue length"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_queue_len gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cleanup_queue_len {}",
|
||||
ip_memory.cleanup_queue_len
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cleanup_total Release cleanups deferred through the cleanup queue"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cleanup_total{{path=\"deferred\"}} {}",
|
||||
ip_memory.cleanup_deferred_releases
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cap_rejects_total New connection rejects caused by global IP tracker caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cap_rejects_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cap_rejects_total{{scope=\"active\"}} {}",
|
||||
ip_memory.active_cap_rejects
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cap_rejects_total{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_cap_rejects
|
||||
);
|
||||
|
||||
let mut user_stats_emitted = 0usize;
|
||||
let mut user_stats_suppressed = 0usize;
|
||||
let mut unique_ip_emitted = 0usize;
|
||||
let mut unique_ip_suppressed = 0usize;
|
||||
|
||||
if user_enabled {
|
||||
for entry in stats.iter_user_stats() {
|
||||
if user_stats_emitted >= USER_LABELED_METRICS_MAX_USERS {
|
||||
user_stats_suppressed = user_stats_suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
let user = entry.key();
|
||||
let s = entry.value();
|
||||
user_stats_emitted = user_stats_emitted.saturating_add(1);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_connections_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.connects.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_connections_current{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.curr_connects.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_octets_from_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.octets_from_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_octets_to_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.octets_to_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_msgs_from_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.msgs_from_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_msgs_to_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.msgs_to_client.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
}
|
||||
|
||||
let ip_stats = ip_tracker.get_stats_snapshot().await;
|
||||
let ip_counts: HashMap<String, usize> = ip_stats
|
||||
.into_iter()
|
||||
.map(|(user, count, _)| (user, count))
|
||||
.collect();
|
||||
|
||||
let mut unique_users = BTreeSet::new();
|
||||
unique_users.extend(config.access.users.keys().cloned());
|
||||
unique_users.extend(config.access.user_max_unique_ips.keys().cloned());
|
||||
unique_users.extend(ip_counts.keys().cloned());
|
||||
let unique_users_vec: Vec<String> = unique_users.iter().cloned().collect();
|
||||
let recent_counts = ip_tracker
|
||||
.get_recent_counts_for_users_snapshot(&unique_users_vec)
|
||||
.await;
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_current Per-user current number of unique active IPs"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_recent_window Per-user unique IPs seen in configured observation window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_recent_window gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_limit Effective per-user unique IP limit (0 means unlimited)"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_limit gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_utilization Per-user unique IP usage ratio (0 for unlimited)"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge");
|
||||
|
||||
for user in unique_users {
|
||||
if unique_ip_emitted >= USER_LABELED_METRICS_MAX_USERS {
|
||||
unique_ip_suppressed = unique_ip_suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
unique_ip_emitted = unique_ip_emitted.saturating_add(1);
|
||||
let current = ip_counts.get(&user).copied().unwrap_or(0);
|
||||
let limit = config
|
||||
.access
|
||||
.user_max_unique_ips
|
||||
.get(&user)
|
||||
.copied()
|
||||
.filter(|limit| *limit > 0)
|
||||
.or((config.access.user_max_unique_ips_global_each > 0)
|
||||
.then_some(config.access.user_max_unique_ips_global_each))
|
||||
.unwrap_or(0);
|
||||
let utilization = if limit > 0 {
|
||||
current as f64 / limit as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_current{{user=\"{}\"}} {}",
|
||||
user, current
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_recent_window{{user=\"{}\"}} {}",
|
||||
user,
|
||||
recent_counts.get(&user).copied().unwrap_or(0)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_limit{{user=\"{}\"}} {}",
|
||||
user, limit
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_utilization{{user=\"{}\"}} {:.6}",
|
||||
user, utilization
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_suppressed {}",
|
||||
if user_enabled && user_stats_suppressed == 0 && unique_ip_suppressed == 0 {
|
||||
0
|
||||
} else {
|
||||
1
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_series_users User-labeled metric users by export status"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_users gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"emitted\"}} {}",
|
||||
user_stats_emitted
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"suppressed\"}} {}",
|
||||
user_stats_suppressed
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"emitted\"}} {}",
|
||||
unique_ip_emitted
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"suppressed\"}} {}",
|
||||
unique_ip_suppressed
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use std::net::IpAddr;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use crate::tls_front::types::{
|
||||
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
|
||||
};
|
||||
|
||||
fn test_web_publication() -> crate::web::control::WebRuntimePublication {
|
||||
let control = crate::web::control::WebRuntimeControl::new();
|
||||
control.subscribe().borrow().clone()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_metrics_format() {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.access
|
||||
.user_max_unique_ips
|
||||
.insert("alice".to_string(), 4);
|
||||
|
||||
stats.increment_connects_all();
|
||||
stats.increment_connects_all();
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
stats.increment_handshake_timeouts();
|
||||
stats.increment_handshake_failure_class("timeout");
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_expensive_checks_total
|
||||
.fetch_add(9, std::sync::atomic::Ordering::Relaxed);
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_budget_exhausted_total
|
||||
.fetch_add(2, std::sync::atomic::Ordering::Relaxed);
|
||||
stats.increment_upstream_connect_attempt_total();
|
||||
stats.increment_upstream_connect_attempt_total();
|
||||
stats.increment_upstream_connect_success_total();
|
||||
stats.increment_upstream_connect_fail_total();
|
||||
stats.increment_upstream_connect_failfast_hard_error_total();
|
||||
stats.observe_upstream_connect_attempts_per_request(2);
|
||||
stats.observe_upstream_connect_duration_ms(220, true);
|
||||
stats.observe_upstream_connect_duration_ms(1500, false);
|
||||
stats.increment_me_rpc_proxy_req_signal_sent_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_failed_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_response_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_close_sent_total();
|
||||
stats.increment_me_idle_close_by_peer_total();
|
||||
stats.increment_relay_idle_soft_mark_total();
|
||||
stats.increment_relay_idle_hard_close_total();
|
||||
stats.increment_relay_pressure_evict_total();
|
||||
stats.increment_relay_protocol_desync_close_total();
|
||||
stats.increment_me_d2c_batches_total();
|
||||
stats.add_me_d2c_batch_frames_total(3);
|
||||
stats.add_me_d2c_batch_bytes_total(2048);
|
||||
stats.increment_me_d2c_flush_reason(crate::stats::MeD2cFlushReason::AckImmediate);
|
||||
stats.increment_me_d2c_data_frames_total();
|
||||
stats.increment_me_d2c_ack_frames_total();
|
||||
stats.add_me_d2c_payload_bytes_total(1800);
|
||||
stats.increment_me_d2c_write_mode(crate::stats::MeD2cWriteMode::Coalesced);
|
||||
stats.increment_me_d2c_quota_reject_total(crate::stats::MeD2cQuotaRejectStage::PostWrite);
|
||||
stats.observe_me_d2c_frame_buf_shrink(4096);
|
||||
stats.increment_me_endpoint_quarantine_total();
|
||||
stats.increment_me_endpoint_quarantine_unexpected_total();
|
||||
stats.increment_me_endpoint_quarantine_draining_suppressed_total();
|
||||
stats.increment_user_connects("alice");
|
||||
stats.increment_user_curr_connects("alice");
|
||||
stats.add_user_octets_from("alice", 1024);
|
||||
stats.add_user_octets_to("alice", 2048);
|
||||
stats.increment_user_msgs_from("alice");
|
||||
stats.increment_user_msgs_to("alice");
|
||||
stats.increment_user_msgs_to("alice");
|
||||
tracker
|
||||
.check_and_add("alice", "203.0.113.10".parse().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
shared_state.as_ref(),
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains(&format!(
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
)));
|
||||
assert!(output.contains("telemt_connections_total 2"));
|
||||
assert!(output.contains("telemt_connections_bad_total 1"));
|
||||
assert!(
|
||||
output.contains(
|
||||
"telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1"
|
||||
)
|
||||
);
|
||||
assert!(output.contains("telemt_handshake_timeouts_total 1"));
|
||||
assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1"));
|
||||
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
|
||||
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
|
||||
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
|
||||
assert!(output.contains("telemt_upstream_connect_success_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_fail_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_failfast_hard_error_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_attempts_per_request{bucket=\"2\"} 1"));
|
||||
assert!(
|
||||
output.contains("telemt_upstream_connect_duration_success_total{bucket=\"101_500ms\"} 1")
|
||||
);
|
||||
assert!(output.contains("telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\"} 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_sent_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_failed_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_skipped_no_meta_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_response_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_close_sent_total 1"));
|
||||
assert!(output.contains("telemt_me_idle_close_by_peer_total 1"));
|
||||
assert!(output.contains("telemt_relay_idle_soft_mark_total 1"));
|
||||
assert!(output.contains("telemt_relay_idle_hard_close_total 1"));
|
||||
assert!(output.contains("telemt_relay_pressure_evict_total 1"));
|
||||
assert!(output.contains("telemt_relay_protocol_desync_close_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_batches_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_batch_frames_total 3"));
|
||||
assert!(output.contains("telemt_me_d2c_batch_bytes_total 2048"));
|
||||
assert!(output.contains("telemt_me_d2c_flush_reason_total{reason=\"ack_immediate\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_data_frames_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_ack_frames_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_payload_bytes_total 1800"));
|
||||
assert!(output.contains("telemt_me_d2c_write_mode_total{mode=\"coalesced\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_quota_reject_total{stage=\"post_write\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_bytes_total 4096"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_total 1"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_unexpected_total 1"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_draining_suppressed_total 1"));
|
||||
assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_octets_from_client_total{user=\"alice\"} 1024"));
|
||||
assert!(output.contains("telemt_user_octets_to_client_total{user=\"alice\"} 2048"));
|
||||
assert!(output.contains("telemt_user_msgs_from_client_total{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_msgs_to_client_total{user=\"alice\"} 2"));
|
||||
assert!(output.contains("telemt_user_unique_ips_current{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_unique_ips_recent_window{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 4"));
|
||||
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.250000"));
|
||||
assert!(output.contains("telemt_ip_tracker_users{scope=\"active\"} 1"));
|
||||
assert!(output.contains("telemt_ip_tracker_entries{scope=\"active\"} 1"));
|
||||
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_tls_front_profile_health() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_domain = "primary.example".to_string();
|
||||
config.censorship.tls_domains = vec!["fallback.example".to_string()];
|
||||
|
||||
let cache = TlsFrontCache::new(
|
||||
&[
|
||||
"primary.example".to_string(),
|
||||
"fallback.example".to_string(),
|
||||
],
|
||||
1024,
|
||||
"tlsfront-profile-health-test",
|
||||
);
|
||||
cache
|
||||
.set(
|
||||
"primary.example",
|
||||
CachedTlsData {
|
||||
server_hello_template: ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: {
|
||||
let mut key_share = vec![0x00, 0x1d, 0x00, 0x20];
|
||||
key_share.resize(36, 0x42);
|
||||
vec![
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x002b,
|
||||
data: vec![0x03, 0x04],
|
||||
},
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x0033,
|
||||
data: key_share,
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
cert_info: None,
|
||||
cert_payload: Some(TlsCertPayload {
|
||||
cert_chain_der: vec![vec![0x30, 0x01]],
|
||||
certificate_message: vec![0x0b, 0x00, 0x00, 0x00],
|
||||
}),
|
||||
app_data_records_sizes: vec![1024, 512],
|
||||
total_app_data_len: 1536,
|
||||
behavior_profile: TlsBehaviorProfile {
|
||||
change_cipher_spec_count: 1,
|
||||
app_data_record_sizes: vec![1024, 512],
|
||||
ticket_record_sizes: vec![69],
|
||||
source: TlsProfileSource::Merged,
|
||||
..TlsBehaviorProfile::default()
|
||||
},
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "primary.example".to_string(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
Some(&cache),
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0"));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"primary.example\",quality=\"raw_strict\",key_share_group=\"x25519\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"fallback.example\",quality=\"fallback\",key_share_group=\"none\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output
|
||||
.contains("telemt_tls_front_profile_server_hello_bytes{domain=\"primary.example\"} 90")
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_server_hello_extensions{domain=\"primary.example\"} 2"
|
||||
));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1")
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1"
|
||||
));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn process_tls_budget_metrics_survive_a_generation_without_tls_cache() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let budget = Arc::new(TlsFullCertBudget::new());
|
||||
let cache = TlsFrontCache::new_with_full_cert_budget(
|
||||
&["example.com".to_string()],
|
||||
1024,
|
||||
"tlsfront-test-cache",
|
||||
Arc::clone(&budget),
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip(
|
||||
"example.com",
|
||||
"127.0.0.1".parse().unwrap(),
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await
|
||||
);
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
budget.as_ref(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_tls_front_full_cert_budget_entries 1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_empty_stats() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
assert!(output.contains("telemt_connections_total 0"));
|
||||
assert!(output.contains("telemt_connections_bad_total 0"));
|
||||
assert!(output.contains("telemt_handshake_timeouts_total 0"));
|
||||
assert!(output.contains("telemt_auth_expensive_checks_total 0"));
|
||||
assert!(output.contains("telemt_auth_budget_exhausted_total 0"));
|
||||
assert!(output.contains("telemt_user_unique_ips_current{user="));
|
||||
assert!(output.contains("telemt_user_unique_ips_recent_window{user="));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_uses_global_each_unique_ip_limit() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
stats.increment_user_connects("alice");
|
||||
stats.increment_user_curr_connects("alice");
|
||||
let tracker = UserIpTracker::new();
|
||||
tracker
|
||||
.check_and_add("alice", "203.0.113.10".parse().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut config = ProxyConfig::default();
|
||||
config.access.user_max_unique_ips_global_each = 2;
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
|
||||
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_has_type_annotations() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
|
||||
assert!(output.contains("# TYPE telemt_connections_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_idle_soft_mark_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_idle_hard_close_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_pressure_evict_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_protocol_desync_close_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_batches_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_flush_reason_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_write_mode_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_batch_frames_bucket_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"));
|
||||
assert!(
|
||||
output.contains("# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter")
|
||||
);
|
||||
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
||||
assert!(
|
||||
output.contains("# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge")
|
||||
);
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_current gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_recent_window gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge"));
|
||||
assert!(output.contains("# TYPE telemt_stats_user_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_telemetry_user_series_users gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_users gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_queue_len gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cap_rejects_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_quality_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_bytes gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_extensions gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_endpoint_integration() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.general.beobachten = true;
|
||||
config.general.beobachten_minutes = 10;
|
||||
let runtime = crate::maestro::generation::test_runtime_generation(1, config);
|
||||
let web_publication = test_web_publication();
|
||||
let tls_full_cert_budget = TlsFullCertBudget::new();
|
||||
runtime.stats.increment_connects_all();
|
||||
runtime.stats.increment_connects_all();
|
||||
runtime.stats.increment_connects_all();
|
||||
|
||||
let req = Request::builder().uri("/metrics").body(()).unwrap();
|
||||
let resp = handle(req, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = resp.into_body().collect().await.unwrap().to_bytes();
|
||||
assert!(
|
||||
std::str::from_utf8(body.as_ref())
|
||||
.unwrap()
|
||||
.contains("telemt_connections_total 3")
|
||||
);
|
||||
assert!(
|
||||
std::str::from_utf8(body.as_ref())
|
||||
.unwrap()
|
||||
.contains(&format!(
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
))
|
||||
);
|
||||
|
||||
runtime.beobachten.record(
|
||||
"TLS-scanner",
|
||||
"203.0.113.10".parse::<IpAddr>().unwrap(),
|
||||
Duration::from_secs(600),
|
||||
);
|
||||
let req_beob = Request::builder().uri("/beobachten").body(()).unwrap();
|
||||
let resp_beob = handle(req_beob, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp_beob.status(), StatusCode::OK);
|
||||
let body_beob = resp_beob.into_body().collect().await.unwrap().to_bytes();
|
||||
let beob_text = std::str::from_utf8(body_beob.as_ref()).unwrap();
|
||||
assert!(beob_text.contains("[TLS-scanner]"));
|
||||
assert!(beob_text.contains("203.0.113.10-1"));
|
||||
|
||||
let req404 = Request::builder().uri("/other").body(()).unwrap();
|
||||
let resp404 = handle(req404, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp404.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
+10
-184
@@ -70,9 +70,7 @@ pub async fn run_probe(
|
||||
) -> Result<NetworkProbe> {
|
||||
let mut probe = NetworkProbe::default();
|
||||
let dns_resolver = Arc::new(
|
||||
crate::network::dns_overrides::GenerationDnsResolver::from_entries(
|
||||
&config.dns_overrides,
|
||||
)?,
|
||||
crate::network::dns_overrides::GenerationDnsResolver::from_entries(&config.dns_overrides)?,
|
||||
);
|
||||
let servers = collect_stun_servers(config);
|
||||
let mut detected_ipv4 = detect_local_ip_v4();
|
||||
@@ -426,185 +424,13 @@ pub fn decide_network_capabilities(
|
||||
}
|
||||
}
|
||||
|
||||
// Local interface discovery and bogon classification.
|
||||
mod local;
|
||||
pub use local::{
|
||||
detect_interface_ipv4, detect_interface_ipv6, is_bogon, is_bogon_v4, is_bogon_v6,
|
||||
log_probe_result,
|
||||
};
|
||||
use local::{detect_local_ip_v4, detect_local_ip_v6};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::NetworkConfig;
|
||||
|
||||
#[test]
|
||||
fn manual_nat_ip_enables_ipv4_me_without_reflection() {
|
||||
let config = NetworkConfig {
|
||||
ipv4: true,
|
||||
..Default::default()
|
||||
};
|
||||
let probe = NetworkProbe {
|
||||
detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)),
|
||||
ipv4_is_bogon: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = decide_network_capabilities(
|
||||
&config,
|
||||
&probe,
|
||||
Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))),
|
||||
);
|
||||
|
||||
assert!(decision.ipv4_me);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_nat_ip_does_not_enable_other_family() {
|
||||
let config = NetworkConfig {
|
||||
ipv4: true,
|
||||
ipv6: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
let probe = NetworkProbe {
|
||||
detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)),
|
||||
detected_ipv6: Some(Ipv6Addr::LOCALHOST),
|
||||
ipv4_is_bogon: true,
|
||||
ipv6_is_bogon: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision = decide_network_capabilities(
|
||||
&config,
|
||||
&probe,
|
||||
Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))),
|
||||
);
|
||||
|
||||
assert!(decision.ipv4_me);
|
||||
assert!(!decision.ipv6_me);
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_local_ip_v4() -> Option<Ipv4Addr> {
|
||||
let socket = UdpSocket::bind("0.0.0.0:0").ok()?;
|
||||
socket.connect("8.8.8.8:80").ok()?;
|
||||
match socket.local_addr().ok()?.ip() {
|
||||
IpAddr::V4(v4) => Some(v4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_local_ip_v6() -> Option<Ipv6Addr> {
|
||||
let socket = UdpSocket::bind("[::]:0").ok()?;
|
||||
socket.connect("[2001:4860:4860::8888]:80").ok()?;
|
||||
match socket.local_addr().ok()?.ip() {
|
||||
IpAddr::V6(v6) => Some(v6),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_interface_ipv4() -> Option<Ipv4Addr> {
|
||||
detect_local_ip_v4()
|
||||
}
|
||||
|
||||
pub fn detect_interface_ipv6() -> Option<Ipv6Addr> {
|
||||
detect_local_ip_v6()
|
||||
}
|
||||
|
||||
pub fn is_bogon(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => is_bogon_v4(v4),
|
||||
IpAddr::V6(v6) => is_bogon_v6(v6),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_bogon_v4(ip: Ipv4Addr) -> bool {
|
||||
let octets = ip.octets();
|
||||
if ip.is_private() || ip.is_loopback() || ip.is_link_local() {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 0 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 100 && (octets[1] & 0xC0) == 64 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 192 && octets[1] == 0 && octets[2] == 0 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 192 && octets[1] == 0 && octets[2] == 2 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 198 && (octets[1] & 0xFE) == 18 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 198 && octets[1] == 51 && octets[2] == 100 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 203 && octets[1] == 0 && octets[2] == 113 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_multicast() {
|
||||
return true;
|
||||
}
|
||||
if octets[0] >= 240 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_broadcast() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn is_bogon_v6(ip: Ipv6Addr) -> bool {
|
||||
if ip.is_unspecified() || ip.is_loopback() || ip.is_unique_local() {
|
||||
return true;
|
||||
}
|
||||
let segs = ip.segments();
|
||||
if (segs[0] & 0xFFC0) == 0xFE80 {
|
||||
return true;
|
||||
}
|
||||
if segs[0..5] == [0, 0, 0, 0, 0] && segs[5] == 0xFFFF {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x0100 && segs[1..4] == [0, 0, 0] {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x2001 && segs[1] == 0x0db8 {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x2002 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_multicast() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn log_probe_result(probe: &NetworkProbe, decision: &NetworkDecision) {
|
||||
info!(
|
||||
ipv4 = probe
|
||||
.detected_ipv4
|
||||
.as_ref()
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
ipv6 = probe
|
||||
.detected_ipv6
|
||||
.as_ref()
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
reflected_v4 = probe
|
||||
.reflected_ipv4
|
||||
.as_ref()
|
||||
.map(|v| v.ip().to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
reflected_v6 = probe
|
||||
.reflected_ipv6
|
||||
.as_ref()
|
||||
.map(|v| v.ip().to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
ipv4_bogon = probe.ipv4_is_bogon,
|
||||
ipv6_bogon = probe.ipv6_is_bogon,
|
||||
ipv4_me = decision.ipv4_me,
|
||||
ipv6_me = decision.ipv6_me,
|
||||
ipv4_dc = decision.ipv4_dc,
|
||||
ipv6_dc = decision.ipv6_dc,
|
||||
prefer = decision.effective_prefer,
|
||||
multipath = decision.effective_multipath,
|
||||
"Network capabilities resolved"
|
||||
);
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn detect_local_ip_v4() -> Option<Ipv4Addr> {
|
||||
let socket = UdpSocket::bind("0.0.0.0:0").ok()?;
|
||||
socket.connect("8.8.8.8:80").ok()?;
|
||||
match socket.local_addr().ok()?.ip() {
|
||||
IpAddr::V4(v4) => Some(v4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn detect_local_ip_v6() -> Option<Ipv6Addr> {
|
||||
let socket = UdpSocket::bind("[::]:0").ok()?;
|
||||
socket.connect("[2001:4860:4860::8888]:80").ok()?;
|
||||
match socket.local_addr().ok()?.ip() {
|
||||
IpAddr::V6(v6) => Some(v6),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_interface_ipv4() -> Option<Ipv4Addr> {
|
||||
detect_local_ip_v4()
|
||||
}
|
||||
|
||||
pub fn detect_interface_ipv6() -> Option<Ipv6Addr> {
|
||||
detect_local_ip_v6()
|
||||
}
|
||||
|
||||
pub fn is_bogon(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => is_bogon_v4(v4),
|
||||
IpAddr::V6(v6) => is_bogon_v6(v6),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_bogon_v4(ip: Ipv4Addr) -> bool {
|
||||
let octets = ip.octets();
|
||||
if ip.is_private() || ip.is_loopback() || ip.is_link_local() {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 0 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 100 && (octets[1] & 0xC0) == 64 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 192 && octets[1] == 0 && octets[2] == 0 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 192 && octets[1] == 0 && octets[2] == 2 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 198 && (octets[1] & 0xFE) == 18 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 198 && octets[1] == 51 && octets[2] == 100 {
|
||||
return true;
|
||||
}
|
||||
if octets[0] == 203 && octets[1] == 0 && octets[2] == 113 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_multicast() {
|
||||
return true;
|
||||
}
|
||||
if octets[0] >= 240 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_broadcast() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn is_bogon_v6(ip: Ipv6Addr) -> bool {
|
||||
if ip.is_unspecified() || ip.is_loopback() || ip.is_unique_local() {
|
||||
return true;
|
||||
}
|
||||
let segs = ip.segments();
|
||||
if (segs[0] & 0xFFC0) == 0xFE80 {
|
||||
return true;
|
||||
}
|
||||
if segs[0..5] == [0, 0, 0, 0, 0] && segs[5] == 0xFFFF {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x0100 && segs[1..4] == [0, 0, 0] {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x2001 && segs[1] == 0x0db8 {
|
||||
return true;
|
||||
}
|
||||
if segs[0] == 0x2002 {
|
||||
return true;
|
||||
}
|
||||
if ip.is_multicast() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn log_probe_result(probe: &NetworkProbe, decision: &NetworkDecision) {
|
||||
info!(
|
||||
ipv4 = probe
|
||||
.detected_ipv4
|
||||
.as_ref()
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
ipv6 = probe
|
||||
.detected_ipv6
|
||||
.as_ref()
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
reflected_v4 = probe
|
||||
.reflected_ipv4
|
||||
.as_ref()
|
||||
.map(|v| v.ip().to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
reflected_v6 = probe
|
||||
.reflected_ipv6
|
||||
.as_ref()
|
||||
.map(|v| v.ip().to_string())
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
ipv4_bogon = probe.ipv4_is_bogon,
|
||||
ipv6_bogon = probe.ipv6_is_bogon,
|
||||
ipv4_me = decision.ipv4_me,
|
||||
ipv6_me = decision.ipv6_me,
|
||||
ipv4_dc = decision.ipv4_dc,
|
||||
ipv6_dc = decision.ipv6_dc,
|
||||
prefer = decision.effective_prefer,
|
||||
multipath = decision.effective_multipath,
|
||||
"Network capabilities resolved"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use super::*;
|
||||
use crate::config::NetworkConfig;
|
||||
|
||||
#[test]
|
||||
fn manual_nat_ip_enables_ipv4_me_without_reflection() {
|
||||
let config = NetworkConfig {
|
||||
ipv4: true,
|
||||
..Default::default()
|
||||
};
|
||||
let probe = NetworkProbe {
|
||||
detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)),
|
||||
ipv4_is_bogon: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision =
|
||||
decide_network_capabilities(&config, &probe, Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))));
|
||||
|
||||
assert!(decision.ipv4_me);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_nat_ip_does_not_enable_other_family() {
|
||||
let config = NetworkConfig {
|
||||
ipv4: true,
|
||||
ipv6: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
let probe = NetworkProbe {
|
||||
detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)),
|
||||
detected_ipv6: Some(Ipv6Addr::LOCALHOST),
|
||||
ipv4_is_bogon: true,
|
||||
ipv6_is_bogon: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let decision =
|
||||
decide_network_capabilities(&config, &probe, Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))));
|
||||
|
||||
assert!(decision.ipv4_me);
|
||||
assert!(!decision.ipv6_me);
|
||||
}
|
||||
+2
-2
@@ -394,8 +394,8 @@ async fn resolve_stun_addr(
|
||||
}
|
||||
|
||||
if let Some((host, port)) = split_host_port(stun_addr)
|
||||
&& let Some(addr) = dns_resolver
|
||||
.and_then(|resolver| resolver.resolve_socket_addr(&host, port))
|
||||
&& let Some(addr) =
|
||||
dns_resolver.and_then(|resolver| resolver.resolve_socket_addr(&host, port))
|
||||
{
|
||||
return Ok(match (addr.is_ipv4(), family) {
|
||||
(true, IpFamily::V4) | (false, IpFamily::V6) => Some(addr),
|
||||
|
||||
+16
-1550
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
/// Main dispatch after successful handshake.
|
||||
/// Two modes:
|
||||
/// - Direct: TCP relay to TG DC (existing behavior)
|
||||
/// - Middle Proxy: RPC multiplex through ME pool (supports CDN DCs)
|
||||
#[cfg(test)]
|
||||
pub(super) async fn handle_authenticated_static<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
local_addr: SocketAddr,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
None,
|
||||
route_runtime,
|
||||
local_addr,
|
||||
peer_addr,
|
||||
ip_tracker,
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_authenticated_static_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
local_addr: SocketAddr,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
run_authenticated(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
ClientRuntimeDeps {
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime,
|
||||
ip_tracker,
|
||||
shared,
|
||||
},
|
||||
local_addr,
|
||||
peer_addr,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn acquire_user_connection_reservation_static(
|
||||
user: &str,
|
||||
config: &ProxyConfig,
|
||||
stats: Arc<Stats>,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
) -> Result<UserConnectionReservation> {
|
||||
acquire_user_connection_reservation(user, config, stats, peer_addr, ip_tracker).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn check_user_limits_static(
|
||||
user: &str,
|
||||
config: &ProxyConfig,
|
||||
stats: &Stats,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: &UserIpTracker,
|
||||
) -> Result<()> {
|
||||
if let Some(expiration) = config.access.user_expirations.get(user)
|
||||
&& chrono::Utc::now() > *expiration
|
||||
{
|
||||
return Err(ProxyError::UserExpired {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(quota) = config.access.user_data_quota.get(user)
|
||||
&& stats.get_user_quota_used(user) >= *quota
|
||||
{
|
||||
return Err(ProxyError::DataQuotaExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let limit = config
|
||||
.access
|
||||
.user_max_tcp_conns
|
||||
.get(user)
|
||||
.copied()
|
||||
.filter(|limit| *limit > 0)
|
||||
.or((config.access.user_max_tcp_conns_global_each > 0)
|
||||
.then_some(config.access.user_max_tcp_conns_global_each))
|
||||
.map(|v| v as u64);
|
||||
if !stats.try_acquire_user_curr_connects(user, limit) {
|
||||
return Err(ProxyError::ConnectionLimitExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
match ip_tracker.check_and_add(user, peer_addr.ip()).await {
|
||||
Ok(()) => {
|
||||
ip_tracker.remove_ip(user, peer_addr.ip()).await;
|
||||
}
|
||||
Err(reason) => {
|
||||
stats.decrement_user_curr_connects(user);
|
||||
warn!(
|
||||
user = %user,
|
||||
ip = %peer_addr.ip(),
|
||||
reason = %reason,
|
||||
"IP limit exceeded"
|
||||
);
|
||||
return Err(ProxyError::ConnectionLimitExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
stats.decrement_user_curr_connects(user);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub(super) async fn handle_direct_client(
|
||||
mut self,
|
||||
first_bytes: [u8; 5],
|
||||
local_addr: SocketAddr,
|
||||
) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
|
||||
if !self.config.general.modes.classic && !self.config.general.modes.secure {
|
||||
debug!(peer = %peer, "Non-TLS modes disabled");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("direct_modes_disabled");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = [0u8; HANDSHAKE_LEN];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
self.stream.read_exact(&mut handshake[5..]).await?;
|
||||
|
||||
let config = self.config.clone();
|
||||
let replay_checker = self.replay_checker.clone();
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&handshake,
|
||||
read_half,
|
||||
write_half,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
false,
|
||||
None,
|
||||
self.shared.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
self.upstream_manager,
|
||||
self.stats,
|
||||
self.config,
|
||||
buffer_pool,
|
||||
self.rng,
|
||||
self.me_pool,
|
||||
self.me_pool_runtime,
|
||||
self.route_runtime.clone(),
|
||||
local_addr,
|
||||
peer,
|
||||
self.ip_tracker,
|
||||
self.shared,
|
||||
),
|
||||
)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn beobachten_ttl(config: &ProxyConfig) -> Duration {
|
||||
const BEOBACHTEN_TTL_MAX_MINUTES: u64 = 24 * 60;
|
||||
let minutes = config.general.beobachten_minutes;
|
||||
if minutes == 0 {
|
||||
static BEOBACHTEN_ZERO_MINUTES_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = BEOBACHTEN_ZERO_MINUTES_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
"general.beobachten_minutes=0 is insecure because entries expire immediately; forcing minimum TTL to 1 minute"
|
||||
);
|
||||
}
|
||||
return Duration::from_secs(60);
|
||||
}
|
||||
|
||||
if minutes > BEOBACHTEN_TTL_MAX_MINUTES {
|
||||
static BEOBACHTEN_OVERSIZED_MINUTES_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = BEOBACHTEN_OVERSIZED_MINUTES_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
configured_minutes = minutes,
|
||||
max_minutes = BEOBACHTEN_TTL_MAX_MINUTES,
|
||||
"general.beobachten_minutes is too large; clamping to secure maximum"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Duration::from_secs(minutes.min(BEOBACHTEN_TTL_MAX_MINUTES).saturating_mul(60))
|
||||
}
|
||||
|
||||
pub(super) fn wrap_tls_application_record(payload: &[u8]) -> Vec<u8> {
|
||||
let chunks = payload.len().div_ceil(u16::MAX as usize).max(1);
|
||||
let mut record = Vec::with_capacity(payload.len() + 5 * chunks);
|
||||
|
||||
if payload.is_empty() {
|
||||
record.push(TLS_RECORD_APPLICATION);
|
||||
record.extend_from_slice(&TLS_VERSION);
|
||||
record.extend_from_slice(&0u16.to_be_bytes());
|
||||
return record;
|
||||
}
|
||||
|
||||
for chunk in payload.chunks(u16::MAX as usize) {
|
||||
record.push(TLS_RECORD_APPLICATION);
|
||||
record.extend_from_slice(&TLS_VERSION);
|
||||
record.extend_from_slice(&(chunk.len() as u16).to_be_bytes());
|
||||
record.extend_from_slice(chunk);
|
||||
}
|
||||
|
||||
record
|
||||
}
|
||||
|
||||
pub(super) fn tls_clienthello_len_in_bounds(tls_len: usize) -> bool {
|
||||
(MIN_TLS_CLIENT_HELLO_SIZE..=MAX_TLS_PLAINTEXT_SIZE).contains(&tls_len)
|
||||
}
|
||||
|
||||
pub(super) async fn read_with_progress<R: AsyncRead + Unpin>(
|
||||
reader: &mut R,
|
||||
mut buf: &mut [u8],
|
||||
) -> std::io::Result<usize> {
|
||||
let mut total = 0usize;
|
||||
while !buf.is_empty() {
|
||||
match reader.read(buf).await {
|
||||
Ok(0) => return Ok(total),
|
||||
Ok(n) => {
|
||||
total += n;
|
||||
let (_, rest) = buf.split_at_mut(n);
|
||||
buf = rest;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
pub(super) async fn maybe_apply_mask_reject_delay(config: &ProxyConfig) {
|
||||
let min = config.censorship.server_hello_delay_min_ms;
|
||||
let max = config.censorship.server_hello_delay_max_ms;
|
||||
if max == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let delay_ms = if min >= max {
|
||||
max
|
||||
} else {
|
||||
rand::rng().random_range(min..=max)
|
||||
};
|
||||
|
||||
if delay_ms > 0 {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn handshake_timeout_with_mask_grace(config: &ProxyConfig) -> Duration {
|
||||
let base = Duration::from_secs(config.timeouts.client_handshake);
|
||||
if config.censorship.mask {
|
||||
base.saturating_add(Duration::from_millis(750))
|
||||
} else {
|
||||
base
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn effective_client_first_byte_idle_secs(
|
||||
config: &ProxyConfig,
|
||||
shared: &ProxySharedState,
|
||||
) -> u64 {
|
||||
let idle_secs = config.timeouts.client_first_byte_idle_secs;
|
||||
if idle_secs == 0 {
|
||||
return 0;
|
||||
}
|
||||
if shared.conntrack_pressure_active() {
|
||||
idle_secs.min(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.client_first_byte_idle_cap_secs(),
|
||||
)
|
||||
} else {
|
||||
idle_secs
|
||||
}
|
||||
}
|
||||
|
||||
const MASK_CLASSIFIER_PREFETCH_WINDOW: usize = 16;
|
||||
#[cfg(test)]
|
||||
pub(super) const MASK_CLASSIFIER_PREFETCH_TIMEOUT: Duration = Duration::from_millis(5);
|
||||
|
||||
pub(super) fn mask_classifier_prefetch_timeout(config: &ProxyConfig) -> Duration {
|
||||
Duration::from_millis(config.censorship.mask_classifier_prefetch_timeout_ms)
|
||||
}
|
||||
|
||||
pub(super) fn should_prefetch_mask_classifier_window(initial_data: &[u8]) -> bool {
|
||||
if initial_data.len() >= MASK_CLASSIFIER_PREFETCH_WINDOW {
|
||||
return false;
|
||||
}
|
||||
|
||||
if initial_data.is_empty() {
|
||||
// Empty initial_data means there is no client probe prefix to refine.
|
||||
// Prefetching in this case can consume fallback relay payload bytes and
|
||||
// accidentally route them through shaping heuristics.
|
||||
return false;
|
||||
}
|
||||
|
||||
if initial_data[0] == 0x16 || initial_data.starts_with(b"SSH-") {
|
||||
return false;
|
||||
}
|
||||
|
||||
initial_data
|
||||
.iter()
|
||||
.all(|b| b.is_ascii_alphabetic() || *b == b' ')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn extend_masking_initial_window<R>(reader: &mut R, initial_data: &mut Vec<u8>)
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
extend_masking_initial_window_with_timeout(
|
||||
reader,
|
||||
initial_data,
|
||||
MASK_CLASSIFIER_PREFETCH_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn extend_masking_initial_window_with_timeout<R>(
|
||||
reader: &mut R,
|
||||
initial_data: &mut Vec<u8>,
|
||||
prefetch_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
if !should_prefetch_mask_classifier_window(initial_data) {
|
||||
return;
|
||||
}
|
||||
|
||||
let need = MASK_CLASSIFIER_PREFETCH_WINDOW.saturating_sub(initial_data.len());
|
||||
if need == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut extra = [0u8; MASK_CLASSIFIER_PREFETCH_WINDOW];
|
||||
if let Ok(Ok(n)) = timeout(prefetch_timeout, reader.read(&mut extra[..need])).await
|
||||
&& n > 0
|
||||
{
|
||||
initial_data.extend_from_slice(&extra[..n]);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn masking_outcome<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: Vec<u8>,
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> HandshakeOutcome
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
HandshakeOutcome::NeedsMasking(Box::pin(async move {
|
||||
let mut reader = reader;
|
||||
let mut initial_data = initial_data;
|
||||
extend_masking_initial_window_with_timeout(
|
||||
&mut reader,
|
||||
&mut initial_data,
|
||||
mask_classifier_prefetch_timeout(&config),
|
||||
)
|
||||
.await;
|
||||
|
||||
crate::proxy::masking::handle_bad_client_with_shared_resolver(
|
||||
reader,
|
||||
writer,
|
||||
&initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
shared.as_ref(),
|
||||
Some(upstream_manager.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn record_beobachten_class(
|
||||
beobachten: &BeobachtenStore,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
class: &str,
|
||||
) {
|
||||
if !config.general.beobachten {
|
||||
return;
|
||||
}
|
||||
beobachten.record(class, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
|
||||
pub(super) fn tls_fingerprint_collection_enabled(config: &ProxyConfig) -> bool {
|
||||
config.general.beobachten || config.server.api.runtime_edge_enabled
|
||||
}
|
||||
|
||||
pub(super) fn observe_tls_client_fingerprint(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
handshake: &[u8],
|
||||
) -> Option<TlsClientFingerprint> {
|
||||
if !tls_fingerprint_collection_enabled(config) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match tls_fingerprint::fingerprint_client_hello(handshake) {
|
||||
Some(fingerprint) => {
|
||||
stats.record_tls_fingerprint_observed(&fingerprint, peer_ip, beobachten_ttl(config));
|
||||
Some(fingerprint)
|
||||
}
|
||||
None => {
|
||||
stats.increment_tls_fingerprint_parse_error();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_tls_fingerprint_auth_success(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
user: &str,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_auth_success(
|
||||
fingerprint,
|
||||
peer_ip,
|
||||
user,
|
||||
beobachten_ttl(config),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_tls_fingerprint_bad_or_probe(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_bad_or_probe(fingerprint, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn classify_expected_64_got_0(kind: std::io::ErrorKind) -> Option<&'static str> {
|
||||
match kind {
|
||||
std::io::ErrorKind::UnexpectedEof => Some("expected_64_got_0_unexpected_eof"),
|
||||
std::io::ErrorKind::ConnectionReset => Some("expected_64_got_0_connection_reset"),
|
||||
std::io::ErrorKind::ConnectionAborted => Some("expected_64_got_0_connection_aborted"),
|
||||
std::io::ErrorKind::BrokenPipe => Some("expected_64_got_0_broken_pipe"),
|
||||
std::io::ErrorKind::NotConnected => Some("expected_64_got_0_not_connected"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn classify_handshake_failure_class(error: &ProxyError) -> &'static str {
|
||||
match error {
|
||||
ProxyError::Io(err) => classify_expected_64_got_0(err.kind()).unwrap_or("other"),
|
||||
ProxyError::Stream(StreamError::UnexpectedEof) => "expected_64_got_0_unexpected_eof",
|
||||
ProxyError::Stream(StreamError::Io(err)) => {
|
||||
classify_expected_64_got_0(err.kind()).unwrap_or("other")
|
||||
}
|
||||
_ => "other",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_handshake_failure_class(
|
||||
beobachten: &BeobachtenStore,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
error: &ProxyError,
|
||||
) {
|
||||
// Keep beobachten buckets stable while detailed per-kind classification
|
||||
// is tracked in API counters.
|
||||
let class = match classify_handshake_failure_class(error) {
|
||||
value if value.starts_with("expected_64_got_0_") => "expected_64_got_0",
|
||||
_ => "other",
|
||||
};
|
||||
record_beobachten_class(beobachten, config, peer_ip, class);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn increment_bad_on_unknown_tls_sni(stats: &Stats, error: &ProxyError) {
|
||||
if matches!(error, ProxyError::UnknownTlsSni) {
|
||||
stats.increment_connects_bad_with_class("unknown_tls_sni");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_trusted_proxy_source(peer_ip: IpAddr, trusted: &[IpNetwork]) -> bool {
|
||||
if trusted.is_empty() {
|
||||
static EMPTY_PROXY_TRUST_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = EMPTY_PROXY_TRUST_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
"PROXY protocol enabled but server.proxy_protocol_trusted_cidrs is empty; rejecting all PROXY headers"
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
trusted.iter().any(|cidr| cidr.contains(peer_ip))
|
||||
}
|
||||
|
||||
pub(super) fn synthetic_local_addr(port: u16) -> SocketAddr {
|
||||
SocketAddr::from(([0, 0, 0, 0], port))
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub async fn run(self) -> Result<()> {
|
||||
self.stats.increment_connects_all();
|
||||
let peer = self.peer;
|
||||
debug!(peer = %peer, "New connection");
|
||||
|
||||
if let Err(e) = configure_client_socket(
|
||||
&self.stream,
|
||||
self.config.timeouts.client_keepalive,
|
||||
self.config.timeouts.client_ack,
|
||||
) {
|
||||
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
let raw_fd = self.raw_fd;
|
||||
let rst_on_close = self.rst_on_close;
|
||||
|
||||
let outcome = match self.do_handshake().await? {
|
||||
Some(outcome) => outcome,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||
match outcome {
|
||||
HandshakeOutcome::NeedsRelay(fut) => {
|
||||
#[cfg(unix)]
|
||||
if matches!(rst_on_close, crate::config::RstOnCloseMode::Errors) {
|
||||
let _ = crate::transport::socket::clear_linger_fd(raw_fd);
|
||||
}
|
||||
fut.await
|
||||
}
|
||||
HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn do_handshake(mut self) -> Result<Option<HandshakeOutcome>> {
|
||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||
|
||||
if self.proxy_protocol_enabled {
|
||||
if !is_trusted_proxy_source(
|
||||
self.peer.ip(),
|
||||
&self.config.server.proxy_protocol_trusted_cidrs,
|
||||
) {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||
warn!(
|
||||
peer = %self.peer,
|
||||
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
|
||||
"Rejecting PROXY protocol header from untrusted source"
|
||||
);
|
||||
record_beobachten_class(&self.beobachten, &self.config, self.peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
|
||||
let proxy_header_timeout =
|
||||
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||
match timeout(
|
||||
proxy_header_timeout,
|
||||
parse_proxy_protocol(&mut self.stream, self.peer),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(info)) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
client = %info.src_addr,
|
||||
version = info.version,
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
self.peer = normalize_ip(info.src_addr);
|
||||
self.real_peer_from_proxy = Some(self.peer);
|
||||
if let Ok(mut slot) = self.real_peer_report.lock() {
|
||||
*slot = Some(self.peer);
|
||||
}
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_invalid_header");
|
||||
warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header");
|
||||
record_beobachten_class(
|
||||
&self.beobachten,
|
||||
&self.config,
|
||||
self.peer.ip(),
|
||||
"other",
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_header_timeout");
|
||||
warn!(
|
||||
peer = %self.peer,
|
||||
timeout_ms = proxy_header_timeout.as_millis(),
|
||||
"PROXY protocol header timeout"
|
||||
);
|
||||
record_beobachten_class(
|
||||
&self.beobachten,
|
||||
&self.config,
|
||||
self.peer.ip(),
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let first_byte_idle_secs =
|
||||
effective_client_first_byte_idle_secs(&self.config, self.shared.as_ref());
|
||||
let first_byte = if first_byte_idle_secs == 0 {
|
||||
None
|
||||
} else {
|
||||
let idle_timeout = Duration::from_secs(first_byte_idle_secs);
|
||||
let mut first_byte = [0u8; 1];
|
||||
match timeout(idle_timeout, self.stream.read(&mut first_byte)).await {
|
||||
Ok(Ok(0)) => {
|
||||
debug!(peer = %self.peer, "Connection closed before first client byte");
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(_)) => Some(first_byte[0]),
|
||||
Ok(Err(e))
|
||||
if matches!(
|
||||
e.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
| std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
) =>
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
error = %e,
|
||||
"Connection closed before first client byte"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
error = %e,
|
||||
"Failed while waiting for first client byte"
|
||||
);
|
||||
return Err(ProxyError::Io(e));
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
idle_secs = first_byte_idle_secs,
|
||||
"Closing idle pooled connection before first client byte"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let handshake_timeout = handshake_timeout_with_mask_grace(&self.config);
|
||||
let stats = self.stats.clone();
|
||||
let config_for_timeout = self.config.clone();
|
||||
let beobachten_for_timeout = self.beobachten.clone();
|
||||
let peer_for_timeout = self.peer.ip();
|
||||
let peer_for_log = self.peer;
|
||||
|
||||
let outcome = match timeout(handshake_timeout, async {
|
||||
let mut first_bytes = [0u8; 5];
|
||||
if let Some(first_byte) = first_byte {
|
||||
first_bytes[0] = first_byte;
|
||||
self.stream.read_exact(&mut first_bytes[1..]).await?;
|
||||
} else {
|
||||
self.stream.read_exact(&mut first_bytes).await?;
|
||||
}
|
||||
|
||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||
let peer = self.peer;
|
||||
|
||||
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
|
||||
|
||||
if is_tls {
|
||||
self.handle_tls_client(first_bytes, local_addr).await
|
||||
} else {
|
||||
self.handle_direct_client(first_bytes, local_addr).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(outcome)) => outcome,
|
||||
Ok(Err(e)) => {
|
||||
debug!(peer = %peer_for_log, error = %e, "Handshake failed");
|
||||
stats.increment_handshake_failure_class(classify_handshake_failure_class(&e));
|
||||
record_handshake_failure_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
&e,
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_handshake_timeouts();
|
||||
stats.increment_handshake_failure_class("timeout");
|
||||
debug!(peer = %peer_for_log, "Handshake timeout");
|
||||
record_beobachten_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::TgHandshakeTimeout);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Some(outcome))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn handle_client_stream<S>(
|
||||
stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_client_stream_with_shared(
|
||||
stream,
|
||||
peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
ProxySharedState::new(),
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(dead_code)]
|
||||
pub async fn handle_client_stream_with_shared<S>(
|
||||
stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_client_stream_with_shared_and_pool_runtime(
|
||||
stream,
|
||||
peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
None,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
shared,
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn handle_client_stream_with_shared_and_pool_runtime<S>(
|
||||
mut stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
stats.increment_connects_all();
|
||||
let mut real_peer = normalize_ip(peer);
|
||||
|
||||
// For non-TCP streams, use a synthetic local address; may be overridden by PROXY protocol dst
|
||||
let mut local_addr = synthetic_local_addr(config.server.port);
|
||||
|
||||
if proxy_protocol_enabled {
|
||||
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs) {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||
warn!(
|
||||
peer = %peer,
|
||||
trusted = ?config.server.proxy_protocol_trusted_cidrs,
|
||||
"Rejecting PROXY protocol header from untrusted source"
|
||||
);
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
|
||||
let proxy_header_timeout =
|
||||
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||
match timeout(
|
||||
proxy_header_timeout,
|
||||
parse_proxy_protocol(&mut stream, peer),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(info)) => {
|
||||
debug!(
|
||||
peer = %peer,
|
||||
client = %info.src_addr,
|
||||
version = info.version,
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
real_peer = normalize_ip(info.src_addr);
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_invalid_header");
|
||||
warn!(peer = %peer, error = %e, "Invalid PROXY protocol header");
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_header_timeout");
|
||||
warn!(peer = %peer, timeout_ms = proxy_header_timeout.as_millis(), "PROXY protocol header timeout");
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(peer = %real_peer, "New connection (generic stream)");
|
||||
|
||||
let first_byte_idle_secs = effective_client_first_byte_idle_secs(&config, shared.as_ref());
|
||||
let first_byte = if first_byte_idle_secs == 0 {
|
||||
None
|
||||
} else {
|
||||
let idle_timeout = Duration::from_secs(first_byte_idle_secs);
|
||||
let mut first_byte = [0u8; 1];
|
||||
match timeout(idle_timeout, stream.read(&mut first_byte)).await {
|
||||
Ok(Ok(0)) => {
|
||||
debug!(peer = %real_peer, "Connection closed before first client byte");
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Ok(_)) => Some(first_byte[0]),
|
||||
Ok(Err(e))
|
||||
if matches!(
|
||||
e.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
| std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
) =>
|
||||
{
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
error = %e,
|
||||
"Connection closed before first client byte"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
error = %e,
|
||||
"Failed while waiting for first client byte"
|
||||
);
|
||||
return Err(ProxyError::Io(e));
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
idle_secs = first_byte_idle_secs,
|
||||
"Closing idle pooled connection before first client byte"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let handshake_timeout = handshake_timeout_with_mask_grace(&config);
|
||||
let stats_for_timeout = stats.clone();
|
||||
let config_for_timeout = config.clone();
|
||||
let beobachten_for_timeout = beobachten.clone();
|
||||
let peer_for_timeout = real_peer.ip();
|
||||
|
||||
// Phase 2: active handshake (with timeout after the first client byte)
|
||||
let outcome = match timeout(handshake_timeout, async {
|
||||
let mut first_bytes = [0u8; 5];
|
||||
if let Some(first_byte) = first_byte {
|
||||
first_bytes[0] = first_byte;
|
||||
stream.read_exact(&mut first_bytes[1..]).await?;
|
||||
} else {
|
||||
stream.read_exact(&mut first_bytes).await?;
|
||||
}
|
||||
|
||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||
debug!(peer = %real_peer, is_tls = is_tls, "Handshake type detected");
|
||||
|
||||
if is_tls {
|
||||
let tls_len = u16::from_be_bytes([first_bytes[3], first_bytes[4]]) as usize;
|
||||
|
||||
// RFC 8446 §5.1: TLS record payload MUST NOT exceed 2^14 (16_384) bytes.
|
||||
// Lower bound is a structural minimum for a valid TLS 1.3 ClientHello
|
||||
// (record header + handshake header + random + session_id + cipher_suites
|
||||
// + compression + at least one extension with SNI). The previous value of
|
||||
// 512 was implicitly coupled to TLS_REQUEST_LENGTH=517 from the official
|
||||
// Telegram MTProxy reference server, leaving only a 5-byte margin and
|
||||
// incorrectly rejecting compact but spec-compliant ClientHellos from
|
||||
// third-party clients or future Telegram versions.
|
||||
if !tls_clienthello_len_in_bounds(tls_len) {
|
||||
debug!(peer = %real_peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = vec![0u8; 5 + tls_len];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
let body_read = match read_with_progress(&mut stream, &mut handshake[5..]).await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
debug!(peer = %real_peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_read_error");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let initial_len = 5;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
if body_read < tls_len {
|
||||
debug!(peer = %real_peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_truncated");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let initial_len = 5 + body_read;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint =
|
||||
observe_tls_client_fingerprint(stats.as_ref(), &config, real_peer.ip(), &handshake);
|
||||
|
||||
let (read_half, write_half) = tokio::io::split(stream);
|
||||
|
||||
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake_with_shared(
|
||||
&handshake, read_half, write_half, real_peer,
|
||||
&config, &replay_checker, &rng, tls_cache.clone(),
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.clone(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
let mtproto_handshake: [u8; HANDSHAKE_LEN] = mtproto_data[..].try_into()
|
||||
.map_err(|_| ProxyError::InvalidHandshake("Short MTProto handshake".into()))?;
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&mtproto_handshake, tls_reader, tls_writer, real_peer,
|
||||
&config, &replay_checker, true, Some(tls_user.as_str()),
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
// MTProto failed after TLS ServerHello was already sent.
|
||||
// Switch fallback relay back to raw transport so the mask
|
||||
// backend receives valid TLS records (not unwrapped payload).
|
||||
let (reader, pending_plaintext) = reader.into_inner_with_pending_plaintext();
|
||||
let writer = writer.into_inner();
|
||||
let pending_record = if pending_plaintext.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
wrap_tls_application_record(&pending_plaintext)
|
||||
};
|
||||
let reader = tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
|
||||
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
Vec::new(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
RunningClientHandler::handle_authenticated_static_with_shared(
|
||||
crypto_reader, crypto_writer, success,
|
||||
upstream_manager, stats, config, buffer_pool, rng, me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime.clone(),
|
||||
local_addr, real_peer, ip_tracker.clone(),
|
||||
shared.clone(),
|
||||
),
|
||||
)))
|
||||
} else {
|
||||
if !config.general.modes.classic && !config.general.modes.secure {
|
||||
debug!(peer = %real_peer, "Non-TLS modes disabled");
|
||||
stats.increment_connects_bad_with_class("direct_modes_disabled");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = [0u8; HANDSHAKE_LEN];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
stream.read_exact(&mut handshake[5..]).await?;
|
||||
|
||||
let (read_half, write_half) = tokio::io::split(stream);
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&handshake, read_half, write_half, real_peer,
|
||||
&config, &replay_checker, false, None,
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
RunningClientHandler::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime.clone(),
|
||||
local_addr,
|
||||
real_peer,
|
||||
ip_tracker.clone(),
|
||||
shared.clone(),
|
||||
)
|
||||
)))
|
||||
}
|
||||
}).await {
|
||||
Ok(Ok(outcome)) => outcome,
|
||||
Ok(Err(e)) => {
|
||||
debug!(peer = %peer, error = %e, "Handshake failed");
|
||||
stats_for_timeout.increment_handshake_failure_class(classify_handshake_failure_class(&e));
|
||||
record_handshake_failure_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
&e,
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats_for_timeout.increment_handshake_timeouts();
|
||||
stats_for_timeout.increment_handshake_failure_class("timeout");
|
||||
debug!(peer = %peer, "Handshake timeout");
|
||||
record_beobachten_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::TgHandshakeTimeout);
|
||||
}
|
||||
};
|
||||
|
||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||
match outcome {
|
||||
HandshakeOutcome::NeedsRelay(fut) | HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub(super) async fn handle_tls_client(
|
||||
mut self,
|
||||
first_bytes: [u8; 5],
|
||||
local_addr: SocketAddr,
|
||||
) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
|
||||
let tls_len = u16::from_be_bytes([first_bytes[3], first_bytes[4]]) as usize;
|
||||
|
||||
debug!(peer = %peer, tls_len = tls_len, "Reading TLS handshake");
|
||||
|
||||
// RFC 8446 §5.1: TLS record payload MUST NOT exceed 2^14 (16_384) bytes.
|
||||
// Lower bound is a structural minimum for a valid TLS 1.3 ClientHello
|
||||
// (record header + handshake header + random + session_id + cipher_suites
|
||||
// + compression + at least one extension with SNI). The previous value of
|
||||
// 512 was implicitly coupled to TLS_REQUEST_LENGTH=517 from the official
|
||||
// Telegram MTProxy reference server, leaving only a 5-byte margin and
|
||||
// incorrectly rejecting compact but spec-compliant ClientHellos from
|
||||
// third-party clients or future Telegram versions.
|
||||
if !tls_clienthello_len_in_bounds(tls_len) {
|
||||
debug!(peer = %peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = vec![0u8; 5 + tls_len];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
let body_read = match read_with_progress(&mut self.stream, &mut handshake[5..]).await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
debug!(peer = %peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_read_error");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..5].to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
if body_read < tls_len {
|
||||
debug!(peer = %peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_truncated");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let initial_len = 5 + body_read;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint = observe_tls_client_fingerprint(
|
||||
self.stats.as_ref(),
|
||||
&self.config,
|
||||
peer.ip(),
|
||||
&handshake,
|
||||
);
|
||||
|
||||
let config = self.config.clone();
|
||||
let replay_checker = self.replay_checker.clone();
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let response_write_options =
|
||||
TlsResponseWriteOptions::tcp(self.raw_fd, self.tls_response_fragment_size);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let response_write_options = TlsResponseWriteOptions::default();
|
||||
|
||||
let (mut tls_reader, tls_writer, tls_user) =
|
||||
match handle_tls_handshake_with_shared_and_options(
|
||||
&handshake,
|
||||
read_half,
|
||||
write_half,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
&self.rng,
|
||||
self.tls_cache.clone(),
|
||||
self.shared.as_ref(),
|
||||
response_write_options,
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.clone(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
let mtproto_handshake: [u8; HANDSHAKE_LEN] = mtproto_data[..]
|
||||
.try_into()
|
||||
.map_err(|_| ProxyError::InvalidHandshake("Short MTProto handshake".into()))?;
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&mtproto_handshake,
|
||||
tls_reader,
|
||||
tls_writer,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
true,
|
||||
Some(tls_user.as_str()),
|
||||
self.shared.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
// MTProto failed after TLS ServerHello was already sent.
|
||||
// Switch fallback relay back to raw transport so the mask
|
||||
// backend receives valid TLS records (not unwrapped payload).
|
||||
let (reader, pending_plaintext) = reader.into_inner_with_pending_plaintext();
|
||||
let writer = writer.into_inner();
|
||||
let pending_record = if pending_plaintext.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
wrap_tls_application_record(&pending_plaintext)
|
||||
};
|
||||
let reader =
|
||||
tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
|
||||
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
Vec::new(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
self.upstream_manager,
|
||||
self.stats,
|
||||
self.config,
|
||||
buffer_pool,
|
||||
self.rng,
|
||||
self.me_pool,
|
||||
self.me_pool_runtime,
|
||||
self.route_runtime.clone(),
|
||||
local_addr,
|
||||
peer,
|
||||
self.ip_tracker,
|
||||
self.shared,
|
||||
),
|
||||
)))
|
||||
}
|
||||
}
|
||||
+12
-395
@@ -36,6 +36,15 @@ use nix::sys::stat::Mode;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
// Direct relay lifecycle and conntrack publication.
|
||||
mod relay;
|
||||
// Telegram DC resolution and upstream handshake.
|
||||
mod routing;
|
||||
|
||||
pub(crate) use relay::{
|
||||
handle_via_direct, handle_via_direct_with_shared, handle_via_direct_with_shared_and_conntrack,
|
||||
};
|
||||
use routing::*;
|
||||
const UNKNOWN_DC_LOG_DISTINCT_LIMIT: usize = 1024;
|
||||
static LOGGED_UNKNOWN_DCS: OnceLock<Mutex<HashSet<i16>>> = OnceLock::new();
|
||||
const MAX_SCOPE_HINT_LEN: usize = 64;
|
||||
@@ -224,401 +233,9 @@ fn clear_unknown_dc_log_cache_for_testing() {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn unknown_dc_test_lock() -> &'static Mutex<()> {
|
||||
static TEST_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
TEST_LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Runs Direct relay with standalone cancellation and shared-state defaults.
|
||||
pub(crate) async fn handle_via_direct<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config.clone(),
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
SocketAddr::from(([0, 0, 0, 0], config.server.port)),
|
||||
CancellationToken::new(),
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay for a kernel-backed TCP client tuple.
|
||||
pub(crate) async fn handle_via_direct_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared_and_conntrack(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel,
|
||||
shared,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay with explicit kernel-conntrack close publication policy.
|
||||
pub(crate) async fn handle_via_direct_with_shared_and_conntrack<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
mut route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
conntrack_close_policy: ConntrackClosePolicy,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let user = &success.user;
|
||||
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
||||
|
||||
debug!(
|
||||
user = %user,
|
||||
peer = %success.peer,
|
||||
dc = success.dc_idx,
|
||||
dc_addr = %dc_addr,
|
||||
proto = ?success.proto_tag,
|
||||
mode = "direct",
|
||||
"Connecting to Telegram DC"
|
||||
);
|
||||
|
||||
let scope_hint = validated_scope_hint(user);
|
||||
if user.starts_with("scope_") && scope_hint.is_none() {
|
||||
warn!(
|
||||
user = %user,
|
||||
"Ignoring invalid scope hint and falling back to default upstream selection"
|
||||
);
|
||||
}
|
||||
let tg_stream = tokio::select! {
|
||||
result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake");
|
||||
|
||||
let (tg_reader, tg_writer) = tokio::select! {
|
||||
result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, "TG handshake complete, starting relay");
|
||||
|
||||
stats.increment_user_connects(user);
|
||||
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
||||
let traffic_lease = shared
|
||||
.traffic_limiter
|
||||
.acquire_lease(user, success.peer.ip());
|
||||
|
||||
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
||||
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
||||
Duration::from_secs(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.direct_activity_timeout_secs(),
|
||||
)
|
||||
} else {
|
||||
Duration::from_secs(1800)
|
||||
};
|
||||
let relay_result = crate::proxy::relay::relay_direct_adaptive(
|
||||
client_reader,
|
||||
client_writer,
|
||||
tg_reader,
|
||||
tg_writer,
|
||||
config.general.direct_relay_copy_buf_c2s_bytes,
|
||||
config.general.direct_relay_copy_buf_s2c_bytes,
|
||||
config.server.max_connections,
|
||||
user,
|
||||
Arc::clone(&stats),
|
||||
config.access.user_data_quota.get(user).copied(),
|
||||
traffic_lease,
|
||||
relay_activity_timeout,
|
||||
session_cancel.clone(),
|
||||
Arc::clone(&shared.direct_buffer_budget),
|
||||
);
|
||||
tokio::pin!(relay_result);
|
||||
let relay_result = loop {
|
||||
if let Some(cutover) =
|
||||
affected_cutover_state(&route_rx, RelayRouteMode::Direct, route_snapshot.generation)
|
||||
{
|
||||
let delay = cutover_stagger_delay(session_id, cutover.generation);
|
||||
warn!(
|
||||
user = %user,
|
||||
target_mode = cutover.mode.as_str(),
|
||||
cutover_generation = cutover.generation,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Cutover affected direct session, closing client connection"
|
||||
);
|
||||
let _cutover_park_lease = stats.acquire_direct_cutover_park_lease();
|
||||
tokio::time::sleep(delay).await;
|
||||
break Err(ProxyError::RouteSwitched);
|
||||
}
|
||||
tokio::select! {
|
||||
result = &mut relay_result => {
|
||||
break result;
|
||||
}
|
||||
changed = route_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break relay_result.await;
|
||||
}
|
||||
}
|
||||
_ = session_cancel.cancelled() => {
|
||||
break Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &relay_result {
|
||||
Ok(()) => debug!(user = %user, "Direct relay completed"),
|
||||
Err(e) => debug!(user = %user, error = %e, "Direct relay ended with error"),
|
||||
}
|
||||
|
||||
let pool_snapshot = buffer_pool_trim.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
pool_snapshot.allocated,
|
||||
pool_snapshot.allocated.saturating_sub(pool_snapshot.pooled),
|
||||
);
|
||||
|
||||
if conntrack_close_policy == ConntrackClosePolicy::Publish {
|
||||
let close_reason = classify_conntrack_close_reason(&relay_result);
|
||||
let publish_result = shared.publish_conntrack_close_event(ConntrackCloseEvent {
|
||||
src: success.peer,
|
||||
dst: local_addr,
|
||||
reason: close_reason,
|
||||
});
|
||||
if !matches!(
|
||||
publish_result,
|
||||
ConntrackClosePublishResult::Sent | ConntrackClosePublishResult::Disabled
|
||||
) {
|
||||
stats.increment_conntrack_close_event_drop_total();
|
||||
}
|
||||
}
|
||||
|
||||
relay_result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(error.kind(), std::io::ErrorKind::TimedOut) =>
|
||||
{
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(crate::error::ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result<SocketAddr> {
|
||||
let prefer_v6 = config.network.prefer == 6 && config.network.ipv6.unwrap_or(true);
|
||||
let datacenters = if prefer_v6 {
|
||||
&*TG_DATACENTERS_V6
|
||||
} else {
|
||||
&*TG_DATACENTERS_V4
|
||||
};
|
||||
|
||||
let num_dcs = datacenters.len();
|
||||
|
||||
let dc_key = dc_idx.to_string();
|
||||
if let Some(addrs) = config.dc_overrides.get(&dc_key) {
|
||||
let mut parsed = Vec::new();
|
||||
for addr_str in addrs {
|
||||
match addr_str.parse::<SocketAddr>() {
|
||||
Ok(addr) => parsed.push(addr),
|
||||
Err(_) => {
|
||||
warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(addr) = parsed
|
||||
.iter()
|
||||
.find(|a| a.is_ipv6() == prefer_v6)
|
||||
.or_else(|| parsed.first())
|
||||
.copied()
|
||||
{
|
||||
debug!(dc_idx = dc_idx, addr = %addr, count = parsed.len(), "Using DC override from config");
|
||||
return Ok(addr);
|
||||
}
|
||||
}
|
||||
|
||||
let abs_dc = dc_idx.unsigned_abs() as usize;
|
||||
if abs_dc >= 1 && abs_dc <= num_dcs {
|
||||
return Ok(SocketAddr::new(datacenters[abs_dc - 1], TG_DATACENTER_PORT));
|
||||
}
|
||||
|
||||
// Unknown DC requested by client without override: log and fall back.
|
||||
if !config.dc_overrides.contains_key(&dc_key) {
|
||||
warn!(
|
||||
dc_idx = dc_idx,
|
||||
"Requested non-standard DC with no override; falling back to default cluster"
|
||||
);
|
||||
if config.general.unknown_dc_file_log_enabled
|
||||
&& let Some(path) = &config.general.unknown_dc_log_path
|
||||
&& let Ok(handle) = tokio::runtime::Handle::try_current()
|
||||
{
|
||||
if let Some(path) = sanitize_unknown_dc_log_path(path) {
|
||||
if should_log_unknown_dc(dc_idx) {
|
||||
handle.spawn_blocking(move || {
|
||||
if unknown_dc_log_path_is_still_safe(&path)
|
||||
&& let Ok(mut file) = open_unknown_dc_log_append_anchored(&path)
|
||||
{
|
||||
let _ = append_unknown_dc_line(&mut file, dc_idx);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
warn!(dc_idx = dc_idx, raw_path = %path, "Rejected unsafe unknown DC log path");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_dc = config.default_dc.unwrap_or(2) as usize;
|
||||
let fallback_idx = if default_dc >= 1 && default_dc <= num_dcs {
|
||||
default_dc - 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
info!(
|
||||
original_dc = dc_idx,
|
||||
fallback_dc = (fallback_idx + 1) as u16,
|
||||
fallback_addr = %datacenters[fallback_idx],
|
||||
"Special DC ---> default_cluster"
|
||||
);
|
||||
|
||||
Ok(SocketAddr::new(
|
||||
datacenters[fallback_idx],
|
||||
TG_DATACENTER_PORT,
|
||||
))
|
||||
}
|
||||
|
||||
async fn do_tg_handshake_static<S>(
|
||||
mut stream: S,
|
||||
success: &HandshakeSuccess,
|
||||
config: &ProxyConfig,
|
||||
rng: &SecureRandom,
|
||||
) -> Result<(CryptoReader<ReadHalf<S>>, CryptoWriter<WriteHalf<S>>)>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin,
|
||||
{
|
||||
let (nonce, _tg_enc_key, _tg_enc_iv, _tg_dec_key, _tg_dec_iv) = generate_tg_nonce(
|
||||
success.proto_tag,
|
||||
success.dc_idx,
|
||||
&success.enc_key,
|
||||
success.enc_iv,
|
||||
rng,
|
||||
config.general.fast_mode,
|
||||
);
|
||||
|
||||
let (encrypted_nonce, tg_encryptor, tg_decryptor) = encrypt_tg_nonce_with_ciphers(&nonce);
|
||||
|
||||
debug!(
|
||||
peer = %success.peer,
|
||||
nonce_head = %hex::encode(&nonce[..16]),
|
||||
"Sending nonce to Telegram"
|
||||
);
|
||||
|
||||
stream.write_all(&encrypted_nonce).await?;
|
||||
stream.flush().await?;
|
||||
|
||||
let (read_half, write_half) = split(stream);
|
||||
|
||||
let max_pending = config.general.crypto_pending_buffer;
|
||||
Ok((
|
||||
CryptoReader::new(read_half, tg_decryptor),
|
||||
CryptoWriter::new(write_half, tg_encryptor, max_pending),
|
||||
))
|
||||
fn unknown_dc_test_lock() -> &'static tokio::sync::Mutex<()> {
|
||||
static TEST_LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
|
||||
TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
use super::*;
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Runs Direct relay with standalone cancellation and shared-state defaults.
|
||||
pub(crate) async fn handle_via_direct<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config.clone(),
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
SocketAddr::from(([0, 0, 0, 0], config.server.port)),
|
||||
CancellationToken::new(),
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay for a kernel-backed TCP client tuple.
|
||||
pub(crate) async fn handle_via_direct_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared_and_conntrack(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel,
|
||||
shared,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay with explicit kernel-conntrack close publication policy.
|
||||
pub(crate) async fn handle_via_direct_with_shared_and_conntrack<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
mut route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
conntrack_close_policy: ConntrackClosePolicy,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let user = &success.user;
|
||||
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
||||
|
||||
debug!(
|
||||
user = %user,
|
||||
peer = %success.peer,
|
||||
dc = success.dc_idx,
|
||||
dc_addr = %dc_addr,
|
||||
proto = ?success.proto_tag,
|
||||
mode = "direct",
|
||||
"Connecting to Telegram DC"
|
||||
);
|
||||
|
||||
let scope_hint = validated_scope_hint(user);
|
||||
if user.starts_with("scope_") && scope_hint.is_none() {
|
||||
warn!(
|
||||
user = %user,
|
||||
"Ignoring invalid scope hint and falling back to default upstream selection"
|
||||
);
|
||||
}
|
||||
let tg_stream = tokio::select! {
|
||||
result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake");
|
||||
|
||||
let (tg_reader, tg_writer) = tokio::select! {
|
||||
result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, "TG handshake complete, starting relay");
|
||||
|
||||
stats.increment_user_connects(user);
|
||||
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
||||
let traffic_lease = shared
|
||||
.traffic_limiter
|
||||
.acquire_lease(user, success.peer.ip());
|
||||
|
||||
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
||||
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
||||
Duration::from_secs(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.direct_activity_timeout_secs(),
|
||||
)
|
||||
} else {
|
||||
Duration::from_secs(1800)
|
||||
};
|
||||
let relay_result = crate::proxy::relay::relay_direct_adaptive(
|
||||
client_reader,
|
||||
client_writer,
|
||||
tg_reader,
|
||||
tg_writer,
|
||||
config.general.direct_relay_copy_buf_c2s_bytes,
|
||||
config.general.direct_relay_copy_buf_s2c_bytes,
|
||||
config.server.max_connections,
|
||||
user,
|
||||
Arc::clone(&stats),
|
||||
config.access.user_data_quota.get(user).copied(),
|
||||
traffic_lease,
|
||||
relay_activity_timeout,
|
||||
session_cancel.clone(),
|
||||
Arc::clone(&shared.direct_buffer_budget),
|
||||
);
|
||||
tokio::pin!(relay_result);
|
||||
let relay_result = loop {
|
||||
if let Some(cutover) =
|
||||
affected_cutover_state(&route_rx, RelayRouteMode::Direct, route_snapshot.generation)
|
||||
{
|
||||
let delay = cutover_stagger_delay(session_id, cutover.generation);
|
||||
warn!(
|
||||
user = %user,
|
||||
target_mode = cutover.mode.as_str(),
|
||||
cutover_generation = cutover.generation,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Cutover affected direct session, closing client connection"
|
||||
);
|
||||
let _cutover_park_lease = stats.acquire_direct_cutover_park_lease();
|
||||
tokio::time::sleep(delay).await;
|
||||
break Err(ProxyError::RouteSwitched);
|
||||
}
|
||||
tokio::select! {
|
||||
result = &mut relay_result => {
|
||||
break result;
|
||||
}
|
||||
changed = route_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break relay_result.await;
|
||||
}
|
||||
}
|
||||
_ = session_cancel.cancelled() => {
|
||||
break Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &relay_result {
|
||||
Ok(()) => debug!(user = %user, "Direct relay completed"),
|
||||
Err(e) => debug!(user = %user, error = %e, "Direct relay ended with error"),
|
||||
}
|
||||
|
||||
let pool_snapshot = buffer_pool_trim.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
pool_snapshot.allocated,
|
||||
pool_snapshot.allocated.saturating_sub(pool_snapshot.pooled),
|
||||
);
|
||||
|
||||
if conntrack_close_policy == ConntrackClosePolicy::Publish {
|
||||
let close_reason = classify_conntrack_close_reason(&relay_result);
|
||||
let publish_result = shared.publish_conntrack_close_event(ConntrackCloseEvent {
|
||||
src: success.peer,
|
||||
dst: local_addr,
|
||||
reason: close_reason,
|
||||
});
|
||||
if !matches!(
|
||||
publish_result,
|
||||
ConntrackClosePublishResult::Sent | ConntrackClosePublishResult::Disabled
|
||||
) {
|
||||
stats.increment_conntrack_close_event_drop_total();
|
||||
}
|
||||
}
|
||||
|
||||
relay_result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(error.kind(), std::io::ErrorKind::TimedOut) =>
|
||||
{
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(crate::error::ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result<SocketAddr> {
|
||||
let prefer_v6 = config.network.prefer == 6 && config.network.ipv6.unwrap_or(true);
|
||||
let datacenters = if prefer_v6 {
|
||||
&*TG_DATACENTERS_V6
|
||||
} else {
|
||||
&*TG_DATACENTERS_V4
|
||||
};
|
||||
|
||||
let num_dcs = datacenters.len();
|
||||
|
||||
let dc_key = dc_idx.to_string();
|
||||
if let Some(addrs) = config.dc_overrides.get(&dc_key) {
|
||||
let mut parsed = Vec::new();
|
||||
for addr_str in addrs {
|
||||
match addr_str.parse::<SocketAddr>() {
|
||||
Ok(addr) => parsed.push(addr),
|
||||
Err(_) => {
|
||||
warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(addr) = parsed
|
||||
.iter()
|
||||
.find(|a| a.is_ipv6() == prefer_v6)
|
||||
.or_else(|| parsed.first())
|
||||
.copied()
|
||||
{
|
||||
debug!(dc_idx = dc_idx, addr = %addr, count = parsed.len(), "Using DC override from config");
|
||||
return Ok(addr);
|
||||
}
|
||||
}
|
||||
|
||||
let abs_dc = dc_idx.unsigned_abs() as usize;
|
||||
if abs_dc >= 1 && abs_dc <= num_dcs {
|
||||
return Ok(SocketAddr::new(datacenters[abs_dc - 1], TG_DATACENTER_PORT));
|
||||
}
|
||||
|
||||
// Unknown DC requested by client without override: log and fall back.
|
||||
if !config.dc_overrides.contains_key(&dc_key) {
|
||||
warn!(
|
||||
dc_idx = dc_idx,
|
||||
"Requested non-standard DC with no override; falling back to default cluster"
|
||||
);
|
||||
if config.general.unknown_dc_file_log_enabled
|
||||
&& let Some(path) = &config.general.unknown_dc_log_path
|
||||
&& let Ok(handle) = tokio::runtime::Handle::try_current()
|
||||
{
|
||||
if let Some(path) = sanitize_unknown_dc_log_path(path) {
|
||||
if should_log_unknown_dc(dc_idx) {
|
||||
handle.spawn_blocking(move || {
|
||||
if unknown_dc_log_path_is_still_safe(&path)
|
||||
&& let Ok(mut file) = open_unknown_dc_log_append_anchored(&path)
|
||||
{
|
||||
let _ = append_unknown_dc_line(&mut file, dc_idx);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
warn!(dc_idx = dc_idx, raw_path = %path, "Rejected unsafe unknown DC log path");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_dc = config.default_dc.unwrap_or(2) as usize;
|
||||
let fallback_idx = if default_dc >= 1 && default_dc <= num_dcs {
|
||||
default_dc - 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
info!(
|
||||
original_dc = dc_idx,
|
||||
fallback_dc = (fallback_idx + 1) as u16,
|
||||
fallback_addr = %datacenters[fallback_idx],
|
||||
"Special DC ---> default_cluster"
|
||||
);
|
||||
|
||||
Ok(SocketAddr::new(
|
||||
datacenters[fallback_idx],
|
||||
TG_DATACENTER_PORT,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn do_tg_handshake_static<S>(
|
||||
mut stream: S,
|
||||
success: &HandshakeSuccess,
|
||||
config: &ProxyConfig,
|
||||
rng: &SecureRandom,
|
||||
) -> Result<(CryptoReader<ReadHalf<S>>, CryptoWriter<WriteHalf<S>>)>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin,
|
||||
{
|
||||
let (nonce, _tg_enc_key, _tg_enc_iv, _tg_dec_key, _tg_dec_iv) = generate_tg_nonce(
|
||||
success.proto_tag,
|
||||
success.dc_idx,
|
||||
&success.enc_key,
|
||||
success.enc_iv,
|
||||
rng,
|
||||
config.general.fast_mode,
|
||||
);
|
||||
|
||||
let (encrypted_nonce, tg_encryptor, tg_decryptor) = encrypt_tg_nonce_with_ciphers(&nonce);
|
||||
|
||||
debug!(
|
||||
peer = %success.peer,
|
||||
nonce_head = %hex::encode(&nonce[..16]),
|
||||
"Sending nonce to Telegram"
|
||||
);
|
||||
|
||||
stream.write_all(&encrypted_nonce).await?;
|
||||
stream.flush().await?;
|
||||
|
||||
let (read_half, write_half) = split(stream);
|
||||
|
||||
let max_pending = config.general.crypto_pending_buffer;
|
||||
Ok((
|
||||
CryptoReader::new(read_half, tg_decryptor),
|
||||
CryptoWriter::new(write_half, tg_encryptor, max_pending),
|
||||
))
|
||||
}
|
||||
@@ -98,7 +98,9 @@ pub(super) fn auth_probe_is_throttled_in(
|
||||
};
|
||||
if auth_probe_state_expired(&entry, now) {
|
||||
drop(entry);
|
||||
state.remove_if(&peer_ip, |_, current| auth_probe_state_expired(current, now));
|
||||
state.remove_if(&peer_ip, |_, current| {
|
||||
auth_probe_state_expired(current, now)
|
||||
});
|
||||
return false;
|
||||
}
|
||||
now < entry.blocked_until
|
||||
@@ -116,7 +118,9 @@ pub(super) fn auth_probe_saturation_grace_exhausted_in(
|
||||
};
|
||||
if auth_probe_state_expired(&entry, now) {
|
||||
drop(entry);
|
||||
state.remove_if(&peer_ip, |_, current| auth_probe_state_expired(current, now));
|
||||
state.remove_if(&peer_ip, |_, current| {
|
||||
auth_probe_state_expired(current, now)
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -264,7 +268,8 @@ pub(super) fn auth_probe_record_failure_with_state_in(
|
||||
}
|
||||
}
|
||||
|
||||
let Some((evict_key, evict_fail_streak, evict_last_seen)) = eviction_candidate else {
|
||||
let Some((evict_key, evict_fail_streak, evict_last_seen)) = eviction_candidate
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if state
|
||||
|
||||
@@ -266,8 +266,7 @@ where
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
}
|
||||
|
||||
let selected_tls_domain =
|
||||
matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
|
||||
let selected_tls_domain = matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
|
||||
let cached_entry = if config.censorship.tls_emulation {
|
||||
if let Some(cache) = tls_cache.as_ref() {
|
||||
let cached_entry = cache.get(selected_tls_domain).await;
|
||||
|
||||
+25
-1298
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn masking_beobachten_ttl(config: &ProxyConfig) -> Duration {
|
||||
let minutes = config.general.beobachten_minutes;
|
||||
let clamped = minutes.clamp(1, 24 * 60);
|
||||
Duration::from_secs(clamped.saturating_mul(60))
|
||||
}
|
||||
|
||||
pub(super) fn build_mask_proxy_header(
|
||||
version: u8,
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
) -> Option<Vec<u8>> {
|
||||
match version {
|
||||
0 => None,
|
||||
2 => Some(
|
||||
ProxyProtocolV2Builder::new()
|
||||
.with_addrs(peer, local_addr)
|
||||
.build(),
|
||||
),
|
||||
_ => {
|
||||
let header = match (peer, local_addr) {
|
||||
(SocketAddr::V4(src), SocketAddr::V4(dst)) => ProxyProtocolV1Builder::new()
|
||||
.tcp4(src.into(), dst.into())
|
||||
.build(),
|
||||
(SocketAddr::V6(src), SocketAddr::V6(dst)) => ProxyProtocolV1Builder::new()
|
||||
.tcp6(src.into(), dst.into())
|
||||
.build(),
|
||||
_ => ProxyProtocolV1Builder::new().build(),
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn configure_mask_backend_socket(stream: &TcpStream) {
|
||||
if let Err(e) = configure_tcp_socket(stream, false, Duration::from_secs(0)) {
|
||||
debug!(error = %e, "Failed to configure mask backend socket");
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles a bad client by forwarding it to the configured mask target.
|
||||
#[cfg(test)]
|
||||
pub async fn handle_bad_client<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let shared = ProxySharedState::new();
|
||||
handle_bad_client_with_shared(
|
||||
reader,
|
||||
writer,
|
||||
initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
config,
|
||||
beobachten,
|
||||
shared.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn mask_copy_read_len(total: usize, byte_cap: usize) -> usize {
|
||||
// Keep short scanner probes on the small baseline buffer and grow only
|
||||
// after the session has proven to be sustained masking relay traffic.
|
||||
let active_buffer_size = if total >= MASK_BUFFER_GROW_AFTER_BYTES {
|
||||
MASK_BUFFER_MAX_SIZE
|
||||
} else {
|
||||
MASK_BUFFER_SIZE
|
||||
};
|
||||
|
||||
if byte_cap == 0 {
|
||||
return active_buffer_size;
|
||||
}
|
||||
|
||||
let remaining_budget = byte_cap.saturating_sub(total);
|
||||
if remaining_budget == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
remaining_budget.min(active_buffer_size)
|
||||
}
|
||||
|
||||
pub(super) async fn copy_with_idle_timeout<R, W>(
|
||||
reader: &mut R,
|
||||
writer: &mut W,
|
||||
byte_cap: usize,
|
||||
shutdown_on_eof: bool,
|
||||
idle_timeout: Duration,
|
||||
) -> CopyOutcome
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
let mut ended_by_eof = false;
|
||||
|
||||
loop {
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let read_res = timeout(idle_timeout, reader.read(&mut buf[..read_len])).await;
|
||||
let n = match read_res {
|
||||
Ok(Ok(n)) => n,
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
};
|
||||
if n == 0 {
|
||||
ended_by_eof = true;
|
||||
if shutdown_on_eof {
|
||||
let _ = timeout(idle_timeout, writer.shutdown()).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
total = total.saturating_add(n);
|
||||
|
||||
let write_res = timeout(idle_timeout, writer.write_all(&buf[..n])).await;
|
||||
match write_res {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
}
|
||||
}
|
||||
CopyOutcome {
|
||||
total,
|
||||
ended_by_eof,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_http_probe(data: &[u8]) -> bool {
|
||||
// RFC 7540 section 3.5: HTTP/2 client preface starts with "PRI ".
|
||||
const HTTP_METHODS: [&[u8]; 10] = [
|
||||
b"GET ", b"POST", b"HEAD", b"PUT ", b"DELETE", b"OPTIONS", b"CONNECT", b"TRACE", b"PATCH",
|
||||
b"PRI ",
|
||||
];
|
||||
|
||||
if data.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let window = &data[..data.len().min(16)];
|
||||
for method in HTTP_METHODS {
|
||||
if data.len() >= method.len() && window.starts_with(method) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (2..=3).contains(&window.len()) && method.starts_with(window) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
pub(super) fn next_mask_shape_bucket(total: usize, floor: usize, cap: usize) -> usize {
|
||||
if total == 0 || floor == 0 || cap < floor {
|
||||
return total;
|
||||
}
|
||||
|
||||
if total >= cap {
|
||||
return total;
|
||||
}
|
||||
|
||||
let mut bucket = floor;
|
||||
while bucket < total {
|
||||
match bucket.checked_mul(2) {
|
||||
Some(next) => bucket = next,
|
||||
None => return total,
|
||||
}
|
||||
if bucket > cap {
|
||||
return cap;
|
||||
}
|
||||
}
|
||||
bucket
|
||||
}
|
||||
|
||||
pub(super) async fn maybe_write_shape_padding<W>(
|
||||
mask_write: &mut W,
|
||||
total_sent: usize,
|
||||
enabled: bool,
|
||||
floor: usize,
|
||||
cap: usize,
|
||||
above_cap_blur: bool,
|
||||
above_cap_blur_max_bytes: usize,
|
||||
aggressive_mode: bool,
|
||||
) where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
if !enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
let target_total = if total_sent >= cap && above_cap_blur && above_cap_blur_max_bytes > 0 {
|
||||
let mut rng = rand::rng();
|
||||
let extra = if aggressive_mode {
|
||||
rng.random_range(1..=above_cap_blur_max_bytes)
|
||||
} else {
|
||||
rng.random_range(0..=above_cap_blur_max_bytes)
|
||||
};
|
||||
total_sent.saturating_add(extra)
|
||||
} else {
|
||||
next_mask_shape_bucket(total_sent, floor, cap)
|
||||
};
|
||||
|
||||
if target_total <= total_sent {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut remaining = target_total - total_sent;
|
||||
let mut pad_chunk = [0u8; 1024];
|
||||
let deadline = Instant::now() + MASK_TIMEOUT;
|
||||
// Use a Send RNG so relay futures remain spawn-safe under Tokio.
|
||||
let mut rng = {
|
||||
let mut seed_source = rand::rng();
|
||||
StdRng::from_rng(&mut seed_source)
|
||||
};
|
||||
|
||||
while remaining > 0 {
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return;
|
||||
}
|
||||
|
||||
let write_len = remaining.min(pad_chunk.len());
|
||||
rng.fill_bytes(&mut pad_chunk[..write_len]);
|
||||
let write_budget = deadline.saturating_duration_since(now);
|
||||
match timeout(write_budget, mask_write.write_all(&pad_chunk[..write_len])).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) | Err(_) => return,
|
||||
}
|
||||
remaining -= write_len;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return;
|
||||
}
|
||||
let flush_budget = deadline.saturating_duration_since(now);
|
||||
let _ = timeout(flush_budget, mask_write.flush()).await;
|
||||
}
|
||||
|
||||
pub(super) async fn write_proxy_header_with_timeout<W>(mask_write: &mut W, header: &[u8]) -> bool
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
match timeout(MASK_TIMEOUT, mask_write.write_all(header)).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(_)) => false,
|
||||
Err(_) => {
|
||||
debug!("Timeout writing proxy protocol header to mask backend");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn consume_client_data_with_timeout_and_cap<R>(
|
||||
reader: R,
|
||||
byte_cap: usize,
|
||||
relay_timeout: Duration,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
consume_client_data(reader, byte_cap, idle_timeout),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Timed out while consuming client data on masking fallback path");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mask_failure_drain_cap(config: &ProxyConfig) -> usize {
|
||||
let configured_cap = config.censorship.mask_relay_max_bytes;
|
||||
if configured_cap == 0 {
|
||||
return MASK_BUFFER_SIZE;
|
||||
}
|
||||
|
||||
configured_cap.min(MASK_BUFFER_SIZE)
|
||||
}
|
||||
|
||||
pub(super) async fn consume_mask_failure_path<R>(
|
||||
reader: R,
|
||||
config: &ProxyConfig,
|
||||
relay_timeout: Duration,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
mask_failure_drain_cap(config),
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_connect_budget(started: Instant) {
|
||||
let elapsed = started.elapsed();
|
||||
if elapsed < MASK_TIMEOUT {
|
||||
tokio::time::sleep(MASK_TIMEOUT - elapsed).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Log-normal sample bounded to [floor, ceiling]. Median = sqrt(floor * ceiling).
|
||||
// Implements Box-Muller transform for standard normal sampling — no external
|
||||
// dependency on rand_distr (which is incompatible with rand 0.10).
|
||||
// sigma is chosen so ~99% of raw samples land inside [floor, ceiling] before clamp.
|
||||
// When floor > ceiling (misconfiguration), returns ceiling (the smaller value).
|
||||
// When floor == ceiling, returns that value. When both are 0, returns 0.
|
||||
@@ -0,0 +1,260 @@
|
||||
use super::*;
|
||||
|
||||
/// Handles a bad client with shared pre-auth fallback admission state.
|
||||
pub(crate) async fn handle_bad_client_with_shared<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
shared: &ProxySharedState,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_bad_client_with_shared_resolver(
|
||||
reader,
|
||||
writer,
|
||||
initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
config,
|
||||
beobachten,
|
||||
shared,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(in crate::proxy) async fn handle_bad_client_with_shared_resolver<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
shared: &ProxySharedState,
|
||||
upstream_manager: Option<&crate::transport::UpstreamManager>,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let client_type = detect_client_type(initial_data);
|
||||
if config.general.beobachten {
|
||||
let ttl = masking_beobachten_ttl(config);
|
||||
beobachten.record(client_type, peer.ip(), ttl);
|
||||
}
|
||||
|
||||
let relay_timeout = Duration::from_millis(config.censorship.mask_relay_timeout_ms);
|
||||
let idle_timeout = Duration::from_millis(config.censorship.mask_relay_idle_timeout_ms);
|
||||
|
||||
if !config.censorship.mask {
|
||||
// Masking disabled, just consume data
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(_masking_permit) = shared.try_acquire_masking_fallback_permit() else {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
"Masking fallback concurrency limit reached"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
};
|
||||
|
||||
let client_sni = tls::extract_sni_from_client_hello(initial_data);
|
||||
let exclusive_tcp_target = client_sni
|
||||
.as_deref()
|
||||
.and_then(|sni| exclusive_mask_target_for_sni(config, sni));
|
||||
|
||||
// Connect via Unix socket or TCP
|
||||
#[cfg(unix)]
|
||||
if exclusive_tcp_target.is_none()
|
||||
&& let Some(ref sock_path) = config.censorship.mask_unix_sock
|
||||
{
|
||||
let outcome_started = Instant::now();
|
||||
let connect_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
sock = %sock_path,
|
||||
data_len = initial_data.len(),
|
||||
"Forwarding bad client to mask unix socket"
|
||||
);
|
||||
|
||||
let connect_result = timeout(MASK_TIMEOUT, UnixStream::connect(sock_path)).await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
let (mask_read, mut mask_write) = stream.into_split();
|
||||
let proxy_header = build_mask_proxy_header(
|
||||
config.censorship.mask_proxy_protocol,
|
||||
peer,
|
||||
local_addr,
|
||||
);
|
||||
if let Some(header) = proxy_header
|
||||
&& !write_proxy_header_with_timeout(&mut mask_write, &header).await
|
||||
{
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
relay_to_mask(
|
||||
reader,
|
||||
writer,
|
||||
mask_read,
|
||||
mask_write,
|
||||
initial_data,
|
||||
config.censorship.mask_shape_hardening,
|
||||
config.censorship.mask_shape_bucket_floor_bytes,
|
||||
config.censorship.mask_shape_bucket_cap_bytes,
|
||||
config.censorship.mask_shape_above_cap_blur,
|
||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
idle_timeout,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Mask relay timed out (unix socket)");
|
||||
}
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask unix socket");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask unix socket");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mask_target = exclusive_tcp_target.unwrap_or_else(|| {
|
||||
default_mask_tcp_target_for_initial_data(config, initial_data, client_sni.as_deref())
|
||||
});
|
||||
let mask_host = mask_target.host;
|
||||
let mask_port = mask_target.port;
|
||||
|
||||
let resolved_mask_addrs =
|
||||
match resolve_mask_target_addrs(mask_host, mask_port, upstream_manager).await {
|
||||
Ok(addrs) => addrs,
|
||||
Err(e) => {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
error = %e,
|
||||
"Failed to resolve mask target"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Fail closed when fallback points at our own listener endpoint.
|
||||
// Self-referential masking can create recursive proxy loops under
|
||||
// misconfiguration and leak distinguishable load spikes to adversaries.
|
||||
if is_mask_target_local_listener_async(mask_host, mask_port, local_addr, &resolved_mask_addrs)
|
||||
.await
|
||||
{
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
local = %local_addr,
|
||||
"Mask target resolves to local listener; refusing self-referential masking fallback"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let outcome_started = Instant::now();
|
||||
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
data_len = initial_data.len(),
|
||||
"Forwarding bad client to mask host"
|
||||
);
|
||||
|
||||
let connect_started = Instant::now();
|
||||
let connect_result = timeout(
|
||||
MASK_TIMEOUT,
|
||||
TcpStream::connect(resolved_mask_addrs.as_slice()),
|
||||
)
|
||||
.await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
configure_mask_backend_socket(&stream);
|
||||
let proxy_header =
|
||||
build_mask_proxy_header(config.censorship.mask_proxy_protocol, peer, local_addr);
|
||||
|
||||
let (mask_read, mut mask_write) = stream.into_split();
|
||||
if let Some(header) = proxy_header
|
||||
&& !write_proxy_header_with_timeout(&mut mask_write, &header).await
|
||||
{
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
relay_to_mask(
|
||||
reader,
|
||||
writer,
|
||||
mask_read,
|
||||
mask_write,
|
||||
initial_data,
|
||||
config.censorship.mask_shape_hardening,
|
||||
config.censorship.mask_shape_bucket_floor_bytes,
|
||||
config.censorship.mask_shape_bucket_cap_bytes,
|
||||
config.censorship.mask_shape_above_cap_blur,
|
||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
idle_timeout,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Mask relay timed out");
|
||||
}
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask host");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask host");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn canonical_ip(ip: IpAddr) -> IpAddr {
|
||||
match ip {
|
||||
IpAddr::V6(v6) => v6
|
||||
.to_ipv4_mapped()
|
||||
.map(IpAddr::V4)
|
||||
.unwrap_or(IpAddr::V6(v6)),
|
||||
IpAddr::V4(v4) => IpAddr::V4(v4),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) fn collect_local_interface_ips() -> Vec<IpAddr> {
|
||||
#[cfg(test)]
|
||||
LOCAL_INTERFACE_ENUMERATIONS.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let mut out = Vec::new();
|
||||
if let Ok(addrs) = getifaddrs() {
|
||||
for iface in addrs {
|
||||
if let Some(address) = iface.address {
|
||||
if let Some(v4) = address.as_sockaddr_in() {
|
||||
out.push(canonical_ip(IpAddr::V4(v4.ip())));
|
||||
} else if let Some(v6) = address.as_sockaddr_in6() {
|
||||
out.push(canonical_ip(IpAddr::V6(v6.ip())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) fn choose_interface_snapshot(
|
||||
previous: &[IpAddr],
|
||||
refreshed: Vec<IpAddr>,
|
||||
) -> Vec<IpAddr> {
|
||||
if refreshed.is_empty() && !previous.is_empty() {
|
||||
return previous.to_vec();
|
||||
}
|
||||
|
||||
refreshed
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[derive(Default)]
|
||||
struct LocalInterfaceCache {
|
||||
ips: Vec<IpAddr>,
|
||||
refreshed_at: Option<StdInstant>,
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
static LOCAL_INTERFACE_CACHE: OnceLock<Mutex<LocalInterfaceCache>> = OnceLock::new();
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) static LOCAL_INTERFACE_REFRESH_LOCK: OnceLock<AsyncMutex<()>> = OnceLock::new();
|
||||
|
||||
#[cfg(all(unix, test))]
|
||||
pub(super) fn local_interface_ips() -> Vec<IpAddr> {
|
||||
let cache = LOCAL_INTERFACE_CACHE.get_or_init(|| Mutex::new(LocalInterfaceCache::default()));
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if stale {
|
||||
let refreshed = collect_local_interface_ips();
|
||||
guard.ips = choose_interface_snapshot(&guard.ips, refreshed);
|
||||
guard.refreshed_at = Some(StdInstant::now());
|
||||
}
|
||||
|
||||
guard.ips.clone()
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) async fn local_interface_ips_async() -> Vec<IpAddr> {
|
||||
let cache = LOCAL_INTERFACE_CACHE.get_or_init(|| Mutex::new(LocalInterfaceCache::default()));
|
||||
|
||||
{
|
||||
let guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if !stale {
|
||||
return guard.ips.clone();
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_lock = LOCAL_INTERFACE_REFRESH_LOCK.get_or_init(|| AsyncMutex::new(()));
|
||||
let _refresh_guard = refresh_lock.lock().await;
|
||||
|
||||
{
|
||||
let guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if !stale {
|
||||
return guard.ips.clone();
|
||||
}
|
||||
}
|
||||
|
||||
let refreshed = tokio::task::spawn_blocking(collect_local_interface_ips)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if stale {
|
||||
guard.ips = choose_interface_snapshot(&guard.ips, refreshed);
|
||||
guard.refreshed_at = Some(StdInstant::now());
|
||||
}
|
||||
|
||||
guard.ips.clone()
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), test))]
|
||||
pub(super) fn local_interface_ips() -> Vec<IpAddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(super) async fn local_interface_ips_async() -> Vec<IpAddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static LOCAL_INTERFACE_ENUMERATIONS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn reset_local_interface_enumerations_for_tests() {
|
||||
LOCAL_INTERFACE_ENUMERATIONS.store(0, Ordering::Relaxed);
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(cache) = LOCAL_INTERFACE_CACHE.get() {
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
guard.ips.clear();
|
||||
guard.refreshed_at = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn local_interface_enumerations_for_tests() -> usize {
|
||||
LOCAL_INTERFACE_ENUMERATIONS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn interface_cache_test_lock() -> &'static tokio::sync::Mutex<()> {
|
||||
static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
|
||||
}
|
||||
|
||||
pub(super) fn is_mask_target_local_listener_with_interfaces(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
interface_ips: &[IpAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let local_ip = canonical_ip(local_addr.ip());
|
||||
let literal_mask_ip = parse_mask_host_ip_literal(mask_host).map(canonical_ip);
|
||||
|
||||
for addr in resolved_addrs {
|
||||
let resolved_ip = canonical_ip(addr.ip());
|
||||
if resolved_ip == local_ip {
|
||||
return true;
|
||||
}
|
||||
|
||||
if local_ip.is_unspecified()
|
||||
&& (resolved_ip.is_loopback()
|
||||
|| resolved_ip.is_unspecified()
|
||||
|| interface_ips.contains(&resolved_ip))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(mask_ip) = literal_mask_ip {
|
||||
if mask_ip == local_ip {
|
||||
return true;
|
||||
}
|
||||
|
||||
if local_ip.is_unspecified()
|
||||
&& (mask_ip.is_loopback()
|
||||
|| mask_ip.is_unspecified()
|
||||
|| interface_ips.contains(&mask_ip))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn is_mask_target_local_listener(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let interfaces = local_interface_ips();
|
||||
is_mask_target_local_listener_with_interfaces(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn is_mask_target_local_listener_async(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let interfaces = local_interface_ips_async().await;
|
||||
is_mask_target_local_listener_with_interfaces(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use super::*;
|
||||
|
||||
/// Relays traffic between the client and mask backend.
|
||||
pub(super) async fn relay_to_mask<R, W, MR, MW>(
|
||||
mut reader: R,
|
||||
mut writer: W,
|
||||
mut mask_read: MR,
|
||||
mut mask_write: MW,
|
||||
initial_data: &[u8],
|
||||
shape_hardening_enabled: bool,
|
||||
shape_bucket_floor_bytes: usize,
|
||||
shape_bucket_cap_bytes: usize,
|
||||
shape_above_cap_blur: bool,
|
||||
shape_above_cap_blur_max_bytes: usize,
|
||||
shape_hardening_aggressive_mode: bool,
|
||||
mask_relay_max_bytes: usize,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
MR: AsyncRead + Unpin + Send + 'static,
|
||||
MW: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
// Send initial data to mask host
|
||||
if mask_write.write_all(initial_data).await.is_err() {
|
||||
return;
|
||||
}
|
||||
if mask_write.flush().await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let (upstream_copy, downstream_copy) = tokio::join!(
|
||||
async {
|
||||
copy_with_idle_timeout(
|
||||
&mut reader,
|
||||
&mut mask_write,
|
||||
mask_relay_max_bytes,
|
||||
!shape_hardening_enabled,
|
||||
idle_timeout,
|
||||
)
|
||||
.await
|
||||
},
|
||||
async {
|
||||
copy_with_idle_timeout(
|
||||
&mut mask_read,
|
||||
&mut writer,
|
||||
mask_relay_max_bytes,
|
||||
true,
|
||||
idle_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
);
|
||||
|
||||
let total_sent = initial_data.len().saturating_add(upstream_copy.total);
|
||||
|
||||
let should_shape = shape_hardening_enabled
|
||||
&& !initial_data.is_empty()
|
||||
&& (upstream_copy.ended_by_eof
|
||||
|| (shape_hardening_aggressive_mode && downstream_copy.total == 0));
|
||||
|
||||
maybe_write_shape_padding(
|
||||
&mut mask_write,
|
||||
total_sent,
|
||||
should_shape,
|
||||
shape_bucket_floor_bytes,
|
||||
shape_bucket_cap_bytes,
|
||||
shape_above_cap_blur,
|
||||
shape_above_cap_blur_max_bytes,
|
||||
shape_hardening_aggressive_mode,
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = mask_write.shutdown().await;
|
||||
let _ = writer.shutdown().await;
|
||||
}
|
||||
|
||||
/// Just consume all data from client without responding.
|
||||
pub(super) async fn consume_client_data<R: AsyncRead + Unpin>(
|
||||
mut reader: R,
|
||||
byte_cap: usize,
|
||||
idle_timeout: Duration,
|
||||
) {
|
||||
// Keep drain path fail-closed under slow-loris stalls.
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
|
||||
loop {
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let n = match timeout(idle_timeout, reader.read(&mut buf[..read_len])).await {
|
||||
Ok(Ok(n)) => n,
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
};
|
||||
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
total = total.saturating_add(n);
|
||||
if byte_cap != 0 && total >= byte_cap {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use super::*;
|
||||
|
||||
/// Detect client type based on initial data.
|
||||
pub(super) fn detect_client_type(data: &[u8]) -> &'static str {
|
||||
// Check for HTTP request
|
||||
if is_http_probe(data) {
|
||||
return "HTTP";
|
||||
}
|
||||
|
||||
// Check for TLS ClientHello (0x16 = handshake, 0x03 0x01-0x03 = TLS version)
|
||||
if data.len() > 3 && data[0] == 0x16 && data[1] == 0x03 {
|
||||
return "TLS-scanner";
|
||||
}
|
||||
|
||||
// Check for SSH
|
||||
if data.starts_with(b"SSH-") {
|
||||
return "SSH";
|
||||
}
|
||||
|
||||
// Port scanner (very short data)
|
||||
if data.len() < 10 {
|
||||
return "port-scanner";
|
||||
}
|
||||
|
||||
"unknown"
|
||||
}
|
||||
|
||||
pub(super) fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
|
||||
if host.starts_with('[') && host.ends_with(']') {
|
||||
return host[1..host.len() - 1].parse::<IpAddr>().ok();
|
||||
}
|
||||
host.parse::<IpAddr>().ok()
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_mask_target_addrs(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
upstream_manager: Option<&crate::transport::UpstreamManager>,
|
||||
) -> std::io::Result<Vec<SocketAddr>> {
|
||||
if let Some(ip) = parse_mask_host_ip_literal(mask_host) {
|
||||
return Ok(vec![SocketAddr::new(ip, mask_port)]);
|
||||
}
|
||||
|
||||
if let Some(upstream_manager) = upstream_manager {
|
||||
return upstream_manager
|
||||
.resolve_all(mask_host, mask_port)
|
||||
.await
|
||||
.map_err(|error| IoError::new(ErrorKind::NotFound, error.to_string()));
|
||||
}
|
||||
|
||||
let addrs = timeout(MASK_TIMEOUT, lookup_host((mask_host, mask_port)))
|
||||
.await
|
||||
.map_err(|_| IoError::new(ErrorKind::TimedOut, "mask target DNS lookup timed out"))??;
|
||||
let addrs = addrs
|
||||
.take(MASK_DNS_RESULT_MAX_ADDRESSES)
|
||||
.collect::<Vec<_>>();
|
||||
if addrs.is_empty() {
|
||||
return Err(IoError::new(
|
||||
ErrorKind::NotFound,
|
||||
"mask target DNS lookup returned no addresses",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(addrs)
|
||||
}
|
||||
|
||||
pub(super) fn matching_tls_domain_for_sni<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
sni: &str,
|
||||
) -> Option<&'a str> {
|
||||
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
|
||||
return Some(config.censorship.tls_domain.as_str());
|
||||
}
|
||||
|
||||
for domain in &config.censorship.tls_domains {
|
||||
if domain.eq_ignore_ascii_case(sni) {
|
||||
return Some(domain.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn parse_exclusive_mask_target(target: &str) -> Option<MaskTcpTarget<'_>> {
|
||||
let target = target.trim();
|
||||
if target.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if target.starts_with('[') {
|
||||
let end = target.find(']')?;
|
||||
if target.get(end + 1..end + 2)? != ":" {
|
||||
return None;
|
||||
}
|
||||
let port = target[end + 2..].parse::<u16>().ok()?;
|
||||
return (port > 0).then_some(MaskTcpTarget {
|
||||
host: &target[..=end],
|
||||
port,
|
||||
});
|
||||
}
|
||||
|
||||
let (host, port) = target.rsplit_once(':')?;
|
||||
if host.is_empty() || host.contains(':') {
|
||||
return None;
|
||||
}
|
||||
let port = port.parse::<u16>().ok()?;
|
||||
(port > 0).then_some(MaskTcpTarget { host, port })
|
||||
}
|
||||
|
||||
pub(super) fn exclusive_mask_target_for_sni<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
sni: &str,
|
||||
) -> Option<MaskTcpTarget<'a>> {
|
||||
if let Some(target) = config.censorship.exclusive_mask_targets.get(sni) {
|
||||
return Some(MaskTcpTarget {
|
||||
host: target.host.as_str(),
|
||||
port: target.port,
|
||||
});
|
||||
}
|
||||
if let Some(target) = config.censorship.exclusive_mask.get(sni) {
|
||||
return parse_exclusive_mask_target(target);
|
||||
}
|
||||
|
||||
if sni.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||
let normalized_sni = sni.to_ascii_lowercase();
|
||||
if let Some(target) = config
|
||||
.censorship
|
||||
.exclusive_mask_targets
|
||||
.get(&normalized_sni)
|
||||
{
|
||||
return Some(MaskTcpTarget {
|
||||
host: target.host.as_str(),
|
||||
port: target.port,
|
||||
});
|
||||
}
|
||||
if let Some(target) = config.censorship.exclusive_mask.get(&normalized_sni) {
|
||||
return parse_exclusive_mask_target(target);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn mask_host_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
) -> &'a str {
|
||||
mask_tcp_target_for_initial_data(config, initial_data).host
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn mask_tcp_target_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
) -> MaskTcpTarget<'a> {
|
||||
let sni = tls::extract_sni_from_client_hello(initial_data);
|
||||
if let Some(target) = sni
|
||||
.as_deref()
|
||||
.and_then(|sni| exclusive_mask_target_for_sni(config, sni))
|
||||
{
|
||||
return target;
|
||||
}
|
||||
|
||||
default_mask_tcp_target_for_initial_data(config, initial_data, sni.as_deref())
|
||||
}
|
||||
|
||||
pub(super) fn default_mask_tcp_target_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
sni: Option<&str>,
|
||||
) -> MaskTcpTarget<'a> {
|
||||
let configured_mask_host = config
|
||||
.censorship
|
||||
.mask_host
|
||||
.as_deref()
|
||||
.unwrap_or(&config.censorship.tls_domain);
|
||||
|
||||
if config.censorship.mask_host.is_none() && config.censorship.mask_dynamic {
|
||||
let extracted_sni = if sni.is_none() {
|
||||
tls::extract_sni_from_client_hello(initial_data)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(host) = sni
|
||||
.or(extracted_sni.as_deref())
|
||||
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
|
||||
{
|
||||
return MaskTcpTarget {
|
||||
host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(mask_host) = config.censorship.mask_host.as_deref() {
|
||||
return MaskTcpTarget {
|
||||
host: mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
|
||||
MaskTcpTarget {
|
||||
host: configured_mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn sample_lognormal_percentile_bounded(
|
||||
floor: u64,
|
||||
ceiling: u64,
|
||||
rng: &mut impl Rng,
|
||||
) -> u64 {
|
||||
if ceiling == 0 && floor == 0 {
|
||||
return 0;
|
||||
}
|
||||
if floor > ceiling {
|
||||
return ceiling;
|
||||
}
|
||||
if floor == ceiling {
|
||||
return floor;
|
||||
}
|
||||
let floor_f = floor.max(1) as f64;
|
||||
let ceiling_f = ceiling.max(1) as f64;
|
||||
let mu = (floor_f.ln() + ceiling_f.ln()) / 2.0;
|
||||
// 4.65 ≈ 2 * 2.326 (double-sided z-score for 99th percentile)
|
||||
let sigma = ((ceiling_f / floor_f).ln() / 4.65).max(0.01);
|
||||
// Box-Muller transform: two uniform samples → one standard normal sample
|
||||
let u1: f64 = rng.random_range(f64::MIN_POSITIVE..1.0);
|
||||
let u2: f64 = rng.random_range(0.0_f64..std::f64::consts::TAU);
|
||||
let normal_sample = (-2.0_f64 * u1.ln()).sqrt() * u2.cos();
|
||||
let raw = (mu + sigma * normal_sample).exp();
|
||||
if raw.is_finite() {
|
||||
(raw as u64).clamp(floor, ceiling)
|
||||
} else {
|
||||
((floor_f * ceiling_f).sqrt()) as u64
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mask_outcome_target_budget(config: &ProxyConfig) -> Duration {
|
||||
if config.censorship.mask_timing_normalization_enabled {
|
||||
let floor = config.censorship.mask_timing_normalization_floor_ms;
|
||||
let ceiling = config.censorship.mask_timing_normalization_ceiling_ms;
|
||||
if floor == 0 {
|
||||
if ceiling == 0 {
|
||||
return Duration::from_millis(0);
|
||||
}
|
||||
// floor=0 stays uniform: log-normal cannot model distribution anchored at zero
|
||||
let mut rng = rand::rng();
|
||||
return Duration::from_millis(rng.random_range(0..=ceiling));
|
||||
}
|
||||
if ceiling > floor {
|
||||
let mut rng = rand::rng();
|
||||
return Duration::from_millis(sample_lognormal_percentile_bounded(
|
||||
floor, ceiling, &mut rng,
|
||||
));
|
||||
}
|
||||
// ceiling <= floor: use the larger value (fail-closed: preserve longer delay)
|
||||
return Duration::from_millis(floor.max(ceiling));
|
||||
}
|
||||
|
||||
MASK_TIMEOUT
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_connect_budget_if_needed(started: Instant, config: &ProxyConfig) {
|
||||
if config.censorship.mask_timing_normalization_enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
wait_mask_connect_budget(started).await;
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) {
|
||||
let target = mask_outcome_target_budget(config);
|
||||
let elapsed = started.elapsed();
|
||||
if elapsed < target {
|
||||
tokio::time::sleep(target - elapsed).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tls_domain_mask_host_tests {
|
||||
use super::{
|
||||
mask_host_for_initial_data, mask_tcp_target_for_initial_data, matching_tls_domain_for_sni,
|
||||
};
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
fn client_hello_with_sni(sni_host: &str) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0x03, 0x03]);
|
||||
body.extend_from_slice(&[0u8; 32]);
|
||||
body.push(32);
|
||||
body.extend_from_slice(&[0x42u8; 32]);
|
||||
body.extend_from_slice(&2u16.to_be_bytes());
|
||||
body.extend_from_slice(&[0x13, 0x01]);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
|
||||
let host_bytes = sni_host.as_bytes();
|
||||
let mut sni_payload = Vec::new();
|
||||
sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes());
|
||||
sni_payload.push(0);
|
||||
sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes());
|
||||
sni_payload.extend_from_slice(host_bytes);
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
extensions.extend_from_slice(&0x0000u16.to_be_bytes());
|
||||
extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes());
|
||||
extensions.extend_from_slice(&sni_payload);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
|
||||
let mut handshake = Vec::new();
|
||||
handshake.push(0x01);
|
||||
let body_len = (body.len() as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&body_len[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
let mut record = Vec::new();
|
||||
record.push(0x16);
|
||||
record.extend_from_slice(&[0x03, 0x01]);
|
||||
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
|
||||
record.extend_from_slice(&handshake);
|
||||
record
|
||||
}
|
||||
|
||||
fn config_with_tls_domains() -> ProxyConfig {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_domain = "a.com".to_string();
|
||||
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
|
||||
config.censorship.mask_host = None;
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() {
|
||||
let config = config_with_tls_domains();
|
||||
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com"));
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com"));
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_preserves_explicit_non_primary_origin() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config.censorship.mask_host = Some("origin.example".to_string());
|
||||
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(
|
||||
mask_host_for_initial_data(&config, &initial_data),
|
||||
"origin.example"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() {
|
||||
let config = config_with_tls_domains();
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_uses_primary_domain_when_dynamic_masking_is_disabled() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config.censorship.mask_dynamic = false;
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "a.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusive_mask_target_overrides_only_matching_sni() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config
|
||||
.censorship
|
||||
.exclusive_mask
|
||||
.insert("b.com".to_string(), "origin-b.example:8443".to_string());
|
||||
let b_initial_data = client_hello_with_sni("B.COM");
|
||||
let c_initial_data = client_hello_with_sni("c.com");
|
||||
|
||||
let b_target = mask_tcp_target_for_initial_data(&config, &b_initial_data);
|
||||
let c_target = mask_tcp_target_for_initial_data(&config, &c_initial_data);
|
||||
|
||||
assert_eq!(b_target.host, "origin-b.example");
|
||||
assert_eq!(b_target.port, 8443);
|
||||
assert_eq!(c_target.host, "c.com");
|
||||
assert_eq!(c_target.port, config.censorship.mask_port);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,12 @@
|
||||
use super::*;
|
||||
|
||||
// Bounded C2ME sender and downstream writer tasks.
|
||||
mod tasks;
|
||||
// Conntrack close classification.
|
||||
mod close_reason;
|
||||
|
||||
use close_reason::classify_conntrack_close_reason;
|
||||
use tasks::{run_c2me_sender, run_me_writer};
|
||||
struct RelayConnLease {
|
||||
connection: Option<ConnLease>,
|
||||
conn_id: u64,
|
||||
@@ -174,47 +181,21 @@ where
|
||||
};
|
||||
let c2me_byte_budget = c2me_queued_permit_budget(c2me_channel_capacity, frame_limit);
|
||||
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
|
||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||
let (c2me_tx, c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||
let me_pool_c2me = me_pool.clone();
|
||||
let mut c2me_sender = tokio::spawn(async move {
|
||||
let mut sent_since_yield = 0usize;
|
||||
while let Some(cmd) = c2me_rx.recv().await {
|
||||
match cmd {
|
||||
C2MeCommand::Data {
|
||||
payload,
|
||||
flags,
|
||||
_permit,
|
||||
} => {
|
||||
me_pool_c2me
|
||||
.send_proxy_req_pooled(
|
||||
let mut c2me_sender = tokio::spawn(run_c2me_sender(
|
||||
c2me_rx,
|
||||
me_pool_c2me,
|
||||
conn_id,
|
||||
success.dc_idx,
|
||||
success,
|
||||
peer,
|
||||
translated_local_addr,
|
||||
payload,
|
||||
_permit,
|
||||
flags,
|
||||
effective_tag_array,
|
||||
)
|
||||
.await?;
|
||||
sent_since_yield = sent_since_yield.saturating_add(1);
|
||||
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
|
||||
sent_since_yield = 0;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
C2MeCommand::Close => {
|
||||
let _ = me_pool_c2me.send_close(conn_id).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
));
|
||||
|
||||
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
|
||||
let (stop_tx, stop_rx) = oneshot::channel::<()>();
|
||||
let flow_cancel = CancellationToken::new();
|
||||
let mut me_rx_task = me_rx;
|
||||
let me_rx_task = me_rx;
|
||||
let stats_clone = stats.clone();
|
||||
let rng_clone = rng.clone();
|
||||
let user_clone = user.clone();
|
||||
@@ -224,361 +205,24 @@ where
|
||||
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
||||
let bytes_me2c_clone = bytes_me2c.clone();
|
||||
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
||||
let mut me_writer = tokio::spawn(async move {
|
||||
let mut writer = crypto_writer;
|
||||
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
|
||||
fn shrink_session_vec(buf: &mut Vec<u8>, threshold: usize) {
|
||||
if buf.capacity() > threshold {
|
||||
buf.clear();
|
||||
buf.shrink_to(threshold);
|
||||
} else {
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
msg = me_rx_task.recv() => {
|
||||
let Some(first) = msg else {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
};
|
||||
|
||||
let mut batch_frames = 0usize;
|
||||
let mut batch_bytes = 0usize;
|
||||
let mut flush_immediately;
|
||||
let mut max_delay_fired = false;
|
||||
|
||||
let first_is_downstream_activity =
|
||||
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
first,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
let mut me_writer = tokio::spawn(run_me_writer(
|
||||
crypto_writer,
|
||||
me_rx_task,
|
||||
stats_clone,
|
||||
rng_clone,
|
||||
user_clone,
|
||||
quota_user_stats_me_writer,
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
false,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if first_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately = immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(next) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
traffic_lease_me_writer,
|
||||
flow_cancel_me_writer,
|
||||
last_downstream_activity_ms_clone,
|
||||
bytes_me2c_clone,
|
||||
d2c_flush_policy,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
session_started_at,
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at =
|
||||
if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ =
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !flush_immediately
|
||||
&& !d2c_flush_policy.max_delay.is_zero()
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
stats_clone.increment_me_d2c_batch_timeout_armed_total();
|
||||
match tokio::time::timeout(d2c_flush_policy.max_delay, me_rx_task.recv()).await {
|
||||
Ok(Some(next)) => {
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(extra) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let extra_is_downstream_activity =
|
||||
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
extra,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if extra_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
}
|
||||
Err(_) => {
|
||||
max_delay_fired = true;
|
||||
stats_clone.increment_me_d2c_batch_timeout_fired_total();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let flush_reason = classify_me_d2c_flush_reason(
|
||||
flush_immediately,
|
||||
batch_frames,
|
||||
d2c_flush_policy.max_frames,
|
||||
batch_bytes,
|
||||
d2c_flush_policy.max_bytes,
|
||||
max_delay_fired,
|
||||
);
|
||||
let physical_flush =
|
||||
me_d2c_flush_reason_requires_client_flush(flush_reason);
|
||||
let flush_started_at = if physical_flush
|
||||
&& stats_clone.telemetry_policy().me_level.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if physical_flush {
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
}
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
flush_reason,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
let shrink_trigger = shrink_threshold
|
||||
.saturating_mul(ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR);
|
||||
if frame_buf.capacity() > shrink_trigger {
|
||||
let cap_before = frame_buf.capacity();
|
||||
frame_buf.shrink_to(shrink_threshold);
|
||||
let cap_after = frame_buf.capacity();
|
||||
let bytes_freed = cap_before.saturating_sub(cap_after) as u64;
|
||||
stats_clone.observe_me_d2c_frame_buf_shrink(bytes_freed);
|
||||
}
|
||||
}
|
||||
_ = &mut stop_rx => {
|
||||
debug!(conn_id, "ME writer stop signal");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
stop_rx,
|
||||
));
|
||||
|
||||
let mut main_result: Result<()> = Ok(());
|
||||
let mut client_closed = false;
|
||||
@@ -875,30 +519,3 @@ where
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(ProxyError::Io(error)) if matches!(error.kind(), std::io::ErrorKind::TimedOut) => {
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(ProxyError::Io(error)) if matches!(error.kind(), std::io::ErrorKind::TimedOut) => {
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
use super::*;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_c2me_sender(
|
||||
mut c2me_rx: mpsc::Receiver<C2MeCommand>,
|
||||
me_pool_c2me: Arc<MePool>,
|
||||
conn_id: u64,
|
||||
success: HandshakeSuccess,
|
||||
peer: SocketAddr,
|
||||
translated_local_addr: SocketAddr,
|
||||
effective_tag_array: Option<[u8; 16]>,
|
||||
) -> Result<()> {
|
||||
let mut sent_since_yield = 0usize;
|
||||
while let Some(cmd) = c2me_rx.recv().await {
|
||||
match cmd {
|
||||
C2MeCommand::Data {
|
||||
payload,
|
||||
flags,
|
||||
_permit,
|
||||
} => {
|
||||
me_pool_c2me
|
||||
.send_proxy_req_pooled(
|
||||
conn_id,
|
||||
success.dc_idx,
|
||||
peer,
|
||||
translated_local_addr,
|
||||
payload,
|
||||
_permit,
|
||||
flags,
|
||||
effective_tag_array,
|
||||
)
|
||||
.await?;
|
||||
sent_since_yield = sent_since_yield.saturating_add(1);
|
||||
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
|
||||
sent_since_yield = 0;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
C2MeCommand::Close => {
|
||||
let _ = me_pool_c2me.send_close(conn_id).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_me_writer<W>(
|
||||
crypto_writer: CryptoWriter<W>,
|
||||
mut me_rx_task: mpsc::Receiver<MeResponse>,
|
||||
stats_clone: Arc<Stats>,
|
||||
rng_clone: Arc<SecureRandom>,
|
||||
user_clone: String,
|
||||
quota_user_stats_me_writer: Option<Arc<UserStats>>,
|
||||
quota_limit: Option<u64>,
|
||||
traffic_lease_me_writer: Option<Arc<TrafficLease>>,
|
||||
flow_cancel_me_writer: CancellationToken,
|
||||
last_downstream_activity_ms_clone: Arc<AtomicU64>,
|
||||
bytes_me2c_clone: Arc<AtomicU64>,
|
||||
d2c_flush_policy: MeD2cFlushPolicy,
|
||||
proto_tag: ProtoTag,
|
||||
session_started_at: Instant,
|
||||
conn_id: u64,
|
||||
mut stop_rx: oneshot::Receiver<()>,
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let mut writer = crypto_writer;
|
||||
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
|
||||
fn shrink_session_vec(buf: &mut Vec<u8>, threshold: usize) {
|
||||
if buf.capacity() > threshold {
|
||||
buf.clear();
|
||||
buf.shrink_to(threshold);
|
||||
} else {
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
msg = me_rx_task.recv() => {
|
||||
let Some(first) = msg else {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
};
|
||||
|
||||
let mut batch_frames = 0usize;
|
||||
let mut batch_bytes = 0usize;
|
||||
let mut flush_immediately;
|
||||
let mut max_delay_fired = false;
|
||||
|
||||
let first_is_downstream_activity =
|
||||
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
first,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
false,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if first_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately = immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(next) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at =
|
||||
if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ =
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !flush_immediately
|
||||
&& !d2c_flush_policy.max_delay.is_zero()
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
stats_clone.increment_me_d2c_batch_timeout_armed_total();
|
||||
match tokio::time::timeout(d2c_flush_policy.max_delay, me_rx_task.recv()).await {
|
||||
Ok(Some(next)) => {
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(extra) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let extra_is_downstream_activity =
|
||||
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
extra,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if extra_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
}
|
||||
Err(_) => {
|
||||
max_delay_fired = true;
|
||||
stats_clone.increment_me_d2c_batch_timeout_fired_total();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let flush_reason = classify_me_d2c_flush_reason(
|
||||
flush_immediately,
|
||||
batch_frames,
|
||||
d2c_flush_policy.max_frames,
|
||||
batch_bytes,
|
||||
d2c_flush_policy.max_bytes,
|
||||
max_delay_fired,
|
||||
);
|
||||
let physical_flush =
|
||||
me_d2c_flush_reason_requires_client_flush(flush_reason);
|
||||
let flush_started_at = if physical_flush
|
||||
&& stats_clone.telemetry_policy().me_level.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if physical_flush {
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
}
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
flush_reason,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
let shrink_trigger = shrink_threshold
|
||||
.saturating_mul(ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR);
|
||||
if frame_buf.capacity() > shrink_trigger {
|
||||
let cap_before = frame_buf.capacity();
|
||||
frame_buf.shrink_to(shrink_threshold);
|
||||
let cap_after = frame_buf.capacity();
|
||||
let bytes_freed = cap_before.saturating_sub(cap_after) as u64;
|
||||
stats_clone.observe_me_d2c_frame_buf_shrink(bytes_freed);
|
||||
}
|
||||
}
|
||||
_ = &mut stop_rx => {
|
||||
debug!(conn_id, "ME writer stop signal");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,358 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-parent-swap-openat-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&base) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&base).expect("stale parent-swap-openat path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&base)
|
||||
.expect("stale parent-swap-openat directory must be removable");
|
||||
}
|
||||
}
|
||||
let moved = base.with_extension("bak");
|
||||
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&moved)
|
||||
.expect("stale parent-swap-openat backup path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&moved)
|
||||
.expect("stale parent-swap-openat backup directory must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-parent-swap-openat-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent swap");
|
||||
fs::write(&sanitized.resolved_path, "seed\n").expect("seed target file must be writable");
|
||||
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"precondition: target should initially pass revalidation"
|
||||
);
|
||||
|
||||
let outside_parent = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-parent-swap-openat-outside-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&outside_parent).expect("outside parent directory must be creatable");
|
||||
let outside_target = outside_parent.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&outside_target);
|
||||
|
||||
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
|
||||
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect_err("anchored open must fail when parent is swapped to symlink");
|
||||
let raw = err.raw_os_error();
|
||||
assert!(
|
||||
matches!(
|
||||
raw,
|
||||
Some(libc::ELOOP) | Some(libc::ENOTDIR) | Some(libc::ENOENT)
|
||||
),
|
||||
"anchored open must fail closed on parent swap race, got raw_os_error={raw:?}"
|
||||
);
|
||||
assert!(
|
||||
!outside_target.exists(),
|
||||
"anchored open must never create a log file in swapped outside parent"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_nix_path_writes_expected_lines() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-open-ok-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-open-ok base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-open-ok-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut first = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must create log file in allowed parent");
|
||||
append_unknown_dc_line(&mut first, 31_200).expect("first append must succeed");
|
||||
|
||||
let mut second = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored reopen must succeed for existing regular file");
|
||||
append_unknown_dc_line(&mut second, 31_201).expect("second append must succeed");
|
||||
|
||||
let content =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("anchored log file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 2, "expected one line per anchored append call");
|
||||
assert!(
|
||||
lines.contains(&"dc_idx=31200") && lines.contains(&"dc_idx=31201"),
|
||||
"anchored append output must contain both expected dc_idx lines"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_parallel_appends_preserve_line_integrity() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-open-parallel-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-open-parallel base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-open-parallel-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut workers = Vec::new();
|
||||
for idx in 0..64i16 {
|
||||
let sanitized = sanitized.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must succeed in worker");
|
||||
append_unknown_dc_line(&mut file, 32_000 + idx).expect("worker append must succeed");
|
||||
}));
|
||||
}
|
||||
|
||||
for worker in workers {
|
||||
worker.join().expect("worker must not panic");
|
||||
}
|
||||
|
||||
let content =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("parallel log file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lines.len(),
|
||||
64,
|
||||
"expected one complete line per worker append"
|
||||
);
|
||||
for line in lines {
|
||||
assert!(
|
||||
line.starts_with("dc_idx="),
|
||||
"line must keep dc_idx prefix and not be interleaved: {line}"
|
||||
);
|
||||
let value = line
|
||||
.strip_prefix("dc_idx=")
|
||||
.expect("prefix checked above")
|
||||
.parse::<i16>();
|
||||
assert!(
|
||||
value.is_ok(),
|
||||
"line payload must remain parseable i16 and not be corrupted: {line}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_creates_private_0600_file_permissions() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-perms-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-perms base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-perms-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must create file with restricted mode");
|
||||
append_unknown_dc_line(&mut file, 31_210).expect("initial append must succeed");
|
||||
drop(file);
|
||||
|
||||
let mode = fs::metadata(&sanitized.resolved_path)
|
||||
.expect("created log file metadata must be readable")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777;
|
||||
assert_eq!(
|
||||
mode, 0o600,
|
||||
"anchored open must create unknown-dc log file with owner-only rw permissions"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_rejects_existing_symlink_target() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-symlink-target-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-symlink-target base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-symlink-target-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-anchored-symlink-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside\n").expect("outside baseline file must be writable");
|
||||
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
symlink(&outside, &sanitized.resolved_path)
|
||||
.expect("target symlink for anchored-open rejection test must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect_err("anchored open must reject symlinked filename target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"anchored open should fail closed with ELOOP on symlinked target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_high_contention_multi_write_preserves_complete_lines() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-contention-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-contention base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-contention-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let workers = 24usize;
|
||||
let rounds = 40usize;
|
||||
let mut threads = Vec::new();
|
||||
|
||||
for worker in 0..workers {
|
||||
let sanitized = sanitized.clone();
|
||||
threads.push(std::thread::spawn(move || {
|
||||
for round in 0..rounds {
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must succeed under contention");
|
||||
let dc_idx = 20_000i16.wrapping_add((worker * rounds + round) as i16);
|
||||
append_unknown_dc_line(&mut file, dc_idx)
|
||||
.expect("each contention append must complete");
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for thread in threads {
|
||||
thread.join().expect("contention worker must not panic");
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(&sanitized.resolved_path)
|
||||
.expect("contention output file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lines.len(),
|
||||
workers * rounds,
|
||||
"every contention append must produce exactly one line"
|
||||
);
|
||||
|
||||
let mut unique = std::collections::HashSet::new();
|
||||
for line in lines {
|
||||
assert!(
|
||||
line.starts_with("dc_idx="),
|
||||
"line must preserve expected prefix under heavy contention: {line}"
|
||||
);
|
||||
let value = line
|
||||
.strip_prefix("dc_idx=")
|
||||
.expect("prefix validated")
|
||||
.parse::<i16>()
|
||||
.expect("line payload must remain parseable i16 under contention");
|
||||
unique.insert(value);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
unique.len(),
|
||||
workers * rounds,
|
||||
"contention output must not lose or duplicate logical writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn append_unknown_dc_line_returns_error_for_read_only_descriptor() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-append-ro-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("append-ro base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-append-ro-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
fs::write(&sanitized.resolved_path, "seed\n").expect("seed file must be writable");
|
||||
|
||||
let mut readonly = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(&sanitized.resolved_path)
|
||||
.expect("readonly file open must succeed");
|
||||
|
||||
append_unknown_dc_line(&mut readonly, 31_222)
|
||||
.expect_err("append on readonly descriptor must fail closed");
|
||||
|
||||
let content_after =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("seed file must remain readable");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&content_after),
|
||||
1,
|
||||
"failed readonly append must not modify persisted unknown-dc log content"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_absolute_log_path_writes_one_entry() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_001;
|
||||
let file_path = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-abs-{}-{}.log",
|
||||
std::process::id(),
|
||||
dc_idx
|
||||
));
|
||||
let _ = fs::remove_file(&file_path);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(
|
||||
file_path
|
||||
.to_str()
|
||||
.expect("temp file path must be valid UTF-8")
|
||||
.to_string(),
|
||||
);
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..20 {
|
||||
if let Ok(text) = fs::read_to_string(&file_path) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("absolute unknown-DC log path must produce exactly one log write");
|
||||
assert!(
|
||||
text.contains(&format!("dc_idx={dc_idx}")),
|
||||
"absolute unknown-DC integration log must contain requested dc_idx"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_safe_relative_log_path_writes_one_entry() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_002;
|
||||
let rel_dir = format!("target/telemt-unknown-dc-int-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("integration test log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..20 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("safe relative path must produce exactly one log write");
|
||||
assert!(
|
||||
text.contains(&format!("dc_idx={dc_idx}")),
|
||||
"unknown-DC integration log must contain requested dc_idx"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_same_index_burst_writes_only_once() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_010;
|
||||
let rel_dir = format!("target/telemt-unknown-dc-same-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir().unwrap().join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("same-index log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
for _ in 0..64 {
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
}
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..30 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("same-index burst must produce at least one log write");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&text),
|
||||
1,
|
||||
"same unknown dc index must be deduplicated to one file line"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_distinct_burst_is_hard_capped_on_file_writes() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-unknown-dc-cap-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir().unwrap().join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("cap log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
for i in 0..(UNKNOWN_DC_LOG_DISTINCT_LIMIT + 128) {
|
||||
let dc_idx = 20_000i16.wrapping_add(i as i16);
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
}
|
||||
|
||||
let mut final_text = String::new();
|
||||
for _ in 0..80 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
final_text = text;
|
||||
if nonempty_line_count(&final_text) >= UNKNOWN_DC_LOG_DISTINCT_LIMIT {
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let line_count = nonempty_line_count(&final_text);
|
||||
assert!(
|
||||
line_count > 0,
|
||||
"distinct unknown-dc burst must write at least one line"
|
||||
);
|
||||
assert!(
|
||||
line_count <= UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"distinct unknown-dc writes must stay within dedup hard cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_symlinked_target_escape_is_not_written_integration() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-no-write-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("integration symlink base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "baseline\n").expect("outside baseline file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("symlink target must be creatable");
|
||||
|
||||
let rel_file = format!(
|
||||
"target/telemt-unknown-dc-no-write-link-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let dc_idx: i16 = 31_050;
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
let before = fs::read_to_string(&outside).expect("must read baseline outside file");
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
tokio::time::sleep(Duration::from_millis(80)).await;
|
||||
let after = fs::read_to_string(&outside).expect("must read outside file after attempt");
|
||||
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"symlink target escape must not be written by unknown-DC logging"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_open_append_rejects_symlink_target_with_nofollow() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-nofollow-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("nofollow base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-nofollow-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = fs::remove_file(&outside);
|
||||
fs::write(&outside, "outside\n").expect("outside file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("symlink target must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&linked_target)
|
||||
.expect_err("O_NOFOLLOW open must fail for symlink target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"symlink target must be rejected with ELOOP when O_NOFOLLOW is applied"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_open_append_rejects_broken_symlink_target_with_nofollow() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-broken-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("broken-link base must be creatable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(base.join("missing-target.log"), &linked_target)
|
||||
.expect("broken symlink target must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&linked_target)
|
||||
.expect_err("O_NOFOLLOW open must fail for broken symlink target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"broken symlink target must be rejected with ELOOP when O_NOFOLLOW is applied"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_unknown_dc_open_append_symlink_flip_never_writes_outside_file() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-symlink-flip-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("symlink-flip base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-symlink-flip-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside-baseline\n").expect("outside baseline file must be writable");
|
||||
let outside_before = fs::read_to_string(&outside).expect("outside baseline must be readable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
for step in 0..1024usize {
|
||||
let _ = fs::remove_file(&target);
|
||||
if step % 2 == 0 {
|
||||
symlink(&outside, &target).expect("symlink creation in flip loop must succeed");
|
||||
}
|
||||
if let Ok(mut file) = open_unknown_dc_log_append(&target) {
|
||||
writeln!(file, "dc_idx={step}").expect("append on regular file must succeed");
|
||||
}
|
||||
}
|
||||
|
||||
let outside_after = fs::read_to_string(&outside).expect("outside file must remain readable");
|
||||
assert_eq!(
|
||||
outside_after, outside_before,
|
||||
"outside file must never be modified under symlink-flip adversarial churn"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_open_append_creates_regular_file() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-open-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("open test base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
{
|
||||
let mut file = open_unknown_dc_log_append(&target)
|
||||
.expect("regular target must be creatable with append open");
|
||||
writeln!(file, "dc_idx=1234").expect("append write must succeed");
|
||||
}
|
||||
|
||||
let meta = fs::symlink_metadata(&target).expect("created target metadata must be readable");
|
||||
assert!(meta.file_type().is_file(), "target must be a regular file");
|
||||
assert!(
|
||||
!meta.file_type().is_symlink(),
|
||||
"regular target open path must not produce symlink artifacts"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_unknown_dc_open_append_regular_file_preserves_line_integrity() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-open-stress-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("stress open base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
let writes = 2048usize;
|
||||
for idx in 0..writes {
|
||||
let mut file = open_unknown_dc_log_append(&target)
|
||||
.expect("stress append open on regular file must succeed");
|
||||
writeln!(file, "dc_idx={idx}").expect("stress append write must succeed");
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(&target).expect("stress output file must be readable");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&content),
|
||||
writes,
|
||||
"regular-file append stress must preserve one logical line per write"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_accepts_regular_existing_target() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-safe-target-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("safe target base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
fs::write(&target, "seed\n").expect("safe target seed write must succeed");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-safe-target-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized =
|
||||
sanitize_unknown_dc_log_path(&rel_candidate).expect("safe candidate must sanitize");
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must allow safe existing regular files"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_deleted_parent_after_sanitize() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-vanish-parent-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("vanish-parent base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-vanish-parent-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent deletion");
|
||||
|
||||
fs::remove_dir_all(&base).expect("test parent directory must be removable");
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must fail when sanitized parent disappears before write"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let parent = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-parent-swap-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&parent) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&parent).expect("stale parent-swap path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&parent).expect("stale parent-swap directory must be removable");
|
||||
}
|
||||
}
|
||||
let moved = parent.with_extension("bak");
|
||||
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&moved).expect("stale parent-swap backup path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&moved)
|
||||
.expect("stale parent-swap backup directory must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-parent-swap-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent swap");
|
||||
|
||||
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
|
||||
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
|
||||
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must fail when canonical parent is swapped to a symlinked target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_check_then_symlink_flip_is_blocked_by_nofollow_open() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let parent = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-check-open-race-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&parent) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&parent).expect("stale check-open-race path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&parent).expect("stale check-open-race parent must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&parent).expect("check-open-race parent must be creatable");
|
||||
|
||||
let target = parent.join("unknown-dc.log");
|
||||
fs::write(&target, "seed\n").expect("seed target file must be writable");
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-check-open-race-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"precondition: target should initially pass revalidation"
|
||||
);
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-check-open-race-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside\n").expect("outside file must be writable");
|
||||
fs::remove_file(&target).expect("target removal before flip must succeed");
|
||||
symlink(&outside, &target).expect("target symlink flip must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&sanitized.resolved_path)
|
||||
.expect_err("nofollow open must fail after symlink flip between check and open");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"symlink flip in check/open window must be neutralized by O_NOFOLLOW"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_dc_never_panics_with_single_dc_list() {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.default_dc = Some(42);
|
||||
|
||||
let addr = get_dc_addr_static(999, &cfg).expect("fallback dc must resolve safely");
|
||||
let expected = SocketAddr::new(TG_DATACENTERS_V6[0], TG_DATACENTER_PORT);
|
||||
assert_eq!(addr, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_abort_midflight_releases_route_gauge() {
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
let _hold_stream = stream;
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "abort-direct-user".to_string(),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: "127.0.0.1:50000".parse().unwrap(),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let relay_task = tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats.clone(),
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xabad1dea,
|
||||
));
|
||||
|
||||
let started = tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
started.is_ok(),
|
||||
"direct relay must increment route gauge before abort"
|
||||
);
|
||||
|
||||
relay_task.abort();
|
||||
let joined = relay_task.await;
|
||||
assert!(
|
||||
joined.is_err(),
|
||||
"aborted direct relay task must return join error"
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"route gauge must be released when direct relay task is aborted mid-flight"
|
||||
);
|
||||
|
||||
drop(client_side);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_cutover_midflight_releases_route_gauge() {
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
let _hold_stream = stream;
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "cutover-direct-user".to_string(),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: "127.0.0.1:50002".parse().unwrap(),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let relay_task = tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats.clone(),
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xface_cafe,
|
||||
));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("direct relay must increment route gauge before cutover");
|
||||
|
||||
assert!(
|
||||
route_runtime.set_mode(RelayRouteMode::Middle).is_some(),
|
||||
"cutover must advance route generation"
|
||||
);
|
||||
|
||||
let relay_result = tokio::time::timeout(Duration::from_secs(6), relay_task)
|
||||
.await
|
||||
.expect("direct relay must terminate after cutover")
|
||||
.expect("direct relay task must not panic");
|
||||
assert!(
|
||||
relay_result.is_err(),
|
||||
"cutover should terminate direct relay session"
|
||||
);
|
||||
assert!(
|
||||
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||
"client-visible cutover error must stay generic and avoid route-internal metadata"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"route gauge must be released when direct relay exits on cutover"
|
||||
);
|
||||
|
||||
drop(client_side);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_releases_gauge() {
|
||||
let session_count = 6usize;
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let mut held_streams = Vec::with_capacity(session_count);
|
||||
for _ in 0..session_count {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
held_streams.push(stream);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
drop(held_streams);
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(session_count);
|
||||
let mut client_sides = Vec::with_capacity(session_count);
|
||||
|
||||
for idx in 0..session_count {
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
client_sides.push(client_side);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: format!("cutover-storm-direct-user-{idx}"),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: SocketAddr::new(
|
||||
std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)),
|
||||
51000 + idx as u16,
|
||||
),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
relay_tasks.push(tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager.clone(),
|
||||
stats.clone(),
|
||||
config.clone(),
|
||||
buffer_pool.clone(),
|
||||
rng.clone(),
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xA000_0000 + idx as u64,
|
||||
)));
|
||||
}
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(4), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == session_count as u64 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all direct sessions must become active before cutover storm");
|
||||
|
||||
let route_runtime_flipper = route_runtime.clone();
|
||||
let flipper = tokio::spawn(async move {
|
||||
for step in 0..64u32 {
|
||||
let mode = if (step & 1) == 0 {
|
||||
RelayRouteMode::Middle
|
||||
} else {
|
||||
RelayRouteMode::Direct
|
||||
};
|
||||
let _ = route_runtime_flipper.set_mode(mode);
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
});
|
||||
|
||||
for relay_task in relay_tasks {
|
||||
let relay_result = tokio::time::timeout(Duration::from_secs(10), relay_task)
|
||||
.await
|
||||
.expect("direct relay task must finish under cutover storm")
|
||||
.expect("direct relay task must not panic");
|
||||
|
||||
assert!(
|
||||
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||
"storm-cutover termination must remain generic for all direct sessions"
|
||||
);
|
||||
}
|
||||
|
||||
flipper.abort();
|
||||
let _ = flipper.await;
|
||||
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"direct route gauge must return to zero after cutover storm"
|
||||
);
|
||||
|
||||
drop(client_sides);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn prefer_v6_override_matrix_prefers_matching_family_then_degrades_safely() {
|
||||
let dc_idx: i16 = 2;
|
||||
|
||||
let mut cfg_a = ProxyConfig::default();
|
||||
cfg_a.network.prefer = 6;
|
||||
cfg_a.network.ipv6 = Some(true);
|
||||
cfg_a.dc_overrides.insert(
|
||||
dc_idx.to_string(),
|
||||
vec![
|
||||
"203.0.113.90:443".to_string(),
|
||||
"[2001:db8::90]:443".to_string(),
|
||||
],
|
||||
);
|
||||
let a = get_dc_addr_static(dc_idx, &cfg_a).expect("v6+v4 override set must resolve");
|
||||
assert!(
|
||||
a.is_ipv6(),
|
||||
"prefer_v6 should choose v6 override when present"
|
||||
);
|
||||
|
||||
let mut cfg_b = ProxyConfig::default();
|
||||
cfg_b.network.prefer = 6;
|
||||
cfg_b.network.ipv6 = Some(true);
|
||||
cfg_b
|
||||
.dc_overrides
|
||||
.insert(dc_idx.to_string(), vec!["203.0.113.91:443".to_string()]);
|
||||
let b = get_dc_addr_static(dc_idx, &cfg_b).expect("v4-only override must still resolve");
|
||||
assert!(
|
||||
b.is_ipv4(),
|
||||
"when no v6 override exists, v4 override must be used"
|
||||
);
|
||||
|
||||
let mut cfg_c = ProxyConfig::default();
|
||||
cfg_c.network.prefer = 6;
|
||||
cfg_c.network.ipv6 = Some(true);
|
||||
let c = get_dc_addr_static(dc_idx, &cfg_c).expect("table fallback must resolve");
|
||||
assert_eq!(
|
||||
c,
|
||||
SocketAddr::new(TG_DATACENTERS_V6[(dc_idx as usize) - 1], TG_DATACENTER_PORT),
|
||||
"without overrides, prefer_v6 path must resolve from static v6 datacenter table"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefer_v6_override_matrix_ignores_invalid_entries_and_keeps_fail_closed_fallback() {
|
||||
let dc_idx: i16 = 3;
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.dc_overrides.insert(
|
||||
dc_idx.to_string(),
|
||||
vec![
|
||||
"not-an-addr".to_string(),
|
||||
"also:bad".to_string(),
|
||||
"203.0.113.55:443".to_string(),
|
||||
],
|
||||
);
|
||||
|
||||
let addr = get_dc_addr_static(dc_idx, &cfg)
|
||||
.expect("at least one valid override must keep resolution alive");
|
||||
assert_eq!(addr, "203.0.113.55:443".parse::<SocketAddr>().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_prefer_v6_override_matrix_is_deterministic_under_mixed_inputs() {
|
||||
for idx in 1..=5i16 {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.dc_overrides.insert(
|
||||
idx.to_string(),
|
||||
vec![
|
||||
format!("203.0.113.{}:443", 100 + idx),
|
||||
format!("[2001:db8::{}]:443", 100 + idx),
|
||||
],
|
||||
);
|
||||
|
||||
let first = get_dc_addr_static(idx, &cfg).expect("first lookup must resolve");
|
||||
let second = get_dc_addr_static(idx, &cfg).expect("second lookup must resolve");
|
||||
assert_eq!(
|
||||
first, second,
|
||||
"override resolution must stay deterministic for dc {idx}"
|
||||
);
|
||||
assert!(first.is_ipv6(), "dc {idx}: v6 override should be preferred");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn negative_direct_relay_dc_connection_refused_fails_fast() {
|
||||
let (client_reader_side, _client_writer_side) = duplex(1024);
|
||||
let (_client_reader_relay, client_writer_side) = duplex(1024);
|
||||
|
||||
let key = [0u8; 32];
|
||||
let iv = 0u128;
|
||||
let client_reader = CryptoReader::new(client_reader_side, AesCtr::new(&key, iv));
|
||||
let client_writer = CryptoWriter::new(client_writer_side, AesCtr::new(&key, iv), 1024);
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(1024, 1));
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let route_runtime = RouteRuntimeController::new(RelayRouteMode::Direct);
|
||||
|
||||
// Reserve an ephemeral port and immediately release it to deterministically
|
||||
// exercise the direct-connect failure path without long-lived hangs.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let dc_addr = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
|
||||
let mut config_with_override = ProxyConfig::default();
|
||||
config_with_override
|
||||
.dc_overrides
|
||||
.insert("1".to_string(), vec![dc_addr.to_string()]);
|
||||
let config = Arc::new(config_with_override);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
enabled: true,
|
||||
weight: 1,
|
||||
scopes: String::new(),
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
100,
|
||||
5000,
|
||||
10,
|
||||
3,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "test-user".to_string(),
|
||||
peer: "127.0.0.1:12345".parse().unwrap(),
|
||||
dc_idx: 1,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
enc_key: key,
|
||||
enc_iv: iv,
|
||||
dec_key: key,
|
||||
dec_iv: iv,
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let result = timeout(
|
||||
TokioDuration::from_secs(2),
|
||||
handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_runtime.snapshot(),
|
||||
0xABCD_1234,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("direct relay must fail fast on connection-refused upstream");
|
||||
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"connection-refused upstream must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adversarial_direct_relay_cutover_integrity() {
|
||||
let (client_reader_side, _client_writer_side) = duplex(1024);
|
||||
let (_client_reader_relay, client_writer_side) = duplex(1024);
|
||||
|
||||
let key = [0u8; 32];
|
||||
let iv = 0u128;
|
||||
let client_reader = CryptoReader::new(client_reader_side, AesCtr::new(&key, iv));
|
||||
let client_writer = CryptoWriter::new(client_writer_side, AesCtr::new(&key, iv), 1024);
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(1024, 1));
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let route_runtime = RouteRuntimeController::new(RelayRouteMode::Direct);
|
||||
|
||||
// Mock upstream server.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let dc_addr = listener.local_addr().unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
// Read handshake nonce.
|
||||
let mut nonce = [0u8; 64];
|
||||
let _ = stream.read_exact(&mut nonce).await;
|
||||
// Keep connection open.
|
||||
tokio::time::sleep(TokioDuration::from_secs(5)).await;
|
||||
});
|
||||
|
||||
let mut config_with_override = ProxyConfig::default();
|
||||
config_with_override
|
||||
.dc_overrides
|
||||
.insert("1".to_string(), vec![dc_addr.to_string()]);
|
||||
let config = Arc::new(config_with_override);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
enabled: true,
|
||||
weight: 1,
|
||||
scopes: String::new(),
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
100,
|
||||
5000,
|
||||
10,
|
||||
3,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "test-user".to_string(),
|
||||
peer: "127.0.0.1:12345".parse().unwrap(),
|
||||
dc_idx: 1,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
enc_key: key,
|
||||
enc_iv: iv,
|
||||
dec_key: key,
|
||||
dec_iv: iv,
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let stats_for_task = stats.clone();
|
||||
let runtime_clone = route_runtime.clone();
|
||||
let session_task = tokio::spawn(async move {
|
||||
handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats_for_task,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
runtime_clone.subscribe(),
|
||||
runtime_clone.snapshot(),
|
||||
0xABCD_1234,
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
timeout(TokioDuration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("direct relay session must start before cutover");
|
||||
|
||||
// Trigger cutover.
|
||||
route_runtime.set_mode(RelayRouteMode::Middle).unwrap();
|
||||
|
||||
// The session should terminate after the staggered delay (1000-2000ms).
|
||||
let result = timeout(TokioDuration::from_secs(5), session_task)
|
||||
.await
|
||||
.expect("Session must terminate after cutover")
|
||||
.expect("Session must not panic");
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::RouteSwitched)),
|
||||
"Session must terminate with route switch error on cutover"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_is_deduplicated_per_dc_idx() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
assert!(should_log_unknown_dc(777));
|
||||
assert!(
|
||||
!should_log_unknown_dc(777),
|
||||
"same unknown dc_idx must not be logged repeatedly"
|
||||
);
|
||||
assert!(
|
||||
should_log_unknown_dc(778),
|
||||
"different unknown dc_idx must still be loggable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_respects_distinct_limit() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
for dc in 1..=UNKNOWN_DC_LOG_DISTINCT_LIMIT {
|
||||
assert!(
|
||||
should_log_unknown_dc(dc as i16),
|
||||
"expected first-time unknown dc_idx to be loggable"
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
!should_log_unknown_dc(i16::MAX),
|
||||
"distinct unknown dc_idx entries above limit must not be logged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_fails_closed_when_dedup_lock_is_poisoned() {
|
||||
let poisoned = Arc::new(std::sync::Mutex::new(
|
||||
std::collections::HashSet::<i16>::new(),
|
||||
));
|
||||
let poisoned_for_thread = poisoned.clone();
|
||||
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poisoned_for_thread
|
||||
.lock()
|
||||
.expect("poison setup lock must be available");
|
||||
panic!("intentional poison for fail-closed regression");
|
||||
})
|
||||
.join();
|
||||
|
||||
assert!(
|
||||
!should_log_unknown_dc_with_set(poisoned.as_ref(), 4242),
|
||||
"poisoned unknown-DC dedup lock must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_unknown_dc_log_path_does_not_consume_dedup_slot() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_123;
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some("../telemt-unknown-dc-unsafe.log".to_string());
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
assert!(
|
||||
should_log_unknown_dc(dc_idx),
|
||||
"rejected unsafe log path must not consume unknown-dc dedup entry"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_unknown_dc_log_concurrent_unique_churn_respects_cap() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let accepted = Arc::new(AtomicUsize::new(0));
|
||||
let mut workers = Vec::new();
|
||||
|
||||
// Adversarial model: many concurrent peers rotate dc_idx values rapidly.
|
||||
for worker in 0..16usize {
|
||||
let accepted = Arc::clone(&accepted);
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let base = (worker * 2048) as i32;
|
||||
for offset in 0..512i32 {
|
||||
let raw = base + offset;
|
||||
let dc = (raw % i16::MAX as i32) as i16;
|
||||
if should_log_unknown_dc(dc) {
|
||||
accepted.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for worker in workers {
|
||||
worker.join().expect("worker thread must not panic");
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
accepted.load(Ordering::Relaxed),
|
||||
UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"concurrent unique churn must never admit more than the configured distinct cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_unknown_dc_log_mixed_duplicates_never_exceeds_cap() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
// Deterministic xorshift sequence for reproducible mixed duplicate fuzzing.
|
||||
let mut s: u64 = 0xA5A5_5A5A_C3C3_3C3C;
|
||||
let mut admitted = 0usize;
|
||||
|
||||
for _ in 0..20_000 {
|
||||
s ^= s << 7;
|
||||
s ^= s >> 9;
|
||||
s ^= s << 8;
|
||||
|
||||
let dc = (s as i16).wrapping_sub(i16::MAX / 2);
|
||||
if should_log_unknown_dc(dc) {
|
||||
admitted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
admitted <= UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"mixed-duplicate fuzzed inputs must not admit more than cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_hint_accepts_ascii_alnum_and_dash_within_limit() {
|
||||
assert_eq!(validated_scope_hint("scope_alpha-1"), Some("alpha-1"));
|
||||
assert_eq!(validated_scope_hint("scope_AZ09"), Some("AZ09"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_hint_rejects_invalid_or_oversized_values() {
|
||||
assert_eq!(validated_scope_hint("plain_user"), None);
|
||||
assert_eq!(validated_scope_hint("scope_"), None);
|
||||
assert_eq!(validated_scope_hint("scope_a/b"), None);
|
||||
assert_eq!(validated_scope_hint("scope_bad space"), None);
|
||||
assert_eq!(validated_scope_hint("scope_bad.dot"), None);
|
||||
|
||||
let oversized = format!("scope_{}", "a".repeat(MAX_SCOPE_HINT_LEN + 1));
|
||||
assert_eq!(validated_scope_hint(&oversized), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_parent_traversal_inputs() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("../unknown-dc.txt").is_none(),
|
||||
"parent traversal paths must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("logs/../unknown-dc.txt").is_none(),
|
||||
"embedded parent traversal must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("./../unknown-dc.txt").is_none(),
|
||||
"relative parent traversal must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_absolute_paths_with_existing_parent() {
|
||||
let absolute = std::env::temp_dir().join("unknown-dc.txt");
|
||||
let absolute_str = absolute
|
||||
.to_str()
|
||||
.expect("temp absolute path must be valid UTF-8");
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(absolute_str)
|
||||
.expect("absolute paths with existing parent must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, absolute);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_absolute_parent_traversal() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("/tmp/../etc/passwd").is_none(),
|
||||
"absolute parent traversal must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_safe_relative_path() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-log-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("temp test directory must be creatable");
|
||||
|
||||
let candidate = base.join("unknown-dc.txt");
|
||||
let candidate_relative = format!(
|
||||
"target/telemt-unknown-dc-log-{}/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&candidate_relative)
|
||||
.expect("safe relative path with existing parent must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, candidate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_empty_or_dot_only_inputs() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("").is_none(),
|
||||
"empty path must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(".").is_none(),
|
||||
"dot-only path without filename must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_directory_only_as_filename_projection() {
|
||||
let sanitized = sanitize_unknown_dc_log_path("target/")
|
||||
.expect("directory-only input is interpreted as filename projection in current sanitizer");
|
||||
assert!(
|
||||
sanitized.resolved_path.ends_with("target"),
|
||||
"directory-only input should resolve to canonical parent plus filename projection"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_dot_prefixed_relative_path() {
|
||||
let rel_dir = format!("target/telemt-unknown-dc-dot-{}", std::process::id());
|
||||
let abs_dir = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("dot-prefixed test directory must be creatable");
|
||||
|
||||
let rel_candidate = format!("./{rel_dir}/unknown-dc.log");
|
||||
let expected = abs_dir.join("unknown-dc.log");
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("dot-prefixed safe path must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_unknown_dc_path_parentdir_inputs_always_rejected() {
|
||||
let mut s: u64 = 0xD00D_BAAD_1234_5678;
|
||||
for _ in 0..4096 {
|
||||
s ^= s << 7;
|
||||
s ^= s >> 9;
|
||||
s ^= s << 8;
|
||||
let a = (s as usize) % 32;
|
||||
let b = ((s >> 8) as usize) % 32;
|
||||
let candidate = format!("target/{a}/../{b}/unknown-dc.log");
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(&candidate).is_none(),
|
||||
"parent-dir candidate must be rejected: {candidate}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_nonexistent_parent_directory() {
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-missing-{}/nested/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(&rel_candidate).is_none(),
|
||||
"path with missing parent must be rejected to avoid implicit directory creation"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_symlinked_parent_inside_workspace() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-log-symlink-internal-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let real_parent = base.join("real_parent");
|
||||
fs::create_dir_all(&real_parent).expect("real parent dir must be creatable");
|
||||
|
||||
let symlink_parent = base.join("internal_link");
|
||||
let _ = fs::remove_file(&symlink_parent);
|
||||
symlink(&real_parent, &symlink_parent).expect("internal symlink must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-log-symlink-internal-{}/internal_link/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("symlinked parent that resolves inside workspace must be accepted");
|
||||
assert!(
|
||||
sanitized.resolved_path.starts_with(&real_parent),
|
||||
"sanitized path must resolve to canonical internal parent"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_symlink_parent_escape_as_canonical_path() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-log-symlink-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("symlink test directory must be creatable");
|
||||
|
||||
let symlink_parent = base.join("escape_link");
|
||||
let _ = fs::remove_file(&symlink_parent);
|
||||
symlink("/tmp", &symlink_parent).expect("symlink parent must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-log-symlink-{}/escape_link/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("symlinked parent must canonicalize to target path");
|
||||
assert!(
|
||||
sanitized.resolved_path.starts_with(Path::new("/tmp")),
|
||||
"sanitized path must resolve to canonical symlink target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_symlinked_target_escape() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-target-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("target-link base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!("telemt-outside-{}", std::process::id()));
|
||||
let _ = fs::remove_file(&outside);
|
||||
fs::write(&outside, "outside").expect("outside file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("target symlink must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-target-link-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate should sanitize before final revalidation");
|
||||
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"final revalidation must reject symlinked target escape"
|
||||
);
|
||||
}
|
||||
@@ -8,9 +8,7 @@ fn nonempty_line_count(text: &str) -> usize {
|
||||
|
||||
#[test]
|
||||
fn subtle_stress_single_unknown_dc_under_concurrency_logs_once() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let winners = Arc::new(AtomicUsize::new(0));
|
||||
@@ -103,9 +101,7 @@ fn subtle_light_fuzz_dc_resolution_never_panics_and_preserves_port() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn subtle_integration_parallel_same_dc_logs_one_line() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-direct-relay-same-{}", std::process::id());
|
||||
@@ -148,9 +144,7 @@ async fn subtle_integration_parallel_same_dc_logs_one_line() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn subtle_integration_parallel_unique_dcs_log_unique_lines() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-direct-relay-unique-{}", std::process::id());
|
||||
|
||||
@@ -13,795 +13,9 @@ use tokio::time::{Duration, Instant};
|
||||
// Probing Indistinguishability (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_probes_indistinguishable_timing() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 80; // Should timeout/refuse
|
||||
|
||||
let peer: SocketAddr = "192.0.2.10:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
// Test different probe types
|
||||
let probes = vec![
|
||||
(b"GET / HTTP/1.1\r\nHost: x\r\n\r\n".to_vec(), "HTTP"),
|
||||
(b"SSH-2.0-probe".to_vec(), "SSH"),
|
||||
(
|
||||
vec![0x16, 0x03, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x01, 0x00],
|
||||
"TLS-scanner",
|
||||
),
|
||||
(vec![0x42; 5], "port-scanner"),
|
||||
];
|
||||
|
||||
for (probe, type_name) in probes {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// We expect any outcome to take roughly MASK_TIMEOUT (50ms in tests)
|
||||
// to mask whether the backend was reachable or refused.
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Probe {type_name} finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Masking Budget Stress Tests (OWASP ASVS 5.1.6)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_budget_stress_under_load() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let peer: SocketAddr = "192.0.2.20:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..50 {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
let config = config.clone();
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
start.elapsed()
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
let elapsed = task.await.unwrap();
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Stress probe finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// detect_client_type Fingerprint Check
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_detect_client_type_boundary_cases() {
|
||||
// 9 bytes = port-scanner
|
||||
assert_eq!(detect_client_type(&[0x42; 9]), "port-scanner");
|
||||
// 10 bytes = unknown
|
||||
assert_eq!(detect_client_type(&[0x42; 10]), "unknown");
|
||||
|
||||
// HTTP verbs without trailing space
|
||||
assert_eq!(detect_client_type(b"GET/"), "port-scanner"); // because len < 10
|
||||
assert_eq!(detect_client_type(b"GET /path"), "HTTP");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Slowloris and Slow Read Attacks (OWASP ASVS 5.1.5)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_slowloris_client_idle_timeout_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let initial = b"GET / HTTP/1.1\r\nHost: front.example\r\n\r\n".to_vec();
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let initial = initial.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; initial.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(observed, initial);
|
||||
|
||||
let mut drip = [0u8; 1];
|
||||
let drip_read =
|
||||
tokio::time::timeout(Duration::from_millis(220), stream.read_exact(&mut drip))
|
||||
.await;
|
||||
assert!(
|
||||
drip_read.is_err() || drip_read.unwrap().is_err(),
|
||||
"backend must not receive post-timeout slowloris drip bytes"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.10:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(1024);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&initial,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(160)).await;
|
||||
let _ = client_writer.write_all(b"X").await;
|
||||
|
||||
handle.await.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Fallback Server Down / Fingerprinting (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_fallback_down_mimics_timeout() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let (server_reader, server_writer) = duplex(1024);
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.12:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
server_reader,
|
||||
server_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
// It should wait for MASK_TIMEOUT (50ms in tests) even if connection was refused immediately
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(40),
|
||||
"Must respect connect budget even on failure: {:?}",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: SSRF Prevention (OWASP ASVS 5.1.2)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_ssrf_resolve_internal_ranges_blocked() {
|
||||
use crate::network::dns_overrides::DnsOverrides;
|
||||
|
||||
let blocked_ips = [
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"10.0.0.1",
|
||||
"192.168.1.1",
|
||||
"0.0.0.0",
|
||||
];
|
||||
let resolver = DnsOverrides::default();
|
||||
|
||||
for ip in blocked_ips {
|
||||
assert!(
|
||||
resolver.resolve_socket_addr(ip, 80).is_none(),
|
||||
"runtime DNS overrides must not resolve unconfigured literal host targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_unknown_proxy_protocol_version_falls_back_to_v1_unknown_header() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
let mut header = [0u8; 15];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(&header, b"PROXY UNKNOWN\r\n");
|
||||
|
||||
let mut payload = [0u8; 5];
|
||||
stream.read_exact(&mut payload).await.unwrap();
|
||||
assert_eq!(&payload, b"probe");
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 255;
|
||||
|
||||
let peer: SocketAddr = "198.51.100.77:50001".parse().unwrap();
|
||||
let local_addr: SocketAddr = "[2001:db8::10]:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_zero_length_initial_data_does_not_hang_or_panic() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut one = [0u8; 1];
|
||||
let n = tokio::time::timeout(Duration::from_millis(150), stream.read(&mut one))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
n, 0,
|
||||
"backend must observe clean EOF for empty initial payload"
|
||||
);
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.70:50002".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (client_reader, client_writer) = duplex(64);
|
||||
drop(client_writer);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_oversized_initial_payload_is_forwarded_verbatim() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let payload = vec![0xA5u8; 32 * 1024];
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let payload = payload.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; payload.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(
|
||||
observed, payload,
|
||||
"large initial payload must stay byte-for-byte"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.71:50003".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(64);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&payload,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_refused_backend_keeps_constantish_timing_floor_under_burst() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.72:50004".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
for _ in 0..16 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(30),
|
||||
"refused-backend path must keep timing floor to reduce fingerprinting"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_backend_half_close_then_client_half_close_completes_without_hang() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut pre = [0u8; 4];
|
||||
stream.read_exact(&mut pre).await.unwrap();
|
||||
assert_eq!(&pre, b"PING");
|
||||
stream.write_all(b"PONG").await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.73:50005".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(256);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"PING",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
client_writer.shutdown().await.unwrap();
|
||||
|
||||
let mut got = [0u8; 4];
|
||||
client_visible_reader.read_exact(&mut got).await.unwrap();
|
||||
assert_eq!(&got, b"PONG");
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking task must terminate after bilateral half-close")
|
||||
.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_concurrently() {
|
||||
const MASKING_SESSIONS: usize = 48;
|
||||
const RELAY_SESSIONS: usize = 48;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..MASKING_SESSIONS {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"masking backend must receive storm reconnect probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.200:55555".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
let mut masking_tasks = Vec::with_capacity(MASKING_SESSIONS);
|
||||
for i in 0..MASKING_SESSIONS {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{i:04} HTTP/1.1\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking reconnect task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(RELAY_SESSIONS);
|
||||
for i in 0..RELAY_SESSIONS {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-storm-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(i as u8).wrapping_add(1); 64];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(i as u8).wrapping_add(17); 96];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(2), relay_task)
|
||||
.await
|
||||
.expect("relay reconnect task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("masking storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("relay storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(3), backend_task)
|
||||
.await
|
||||
.expect("masking backend accept loop must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn read_env_usize_or_default(name: &str, default: usize) -> usize {
|
||||
match std::env::var(name) {
|
||||
Ok(raw) => match raw.parse::<usize>() {
|
||||
Ok(parsed) if parsed > 0 => parsed,
|
||||
_ => default,
|
||||
},
|
||||
Err(_) => default,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_multiwave_soak() {
|
||||
let waves = read_env_usize_or_default("CHAOS_WAVES", 4);
|
||||
let masking_per_wave = read_env_usize_or_default("CHAOS_MASKING_PER_WAVE", 160);
|
||||
let relay_per_wave = read_env_usize_or_default("CHAOS_RELAY_PER_WAVE", 160);
|
||||
let total_masking = waves * masking_per_wave;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..total_masking {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"mask backend must only receive storm probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.201:56565".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
for wave in 0..waves {
|
||||
let mut masking_tasks = Vec::with_capacity(masking_per_wave);
|
||||
for i in 0..masking_per_wave {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{wave:02}-{i:03}\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(3), handle)
|
||||
.await
|
||||
.expect("masking storm task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(relay_per_wave);
|
||||
for i in 0..relay_per_wave {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-multiwave-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(1); 32];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(17); 48];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(3), relay_task)
|
||||
.await
|
||||
.expect("relay storm task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("masking wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("relay wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(8), backend_task)
|
||||
.await
|
||||
.expect("mask backend must complete all accepted storm sessions")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn masking_timing_bucket_soak_refused_backend_stays_within_narrow_band() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.74:50006".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let mut samples = Vec::with_capacity(128);
|
||||
for _ in 0..128 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
samples.push(started.elapsed().as_millis());
|
||||
}
|
||||
|
||||
samples.sort_unstable();
|
||||
let p10 = samples[samples.len() / 10];
|
||||
let p90 = samples[(samples.len() * 9) / 10];
|
||||
assert!(
|
||||
p90.saturating_sub(p10) <= 40,
|
||||
"timing spread too wide for refused-backend masking path: p10={p10}ms p90={p90}ms"
|
||||
);
|
||||
}
|
||||
// Masking timing, fallback, and relay boundary cases.
|
||||
#[path = "masking_adversarial_tests/boundaries.rs"]
|
||||
mod boundaries;
|
||||
// Concurrent reconnect storms and manual soak cases.
|
||||
#[path = "masking_adversarial_tests/chaos.rs"]
|
||||
mod chaos;
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_probes_indistinguishable_timing() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 80; // Should timeout/refuse
|
||||
|
||||
let peer: SocketAddr = "192.0.2.10:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
// Test different probe types
|
||||
let probes = vec![
|
||||
(b"GET / HTTP/1.1\r\nHost: x\r\n\r\n".to_vec(), "HTTP"),
|
||||
(b"SSH-2.0-probe".to_vec(), "SSH"),
|
||||
(
|
||||
vec![0x16, 0x03, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x01, 0x00],
|
||||
"TLS-scanner",
|
||||
),
|
||||
(vec![0x42; 5], "port-scanner"),
|
||||
];
|
||||
|
||||
for (probe, type_name) in probes {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// We expect any outcome to take roughly MASK_TIMEOUT (50ms in tests)
|
||||
// to mask whether the backend was reachable or refused.
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Probe {type_name} finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Masking Budget Stress Tests (OWASP ASVS 5.1.6)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_budget_stress_under_load() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let peer: SocketAddr = "192.0.2.20:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..50 {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
let config = config.clone();
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
start.elapsed()
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
let elapsed = task.await.unwrap();
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Stress probe finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// detect_client_type Fingerprint Check
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_detect_client_type_boundary_cases() {
|
||||
// 9 bytes = port-scanner
|
||||
assert_eq!(detect_client_type(&[0x42; 9]), "port-scanner");
|
||||
// 10 bytes = unknown
|
||||
assert_eq!(detect_client_type(&[0x42; 10]), "unknown");
|
||||
|
||||
// HTTP verbs without trailing space
|
||||
assert_eq!(detect_client_type(b"GET/"), "port-scanner"); // because len < 10
|
||||
assert_eq!(detect_client_type(b"GET /path"), "HTTP");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Slowloris and Slow Read Attacks (OWASP ASVS 5.1.5)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_slowloris_client_idle_timeout_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let initial = b"GET / HTTP/1.1\r\nHost: front.example\r\n\r\n".to_vec();
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let initial = initial.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; initial.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(observed, initial);
|
||||
|
||||
let mut drip = [0u8; 1];
|
||||
let drip_read =
|
||||
tokio::time::timeout(Duration::from_millis(220), stream.read_exact(&mut drip))
|
||||
.await;
|
||||
assert!(
|
||||
drip_read.is_err() || drip_read.unwrap().is_err(),
|
||||
"backend must not receive post-timeout slowloris drip bytes"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.10:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(1024);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&initial,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(160)).await;
|
||||
let _ = client_writer.write_all(b"X").await;
|
||||
|
||||
handle.await.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Fallback Server Down / Fingerprinting (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_fallback_down_mimics_timeout() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let (server_reader, server_writer) = duplex(1024);
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.12:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
server_reader,
|
||||
server_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
// It should wait for MASK_TIMEOUT (50ms in tests) even if connection was refused immediately
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(40),
|
||||
"Must respect connect budget even on failure: {:?}",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: SSRF Prevention (OWASP ASVS 5.1.2)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_ssrf_resolve_internal_ranges_blocked() {
|
||||
use crate::network::dns_overrides::DnsOverrides;
|
||||
|
||||
let blocked_ips = [
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"10.0.0.1",
|
||||
"192.168.1.1",
|
||||
"0.0.0.0",
|
||||
];
|
||||
let resolver = DnsOverrides::default();
|
||||
|
||||
for ip in blocked_ips {
|
||||
assert!(
|
||||
resolver.resolve_socket_addr(ip, 80).is_none(),
|
||||
"runtime DNS overrides must not resolve unconfigured literal host targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_unknown_proxy_protocol_version_falls_back_to_v1_unknown_header() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
let mut header = [0u8; 15];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(&header, b"PROXY UNKNOWN\r\n");
|
||||
|
||||
let mut payload = [0u8; 5];
|
||||
stream.read_exact(&mut payload).await.unwrap();
|
||||
assert_eq!(&payload, b"probe");
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 255;
|
||||
|
||||
let peer: SocketAddr = "198.51.100.77:50001".parse().unwrap();
|
||||
let local_addr: SocketAddr = "[2001:db8::10]:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_zero_length_initial_data_does_not_hang_or_panic() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut one = [0u8; 1];
|
||||
let n = tokio::time::timeout(Duration::from_millis(150), stream.read(&mut one))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
n, 0,
|
||||
"backend must observe clean EOF for empty initial payload"
|
||||
);
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.70:50002".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (client_reader, client_writer) = duplex(64);
|
||||
drop(client_writer);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_oversized_initial_payload_is_forwarded_verbatim() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let payload = vec![0xA5u8; 32 * 1024];
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let payload = payload.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; payload.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(
|
||||
observed, payload,
|
||||
"large initial payload must stay byte-for-byte"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.71:50003".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(64);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&payload,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_refused_backend_keeps_constantish_timing_floor_under_burst() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.72:50004".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
for _ in 0..16 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(30),
|
||||
"refused-backend path must keep timing floor to reduce fingerprinting"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_backend_half_close_then_client_half_close_completes_without_hang() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut pre = [0u8; 4];
|
||||
stream.read_exact(&mut pre).await.unwrap();
|
||||
assert_eq!(&pre, b"PING");
|
||||
stream.write_all(b"PONG").await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.73:50005".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(256);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"PING",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
client_writer.shutdown().await.unwrap();
|
||||
|
||||
let mut got = [0u8; 4];
|
||||
client_visible_reader.read_exact(&mut got).await.unwrap();
|
||||
assert_eq!(&got, b"PONG");
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking task must terminate after bilateral half-close")
|
||||
.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_concurrently() {
|
||||
const MASKING_SESSIONS: usize = 48;
|
||||
const RELAY_SESSIONS: usize = 48;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..MASKING_SESSIONS {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"masking backend must receive storm reconnect probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.200:55555".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
let mut masking_tasks = Vec::with_capacity(MASKING_SESSIONS);
|
||||
for i in 0..MASKING_SESSIONS {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{i:04} HTTP/1.1\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking reconnect task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(RELAY_SESSIONS);
|
||||
for i in 0..RELAY_SESSIONS {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-storm-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(i as u8).wrapping_add(1); 64];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(i as u8).wrapping_add(17); 96];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(2), relay_task)
|
||||
.await
|
||||
.expect("relay reconnect task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("masking storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("relay storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(3), backend_task)
|
||||
.await
|
||||
.expect("masking backend accept loop must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn read_env_usize_or_default(name: &str, default: usize) -> usize {
|
||||
match std::env::var(name) {
|
||||
Ok(raw) => match raw.parse::<usize>() {
|
||||
Ok(parsed) if parsed > 0 => parsed,
|
||||
_ => default,
|
||||
},
|
||||
Err(_) => default,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_multiwave_soak() {
|
||||
let waves = read_env_usize_or_default("CHAOS_WAVES", 4);
|
||||
let masking_per_wave = read_env_usize_or_default("CHAOS_MASKING_PER_WAVE", 160);
|
||||
let relay_per_wave = read_env_usize_or_default("CHAOS_RELAY_PER_WAVE", 160);
|
||||
let total_masking = waves * masking_per_wave;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..total_masking {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"mask backend must only receive storm probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.201:56565".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
for wave in 0..waves {
|
||||
let mut masking_tasks = Vec::with_capacity(masking_per_wave);
|
||||
for i in 0..masking_per_wave {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{wave:02}-{i:03}\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(3), handle)
|
||||
.await
|
||||
.expect("masking storm task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(relay_per_wave);
|
||||
for i in 0..relay_per_wave {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-multiwave-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(1); 32];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(17); 48];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(3), relay_task)
|
||||
.await
|
||||
.expect("relay storm task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("masking wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("relay wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(8), backend_task)
|
||||
.await
|
||||
.expect("mask backend must complete all accepted storm sessions")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn masking_timing_bucket_soak_refused_backend_stays_within_narrow_band() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.74:50006".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let mut samples = Vec::with_capacity(128);
|
||||
for _ in 0..128 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
samples.push(started.elapsed().as_millis());
|
||||
}
|
||||
|
||||
samples.sort_unstable();
|
||||
let p10 = samples[samples.len() / 10];
|
||||
let p90 = samples[(samples.len() * 9) / 10];
|
||||
assert!(
|
||||
p90.saturating_sub(p10) <= 40,
|
||||
"timing spread too wide for refused-backend masking path: p10={p10}ms p90={p90}ms"
|
||||
);
|
||||
}
|
||||
@@ -86,8 +86,7 @@ async fn connect_failure_refusal_close_behavior_matrix() {
|
||||
let peer: SocketAddr = format!("203.0.113.210:{}", 54100 + idx as u16)
|
||||
.parse()
|
||||
.unwrap();
|
||||
let elapsed =
|
||||
run_connect_failure_case(
|
||||
let elapsed = run_connect_failure_case(
|
||||
"127.0.0.1",
|
||||
unused_port,
|
||||
timing_normalization_enabled,
|
||||
|
||||
@@ -1,19 +1,11 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::*;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
fn interface_cache_test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn adversarial_parallel_cold_miss_performs_single_interface_refresh() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::*;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
fn interface_cache_test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tdd_repeated_local_listener_checks_do_not_repeat_interface_enumeration_within_window() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
@@ -29,9 +21,7 @@ async fn tdd_repeated_local_listener_checks_do_not_repeat_interface_enumeration_
|
||||
|
||||
#[tokio::test]
|
||||
async fn tdd_non_local_port_short_circuit_does_not_enumerate_interfaces() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
|
||||
@@ -95,710 +95,12 @@ fn simulate_tiny_debt_pattern(pattern: &[bool], max_steps: usize) -> (Option<usi
|
||||
(None, debt, reals)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_constants_match_security_budget_expectations() {
|
||||
assert_eq!(TINY_FRAME_DEBT_PER_TINY, 8);
|
||||
assert_eq!(TINY_FRAME_DEBT_LIMIT, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relay_client_idle_state_initial_debt_is_zero() {
|
||||
let state = RelayClientIdleState::new(Instant::now());
|
||||
assert_eq!(state.tiny_frame_debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_client_frame_does_not_reset_tiny_frame_debt() {
|
||||
let now = Instant::now();
|
||||
let mut state = RelayClientIdleState::new(now);
|
||||
state.tiny_frame_debt = 77;
|
||||
state.on_client_frame(now);
|
||||
assert_eq!(state.tiny_frame_debt, 77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_increment_is_saturating() {
|
||||
let mut debt = u32::MAX - 1;
|
||||
debt = debt.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
||||
assert_eq!(debt, u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_decrement_is_saturating() {
|
||||
let mut debt = 0u32;
|
||||
debt = debt.saturating_sub(1);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_tiny_frames_close_exactly_at_threshold() {
|
||||
let max_tiny_without_close = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize;
|
||||
let pattern = vec![true; max_tiny_without_close];
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, Some(max_tiny_without_close));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_less_than_threshold_tiny_frames_do_not_close() {
|
||||
let tiny_count = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize - 1;
|
||||
let pattern = vec![true; tiny_count];
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_one_closes_with_bounded_real_frame_count() {
|
||||
let mut pattern = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
pattern.push(true);
|
||||
pattern.push(false);
|
||||
}
|
||||
let (closed_at, _, reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(closed_at.is_some());
|
||||
assert!(
|
||||
reals <= 80,
|
||||
"expected bounded real frames before close, got {reals}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_eight_is_stable_for_long_runs() {
|
||||
let mut pattern = Vec::with_capacity(9 * 5000);
|
||||
for _ in 0..5000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..8 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt <= TINY_FRAME_DEBT_PER_TINY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_seven_eventually_closes() {
|
||||
let mut pattern = Vec::with_capacity(8 * 2000);
|
||||
for _ in 0..2000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..7 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(
|
||||
closed_at.is_some(),
|
||||
"1:7 tiny-to-real must eventually close"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_tiny_one_real_closes_faster_than_one_to_one() {
|
||||
let mut one_to_one = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
one_to_one.push(true);
|
||||
one_to_one.push(false);
|
||||
}
|
||||
|
||||
let mut two_to_one = Vec::with_capacity(768);
|
||||
for _ in 0..256 {
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(false);
|
||||
}
|
||||
|
||||
let (a_close, _, _) = simulate_tiny_debt_pattern(&one_to_one, one_to_one.len());
|
||||
let (b_close, _, _) = simulate_tiny_debt_pattern(&two_to_one, two_to_one.len());
|
||||
assert!(a_close.is_some() && b_close.is_some());
|
||||
assert!(b_close.unwrap_or(usize::MAX) < a_close.unwrap_or(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn burst_then_drain_can_recover_without_close() {
|
||||
let burst_tiny = ((TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) / 2) as usize;
|
||||
let mut pattern = Vec::with_capacity(burst_tiny + 600);
|
||||
for _ in 0..burst_tiny {
|
||||
pattern.push(true);
|
||||
}
|
||||
pattern.extend(std::iter::repeat_n(false, 600));
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_tiny_frame_debt_model_stays_within_bounds() {
|
||||
let mut seed = 0xA5A5_91C3_2026_0322u64;
|
||||
for _case in 0..128 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let len = 512 + ((seed as usize) & 0x3ff);
|
||||
let mut pattern = Vec::with_capacity(len);
|
||||
let mut local_seed = seed;
|
||||
for _ in 0..len {
|
||||
local_seed ^= local_seed << 7;
|
||||
local_seed ^= local_seed >> 9;
|
||||
local_seed ^= local_seed << 8;
|
||||
pattern.push((local_seed & 1) == 0);
|
||||
}
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
if closed_at.is_none() {
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
assert!(debt <= u32::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_many_independent_simulations_keep_isolated_debt_state() {
|
||||
for idx in 0..2048usize {
|
||||
let mut pattern = Vec::with_capacity(64);
|
||||
for j in 0..64usize {
|
||||
pattern.push(((idx ^ j) & 3) == 0);
|
||||
}
|
||||
let (_closed_at, debt, _reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_intermediate_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(11, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_secure_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(12, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Secure,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intermediate_alternating_zero_and_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(13, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(3000);
|
||||
for idx in 0..160u8 {
|
||||
plaintext.extend_from_slice(&0u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&4u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x11, idx ^ 0x22, idx ^ 0x33]);
|
||||
}
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..220 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some(_)) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(other) => panic!("unexpected error while probing alternating close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(closed, "intermediate alternating attack must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn small_tiny_burst_followed_by_real_frame_does_not_spuriously_close() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(14, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(64);
|
||||
for _ in 0..8 {
|
||||
plaintext.push(0x00);
|
||||
}
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[1, 2, 3, 4]);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
|
||||
let first = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match first {
|
||||
Ok(Some((payload, _))) => assert_eq!(payload.as_ref(), &[1, 2, 3, 4]),
|
||||
Err(e) => panic!("unexpected close after small tiny burst: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before real frame"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(1, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 1024];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer
|
||||
.write_all(&flood_encrypted)
|
||||
.await
|
||||
.expect("zero-length flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"idle policy enabled must fail closed for pure zero-length flood"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_alternating_tiny_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(2, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(256 * 6);
|
||||
for idx in 0..=255u8 {
|
||||
plaintext.push(0x00);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x55, idx ^ 0xAA, 0x11]);
|
||||
}
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("alternating flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let mut saw_proxy_close = false;
|
||||
for _ in 0..300 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some((_payload, _quickack))) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
saw_proxy_close = true;
|
||||
break;
|
||||
}
|
||||
Err(ProxyError::Io(e)) => panic!("unexpected IO error before close: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before debt-based closure"),
|
||||
Err(other) => panic!("unexpected error before close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_proxy_close,
|
||||
"alternating tiny/real sequence must eventually fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_idle_policy_valid_nonzero_frame_still_passes() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(3, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let payload = [7u8, 8, 9, 10];
|
||||
let mut plaintext = Vec::with_capacity(1 + payload.len());
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&payload);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("nonzero frame must be writable");
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
.expect("valid frame should decode")
|
||||
.expect("valid frame should return payload");
|
||||
|
||||
assert_eq!(result.0.as_ref(), &payload);
|
||||
assert!(!result.1);
|
||||
assert_eq!(frame_counter, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_quickack_tiny_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(21, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0x80u8; 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"quickack-marked zero-length flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_extended_zero_len_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(22, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut flood_plaintext = Vec::with_capacity(4 * 256);
|
||||
for _ in 0..256 {
|
||||
flood_plaintext.extend_from_slice(&[0x7f, 0x00, 0x00, 0x00]);
|
||||
}
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"extended zero-length abridged flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_to_eight_abridged_wire_pattern_survives_without_false_positive_close() {
|
||||
let mut plaintext = Vec::with_capacity(9 * 300);
|
||||
for idx in 0..300usize {
|
||||
plaintext.push(0x00);
|
||||
for _ in 0..8 {
|
||||
let b = idx as u8;
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x11, b ^ 0x22, b ^ 0x33]);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the test single-task and deterministic: make duplex capacity larger than the
|
||||
// generated ciphertext so write_all cannot block waiting for a concurrent reader.
|
||||
let duplex_capacity = plaintext.len().saturating_add(1024);
|
||||
let (reader, mut writer) = duplex(duplex_capacity);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(23, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..3000 {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected error in 1:8 wire test: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!closed,
|
||||
"wire-level 1:8 tiny-to-real pattern should not trigger debt close"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deterministic_light_fuzz_abridged_wire_behavior_matches_model() {
|
||||
let mut seed = 0xD1CE_BAAD_2026_0322u64;
|
||||
|
||||
for case_idx in 0..32u64 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let events = 300 + ((seed as usize) & 0xff);
|
||||
let mut pattern = Vec::with_capacity(events);
|
||||
let mut local = seed;
|
||||
for _ in 0..events {
|
||||
local ^= local << 7;
|
||||
local ^= local >> 9;
|
||||
local ^= local << 8;
|
||||
pattern.push((local & 0x03) == 0);
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::with_capacity(events * 6);
|
||||
for (idx, tiny) in pattern.iter().copied().enumerate() {
|
||||
if tiny {
|
||||
plaintext.push(0x00);
|
||||
} else {
|
||||
let b = (idx as u8) ^ (case_idx as u8);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x1F, b ^ 0x7A, b ^ 0xC3]);
|
||||
}
|
||||
}
|
||||
|
||||
let (reader, mut writer) = duplex(16 * 1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(500 + case_idx, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
writer
|
||||
.write_all(&encrypt_for_reader(&plaintext))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let (expected_close, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
let mut observed_close = false;
|
||||
|
||||
for _ in 0..(events + 8) {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
observed_close = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected fuzz error: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
observed_close,
|
||||
expected_close.is_some(),
|
||||
"wire parser behavior must match debt model for case {case_idx}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Pure tiny-frame debt model invariants.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/model.rs"]
|
||||
mod model;
|
||||
// Intermediate and secure transport debt behavior.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/transport.rs"]
|
||||
mod transport;
|
||||
// Abridged framing debt behavior.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/abridged.rs"]
|
||||
mod abridged;
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_quickack_tiny_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(21, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0x80u8; 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"quickack-marked zero-length flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_extended_zero_len_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(22, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut flood_plaintext = Vec::with_capacity(4 * 256);
|
||||
for _ in 0..256 {
|
||||
flood_plaintext.extend_from_slice(&[0x7f, 0x00, 0x00, 0x00]);
|
||||
}
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"extended zero-length abridged flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_to_eight_abridged_wire_pattern_survives_without_false_positive_close() {
|
||||
let mut plaintext = Vec::with_capacity(9 * 300);
|
||||
for idx in 0..300usize {
|
||||
plaintext.push(0x00);
|
||||
for _ in 0..8 {
|
||||
let b = idx as u8;
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x11, b ^ 0x22, b ^ 0x33]);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the test single-task and deterministic: make duplex capacity larger than the
|
||||
// generated ciphertext so write_all cannot block waiting for a concurrent reader.
|
||||
let duplex_capacity = plaintext.len().saturating_add(1024);
|
||||
let (reader, mut writer) = duplex(duplex_capacity);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(23, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..3000 {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected error in 1:8 wire test: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!closed,
|
||||
"wire-level 1:8 tiny-to-real pattern should not trigger debt close"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deterministic_light_fuzz_abridged_wire_behavior_matches_model() {
|
||||
let mut seed = 0xD1CE_BAAD_2026_0322u64;
|
||||
|
||||
for case_idx in 0..32u64 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let events = 300 + ((seed as usize) & 0xff);
|
||||
let mut pattern = Vec::with_capacity(events);
|
||||
let mut local = seed;
|
||||
for _ in 0..events {
|
||||
local ^= local << 7;
|
||||
local ^= local >> 9;
|
||||
local ^= local << 8;
|
||||
pattern.push((local & 0x03) == 0);
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::with_capacity(events * 6);
|
||||
for (idx, tiny) in pattern.iter().copied().enumerate() {
|
||||
if tiny {
|
||||
plaintext.push(0x00);
|
||||
} else {
|
||||
let b = (idx as u8) ^ (case_idx as u8);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x1F, b ^ 0x7A, b ^ 0xC3]);
|
||||
}
|
||||
}
|
||||
|
||||
let (reader, mut writer) = duplex(16 * 1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(500 + case_idx, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
writer
|
||||
.write_all(&encrypt_for_reader(&plaintext))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let (expected_close, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
let mut observed_close = false;
|
||||
|
||||
for _ in 0..(events + 8) {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
observed_close = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected fuzz error: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
observed_close,
|
||||
expected_close.is_some(),
|
||||
"wire parser behavior must match debt model for case {case_idx}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_constants_match_security_budget_expectations() {
|
||||
assert_eq!(TINY_FRAME_DEBT_PER_TINY, 8);
|
||||
assert_eq!(TINY_FRAME_DEBT_LIMIT, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relay_client_idle_state_initial_debt_is_zero() {
|
||||
let state = RelayClientIdleState::new(Instant::now());
|
||||
assert_eq!(state.tiny_frame_debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_client_frame_does_not_reset_tiny_frame_debt() {
|
||||
let now = Instant::now();
|
||||
let mut state = RelayClientIdleState::new(now);
|
||||
state.tiny_frame_debt = 77;
|
||||
state.on_client_frame(now);
|
||||
assert_eq!(state.tiny_frame_debt, 77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_increment_is_saturating() {
|
||||
let mut debt = u32::MAX - 1;
|
||||
debt = debt.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
||||
assert_eq!(debt, u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_decrement_is_saturating() {
|
||||
let mut debt = 0u32;
|
||||
debt = debt.saturating_sub(1);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_tiny_frames_close_exactly_at_threshold() {
|
||||
let max_tiny_without_close = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize;
|
||||
let pattern = vec![true; max_tiny_without_close];
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, Some(max_tiny_without_close));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_less_than_threshold_tiny_frames_do_not_close() {
|
||||
let tiny_count = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize - 1;
|
||||
let pattern = vec![true; tiny_count];
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_one_closes_with_bounded_real_frame_count() {
|
||||
let mut pattern = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
pattern.push(true);
|
||||
pattern.push(false);
|
||||
}
|
||||
let (closed_at, _, reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(closed_at.is_some());
|
||||
assert!(
|
||||
reals <= 80,
|
||||
"expected bounded real frames before close, got {reals}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_eight_is_stable_for_long_runs() {
|
||||
let mut pattern = Vec::with_capacity(9 * 5000);
|
||||
for _ in 0..5000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..8 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt <= TINY_FRAME_DEBT_PER_TINY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_seven_eventually_closes() {
|
||||
let mut pattern = Vec::with_capacity(8 * 2000);
|
||||
for _ in 0..2000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..7 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(
|
||||
closed_at.is_some(),
|
||||
"1:7 tiny-to-real must eventually close"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_tiny_one_real_closes_faster_than_one_to_one() {
|
||||
let mut one_to_one = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
one_to_one.push(true);
|
||||
one_to_one.push(false);
|
||||
}
|
||||
|
||||
let mut two_to_one = Vec::with_capacity(768);
|
||||
for _ in 0..256 {
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(false);
|
||||
}
|
||||
|
||||
let (a_close, _, _) = simulate_tiny_debt_pattern(&one_to_one, one_to_one.len());
|
||||
let (b_close, _, _) = simulate_tiny_debt_pattern(&two_to_one, two_to_one.len());
|
||||
assert!(a_close.is_some() && b_close.is_some());
|
||||
assert!(b_close.unwrap_or(usize::MAX) < a_close.unwrap_or(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn burst_then_drain_can_recover_without_close() {
|
||||
let burst_tiny = ((TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) / 2) as usize;
|
||||
let mut pattern = Vec::with_capacity(burst_tiny + 600);
|
||||
for _ in 0..burst_tiny {
|
||||
pattern.push(true);
|
||||
}
|
||||
pattern.extend(std::iter::repeat_n(false, 600));
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_tiny_frame_debt_model_stays_within_bounds() {
|
||||
let mut seed = 0xA5A5_91C3_2026_0322u64;
|
||||
for _case in 0..128 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let len = 512 + ((seed as usize) & 0x3ff);
|
||||
let mut pattern = Vec::with_capacity(len);
|
||||
let mut local_seed = seed;
|
||||
for _ in 0..len {
|
||||
local_seed ^= local_seed << 7;
|
||||
local_seed ^= local_seed >> 9;
|
||||
local_seed ^= local_seed << 8;
|
||||
pattern.push((local_seed & 1) == 0);
|
||||
}
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
if closed_at.is_none() {
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_many_independent_simulations_keep_isolated_debt_state() {
|
||||
for idx in 0..2048usize {
|
||||
let mut pattern = Vec::with_capacity(64);
|
||||
for j in 0..64usize {
|
||||
pattern.push(((idx ^ j) & 3) == 0);
|
||||
}
|
||||
let (_closed_at, debt, _reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_intermediate_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(11, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_secure_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(12, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Secure,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intermediate_alternating_zero_and_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(13, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(3000);
|
||||
for idx in 0..160u8 {
|
||||
plaintext.extend_from_slice(&0u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&4u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x11, idx ^ 0x22, idx ^ 0x33]);
|
||||
}
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..220 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some(_)) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(other) => panic!("unexpected error while probing alternating close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(closed, "intermediate alternating attack must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn small_tiny_burst_followed_by_real_frame_does_not_spuriously_close() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(14, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(64);
|
||||
for _ in 0..8 {
|
||||
plaintext.push(0x00);
|
||||
}
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[1, 2, 3, 4]);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
|
||||
let first = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match first {
|
||||
Ok(Some((payload, _))) => assert_eq!(payload.as_ref(), &[1, 2, 3, 4]),
|
||||
Err(e) => panic!("unexpected close after small tiny burst: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before real frame"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(1, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 1024];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer
|
||||
.write_all(&flood_encrypted)
|
||||
.await
|
||||
.expect("zero-length flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"idle policy enabled must fail closed for pure zero-length flood"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_alternating_tiny_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(2, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(256 * 6);
|
||||
for idx in 0..=255u8 {
|
||||
plaintext.push(0x00);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x55, idx ^ 0xAA, 0x11]);
|
||||
}
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("alternating flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let mut saw_proxy_close = false;
|
||||
for _ in 0..300 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some((_payload, _quickack))) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
saw_proxy_close = true;
|
||||
break;
|
||||
}
|
||||
Err(ProxyError::Io(e)) => panic!("unexpected IO error before close: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before debt-based closure"),
|
||||
Err(other) => panic!("unexpected error before close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_proxy_close,
|
||||
"alternating tiny/real sequence must eventually fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_idle_policy_valid_nonzero_frame_still_passes() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(3, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let payload = [7u8, 8, 9, 10];
|
||||
let mut plaintext = Vec::with_capacity(1 + payload.len());
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&payload);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("nonzero frame must be writable");
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
.expect("valid frame should decode")
|
||||
.expect("valid frame should return payload");
|
||||
|
||||
assert_eq!(result.0.as_ref(), &payload);
|
||||
assert!(!result.1);
|
||||
assert_eq!(frame_counter, 1);
|
||||
}
|
||||
+20
-849
@@ -1,17 +1,33 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use dashmap::DashMap;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::{CidrRateLimitKey, RateLimitBps};
|
||||
use crate::config::RateLimitBps;
|
||||
|
||||
// Atomic per-user and per-CIDR accounting.
|
||||
mod buckets;
|
||||
// Immutable policy matching and sharded registries.
|
||||
mod policy;
|
||||
// Traffic lease accounting and cleanup.
|
||||
mod lease;
|
||||
// Runtime policy application and admission.
|
||||
mod limiter;
|
||||
// Epoch and arithmetic helpers.
|
||||
mod helpers;
|
||||
|
||||
pub use helpers::next_refill_delay;
|
||||
use helpers::{
|
||||
auto_cidr_bucket_key, bytes_per_epoch, current_epoch, decrement_atomic_saturating,
|
||||
now_epoch_secs,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
const REGISTRY_SHARDS: usize = 64;
|
||||
const FAIR_EPOCH_MS: u64 = 20;
|
||||
const MAX_BORROW_CHUNK_BYTES: u64 = 32 * 1024;
|
||||
@@ -56,114 +72,18 @@ struct ScopeMetrics {
|
||||
policy_entries: AtomicU64,
|
||||
}
|
||||
|
||||
impl ScopeMetrics {
|
||||
fn throttle(&self, direction: RateDirection) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.throttle_up_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.throttle_down_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_ms(&self, direction: RateDirection, wait_ms: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.wait_up_ms_total.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.wait_down_ms_total
|
||||
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AtomicRatePair {
|
||||
up_bps: AtomicU64,
|
||||
down_bps: AtomicU64,
|
||||
}
|
||||
|
||||
impl AtomicRatePair {
|
||||
fn set(&self, limits: RateLimitBps) {
|
||||
self.up_bps.store(limits.up_bps, Ordering::Relaxed);
|
||||
self.down_bps.store(limits.down_bps, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn get(&self, direction: RateDirection) -> u64 {
|
||||
match direction {
|
||||
RateDirection::Up => self.up_bps.load(Ordering::Relaxed),
|
||||
RateDirection::Down => self.down_bps.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DirectionBucket {
|
||||
epoch: AtomicU64,
|
||||
used: AtomicU64,
|
||||
}
|
||||
|
||||
impl DirectionBucket {
|
||||
fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn try_consume(&self, cap_bps: u64, requested: u64) -> u64 {
|
||||
if requested == 0 {
|
||||
return 0;
|
||||
}
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
|
||||
loop {
|
||||
let used = self.used.load(Ordering::Relaxed);
|
||||
if used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let remaining = cap_epoch.saturating_sub(used);
|
||||
let grant = requested.min(remaining);
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
let next = used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(used, next, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
struct UserBucket {
|
||||
rates: AtomicRatePair,
|
||||
up: DirectionBucket,
|
||||
@@ -171,38 +91,6 @@ struct UserBucket {
|
||||
active_leases: AtomicU64,
|
||||
}
|
||||
|
||||
impl UserBucket {
|
||||
fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: DirectionBucket::default(),
|
||||
down: DirectionBucket::default(),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
fn try_consume(&self, direction: RateDirection, requested: u64) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(cap_bps, requested),
|
||||
RateDirection::Down => self.down.try_consume(cap_bps, requested),
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => self.up.refund(bytes),
|
||||
RateDirection::Down => self.down.refund(bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CidrDirectionBucket {
|
||||
epoch: AtomicU64,
|
||||
@@ -210,125 +98,18 @@ struct CidrDirectionBucket {
|
||||
active_users: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrDirectionBucket {
|
||||
fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
self.active_users.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn try_consume(
|
||||
&self,
|
||||
user_state: &CidrUserDirectionState,
|
||||
cap_epoch: u64,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
if requested == 0 || cap_epoch == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
user_state.sync_epoch_and_mark_active(epoch, &self.active_users);
|
||||
let active_users = self.active_users.load(Ordering::Relaxed).max(1);
|
||||
let fair_share = cap_epoch.saturating_div(active_users).max(1);
|
||||
|
||||
loop {
|
||||
let total_used = self.used.load(Ordering::Relaxed);
|
||||
if total_used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let total_remaining = cap_epoch.saturating_sub(total_used);
|
||||
let user_used = user_state.used.load(Ordering::Relaxed);
|
||||
let guaranteed_remaining = fair_share.saturating_sub(user_used);
|
||||
|
||||
let grant = if guaranteed_remaining > 0 {
|
||||
requested.min(guaranteed_remaining).min(total_remaining)
|
||||
} else {
|
||||
requested.min(total_remaining).min(MAX_BORROW_CHUNK_BYTES)
|
||||
};
|
||||
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let next_total = total_used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(total_used, next_total, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
user_state.used.fetch_add(grant, Ordering::Relaxed);
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CidrUserDirectionState {
|
||||
epoch: AtomicU64,
|
||||
used: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrUserDirectionState {
|
||||
fn sync_epoch_and_mark_active(&self, epoch: u64, active_users: &AtomicU64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
active_users.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
struct CidrUserShare {
|
||||
active_conns: AtomicU64,
|
||||
up: CidrUserDirectionState,
|
||||
down: CidrUserDirectionState,
|
||||
}
|
||||
|
||||
impl CidrUserShare {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
active_conns: AtomicU64::new(0),
|
||||
up: CidrUserDirectionState::default(),
|
||||
down: CidrUserDirectionState::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CidrBucket {
|
||||
rates: AtomicRatePair,
|
||||
up: CidrDirectionBucket,
|
||||
@@ -337,75 +118,6 @@ struct CidrBucket {
|
||||
active_leases: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrBucket {
|
||||
fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: CidrDirectionBucket::default(),
|
||||
down: CidrDirectionBucket::default(),
|
||||
users: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
fn acquire_user_share(&self, user: &str) -> Arc<CidrUserShare> {
|
||||
self.users
|
||||
.get_or_insert_with(user, CidrUserShare::new, |share| {
|
||||
share.active_conns.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
fn release_user_share(&self, user: &str, share: &Arc<CidrUserShare>) {
|
||||
decrement_atomic_saturating(&share.active_conns, 1);
|
||||
let share_for_remove = Arc::clone(share);
|
||||
let _ = self.users.remove_if(user, |candidate| {
|
||||
Arc::ptr_eq(candidate, &share_for_remove)
|
||||
&& candidate.active_conns.load(Ordering::Relaxed) == 0
|
||||
});
|
||||
}
|
||||
|
||||
fn try_consume_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(&share.up, cap_epoch, requested),
|
||||
RateDirection::Down => self.down.try_consume(&share.down, cap_epoch, requested),
|
||||
}
|
||||
}
|
||||
|
||||
fn refund_for_user(&self, direction: RateDirection, share: &CidrUserShare, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.up.refund(bytes);
|
||||
share.up.refund(bytes);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.down.refund(bytes);
|
||||
share.down.refund(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_idle_users(&self) {
|
||||
self.users
|
||||
.retain(|_, share| share.active_conns.load(Ordering::Relaxed) > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CidrRule {
|
||||
key: String,
|
||||
@@ -435,97 +147,11 @@ struct PolicySnapshot {
|
||||
cidr_rule_keys: HashSet<String>,
|
||||
}
|
||||
|
||||
impl PolicySnapshot {
|
||||
fn match_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
match ip {
|
||||
IpAddr::V4(_) => self
|
||||
.cidr_rules_v4
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
IpAddr::V6(_) => self
|
||||
.cidr_rules_v6
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
}
|
||||
.map(CidrPolicyMatch::Explicit)
|
||||
.or_else(|| self.match_auto_cidr(ip))
|
||||
}
|
||||
|
||||
fn match_auto_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
let rule = match ip {
|
||||
IpAddr::V4(_) => self.cidr_auto_rules_v4.first()?,
|
||||
IpAddr::V6(_) => self.cidr_auto_rules_v6.first()?,
|
||||
};
|
||||
let key = auto_cidr_bucket_key(ip, rule.prefix_len)?;
|
||||
Some(CidrPolicyMatch::Auto {
|
||||
key,
|
||||
limits: rule.limits,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ShardedRegistry<T> {
|
||||
shards: Box<[DashMap<String, Arc<T>>]>,
|
||||
mask: usize,
|
||||
}
|
||||
|
||||
impl<T> ShardedRegistry<T> {
|
||||
fn new(shards: usize) -> Self {
|
||||
let shard_count = shards.max(1).next_power_of_two();
|
||||
let mut items = Vec::with_capacity(shard_count);
|
||||
for _ in 0..shard_count {
|
||||
items.push(DashMap::<String, Arc<T>>::new());
|
||||
}
|
||||
Self {
|
||||
shards: items.into_boxed_slice(),
|
||||
mask: shard_count.saturating_sub(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn shard_index(&self, key: &str) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
key.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.mask
|
||||
}
|
||||
|
||||
fn get_or_insert_with<F, A>(&self, key: &str, make: F, activate: A) -> Arc<T>
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
A: FnOnce(&Arc<T>),
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
match shard.entry(key.to_string()) {
|
||||
dashmap::mapref::entry::Entry::Occupied(entry) => {
|
||||
activate(entry.get());
|
||||
Arc::clone(entry.get())
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Vacant(slot) => {
|
||||
let value = Arc::new(make());
|
||||
activate(&value);
|
||||
slot.insert(Arc::clone(&value));
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn retain<F>(&self, predicate: F)
|
||||
where
|
||||
F: Fn(&String, &Arc<T>) -> bool + Copy,
|
||||
{
|
||||
for shard in &*self.shards {
|
||||
shard.retain(|key, value| predicate(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_if<F>(&self, key: &str, predicate: F) -> bool
|
||||
where
|
||||
F: Fn(&Arc<T>) -> bool,
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
shard.remove_if(key, |_, value| predicate(value)).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TrafficLease {
|
||||
limiter: Arc<TrafficLimiter>,
|
||||
user_bucket: Option<Arc<UserBucket>>,
|
||||
@@ -534,107 +160,6 @@ pub struct TrafficLease {
|
||||
cidr_user_share: Option<Arc<CidrUserShare>>,
|
||||
}
|
||||
|
||||
impl TrafficLease {
|
||||
pub fn try_consume(&self, direction: RateDirection, requested: u64) -> TrafficConsumeResult {
|
||||
if requested == 0 {
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
|
||||
let mut granted = requested;
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
let user_granted = user_bucket.try_consume(direction, granted);
|
||||
if user_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, true, false);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: true,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
granted = user_granted;
|
||||
}
|
||||
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
let cidr_granted =
|
||||
cidr_bucket.try_consume_for_user(direction, cidr_user_share, granted);
|
||||
if cidr_granted < granted
|
||||
&& let Some(user_bucket) = self.user_bucket.as_ref()
|
||||
{
|
||||
user_bucket.refund(direction, granted.saturating_sub(cidr_granted));
|
||||
}
|
||||
if cidr_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, false, true);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: true,
|
||||
};
|
||||
}
|
||||
granted = cidr_granted;
|
||||
}
|
||||
|
||||
TrafficConsumeResult {
|
||||
granted,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
user_bucket.refund(direction, bytes);
|
||||
}
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
cidr_bucket.refund_for_user(direction, cidr_user_share, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe_wait_ms(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if wait_ms == 0 {
|
||||
return;
|
||||
}
|
||||
self.limiter
|
||||
.observe_wait(direction, blocked_user, blocked_cidr, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TrafficLease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(bucket) = self.user_bucket.as_ref() {
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.user_scope.active_leases, 1);
|
||||
}
|
||||
|
||||
if let Some(bucket) = self.cidr_bucket.as_ref() {
|
||||
if let (Some(user_key), Some(share)) =
|
||||
(self.cidr_user_key.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
bucket.release_user_share(user_key, share);
|
||||
}
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.cidr_scope.active_leases, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TrafficLimiter {
|
||||
policy: ArcSwap<PolicySnapshot>,
|
||||
user_buckets: ShardedRegistry<UserBucket>,
|
||||
@@ -643,357 +168,3 @@ pub struct TrafficLimiter {
|
||||
cidr_scope: ScopeMetrics,
|
||||
last_cleanup_epoch_secs: AtomicU64,
|
||||
}
|
||||
|
||||
impl TrafficLimiter {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
policy: ArcSwap::from_pointee(PolicySnapshot::default()),
|
||||
user_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
cidr_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
user_scope: ScopeMetrics::default(),
|
||||
cidr_scope: ScopeMetrics::default(),
|
||||
last_cleanup_epoch_secs: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_policy(
|
||||
&self,
|
||||
user_limits: HashMap<String, RateLimitBps>,
|
||||
cidr_limits: HashMap<CidrRateLimitKey, RateLimitBps>,
|
||||
) {
|
||||
let filtered_users = user_limits
|
||||
.into_iter()
|
||||
.filter(|(_, limit)| limit.up_bps > 0 || limit.down_bps > 0)
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut cidr_rules_v4 = Vec::new();
|
||||
let mut cidr_rules_v6 = Vec::new();
|
||||
let mut cidr_auto_rules_v4 = Vec::new();
|
||||
let mut cidr_auto_rules_v6 = Vec::new();
|
||||
let mut cidr_rule_keys = HashSet::new();
|
||||
for (key, limits) in cidr_limits {
|
||||
if limits.up_bps == 0 && limits.down_bps == 0 {
|
||||
continue;
|
||||
}
|
||||
match key {
|
||||
CidrRateLimitKey::Network(cidr) => {
|
||||
let key = cidr.to_string();
|
||||
let rule = CidrRule {
|
||||
key: key.clone(),
|
||||
cidr,
|
||||
limits,
|
||||
prefix_len: cidr.prefix(),
|
||||
};
|
||||
cidr_rule_keys.insert(key);
|
||||
match rule.cidr {
|
||||
IpNetwork::V4(_) => cidr_rules_v4.push(rule),
|
||||
IpNetwork::V6(_) => cidr_rules_v6.push(rule),
|
||||
}
|
||||
}
|
||||
CidrRateLimitKey::AutoV4(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoV6(prefix_len) => {
|
||||
cidr_auto_rules_v6.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoDual(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
cidr_auto_rules_v6.push(CidrAutoRule {
|
||||
prefix_len: prefix_len.saturating_mul(4),
|
||||
limits,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cidr_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
let cidr_policy_entries =
|
||||
cidr_rule_keys.len() + cidr_auto_rules_v4.len() + cidr_auto_rules_v6.len();
|
||||
|
||||
self.user_scope
|
||||
.policy_entries
|
||||
.store(filtered_users.len() as u64, Ordering::Relaxed);
|
||||
self.cidr_scope
|
||||
.policy_entries
|
||||
.store(cidr_policy_entries as u64, Ordering::Relaxed);
|
||||
|
||||
self.policy.store(Arc::new(PolicySnapshot {
|
||||
user_limits: filtered_users,
|
||||
cidr_rules_v4,
|
||||
cidr_rules_v6,
|
||||
cidr_auto_rules_v4,
|
||||
cidr_auto_rules_v6,
|
||||
cidr_rule_keys,
|
||||
}));
|
||||
|
||||
self.maybe_cleanup();
|
||||
}
|
||||
|
||||
pub fn acquire_lease(
|
||||
self: &Arc<Self>,
|
||||
user: &str,
|
||||
client_ip: IpAddr,
|
||||
) -> Option<Arc<TrafficLease>> {
|
||||
let policy = self.policy.load_full();
|
||||
let mut user_bucket = None;
|
||||
if let Some(limit) = policy.user_limits.get(user).copied() {
|
||||
let bucket = self
|
||||
.user_buckets
|
||||
.get_or_insert_with(user, || UserBucket::new(limit), |bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
bucket.set_rates(limit);
|
||||
self.user_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
user_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
let mut cidr_bucket = None;
|
||||
let mut cidr_user_key = None;
|
||||
let mut cidr_user_share = None;
|
||||
if let Some(rule_match) = policy.match_cidr(client_ip) {
|
||||
let (key, limits) = match &rule_match {
|
||||
CidrPolicyMatch::Explicit(rule) => (rule.key.as_str(), rule.limits),
|
||||
CidrPolicyMatch::Auto { key, limits } => (key.as_str(), *limits),
|
||||
};
|
||||
let bucket = self
|
||||
.cidr_buckets
|
||||
.get_or_insert_with(key, || CidrBucket::new(limits), |bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
bucket.set_rates(limits);
|
||||
self.cidr_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
let share = bucket.acquire_user_share(user);
|
||||
cidr_user_key = Some(user.to_string());
|
||||
cidr_user_share = Some(share);
|
||||
cidr_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
if user_bucket.is_none() && cidr_bucket.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.maybe_cleanup();
|
||||
Some(Arc::new(TrafficLease {
|
||||
limiter: Arc::clone(self),
|
||||
user_bucket,
|
||||
cidr_bucket,
|
||||
cidr_user_key,
|
||||
cidr_user_share,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn metrics_snapshot(&self) -> TrafficLimiterMetricsSnapshot {
|
||||
TrafficLimiterMetricsSnapshot {
|
||||
user_throttle_up_total: self.user_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
user_throttle_down_total: self.user_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_up_total: self.cidr_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_down_total: self.cidr_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
user_wait_up_ms_total: self.user_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
user_wait_down_ms_total: self.user_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_up_ms_total: self.cidr_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_down_ms_total: self.cidr_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
user_active_leases: self.user_scope.active_leases.load(Ordering::Relaxed),
|
||||
cidr_active_leases: self.cidr_scope.active_leases.load(Ordering::Relaxed),
|
||||
user_policy_entries: self.user_scope.policy_entries.load(Ordering::Relaxed),
|
||||
cidr_policy_entries: self.cidr_scope.policy_entries.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_throttle(&self, direction: RateDirection, blocked_user: bool, blocked_cidr: bool) {
|
||||
if blocked_user {
|
||||
self.user_scope.throttle(direction);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.throttle(direction);
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_wait(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_cleanup(&self) {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed);
|
||||
if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS {
|
||||
return;
|
||||
}
|
||||
if self
|
||||
.last_cleanup_epoch_secs
|
||||
.compare_exchange(last, now_epoch_secs, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let policy = self.policy.load_full();
|
||||
self.user_buckets.retain(|user, bucket| {
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.user_limits.contains_key(user)
|
||||
});
|
||||
self.cidr_buckets.retain(|cidr_key, bucket| {
|
||||
bucket.cleanup_idle_users();
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.cidr_rule_keys.contains(cidr_key)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_refill_delay() -> Duration {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
let epoch_pos = elapsed_ms % FAIR_EPOCH_MS;
|
||||
let wait_ms = FAIR_EPOCH_MS.saturating_sub(epoch_pos).max(1);
|
||||
Duration::from_millis(wait_ms)
|
||||
}
|
||||
|
||||
fn decrement_atomic_saturating(counter: &AtomicU64, by: u64) {
|
||||
if by == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current == 0 {
|
||||
return;
|
||||
}
|
||||
let next = current.saturating_sub(by);
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn bytes_per_epoch(bps: u64) -> u64 {
|
||||
if bps == 0 {
|
||||
return 0;
|
||||
}
|
||||
let numerator = bps.saturating_mul(FAIR_EPOCH_MS);
|
||||
let bytes = numerator.saturating_div(8_000);
|
||||
bytes.max(1)
|
||||
}
|
||||
|
||||
fn auto_cidr_bucket_key(ip: IpAddr, prefix_len: u8) -> Option<String> {
|
||||
let cidr = IpNetwork::new(ip, prefix_len).ok()?;
|
||||
let network = IpNetwork::new(cidr.network(), prefix_len).ok()?;
|
||||
let family = match network {
|
||||
IpNetwork::V4(_) => "4",
|
||||
IpNetwork::V6(_) => "6",
|
||||
};
|
||||
Some(format!("auto:{family}:{network}"))
|
||||
}
|
||||
|
||||
fn current_epoch() -> u64 {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
elapsed_ms / FAIR_EPOCH_MS
|
||||
}
|
||||
|
||||
fn limiter_epoch_start() -> &'static Instant {
|
||||
static START: OnceLock<Instant> = OnceLock::new();
|
||||
START.get_or_init(Instant::now)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rate(up_bps: u64, down_bps: u64) -> RateLimitBps {
|
||||
RateLimitBps { up_bps, down_bps }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_cidr_rule_wins_over_auto_template() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(
|
||||
CidrRateLimitKey::Network("203.0.113.7/32".parse().unwrap()),
|
||||
rate(2_000, 0),
|
||||
);
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.7".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Explicit(rule) => assert_eq!(rule.key.as_str(), "203.0.113.7/32"),
|
||||
CidrPolicyMatch::Auto { .. } => panic!("explicit CIDR must have priority"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_template_uses_longest_prefix() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(32), rate(2_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.129".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, limits } => {
|
||||
assert_eq!(key, "auto:4:203.0.113.129/32");
|
||||
assert_eq!(limits.up_bps, 2_000);
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dual_auto_template_maps_v6_prefix_by_four() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoDual(32), rate(1_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("2001:db8::1".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, .. } => {
|
||||
assert_eq!(key, "auto:6:2001:db8::1/128");
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_cidr_bucket_key_canonicalizes_network_address() {
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("203.0.113.129".parse().unwrap(), 24).unwrap(),
|
||||
"auto:4:203.0.113.0/24"
|
||||
);
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("2001:db8::abcd".parse().unwrap(), 64).unwrap(),
|
||||
"auto:6:2001:db8::/64"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
use super::*;
|
||||
|
||||
impl ScopeMetrics {
|
||||
pub(super) fn throttle(&self, direction: RateDirection) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.throttle_up_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.throttle_down_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn wait_ms(&self, direction: RateDirection, wait_ms: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.wait_up_ms_total.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.wait_down_ms_total
|
||||
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AtomicRatePair {
|
||||
pub(super) fn set(&self, limits: RateLimitBps) {
|
||||
self.up_bps.store(limits.up_bps, Ordering::Relaxed);
|
||||
self.down_bps.store(limits.down_bps, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(super) fn get(&self, direction: RateDirection) -> u64 {
|
||||
match direction {
|
||||
RateDirection::Up => self.up_bps.load(Ordering::Relaxed),
|
||||
RateDirection::Down => self.down_bps.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DirectionBucket {
|
||||
pub(super) fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(&self, cap_bps: u64, requested: u64) -> u64 {
|
||||
if requested == 0 {
|
||||
return 0;
|
||||
}
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
|
||||
loop {
|
||||
let used = self.used.load(Ordering::Relaxed);
|
||||
if used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let remaining = cap_epoch.saturating_sub(used);
|
||||
let grant = requested.min(remaining);
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
let next = used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(used, next, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl UserBucket {
|
||||
pub(super) fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: DirectionBucket::default(),
|
||||
down: DirectionBucket::default(),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(&self, direction: RateDirection, requested: u64) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(cap_bps, requested),
|
||||
RateDirection::Down => self.down.try_consume(cap_bps, requested),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => self.up.refund(bytes),
|
||||
RateDirection::Down => self.down.refund(bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrDirectionBucket {
|
||||
pub(super) fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
self.active_users.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(
|
||||
&self,
|
||||
user_state: &CidrUserDirectionState,
|
||||
cap_epoch: u64,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
if requested == 0 || cap_epoch == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
user_state.sync_epoch_and_mark_active(epoch, &self.active_users);
|
||||
let active_users = self.active_users.load(Ordering::Relaxed).max(1);
|
||||
let fair_share = cap_epoch.saturating_div(active_users).max(1);
|
||||
|
||||
loop {
|
||||
let total_used = self.used.load(Ordering::Relaxed);
|
||||
if total_used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let total_remaining = cap_epoch.saturating_sub(total_used);
|
||||
let user_used = user_state.used.load(Ordering::Relaxed);
|
||||
let guaranteed_remaining = fair_share.saturating_sub(user_used);
|
||||
|
||||
let grant = if guaranteed_remaining > 0 {
|
||||
requested.min(guaranteed_remaining).min(total_remaining)
|
||||
} else {
|
||||
requested.min(total_remaining).min(MAX_BORROW_CHUNK_BYTES)
|
||||
};
|
||||
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let next_total = total_used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(total_used, next_total, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
user_state.used.fetch_add(grant, Ordering::Relaxed);
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrUserDirectionState {
|
||||
pub(super) fn sync_epoch_and_mark_active(&self, epoch: u64, active_users: &AtomicU64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
active_users.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrUserShare {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
active_conns: AtomicU64::new(0),
|
||||
up: CidrUserDirectionState::default(),
|
||||
down: CidrUserDirectionState::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrBucket {
|
||||
pub(super) fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: CidrDirectionBucket::default(),
|
||||
down: CidrDirectionBucket::default(),
|
||||
users: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
pub(super) fn acquire_user_share(&self, user: &str) -> Arc<CidrUserShare> {
|
||||
self.users
|
||||
.get_or_insert_with(user, CidrUserShare::new, |share| {
|
||||
share.active_conns.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn release_user_share(&self, user: &str, share: &Arc<CidrUserShare>) {
|
||||
decrement_atomic_saturating(&share.active_conns, 1);
|
||||
let share_for_remove = Arc::clone(share);
|
||||
let _ = self.users.remove_if(user, |candidate| {
|
||||
Arc::ptr_eq(candidate, &share_for_remove)
|
||||
&& candidate.active_conns.load(Ordering::Relaxed) == 0
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn try_consume_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(&share.up, cap_epoch, requested),
|
||||
RateDirection::Down => self.down.try_consume(&share.down, cap_epoch, requested),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
bytes: u64,
|
||||
) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.up.refund(bytes);
|
||||
share.up.refund(bytes);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.down.refund(bytes);
|
||||
share.down.refund(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cleanup_idle_users(&self) {
|
||||
self.users
|
||||
.retain(|_, share| share.active_conns.load(Ordering::Relaxed) > 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::*;
|
||||
pub fn next_refill_delay() -> Duration {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
let epoch_pos = elapsed_ms % FAIR_EPOCH_MS;
|
||||
let wait_ms = FAIR_EPOCH_MS.saturating_sub(epoch_pos).max(1);
|
||||
Duration::from_millis(wait_ms)
|
||||
}
|
||||
|
||||
pub(super) fn decrement_atomic_saturating(counter: &AtomicU64, by: u64) {
|
||||
if by == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current == 0 {
|
||||
return;
|
||||
}
|
||||
let next = current.saturating_sub(by);
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
pub(super) fn bytes_per_epoch(bps: u64) -> u64 {
|
||||
if bps == 0 {
|
||||
return 0;
|
||||
}
|
||||
let numerator = bps.saturating_mul(FAIR_EPOCH_MS);
|
||||
let bytes = numerator.saturating_div(8_000);
|
||||
bytes.max(1)
|
||||
}
|
||||
|
||||
pub(super) fn auto_cidr_bucket_key(ip: IpAddr, prefix_len: u8) -> Option<String> {
|
||||
let cidr = IpNetwork::new(ip, prefix_len).ok()?;
|
||||
let network = IpNetwork::new(cidr.network(), prefix_len).ok()?;
|
||||
let family = match network {
|
||||
IpNetwork::V4(_) => "4",
|
||||
IpNetwork::V6(_) => "6",
|
||||
};
|
||||
Some(format!("auto:{family}:{network}"))
|
||||
}
|
||||
|
||||
pub(super) fn current_epoch() -> u64 {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
elapsed_ms / FAIR_EPOCH_MS
|
||||
}
|
||||
|
||||
pub(super) fn limiter_epoch_start() -> &'static Instant {
|
||||
static START: OnceLock<Instant> = OnceLock::new();
|
||||
START.get_or_init(Instant::now)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use super::*;
|
||||
|
||||
impl TrafficLease {
|
||||
pub fn try_consume(&self, direction: RateDirection, requested: u64) -> TrafficConsumeResult {
|
||||
if requested == 0 {
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
|
||||
let mut granted = requested;
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
let user_granted = user_bucket.try_consume(direction, granted);
|
||||
if user_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, true, false);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: true,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
granted = user_granted;
|
||||
}
|
||||
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
let cidr_granted =
|
||||
cidr_bucket.try_consume_for_user(direction, cidr_user_share, granted);
|
||||
if cidr_granted < granted
|
||||
&& let Some(user_bucket) = self.user_bucket.as_ref()
|
||||
{
|
||||
user_bucket.refund(direction, granted.saturating_sub(cidr_granted));
|
||||
}
|
||||
if cidr_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, false, true);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: true,
|
||||
};
|
||||
}
|
||||
granted = cidr_granted;
|
||||
}
|
||||
|
||||
TrafficConsumeResult {
|
||||
granted,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
user_bucket.refund(direction, bytes);
|
||||
}
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
cidr_bucket.refund_for_user(direction, cidr_user_share, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe_wait_ms(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if wait_ms == 0 {
|
||||
return;
|
||||
}
|
||||
self.limiter
|
||||
.observe_wait(direction, blocked_user, blocked_cidr, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TrafficLease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(bucket) = self.user_bucket.as_ref() {
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.user_scope.active_leases, 1);
|
||||
}
|
||||
|
||||
if let Some(bucket) = self.cidr_bucket.as_ref() {
|
||||
if let (Some(user_key), Some(share)) =
|
||||
(self.cidr_user_key.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
bucket.release_user_share(user_key, share);
|
||||
}
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.cidr_scope.active_leases, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
use crate::config::CidrRateLimitKey;
|
||||
|
||||
use super::*;
|
||||
impl TrafficLimiter {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
policy: ArcSwap::from_pointee(PolicySnapshot::default()),
|
||||
user_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
cidr_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
user_scope: ScopeMetrics::default(),
|
||||
cidr_scope: ScopeMetrics::default(),
|
||||
last_cleanup_epoch_secs: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_policy(
|
||||
&self,
|
||||
user_limits: HashMap<String, RateLimitBps>,
|
||||
cidr_limits: HashMap<CidrRateLimitKey, RateLimitBps>,
|
||||
) {
|
||||
let filtered_users = user_limits
|
||||
.into_iter()
|
||||
.filter(|(_, limit)| limit.up_bps > 0 || limit.down_bps > 0)
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut cidr_rules_v4 = Vec::new();
|
||||
let mut cidr_rules_v6 = Vec::new();
|
||||
let mut cidr_auto_rules_v4 = Vec::new();
|
||||
let mut cidr_auto_rules_v6 = Vec::new();
|
||||
let mut cidr_rule_keys = HashSet::new();
|
||||
for (key, limits) in cidr_limits {
|
||||
if limits.up_bps == 0 && limits.down_bps == 0 {
|
||||
continue;
|
||||
}
|
||||
match key {
|
||||
CidrRateLimitKey::Network(cidr) => {
|
||||
let key = cidr.to_string();
|
||||
let rule = CidrRule {
|
||||
key: key.clone(),
|
||||
cidr,
|
||||
limits,
|
||||
prefix_len: cidr.prefix(),
|
||||
};
|
||||
cidr_rule_keys.insert(key);
|
||||
match rule.cidr {
|
||||
IpNetwork::V4(_) => cidr_rules_v4.push(rule),
|
||||
IpNetwork::V6(_) => cidr_rules_v6.push(rule),
|
||||
}
|
||||
}
|
||||
CidrRateLimitKey::AutoV4(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoV6(prefix_len) => {
|
||||
cidr_auto_rules_v6.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoDual(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
cidr_auto_rules_v6.push(CidrAutoRule {
|
||||
prefix_len: prefix_len.saturating_mul(4),
|
||||
limits,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cidr_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
let cidr_policy_entries =
|
||||
cidr_rule_keys.len() + cidr_auto_rules_v4.len() + cidr_auto_rules_v6.len();
|
||||
|
||||
self.user_scope
|
||||
.policy_entries
|
||||
.store(filtered_users.len() as u64, Ordering::Relaxed);
|
||||
self.cidr_scope
|
||||
.policy_entries
|
||||
.store(cidr_policy_entries as u64, Ordering::Relaxed);
|
||||
|
||||
self.policy.store(Arc::new(PolicySnapshot {
|
||||
user_limits: filtered_users,
|
||||
cidr_rules_v4,
|
||||
cidr_rules_v6,
|
||||
cidr_auto_rules_v4,
|
||||
cidr_auto_rules_v6,
|
||||
cidr_rule_keys,
|
||||
}));
|
||||
|
||||
self.maybe_cleanup();
|
||||
}
|
||||
|
||||
pub fn acquire_lease(
|
||||
self: &Arc<Self>,
|
||||
user: &str,
|
||||
client_ip: IpAddr,
|
||||
) -> Option<Arc<TrafficLease>> {
|
||||
let policy = self.policy.load_full();
|
||||
let mut user_bucket = None;
|
||||
if let Some(limit) = policy.user_limits.get(user).copied() {
|
||||
let bucket = self.user_buckets.get_or_insert_with(
|
||||
user,
|
||||
|| UserBucket::new(limit),
|
||||
|bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
},
|
||||
);
|
||||
bucket.set_rates(limit);
|
||||
self.user_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
user_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
let mut cidr_bucket = None;
|
||||
let mut cidr_user_key = None;
|
||||
let mut cidr_user_share = None;
|
||||
if let Some(rule_match) = policy.match_cidr(client_ip) {
|
||||
let (key, limits) = match &rule_match {
|
||||
CidrPolicyMatch::Explicit(rule) => (rule.key.as_str(), rule.limits),
|
||||
CidrPolicyMatch::Auto { key, limits } => (key.as_str(), *limits),
|
||||
};
|
||||
let bucket = self.cidr_buckets.get_or_insert_with(
|
||||
key,
|
||||
|| CidrBucket::new(limits),
|
||||
|bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
},
|
||||
);
|
||||
bucket.set_rates(limits);
|
||||
self.cidr_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
let share = bucket.acquire_user_share(user);
|
||||
cidr_user_key = Some(user.to_string());
|
||||
cidr_user_share = Some(share);
|
||||
cidr_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
if user_bucket.is_none() && cidr_bucket.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.maybe_cleanup();
|
||||
Some(Arc::new(TrafficLease {
|
||||
limiter: Arc::clone(self),
|
||||
user_bucket,
|
||||
cidr_bucket,
|
||||
cidr_user_key,
|
||||
cidr_user_share,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn metrics_snapshot(&self) -> TrafficLimiterMetricsSnapshot {
|
||||
TrafficLimiterMetricsSnapshot {
|
||||
user_throttle_up_total: self.user_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
user_throttle_down_total: self.user_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_up_total: self.cidr_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_down_total: self.cidr_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
user_wait_up_ms_total: self.user_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
user_wait_down_ms_total: self.user_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_up_ms_total: self.cidr_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_down_ms_total: self.cidr_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
user_active_leases: self.user_scope.active_leases.load(Ordering::Relaxed),
|
||||
cidr_active_leases: self.cidr_scope.active_leases.load(Ordering::Relaxed),
|
||||
user_policy_entries: self.user_scope.policy_entries.load(Ordering::Relaxed),
|
||||
cidr_policy_entries: self.cidr_scope.policy_entries.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn observe_throttle(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.throttle(direction);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.throttle(direction);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn observe_wait(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn maybe_cleanup(&self) {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed);
|
||||
if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS {
|
||||
return;
|
||||
}
|
||||
if self
|
||||
.last_cleanup_epoch_secs
|
||||
.compare_exchange(last, now_epoch_secs, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let policy = self.policy.load_full();
|
||||
self.user_buckets.retain(|user, bucket| {
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.user_limits.contains_key(user)
|
||||
});
|
||||
self.cidr_buckets.retain(|cidr_key, bucket| {
|
||||
bucket.cleanup_idle_users();
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.cidr_rule_keys.contains(cidr_key)
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
use super::*;
|
||||
impl PolicySnapshot {
|
||||
pub(super) fn match_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
match ip {
|
||||
IpAddr::V4(_) => self
|
||||
.cidr_rules_v4
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
IpAddr::V6(_) => self
|
||||
.cidr_rules_v6
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
}
|
||||
.map(CidrPolicyMatch::Explicit)
|
||||
.or_else(|| self.match_auto_cidr(ip))
|
||||
}
|
||||
|
||||
pub(super) fn match_auto_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
let rule = match ip {
|
||||
IpAddr::V4(_) => self.cidr_auto_rules_v4.first()?,
|
||||
IpAddr::V6(_) => self.cidr_auto_rules_v6.first()?,
|
||||
};
|
||||
let key = auto_cidr_bucket_key(ip, rule.prefix_len)?;
|
||||
Some(CidrPolicyMatch::Auto {
|
||||
key,
|
||||
limits: rule.limits,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ShardedRegistry<T> {
|
||||
pub(super) fn new(shards: usize) -> Self {
|
||||
let shard_count = shards.max(1).next_power_of_two();
|
||||
let mut items = Vec::with_capacity(shard_count);
|
||||
for _ in 0..shard_count {
|
||||
items.push(DashMap::<String, Arc<T>>::new());
|
||||
}
|
||||
Self {
|
||||
shards: items.into_boxed_slice(),
|
||||
mask: shard_count.saturating_sub(1),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn shard_index(&self, key: &str) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
key.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.mask
|
||||
}
|
||||
|
||||
pub(super) fn get_or_insert_with<F, A>(&self, key: &str, make: F, activate: A) -> Arc<T>
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
A: FnOnce(&Arc<T>),
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
match shard.entry(key.to_string()) {
|
||||
dashmap::mapref::entry::Entry::Occupied(entry) => {
|
||||
activate(entry.get());
|
||||
Arc::clone(entry.get())
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Vacant(slot) => {
|
||||
let value = Arc::new(make());
|
||||
activate(&value);
|
||||
slot.insert(Arc::clone(&value));
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn retain<F>(&self, predicate: F)
|
||||
where
|
||||
F: Fn(&String, &Arc<T>) -> bool + Copy,
|
||||
{
|
||||
for shard in &*self.shards {
|
||||
shard.retain(|key, value| predicate(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remove_if<F>(&self, key: &str, predicate: F) -> bool
|
||||
where
|
||||
F: Fn(&Arc<T>) -> bool,
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
shard.remove_if(key, |_, value| predicate(value)).is_some()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use super::*;
|
||||
use crate::config::CidrRateLimitKey;
|
||||
|
||||
fn rate(up_bps: u64, down_bps: u64) -> RateLimitBps {
|
||||
RateLimitBps { up_bps, down_bps }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_cidr_rule_wins_over_auto_template() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(
|
||||
CidrRateLimitKey::Network("203.0.113.7/32".parse().unwrap()),
|
||||
rate(2_000, 0),
|
||||
);
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.7".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Explicit(rule) => assert_eq!(rule.key.as_str(), "203.0.113.7/32"),
|
||||
CidrPolicyMatch::Auto { .. } => panic!("explicit CIDR must have priority"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_template_uses_longest_prefix() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(32), rate(2_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.129".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, limits } => {
|
||||
assert_eq!(key, "auto:4:203.0.113.129/32");
|
||||
assert_eq!(limits.up_bps, 2_000);
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dual_auto_template_maps_v6_prefix_by_four() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoDual(32), rate(1_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("2001:db8::1".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, .. } => {
|
||||
assert_eq!(key, "auto:6:2001:db8::1/128");
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_cidr_bucket_key_canonicalizes_network_address() {
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("203.0.113.129".parse().unwrap(), 24).unwrap(),
|
||||
"auto:4:203.0.113.0/24"
|
||||
);
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("2001:db8::abcd".parse().unwrap(), 64).unwrap(),
|
||||
"auto:6:2001:db8::/64"
|
||||
);
|
||||
}
|
||||
+3
-7
@@ -83,10 +83,7 @@ impl QuotaStateOwner {
|
||||
}
|
||||
|
||||
/// Persists a checkpoint filtered to the active configured user set.
|
||||
pub(crate) async fn save(
|
||||
&self,
|
||||
configured_users: &BTreeSet<String>,
|
||||
) -> std::io::Result<()> {
|
||||
pub(crate) async fn save(&self, configured_users: &BTreeSet<String>) -> std::io::Result<()> {
|
||||
let guard = Arc::clone(&self.mutation).lock_owned().await;
|
||||
let state = self.state_for_users(configured_users, None);
|
||||
let path = self.path.clone();
|
||||
@@ -313,9 +310,8 @@ mod tests {
|
||||
|
||||
let alice_owner = owner.clone();
|
||||
let alice_users = configured.clone();
|
||||
let alice = tokio::spawn(async move {
|
||||
alice_owner.reset_user(&alice_users, "alice").await
|
||||
});
|
||||
let alice =
|
||||
tokio::spawn(async move { alice_owner.reset_user(&alice_users, "alice").await });
|
||||
let bob_owner = owner.clone();
|
||||
let bob_users = configured.clone();
|
||||
let bob = tokio::spawn(async move { bob_owner.reset_user(&bob_users, "bob").await });
|
||||
|
||||
@@ -82,9 +82,7 @@ impl Stats {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut slots = self
|
||||
.me_handshake_error_code_slots
|
||||
.load(Ordering::Acquire);
|
||||
let mut slots = self.me_handshake_error_code_slots.load(Ordering::Acquire);
|
||||
loop {
|
||||
if slots >= ME_HANDSHAKE_ERROR_CODE_MAX {
|
||||
if let Some(entry) = self.me_handshake_error_codes.get(&code) {
|
||||
@@ -508,9 +506,7 @@ mod tests {
|
||||
(WORKERS * CODES_PER_WORKER) as u64
|
||||
);
|
||||
assert_eq!(
|
||||
stats
|
||||
.me_handshake_error_code_slots
|
||||
.load(Ordering::Acquire),
|
||||
stats.me_handshake_error_code_slots.load(Ordering::Acquire),
|
||||
counts.len()
|
||||
);
|
||||
}
|
||||
|
||||
+14
-745
@@ -1,22 +1,28 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::net::IpAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::tls_front::types::{
|
||||
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, TlsProfileQuality,
|
||||
TlsProfileSource,
|
||||
};
|
||||
|
||||
// Runtime TLS profile cache operations.
|
||||
mod runtime;
|
||||
// Bounded disk-cache reads and domain matching.
|
||||
mod disk;
|
||||
|
||||
use disk::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 1;
|
||||
const FULL_CERT_SENT_MAX_ENTRIES: usize = 65_536;
|
||||
const FULL_CERT_SENT_SHARDS: usize = 64;
|
||||
@@ -97,8 +103,7 @@ impl TlsFullCertBudget {
|
||||
}
|
||||
|
||||
async fn sweep_one_shard(&self, now: Instant) {
|
||||
let shard_index = self.sweep_cursor.fetch_add(1, Ordering::Relaxed)
|
||||
% FULL_CERT_SENT_SHARDS;
|
||||
let shard_index = self.sweep_cursor.fetch_add(1, Ordering::Relaxed) % FULL_CERT_SENT_SHARDS;
|
||||
let mut guard = self.shards[shard_index].write().await;
|
||||
let before = guard.len();
|
||||
guard.retain(|_, entry| entry.expires_at.map_or(true, |expires_at| expires_at > now));
|
||||
@@ -113,9 +118,7 @@ impl TlsFullCertBudget {
|
||||
let should_sweep = self
|
||||
.last_sweep_epoch_secs
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |last_sweep| {
|
||||
if now_epoch_secs.saturating_sub(last_sweep)
|
||||
>= FULL_CERT_SENT_SWEEP_INTERVAL_SECS
|
||||
{
|
||||
if now_epoch_secs.saturating_sub(last_sweep) >= FULL_CERT_SENT_SWEEP_INTERVAL_SECS {
|
||||
Some(now_epoch_secs)
|
||||
} else {
|
||||
None
|
||||
@@ -149,9 +152,7 @@ impl TlsFullCertBudget {
|
||||
|
||||
if guard.len() >= FULL_CERT_SENT_MAX_ENTRIES_PER_SHARD {
|
||||
let before = guard.len();
|
||||
guard.retain(|_, entry| {
|
||||
entry.expires_at.map_or(true, |expires_at| expires_at > now)
|
||||
});
|
||||
guard.retain(|_, entry| entry.expires_at.map_or(true, |expires_at| expires_at > now));
|
||||
self.decrement_entries(before.saturating_sub(guard.len()));
|
||||
}
|
||||
if guard.len() >= FULL_CERT_SENT_MAX_ENTRIES_PER_SHARD || !self.try_reserve_entry() {
|
||||
@@ -228,735 +229,3 @@ fn key_share_group_label(group: Option<u16>) -> &'static str {
|
||||
None => "none",
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TlsFrontCache {
|
||||
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
|
||||
Self::new_with_full_cert_budget(
|
||||
domains,
|
||||
default_len,
|
||||
disk_path,
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a generation-local cache backed by the process-owned full-cert budget.
|
||||
pub(crate) fn new_with_full_cert_budget(
|
||||
domains: &[String],
|
||||
default_len: usize,
|
||||
disk_path: impl AsRef<Path>,
|
||||
full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
) -> Self {
|
||||
let default_template = ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: Vec::new(),
|
||||
};
|
||||
|
||||
let default = Arc::new(CachedTlsData {
|
||||
server_hello_template: default_template,
|
||||
cert_info: None,
|
||||
cert_payload: None,
|
||||
app_data_records_sizes: vec![default_len],
|
||||
total_app_data_len: default_len,
|
||||
behavior_profile: TlsBehaviorProfile::default(),
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "default".to_string(),
|
||||
});
|
||||
|
||||
let mut map = HashMap::new();
|
||||
let mut full_cert_domain_keys = HashMap::new();
|
||||
let mut disk_entry_names = HashSet::new();
|
||||
for d in domains {
|
||||
map.insert(d.clone(), default.clone());
|
||||
disk_entry_names.insert(format!("{}.json", d.replace(['/', '\\'], "_")));
|
||||
let canonical: Arc<str> = Arc::from(normalize_dns_name(d));
|
||||
full_cert_domain_keys.insert(d.clone(), canonical.clone());
|
||||
full_cert_domain_keys
|
||||
.entry(canonical.to_string())
|
||||
.or_insert(canonical);
|
||||
}
|
||||
|
||||
Self {
|
||||
memory: RwLock::new(map),
|
||||
default,
|
||||
full_cert_budget,
|
||||
full_cert_domain_keys,
|
||||
disk_entry_names,
|
||||
disk_path: disk_path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(&self, sni: &str) -> Arc<CachedTlsData> {
|
||||
let guard = self.memory.read().await;
|
||||
guard
|
||||
.get(sni)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.default.clone())
|
||||
}
|
||||
|
||||
pub async fn contains_domain(&self, domain: &str) -> bool {
|
||||
self.memory.read().await.contains_key(domain)
|
||||
}
|
||||
|
||||
pub(crate) async fn profile_health_snapshot(
|
||||
&self,
|
||||
domains: &[String],
|
||||
max_domains: usize,
|
||||
) -> (Vec<TlsFrontProfileHealth>, usize) {
|
||||
let guard = self.memory.read().await;
|
||||
let now = SystemTime::now();
|
||||
let mut snapshot = Vec::with_capacity(domains.len().min(max_domains));
|
||||
let mut suppressed = 0usize;
|
||||
|
||||
for domain in domains {
|
||||
if snapshot.len() >= max_domains {
|
||||
suppressed = suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
let cached = guard
|
||||
.get(domain)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.default.clone());
|
||||
let mut behavior = cached.behavior_profile.clone();
|
||||
behavior.refresh_server_hello_summary(&cached.server_hello_template);
|
||||
let age_seconds = now
|
||||
.duration_since(cached.fetched_at)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or(0);
|
||||
|
||||
snapshot.push(TlsFrontProfileHealth {
|
||||
domain: domain.clone(),
|
||||
source: profile_source_label(behavior.source),
|
||||
quality: profile_quality_label(behavior.quality),
|
||||
key_share_group: key_share_group_label(behavior.server_hello_key_share_group),
|
||||
age_seconds,
|
||||
is_default: cached.domain == "default",
|
||||
has_cert_info: cached.cert_info.is_some(),
|
||||
has_cert_payload: cached.cert_payload.is_some(),
|
||||
server_hello_record_len: behavior.server_hello_record_len,
|
||||
server_hello_extensions: behavior.server_hello_extension_types.len(),
|
||||
app_data_records: cached
|
||||
.app_data_records_sizes
|
||||
.len()
|
||||
.max(behavior.app_data_record_sizes.len()),
|
||||
ticket_records: behavior.ticket_record_sizes.len(),
|
||||
change_cipher_spec_count: behavior.change_cipher_spec_count,
|
||||
total_app_data_len: cached.total_app_data_len,
|
||||
});
|
||||
}
|
||||
|
||||
(snapshot, suppressed)
|
||||
}
|
||||
|
||||
/// Returns configured domains that still resolve to the synthetic default profile.
|
||||
pub(crate) async fn default_profile_domains(&self, domains: &[String]) -> Vec<String> {
|
||||
let guard = self.memory.read().await;
|
||||
domains
|
||||
.iter()
|
||||
.filter(|domain| {
|
||||
guard.get(domain.as_str()).unwrap_or(&self.default).domain == "default"
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn full_cert_domain_key(&self, domain: &str) -> Arc<str> {
|
||||
self.full_cert_domain_keys
|
||||
.get(domain)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Arc::from(normalize_dns_name(domain)))
|
||||
}
|
||||
|
||||
/// Returns true when the selected domain and client IP may receive a full cert payload.
|
||||
pub async fn take_full_cert_budget_for_ip(
|
||||
&self,
|
||||
domain: &str,
|
||||
client_ip: IpAddr,
|
||||
ttl: Duration,
|
||||
) -> bool {
|
||||
self.full_cert_budget
|
||||
.take(self.full_cert_domain_key(domain), client_ip, ttl)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Returns the current process-owned full-cert budget entry count.
|
||||
pub(crate) fn full_cert_budget_entries_for_metrics(&self) -> u64 {
|
||||
self.full_cert_budget.entries_for_metrics()
|
||||
}
|
||||
|
||||
/// Returns the cumulative process-owned full-cert budget cap drops.
|
||||
pub(crate) fn full_cert_budget_cap_drops_for_metrics(&self) -> u64 {
|
||||
self.full_cert_budget.cap_drops_for_metrics()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn insert_full_cert_sent_for_tests(
|
||||
&self,
|
||||
domain: &str,
|
||||
client_ip: IpAddr,
|
||||
expires_at: Instant,
|
||||
) {
|
||||
let key = FullCertBudgetKey {
|
||||
domain: self.full_cert_domain_key(domain),
|
||||
client_ip,
|
||||
};
|
||||
let shard_index = self.full_cert_budget.shard_index(&key);
|
||||
let mut guard = self.full_cert_budget.shards[shard_index].write().await;
|
||||
if guard
|
||||
.insert(
|
||||
key,
|
||||
FullCertBudgetEntry {
|
||||
expires_at: Some(expires_at),
|
||||
},
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
self.full_cert_budget
|
||||
.entries
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn full_cert_sent_is_empty_for_tests(&self) -> bool {
|
||||
for shard in &self.full_cert_budget.shards {
|
||||
if !shard.read().await.is_empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn full_cert_sent_contains_for_tests(&self, domain: &str, client_ip: IpAddr) -> bool {
|
||||
let key = FullCertBudgetKey {
|
||||
domain: self.full_cert_domain_key(domain),
|
||||
client_ip,
|
||||
};
|
||||
let shard_index = self.full_cert_budget.shard_index(&key);
|
||||
self.full_cert_budget.shards[shard_index]
|
||||
.read()
|
||||
.await
|
||||
.contains_key(&key)
|
||||
}
|
||||
|
||||
pub async fn set(&self, domain: &str, data: CachedTlsData) {
|
||||
let mut guard = self.memory.write().await;
|
||||
guard.insert(domain.to_string(), Arc::new(data));
|
||||
}
|
||||
|
||||
pub async fn load_from_disk(&self) {
|
||||
let path = self.disk_path.clone();
|
||||
if tokio::fs::create_dir_all(&path).await.is_err() {
|
||||
return;
|
||||
}
|
||||
let mut loaded = 0usize;
|
||||
for name in &self.disk_entry_names {
|
||||
let entry_path = path.join(name);
|
||||
let Ok(metadata) = tokio::fs::symlink_metadata(&entry_path).await else {
|
||||
continue;
|
||||
};
|
||||
if !metadata.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
if let Ok(data) = read_disk_entry_bounded(&entry_path).await
|
||||
&& let Ok(mut cached) = serde_json::from_slice::<CachedTlsData>(&data)
|
||||
{
|
||||
if cached.domain.is_empty()
|
||||
|| cached.domain.len() > 255
|
||||
|| !cached
|
||||
.domain
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
|
||||
{
|
||||
warn!(file = %name, "Skipping TLS cache entry with invalid domain");
|
||||
continue;
|
||||
}
|
||||
if !self.full_cert_domain_keys.contains_key(&cached.domain) {
|
||||
warn!(
|
||||
file = %name,
|
||||
domain = %cached.domain,
|
||||
"Skipping TLS cache entry outside configured domains"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !cert_info_matches_domain(&cached) {
|
||||
warn!(
|
||||
file = %name,
|
||||
domain = %cached.domain,
|
||||
"Skipping TLS cache entry with mismatched certificate metadata"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// fetched_at is skipped during deserialization; approximate with file mtime if available.
|
||||
if let Ok(modified) = metadata.modified() {
|
||||
cached.fetched_at = modified;
|
||||
}
|
||||
// Drop entries older than 72h
|
||||
if let Ok(age) = cached.fetched_at.elapsed()
|
||||
&& age > Duration::from_secs(72 * 3600)
|
||||
{
|
||||
warn!(domain = %cached.domain, "Skipping stale TLS cache entry (>72h)");
|
||||
continue;
|
||||
}
|
||||
cached
|
||||
.behavior_profile
|
||||
.refresh_server_hello_summary(&cached.server_hello_template);
|
||||
let domain = cached.domain.clone();
|
||||
self.set(&domain, cached).await;
|
||||
loaded += 1;
|
||||
}
|
||||
}
|
||||
if loaded > 0 {
|
||||
info!(count = loaded, "Loaded TLS cache entries from disk");
|
||||
}
|
||||
}
|
||||
|
||||
async fn persist(&self, domain: &str, data: &CachedTlsData) {
|
||||
if tokio::fs::create_dir_all(&self.disk_path).await.is_err() {
|
||||
return;
|
||||
}
|
||||
let fname = format!("{}.json", domain.replace(['/', '\\'], "_"));
|
||||
let path = self.disk_path.join(fname);
|
||||
if let Ok(json) = serde_json::to_vec_pretty(data) {
|
||||
if json.len() as u64 > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
warn!(
|
||||
domain,
|
||||
bytes = json.len(),
|
||||
"Skipping oversized TLS cache persistence"
|
||||
);
|
||||
return;
|
||||
}
|
||||
// best-effort write
|
||||
let _ = tokio::fs::write(path, json).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn background updater that periodically refreshes cached domains using provided fetcher.
|
||||
pub fn spawn_updater<F>(self: Arc<Self>, domains: Vec<String>, interval: Duration, fetcher: F)
|
||||
where
|
||||
F: Fn(String) -> tokio::task::JoinHandle<()> + Send + Sync + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
for domain in &domains {
|
||||
let _ = fetcher(domain.clone()).await;
|
||||
}
|
||||
sleep(interval).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Replace cached entry from a fetch result.
|
||||
pub async fn update_from_fetch(&self, domain: &str, fetched: TlsFetchResult) {
|
||||
let TlsFetchResult {
|
||||
server_hello_parsed,
|
||||
app_data_records_sizes,
|
||||
total_app_data_len,
|
||||
mut behavior_profile,
|
||||
cert_info,
|
||||
cert_payload,
|
||||
} = fetched;
|
||||
behavior_profile.refresh_server_hello_summary(&server_hello_parsed);
|
||||
let quality = behavior_profile.quality;
|
||||
let data = CachedTlsData {
|
||||
server_hello_template: server_hello_parsed,
|
||||
cert_info,
|
||||
cert_payload,
|
||||
app_data_records_sizes: app_data_records_sizes.clone(),
|
||||
total_app_data_len,
|
||||
behavior_profile,
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: domain.to_string(),
|
||||
};
|
||||
|
||||
self.set(domain, data.clone()).await;
|
||||
self.persist(domain, &data).await;
|
||||
if quality == TlsProfileQuality::RawStrict {
|
||||
debug!(domain = %domain, len = total_app_data_len, "TLS cache updated");
|
||||
} else {
|
||||
warn!(
|
||||
domain = %domain,
|
||||
quality = profile_quality_label(quality),
|
||||
len = total_app_data_len,
|
||||
"TLS cache updated with non-strict front profile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_entry(&self) -> Arc<CachedTlsData> {
|
||||
self.default.clone()
|
||||
}
|
||||
|
||||
pub fn disk_path(&self) -> &Path {
|
||||
&self.disk_path
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_disk_entry_bounded(path: &Path) -> std::io::Result<Vec<u8>> {
|
||||
let file = tokio::fs::File::open(path).await?;
|
||||
if file.metadata().await?.len() > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"TLS cache entry exceeds the 1 MiB limit",
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
file.take(TLS_FRONT_DISK_ENTRY_MAX_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut bytes)
|
||||
.await?;
|
||||
if bytes.len() as u64 > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"TLS cache entry grew beyond the 1 MiB limit while reading",
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn cert_info_matches_domain(cached: &CachedTlsData) -> bool {
|
||||
let Some(cert_info) = cached.cert_info.as_ref() else {
|
||||
return true;
|
||||
};
|
||||
if !cert_info.san_names.is_empty() {
|
||||
return cert_info
|
||||
.san_names
|
||||
.iter()
|
||||
.any(|name| dns_name_matches_domain(name, &cached.domain));
|
||||
}
|
||||
cert_info
|
||||
.subject_cn
|
||||
.as_deref()
|
||||
.map_or(true, |name| dns_name_matches_domain(name, &cached.domain))
|
||||
}
|
||||
|
||||
fn dns_name_matches_domain(pattern: &str, domain: &str) -> bool {
|
||||
let pattern = normalize_dns_name(pattern);
|
||||
let domain = normalize_dns_name(domain);
|
||||
if pattern == domain {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some(suffix) = pattern.strip_prefix("*.") else {
|
||||
return false;
|
||||
};
|
||||
let Some(prefix) = domain.strip_suffix(suffix) else {
|
||||
return false;
|
||||
};
|
||||
prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.')
|
||||
}
|
||||
|
||||
fn normalize_dns_name(value: &str) -> String {
|
||||
value.trim().trim_end_matches('.').to_ascii_lowercase()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cached_with_cert_info(
|
||||
domain: &str,
|
||||
subject_cn: Option<&str>,
|
||||
san_names: Vec<&str>,
|
||||
) -> CachedTlsData {
|
||||
CachedTlsData {
|
||||
server_hello_template: ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: Vec::new(),
|
||||
},
|
||||
cert_info: Some(crate::tls_front::types::ParsedCertificateInfo {
|
||||
not_after_unix: None,
|
||||
not_before_unix: None,
|
||||
issuer_cn: None,
|
||||
subject_cn: subject_cn.map(str::to_string),
|
||||
san_names: san_names.into_iter().map(str::to_string).collect(),
|
||||
}),
|
||||
cert_payload: None,
|
||||
app_data_records_sizes: vec![1024],
|
||||
total_app_data_len: 1024,
|
||||
behavior_profile: TlsBehaviorProfile::default(),
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: domain.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_accepts_exact_san() {
|
||||
let cached = cached_with_cert_info("b.com", Some("a.com"), vec!["b.com"]);
|
||||
assert!(cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_rejects_wrong_san() {
|
||||
let cached = cached_with_cert_info("b.com", Some("b.com"), vec!["a.com"]);
|
||||
assert!(!cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_accepts_single_label_wildcard_san() {
|
||||
let cached = cached_with_cert_info("api.b.com", None, vec!["*.b.com"]);
|
||||
assert!(cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_rejects_multi_label_wildcard_san() {
|
||||
let cached = cached_with_cert_info("deep.api.b.com", None, vec!["*.b.com"]);
|
||||
assert!(!cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_profile_domains_reports_only_unprepared_entries() {
|
||||
let domains = vec!["ready.example".to_string(), "pending.example".to_string()];
|
||||
let cache = TlsFrontCache::new(&domains, 1024, "tlsfront-test-cache");
|
||||
cache
|
||||
.set(
|
||||
"ready.example",
|
||||
cached_with_cert_info("ready.example", None, Vec::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
cache.default_profile_domains(&domains).await,
|
||||
vec!["pending.example".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_uses_ttl() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let ttl = Duration::from_millis(80);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
!cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(90)).await;
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_zero_ttl_always_allows_full_payload() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ttl = Duration::ZERO;
|
||||
|
||||
for idx in 0..100_000u32 {
|
||||
let ip = IpAddr::V4(std::net::Ipv4Addr::new(
|
||||
10,
|
||||
((idx >> 16) & 0xff) as u8,
|
||||
((idx >> 8) & 0xff) as u8,
|
||||
(idx & 0xff) as u8,
|
||||
));
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
assert!(cache.full_cert_sent_is_empty_for_tests().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_sweeps_expired_entries_when_due() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(1);
|
||||
let stale_expires_at = Instant::now()
|
||||
.checked_sub(Duration::from_secs(1))
|
||||
.unwrap_or_else(Instant::now);
|
||||
|
||||
cache
|
||||
.insert_full_cert_sent_for_tests("example.com", stale_ip, stale_expires_at)
|
||||
.await;
|
||||
let stale_key = FullCertBudgetKey {
|
||||
domain: cache.full_cert_domain_key("example.com"),
|
||||
client_ip: stale_ip,
|
||||
};
|
||||
cache.full_cert_budget.sweep_cursor.store(
|
||||
cache.full_cert_budget.shard_index(&stale_key),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
cache
|
||||
.full_cert_budget
|
||||
.last_sweep_epoch_secs
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", new_ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
assert!(
|
||||
!cache
|
||||
.full_cert_sent_contains_for_tests("example.com", stale_ip)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", new_ip)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_does_not_sweep_every_call() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(1);
|
||||
let stale_expires_at = Instant::now()
|
||||
.checked_sub(Duration::from_secs(1))
|
||||
.unwrap_or_else(Instant::now);
|
||||
let now_epoch_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
cache
|
||||
.insert_full_cert_sent_for_tests("example.com", stale_ip, stale_expires_at)
|
||||
.await;
|
||||
cache
|
||||
.full_cert_budget
|
||||
.last_sweep_epoch_secs
|
||||
.store(now_epoch_secs, Ordering::Relaxed);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", new_ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", stale_ip)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", new_ip)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cert_budget_is_shared_across_cache_generations_and_scoped_by_domain() {
|
||||
let budget = Arc::new(TlsFullCertBudget::new());
|
||||
let domains = ["one.example".to_string(), "two.example".to_string()];
|
||||
let first = TlsFrontCache::new_with_full_cert_budget(
|
||||
&domains,
|
||||
1024,
|
||||
"tlsfront-test-cache",
|
||||
budget.clone(),
|
||||
);
|
||||
let second = TlsFrontCache::new_with_full_cert_budget(
|
||||
&domains,
|
||||
1024,
|
||||
"tlsfront-test-cache",
|
||||
budget,
|
||||
);
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(60);
|
||||
|
||||
assert!(
|
||||
first
|
||||
.take_full_cert_budget_for_ip("one.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
!second
|
||||
.take_full_cert_budget_for_ip("one.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
second
|
||||
.take_full_cert_budget_for_ip("two.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert_eq!(second.full_cert_budget_entries_for_metrics(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_full_cert_entry_keeps_its_own_expiry_after_ttl_change() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(80))
|
||||
.await
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
assert!(
|
||||
!cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(1))
|
||||
.await
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(70)).await;
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(1))
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_reader_rejects_an_entry_above_the_hard_limit() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("oversized.json");
|
||||
tokio::fs::write(
|
||||
&path,
|
||||
vec![0u8; TLS_FRONT_DISK_ENTRY_MAX_BYTES as usize + 1],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = read_disk_entry_bounded(&path).await.unwrap_err();
|
||||
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn disk_loader_does_not_follow_a_configured_name_symlink() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let target = directory.path().join("outside.json");
|
||||
let cached = cached_with_cert_info("example.com", None, Vec::new());
|
||||
tokio::fs::write(&target, serde_json::to_vec(&cached).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
std::os::unix::fs::symlink(&target, directory.path().join("example.com.json")).unwrap();
|
||||
let cache = TlsFrontCache::new(
|
||||
&["example.com".to_string()],
|
||||
1024,
|
||||
directory.path(),
|
||||
);
|
||||
|
||||
cache.load_from_disk().await;
|
||||
|
||||
assert_eq!(cache.get("example.com").await.domain, "default");
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
use std::path::Path;
|
||||
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
use super::*;
|
||||
pub(super) async fn read_disk_entry_bounded(path: &Path) -> std::io::Result<Vec<u8>> {
|
||||
let file = tokio::fs::File::open(path).await?;
|
||||
if file.metadata().await?.len() > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"TLS cache entry exceeds the 1 MiB limit",
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
file.take(TLS_FRONT_DISK_ENTRY_MAX_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut bytes)
|
||||
.await?;
|
||||
if bytes.len() as u64 > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"TLS cache entry grew beyond the 1 MiB limit while reading",
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(super) fn cert_info_matches_domain(cached: &CachedTlsData) -> bool {
|
||||
let Some(cert_info) = cached.cert_info.as_ref() else {
|
||||
return true;
|
||||
};
|
||||
if !cert_info.san_names.is_empty() {
|
||||
return cert_info
|
||||
.san_names
|
||||
.iter()
|
||||
.any(|name| dns_name_matches_domain(name, &cached.domain));
|
||||
}
|
||||
cert_info
|
||||
.subject_cn
|
||||
.as_deref()
|
||||
.map_or(true, |name| dns_name_matches_domain(name, &cached.domain))
|
||||
}
|
||||
|
||||
pub(super) fn dns_name_matches_domain(pattern: &str, domain: &str) -> bool {
|
||||
let pattern = normalize_dns_name(pattern);
|
||||
let domain = normalize_dns_name(domain);
|
||||
if pattern == domain {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some(suffix) = pattern.strip_prefix("*.") else {
|
||||
return false;
|
||||
};
|
||||
let Some(prefix) = domain.strip_suffix(suffix) else {
|
||||
return false;
|
||||
};
|
||||
prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.')
|
||||
}
|
||||
|
||||
pub(super) fn normalize_dns_name(value: &str) -> String {
|
||||
value.trim().trim_end_matches('.').to_ascii_lowercase()
|
||||
}
|
||||
Vendored
+378
@@ -0,0 +1,378 @@
|
||||
use std::path::Path;
|
||||
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::*;
|
||||
#[allow(dead_code)]
|
||||
impl TlsFrontCache {
|
||||
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
|
||||
Self::new_with_full_cert_budget(
|
||||
domains,
|
||||
default_len,
|
||||
disk_path,
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a generation-local cache backed by the process-owned full-cert budget.
|
||||
pub(crate) fn new_with_full_cert_budget(
|
||||
domains: &[String],
|
||||
default_len: usize,
|
||||
disk_path: impl AsRef<Path>,
|
||||
full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
) -> Self {
|
||||
let default_template = ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: Vec::new(),
|
||||
};
|
||||
|
||||
let default = Arc::new(CachedTlsData {
|
||||
server_hello_template: default_template,
|
||||
cert_info: None,
|
||||
cert_payload: None,
|
||||
app_data_records_sizes: vec![default_len],
|
||||
total_app_data_len: default_len,
|
||||
behavior_profile: TlsBehaviorProfile::default(),
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "default".to_string(),
|
||||
});
|
||||
|
||||
let mut map = HashMap::new();
|
||||
let mut full_cert_domain_keys = HashMap::new();
|
||||
let mut disk_entry_names = HashSet::new();
|
||||
for d in domains {
|
||||
map.insert(d.clone(), default.clone());
|
||||
disk_entry_names.insert(format!("{}.json", d.replace(['/', '\\'], "_")));
|
||||
let canonical: Arc<str> = Arc::from(normalize_dns_name(d));
|
||||
full_cert_domain_keys.insert(d.clone(), canonical.clone());
|
||||
full_cert_domain_keys
|
||||
.entry(canonical.to_string())
|
||||
.or_insert(canonical);
|
||||
}
|
||||
|
||||
Self {
|
||||
memory: RwLock::new(map),
|
||||
default,
|
||||
full_cert_budget,
|
||||
full_cert_domain_keys,
|
||||
disk_entry_names,
|
||||
disk_path: disk_path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(&self, sni: &str) -> Arc<CachedTlsData> {
|
||||
let guard = self.memory.read().await;
|
||||
guard
|
||||
.get(sni)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.default.clone())
|
||||
}
|
||||
|
||||
pub async fn contains_domain(&self, domain: &str) -> bool {
|
||||
self.memory.read().await.contains_key(domain)
|
||||
}
|
||||
|
||||
pub(crate) async fn profile_health_snapshot(
|
||||
&self,
|
||||
domains: &[String],
|
||||
max_domains: usize,
|
||||
) -> (Vec<TlsFrontProfileHealth>, usize) {
|
||||
let guard = self.memory.read().await;
|
||||
let now = SystemTime::now();
|
||||
let mut snapshot = Vec::with_capacity(domains.len().min(max_domains));
|
||||
let mut suppressed = 0usize;
|
||||
|
||||
for domain in domains {
|
||||
if snapshot.len() >= max_domains {
|
||||
suppressed = suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
let cached = guard
|
||||
.get(domain)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.default.clone());
|
||||
let mut behavior = cached.behavior_profile.clone();
|
||||
behavior.refresh_server_hello_summary(&cached.server_hello_template);
|
||||
let age_seconds = now
|
||||
.duration_since(cached.fetched_at)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or(0);
|
||||
|
||||
snapshot.push(TlsFrontProfileHealth {
|
||||
domain: domain.clone(),
|
||||
source: profile_source_label(behavior.source),
|
||||
quality: profile_quality_label(behavior.quality),
|
||||
key_share_group: key_share_group_label(behavior.server_hello_key_share_group),
|
||||
age_seconds,
|
||||
is_default: cached.domain == "default",
|
||||
has_cert_info: cached.cert_info.is_some(),
|
||||
has_cert_payload: cached.cert_payload.is_some(),
|
||||
server_hello_record_len: behavior.server_hello_record_len,
|
||||
server_hello_extensions: behavior.server_hello_extension_types.len(),
|
||||
app_data_records: cached
|
||||
.app_data_records_sizes
|
||||
.len()
|
||||
.max(behavior.app_data_record_sizes.len()),
|
||||
ticket_records: behavior.ticket_record_sizes.len(),
|
||||
change_cipher_spec_count: behavior.change_cipher_spec_count,
|
||||
total_app_data_len: cached.total_app_data_len,
|
||||
});
|
||||
}
|
||||
|
||||
(snapshot, suppressed)
|
||||
}
|
||||
|
||||
/// Returns configured domains that still resolve to the synthetic default profile.
|
||||
pub(crate) async fn default_profile_domains(&self, domains: &[String]) -> Vec<String> {
|
||||
let guard = self.memory.read().await;
|
||||
domains
|
||||
.iter()
|
||||
.filter(|domain| {
|
||||
guard.get(domain.as_str()).unwrap_or(&self.default).domain == "default"
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn full_cert_domain_key(&self, domain: &str) -> Arc<str> {
|
||||
self.full_cert_domain_keys
|
||||
.get(domain)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Arc::from(normalize_dns_name(domain)))
|
||||
}
|
||||
|
||||
/// Returns true when the selected domain and client IP may receive a full cert payload.
|
||||
pub async fn take_full_cert_budget_for_ip(
|
||||
&self,
|
||||
domain: &str,
|
||||
client_ip: IpAddr,
|
||||
ttl: Duration,
|
||||
) -> bool {
|
||||
self.full_cert_budget
|
||||
.take(self.full_cert_domain_key(domain), client_ip, ttl)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Returns the current process-owned full-cert budget entry count.
|
||||
pub(crate) fn full_cert_budget_entries_for_metrics(&self) -> u64 {
|
||||
self.full_cert_budget.entries_for_metrics()
|
||||
}
|
||||
|
||||
/// Returns the cumulative process-owned full-cert budget cap drops.
|
||||
pub(crate) fn full_cert_budget_cap_drops_for_metrics(&self) -> u64 {
|
||||
self.full_cert_budget.cap_drops_for_metrics()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn insert_full_cert_sent_for_tests(
|
||||
&self,
|
||||
domain: &str,
|
||||
client_ip: IpAddr,
|
||||
expires_at: Instant,
|
||||
) {
|
||||
let key = FullCertBudgetKey {
|
||||
domain: self.full_cert_domain_key(domain),
|
||||
client_ip,
|
||||
};
|
||||
let shard_index = self.full_cert_budget.shard_index(&key);
|
||||
let mut guard = self.full_cert_budget.shards[shard_index].write().await;
|
||||
if guard
|
||||
.insert(
|
||||
key,
|
||||
FullCertBudgetEntry {
|
||||
expires_at: Some(expires_at),
|
||||
},
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
self.full_cert_budget
|
||||
.entries
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn full_cert_sent_is_empty_for_tests(&self) -> bool {
|
||||
for shard in &self.full_cert_budget.shards {
|
||||
if !shard.read().await.is_empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn full_cert_sent_contains_for_tests(
|
||||
&self,
|
||||
domain: &str,
|
||||
client_ip: IpAddr,
|
||||
) -> bool {
|
||||
let key = FullCertBudgetKey {
|
||||
domain: self.full_cert_domain_key(domain),
|
||||
client_ip,
|
||||
};
|
||||
let shard_index = self.full_cert_budget.shard_index(&key);
|
||||
self.full_cert_budget.shards[shard_index]
|
||||
.read()
|
||||
.await
|
||||
.contains_key(&key)
|
||||
}
|
||||
|
||||
pub async fn set(&self, domain: &str, data: CachedTlsData) {
|
||||
let mut guard = self.memory.write().await;
|
||||
guard.insert(domain.to_string(), Arc::new(data));
|
||||
}
|
||||
|
||||
pub async fn load_from_disk(&self) {
|
||||
let path = self.disk_path.clone();
|
||||
if tokio::fs::create_dir_all(&path).await.is_err() {
|
||||
return;
|
||||
}
|
||||
let mut loaded = 0usize;
|
||||
for name in &self.disk_entry_names {
|
||||
let entry_path = path.join(name);
|
||||
let Ok(metadata) = tokio::fs::symlink_metadata(&entry_path).await else {
|
||||
continue;
|
||||
};
|
||||
if !metadata.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
if let Ok(data) = read_disk_entry_bounded(&entry_path).await
|
||||
&& let Ok(mut cached) = serde_json::from_slice::<CachedTlsData>(&data)
|
||||
{
|
||||
if cached.domain.is_empty()
|
||||
|| cached.domain.len() > 255
|
||||
|| !cached
|
||||
.domain
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
|
||||
{
|
||||
warn!(file = %name, "Skipping TLS cache entry with invalid domain");
|
||||
continue;
|
||||
}
|
||||
if !self.full_cert_domain_keys.contains_key(&cached.domain) {
|
||||
warn!(
|
||||
file = %name,
|
||||
domain = %cached.domain,
|
||||
"Skipping TLS cache entry outside configured domains"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !cert_info_matches_domain(&cached) {
|
||||
warn!(
|
||||
file = %name,
|
||||
domain = %cached.domain,
|
||||
"Skipping TLS cache entry with mismatched certificate metadata"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// fetched_at is skipped during deserialization; approximate with file mtime if available.
|
||||
if let Ok(modified) = metadata.modified() {
|
||||
cached.fetched_at = modified;
|
||||
}
|
||||
// Drop entries older than 72h
|
||||
if let Ok(age) = cached.fetched_at.elapsed()
|
||||
&& age > Duration::from_secs(72 * 3600)
|
||||
{
|
||||
warn!(domain = %cached.domain, "Skipping stale TLS cache entry (>72h)");
|
||||
continue;
|
||||
}
|
||||
cached
|
||||
.behavior_profile
|
||||
.refresh_server_hello_summary(&cached.server_hello_template);
|
||||
let domain = cached.domain.clone();
|
||||
self.set(&domain, cached).await;
|
||||
loaded += 1;
|
||||
}
|
||||
}
|
||||
if loaded > 0 {
|
||||
info!(count = loaded, "Loaded TLS cache entries from disk");
|
||||
}
|
||||
}
|
||||
|
||||
async fn persist(&self, domain: &str, data: &CachedTlsData) {
|
||||
if tokio::fs::create_dir_all(&self.disk_path).await.is_err() {
|
||||
return;
|
||||
}
|
||||
let fname = format!("{}.json", domain.replace(['/', '\\'], "_"));
|
||||
let path = self.disk_path.join(fname);
|
||||
if let Ok(json) = serde_json::to_vec_pretty(data) {
|
||||
if json.len() as u64 > TLS_FRONT_DISK_ENTRY_MAX_BYTES {
|
||||
warn!(
|
||||
domain,
|
||||
bytes = json.len(),
|
||||
"Skipping oversized TLS cache persistence"
|
||||
);
|
||||
return;
|
||||
}
|
||||
// best-effort write
|
||||
let _ = tokio::fs::write(path, json).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn background updater that periodically refreshes cached domains using provided fetcher.
|
||||
pub fn spawn_updater<F>(self: Arc<Self>, domains: Vec<String>, interval: Duration, fetcher: F)
|
||||
where
|
||||
F: Fn(String) -> tokio::task::JoinHandle<()> + Send + Sync + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
for domain in &domains {
|
||||
let _ = fetcher(domain.clone()).await;
|
||||
}
|
||||
sleep(interval).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Replace cached entry from a fetch result.
|
||||
pub async fn update_from_fetch(&self, domain: &str, fetched: TlsFetchResult) {
|
||||
let TlsFetchResult {
|
||||
server_hello_parsed,
|
||||
app_data_records_sizes,
|
||||
total_app_data_len,
|
||||
mut behavior_profile,
|
||||
cert_info,
|
||||
cert_payload,
|
||||
} = fetched;
|
||||
behavior_profile.refresh_server_hello_summary(&server_hello_parsed);
|
||||
let quality = behavior_profile.quality;
|
||||
let data = CachedTlsData {
|
||||
server_hello_template: server_hello_parsed,
|
||||
cert_info,
|
||||
cert_payload,
|
||||
app_data_records_sizes: app_data_records_sizes.clone(),
|
||||
total_app_data_len,
|
||||
behavior_profile,
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: domain.to_string(),
|
||||
};
|
||||
|
||||
self.set(domain, data.clone()).await;
|
||||
self.persist(domain, &data).await;
|
||||
if quality == TlsProfileQuality::RawStrict {
|
||||
debug!(domain = %domain, len = total_app_data_len, "TLS cache updated");
|
||||
} else {
|
||||
warn!(
|
||||
domain = %domain,
|
||||
quality = profile_quality_label(quality),
|
||||
len = total_app_data_len,
|
||||
"TLS cache updated with non-strict front profile"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_entry(&self) -> Arc<CachedTlsData> {
|
||||
self.default.clone()
|
||||
}
|
||||
|
||||
pub fn disk_path(&self) -> &Path {
|
||||
&self.disk_path
|
||||
}
|
||||
}
|
||||
Vendored
+294
@@ -0,0 +1,294 @@
|
||||
use super::*;
|
||||
|
||||
fn cached_with_cert_info(
|
||||
domain: &str,
|
||||
subject_cn: Option<&str>,
|
||||
san_names: Vec<&str>,
|
||||
) -> CachedTlsData {
|
||||
CachedTlsData {
|
||||
server_hello_template: ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: Vec::new(),
|
||||
},
|
||||
cert_info: Some(crate::tls_front::types::ParsedCertificateInfo {
|
||||
not_after_unix: None,
|
||||
not_before_unix: None,
|
||||
issuer_cn: None,
|
||||
subject_cn: subject_cn.map(str::to_string),
|
||||
san_names: san_names.into_iter().map(str::to_string).collect(),
|
||||
}),
|
||||
cert_payload: None,
|
||||
app_data_records_sizes: vec![1024],
|
||||
total_app_data_len: 1024,
|
||||
behavior_profile: TlsBehaviorProfile::default(),
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: domain.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_accepts_exact_san() {
|
||||
let cached = cached_with_cert_info("b.com", Some("a.com"), vec!["b.com"]);
|
||||
assert!(cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_rejects_wrong_san() {
|
||||
let cached = cached_with_cert_info("b.com", Some("b.com"), vec!["a.com"]);
|
||||
assert!(!cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_accepts_single_label_wildcard_san() {
|
||||
let cached = cached_with_cert_info("api.b.com", None, vec!["*.b.com"]);
|
||||
assert!(cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cert_info_domain_match_rejects_multi_label_wildcard_san() {
|
||||
let cached = cached_with_cert_info("deep.api.b.com", None, vec!["*.b.com"]);
|
||||
assert!(!cert_info_matches_domain(&cached));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_profile_domains_reports_only_unprepared_entries() {
|
||||
let domains = vec!["ready.example".to_string(), "pending.example".to_string()];
|
||||
let cache = TlsFrontCache::new(&domains, 1024, "tlsfront-test-cache");
|
||||
cache
|
||||
.set(
|
||||
"ready.example",
|
||||
cached_with_cert_info("ready.example", None, Vec::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
cache.default_profile_domains(&domains).await,
|
||||
vec!["pending.example".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_uses_ttl() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let ttl = Duration::from_millis(80);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
!cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(90)).await;
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_zero_ttl_always_allows_full_payload() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ttl = Duration::ZERO;
|
||||
|
||||
for idx in 0..100_000u32 {
|
||||
let ip = IpAddr::V4(std::net::Ipv4Addr::new(
|
||||
10,
|
||||
((idx >> 16) & 0xff) as u8,
|
||||
((idx >> 8) & 0xff) as u8,
|
||||
(idx & 0xff) as u8,
|
||||
));
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, ttl)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
assert!(cache.full_cert_sent_is_empty_for_tests().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_sweeps_expired_entries_when_due() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(1);
|
||||
let stale_expires_at = Instant::now()
|
||||
.checked_sub(Duration::from_secs(1))
|
||||
.unwrap_or_else(Instant::now);
|
||||
|
||||
cache
|
||||
.insert_full_cert_sent_for_tests("example.com", stale_ip, stale_expires_at)
|
||||
.await;
|
||||
let stale_key = FullCertBudgetKey {
|
||||
domain: cache.full_cert_domain_key("example.com"),
|
||||
client_ip: stale_ip,
|
||||
};
|
||||
cache.full_cert_budget.sweep_cursor.store(
|
||||
cache.full_cert_budget.shard_index(&stale_key),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
cache
|
||||
.full_cert_budget
|
||||
.last_sweep_epoch_secs
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", new_ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
assert!(
|
||||
!cache
|
||||
.full_cert_sent_contains_for_tests("example.com", stale_ip)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", new_ip)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_take_full_cert_budget_for_ip_does_not_sweep_every_call() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(1);
|
||||
let stale_expires_at = Instant::now()
|
||||
.checked_sub(Duration::from_secs(1))
|
||||
.unwrap_or_else(Instant::now);
|
||||
let now_epoch_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
cache
|
||||
.insert_full_cert_sent_for_tests("example.com", stale_ip, stale_expires_at)
|
||||
.await;
|
||||
cache
|
||||
.full_cert_budget
|
||||
.last_sweep_epoch_secs
|
||||
.store(now_epoch_secs, Ordering::Relaxed);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", new_ip, ttl)
|
||||
.await
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", stale_ip)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.full_cert_sent_contains_for_tests("example.com", new_ip)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cert_budget_is_shared_across_cache_generations_and_scoped_by_domain() {
|
||||
let budget = Arc::new(TlsFullCertBudget::new());
|
||||
let domains = ["one.example".to_string(), "two.example".to_string()];
|
||||
let first = TlsFrontCache::new_with_full_cert_budget(
|
||||
&domains,
|
||||
1024,
|
||||
"tlsfront-test-cache",
|
||||
budget.clone(),
|
||||
);
|
||||
let second =
|
||||
TlsFrontCache::new_with_full_cert_budget(&domains, 1024, "tlsfront-test-cache", budget);
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
let ttl = Duration::from_secs(60);
|
||||
|
||||
assert!(
|
||||
first
|
||||
.take_full_cert_budget_for_ip("one.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
!second
|
||||
.take_full_cert_budget_for_ip("one.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert!(
|
||||
second
|
||||
.take_full_cert_budget_for_ip("two.example", ip, ttl)
|
||||
.await
|
||||
);
|
||||
assert_eq!(second.full_cert_budget_entries_for_metrics(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_full_cert_entry_keeps_its_own_expiry_after_ttl_change() {
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(80))
|
||||
.await
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
assert!(
|
||||
!cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(1))
|
||||
.await
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(70)).await;
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip("example.com", ip, Duration::from_millis(1))
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_reader_rejects_an_entry_above_the_hard_limit() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("oversized.json");
|
||||
tokio::fs::write(
|
||||
&path,
|
||||
vec![0u8; TLS_FRONT_DISK_ENTRY_MAX_BYTES as usize + 1],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = read_disk_entry_bounded(&path).await.unwrap_err();
|
||||
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn disk_loader_does_not_follow_a_configured_name_symlink() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let target = directory.path().join("outside.json");
|
||||
let cached = cached_with_cert_info("example.com", None, Vec::new());
|
||||
tokio::fs::write(&target, serde_json::to_vec(&cached).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
std::os::unix::fs::symlink(&target, directory.path().join("example.com.json")).unwrap();
|
||||
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, directory.path());
|
||||
|
||||
cache.load_from_disk().await;
|
||||
|
||||
assert_eq!(cache.get("example.com").await.domain, "default");
|
||||
}
|
||||
+20
-1651
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,387 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn build_client_config(alpn_protocols: &[&[u8]]) -> Arc<ClientConfig> {
|
||||
let root = rustls::RootCertStore::empty();
|
||||
|
||||
let provider = rustls::crypto::ring::default_provider();
|
||||
let mut config = ClientConfig::builder_with_provider(Arc::new(provider))
|
||||
.with_protocol_versions(&[&rustls::version::TLS13, &rustls::version::TLS12])
|
||||
.expect("protocol versions")
|
||||
.with_root_certificates(root)
|
||||
.with_no_client_auth();
|
||||
|
||||
config
|
||||
.dangerous()
|
||||
.set_certificate_verifier(Arc::new(NoVerify));
|
||||
config.alpn_protocols = alpn_protocols.iter().map(|proto| proto.to_vec()).collect();
|
||||
|
||||
Arc::new(config)
|
||||
}
|
||||
|
||||
pub(super) fn deterministic_bytes(seed: &str, len: usize) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len);
|
||||
let mut counter: u32 = 0;
|
||||
while out.len() < len {
|
||||
let mut chunk_seed = Vec::with_capacity(seed.len() + std::mem::size_of::<u32>());
|
||||
chunk_seed.extend_from_slice(seed.as_bytes());
|
||||
chunk_seed.extend_from_slice(&counter.to_le_bytes());
|
||||
out.extend_from_slice(&sha256(&chunk_seed));
|
||||
counter = counter.wrapping_add(1);
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) fn profile_cipher_suites(profile: TlsFetchProfile) -> &'static [u16] {
|
||||
const MODERN_CHROME: &[u16] = &[
|
||||
0x1301, 0x1302, 0x1303, 0xc02b, 0xc02c, 0xcca9, 0xc02f, 0xc030, 0xcca8, 0x009e, 0x00ff,
|
||||
];
|
||||
const MODERN_FIREFOX: &[u16] = &[
|
||||
0x1301, 0x1303, 0x1302, 0xc02b, 0xcca9, 0xc02c, 0xc02f, 0xcca8, 0xc030, 0x009e, 0x00ff,
|
||||
];
|
||||
const COMPAT_TLS12: &[u16] = &[
|
||||
0xc02b, 0xc02c, 0xc02f, 0xc030, 0xcca9, 0xcca8, 0x1301, 0x1302, 0x1303, 0x009e, 0x00ff,
|
||||
];
|
||||
const LEGACY_MINIMAL: &[u16] = &[0xc02b, 0xc02f, 0x1301, 0x1302, 0x00ff];
|
||||
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike => MODERN_CHROME,
|
||||
TlsFetchProfile::ModernFirefoxLike => MODERN_FIREFOX,
|
||||
TlsFetchProfile::CompatTls12 => COMPAT_TLS12,
|
||||
TlsFetchProfile::LegacyMinimal => LEGACY_MINIMAL,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_groups(profile: TlsFetchProfile) -> &'static [u16] {
|
||||
const MODERN: &[u16] = &[
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
TLS_NAMED_GROUP_X25519,
|
||||
0x0017,
|
||||
0x0018,
|
||||
];
|
||||
const COMPAT: &[u16] = &[TLS_NAMED_GROUP_X25519, 0x0017];
|
||||
const LEGACY: &[u16] = &[0x0017];
|
||||
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => MODERN,
|
||||
TlsFetchProfile::CompatTls12 => COMPAT,
|
||||
TlsFetchProfile::LegacyMinimal => LEGACY,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_sig_algs(profile: TlsFetchProfile) -> &'static [u16] {
|
||||
const MODERN: &[u16] = &[0x0804, 0x0805, 0x0403, 0x0503, 0x0806];
|
||||
const COMPAT: &[u16] = &[0x0403, 0x0503, 0x0804, 0x0805];
|
||||
const LEGACY: &[u16] = &[0x0403, 0x0804];
|
||||
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => MODERN,
|
||||
TlsFetchProfile::CompatTls12 => COMPAT,
|
||||
TlsFetchProfile::LegacyMinimal => LEGACY,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_alpn(profile: TlsFetchProfile) -> &'static [&'static [u8]] {
|
||||
const H2_HTTP11: &[&[u8]] = &[b"h2", b"http/1.1"];
|
||||
const HTTP11: &[&[u8]] = &[b"http/1.1"];
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => H2_HTTP11,
|
||||
TlsFetchProfile::CompatTls12 | TlsFetchProfile::LegacyMinimal => HTTP11,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_alpn_labels(profile: TlsFetchProfile) -> &'static [&'static str] {
|
||||
const H2_HTTP11: &[&str] = &["h2", "http/1.1"];
|
||||
const HTTP11: &[&str] = &["http/1.1"];
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => H2_HTTP11,
|
||||
TlsFetchProfile::CompatTls12 | TlsFetchProfile::LegacyMinimal => HTTP11,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_session_id_len(profile: TlsFetchProfile) -> usize {
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => 32,
|
||||
TlsFetchProfile::CompatTls12 | TlsFetchProfile::LegacyMinimal => 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_supported_versions(profile: TlsFetchProfile) -> &'static [u16] {
|
||||
const MODERN: &[u16] = &[0x0304, 0x0303];
|
||||
const COMPAT: &[u16] = &[0x0303, 0x0304];
|
||||
const LEGACY: &[u16] = &[0x0303];
|
||||
match profile {
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => MODERN,
|
||||
TlsFetchProfile::CompatTls12 => COMPAT,
|
||||
TlsFetchProfile::LegacyMinimal => LEGACY,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn profile_padding_target(profile: TlsFetchProfile) -> usize {
|
||||
match profile {
|
||||
// X25519MLKEM768 makes the Chrome-like ClientHello much larger than
|
||||
// legacy pre-hybrid profiles; keep enough headroom for padding.
|
||||
TlsFetchProfile::ModernChromeLike => 1450,
|
||||
TlsFetchProfile::ModernFirefoxLike => 200,
|
||||
TlsFetchProfile::CompatTls12 => 180,
|
||||
TlsFetchProfile::LegacyMinimal => 64,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn grease_value(rng: &SecureRandom, deterministic: bool, seed: &str) -> u16 {
|
||||
const GREASE_VALUES: [u16; 16] = [
|
||||
0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, 0x8a8a, 0x9a9a, 0xaaaa,
|
||||
0xbaba, 0xcaca, 0xdada, 0xeaea, 0xfafa,
|
||||
];
|
||||
if deterministic {
|
||||
let idx = deterministic_bytes(seed, 1)[0] as usize % GREASE_VALUES.len();
|
||||
GREASE_VALUES[idx]
|
||||
} else {
|
||||
let idx = (rng.bytes(1)[0] as usize) % GREASE_VALUES.len();
|
||||
GREASE_VALUES[idx]
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn gen_mlkem768_client_encapsulation_key(
|
||||
rng: &SecureRandom,
|
||||
deterministic: bool,
|
||||
seed: &str,
|
||||
) -> Option<Vec<u8>> {
|
||||
let seed_bytes = if deterministic {
|
||||
deterministic_bytes(seed, 64)
|
||||
} else {
|
||||
rng.bytes(64)
|
||||
};
|
||||
let seed = MlKemSeed::try_from(seed_bytes.as_slice()).ok()?;
|
||||
let decapsulation_key = MlKemDecapsulationKey::<MlKem768>::from_seed(seed);
|
||||
let encapsulation_key = decapsulation_key.encapsulation_key().to_bytes();
|
||||
let bytes = encapsulation_key.as_slice();
|
||||
if bytes.len() == MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN {
|
||||
Some(bytes.to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn gen_x25519mlkem768_client_key_share(
|
||||
rng: &SecureRandom,
|
||||
deterministic: bool,
|
||||
seed: &str,
|
||||
) -> Option<Vec<u8>> {
|
||||
let mlkem_key =
|
||||
gen_mlkem768_client_encapsulation_key(rng, deterministic, &format!("{seed}:mlkem768"))?;
|
||||
let x25519_key = gen_key_share(rng, deterministic, &format!("{seed}:x25519"));
|
||||
let mut key_share =
|
||||
Vec::with_capacity(MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN + x25519_key.len());
|
||||
key_share.extend_from_slice(&mlkem_key);
|
||||
key_share.extend_from_slice(&x25519_key);
|
||||
Some(key_share)
|
||||
}
|
||||
|
||||
pub(super) fn push_client_key_share_entry(keyshare: &mut Vec<u8>, group: u16, key: &[u8]) {
|
||||
keyshare.extend_from_slice(&group.to_be_bytes());
|
||||
keyshare.extend_from_slice(&(key.len() as u16).to_be_bytes());
|
||||
keyshare.extend_from_slice(key);
|
||||
}
|
||||
|
||||
pub(super) fn build_client_hello(
|
||||
sni: &str,
|
||||
rng: &SecureRandom,
|
||||
profile: TlsFetchProfile,
|
||||
grease_enabled: bool,
|
||||
deterministic: bool,
|
||||
) -> Vec<u8> {
|
||||
// === ClientHello body ===
|
||||
let mut body = Vec::new();
|
||||
|
||||
// Legacy version (TLS 1.0) as in real ClientHello headers
|
||||
body.extend_from_slice(&[0x03, 0x03]);
|
||||
|
||||
// Random
|
||||
if deterministic {
|
||||
body.extend_from_slice(&deterministic_bytes(&format!("tls-fetch-random:{sni}"), 32));
|
||||
} else {
|
||||
body.extend_from_slice(&rng.bytes(32));
|
||||
}
|
||||
|
||||
// Use non-empty Session ID for modern TLS 1.3-like profiles to reduce middlebox friction.
|
||||
let session_id_len = profile_session_id_len(profile);
|
||||
let session_id = if session_id_len == 0 {
|
||||
Vec::new()
|
||||
} else if deterministic {
|
||||
deterministic_bytes(
|
||||
&format!("tls-fetch-session:{sni}:{}", profile.as_str()),
|
||||
session_id_len,
|
||||
)
|
||||
} else {
|
||||
rng.bytes(session_id_len)
|
||||
};
|
||||
body.push(session_id.len() as u8);
|
||||
body.extend_from_slice(&session_id);
|
||||
|
||||
let mut cipher_suites = profile_cipher_suites(profile).to_vec();
|
||||
if grease_enabled {
|
||||
let grease = grease_value(rng, deterministic, &format!("cipher:{sni}"));
|
||||
cipher_suites.insert(0, grease);
|
||||
}
|
||||
body.extend_from_slice(&((cipher_suites.len() * 2) as u16).to_be_bytes());
|
||||
for suite in cipher_suites {
|
||||
body.extend_from_slice(&suite.to_be_bytes());
|
||||
}
|
||||
|
||||
// Compression methods: null only
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
|
||||
// === Extensions ===
|
||||
let mut exts = Vec::new();
|
||||
|
||||
let mut push_extension = |ext_type: u16, data: &[u8]| {
|
||||
exts.extend_from_slice(&ext_type.to_be_bytes());
|
||||
exts.extend_from_slice(&(data.len() as u16).to_be_bytes());
|
||||
exts.extend_from_slice(data);
|
||||
};
|
||||
|
||||
// server_name (SNI)
|
||||
let sni_bytes = sni.as_bytes();
|
||||
let mut sni_ext = Vec::with_capacity(5 + sni_bytes.len());
|
||||
sni_ext.extend_from_slice(&(sni_bytes.len() as u16 + 3).to_be_bytes());
|
||||
sni_ext.push(0);
|
||||
sni_ext.extend_from_slice(&(sni_bytes.len() as u16).to_be_bytes());
|
||||
sni_ext.extend_from_slice(sni_bytes);
|
||||
push_extension(0x0000, &sni_ext);
|
||||
|
||||
// Chrome-like profile keeps browser-like ordering and extension set.
|
||||
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
|
||||
// ec_point_formats: uncompressed only.
|
||||
push_extension(0x000b, &[0x01, 0x00]);
|
||||
}
|
||||
|
||||
// supported_groups
|
||||
let mut groups = profile_groups(profile).to_vec();
|
||||
if grease_enabled {
|
||||
let grease = grease_value(rng, deterministic, &format!("group:{sni}"));
|
||||
groups.insert(0, grease);
|
||||
}
|
||||
let mut groups_ext = Vec::with_capacity(2 + groups.len() * 2);
|
||||
groups_ext.extend_from_slice(&(groups.len() as u16 * 2).to_be_bytes());
|
||||
for g in groups {
|
||||
groups_ext.extend_from_slice(&g.to_be_bytes());
|
||||
}
|
||||
push_extension(0x000a, &groups_ext);
|
||||
|
||||
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
|
||||
// session_ticket
|
||||
push_extension(0x0023, &[]);
|
||||
}
|
||||
|
||||
// signature_algorithms
|
||||
let mut sig_algs = profile_sig_algs(profile).to_vec();
|
||||
if grease_enabled {
|
||||
let grease = grease_value(rng, deterministic, &format!("sigalg:{sni}"));
|
||||
sig_algs.insert(0, grease);
|
||||
}
|
||||
let mut sig_algs_ext = Vec::with_capacity(2 + sig_algs.len() * 2);
|
||||
sig_algs_ext.extend_from_slice(&(sig_algs.len() as u16 * 2).to_be_bytes());
|
||||
for a in sig_algs {
|
||||
sig_algs_ext.extend_from_slice(&a.to_be_bytes());
|
||||
}
|
||||
push_extension(0x000d, &sig_algs_ext);
|
||||
|
||||
// supported_versions
|
||||
let mut versions = profile_supported_versions(profile).to_vec();
|
||||
if grease_enabled {
|
||||
let grease = grease_value(rng, deterministic, &format!("version:{sni}"));
|
||||
versions.insert(0, grease);
|
||||
}
|
||||
let mut versions_ext = Vec::with_capacity(1 + versions.len() * 2);
|
||||
versions_ext.push((versions.len() * 2) as u8);
|
||||
for v in versions {
|
||||
versions_ext.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
push_extension(0x002b, &versions_ext);
|
||||
|
||||
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
|
||||
// psk_key_exchange_modes: psk_dhe_ke
|
||||
push_extension(0x002d, &[0x01, 0x01]);
|
||||
}
|
||||
|
||||
// key_share
|
||||
let key_share_seed = format!("tls-fetch-keyshare:{sni}:{}", profile.as_str());
|
||||
let mut keyshare = Vec::new();
|
||||
if matches!(
|
||||
profile,
|
||||
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike
|
||||
) {
|
||||
if let Some(key) = gen_x25519mlkem768_client_key_share(rng, deterministic, &key_share_seed)
|
||||
{
|
||||
push_client_key_share_entry(&mut keyshare, TLS_NAMED_GROUP_X25519MLKEM768, &key);
|
||||
}
|
||||
}
|
||||
let key = gen_key_share(rng, deterministic, &key_share_seed);
|
||||
push_client_key_share_entry(&mut keyshare, TLS_NAMED_GROUP_X25519, &key);
|
||||
let mut keyshare_ext = Vec::with_capacity(2 + keyshare.len());
|
||||
keyshare_ext.extend_from_slice(&(keyshare.len() as u16).to_be_bytes());
|
||||
keyshare_ext.extend_from_slice(&keyshare);
|
||||
push_extension(0x0033, &keyshare_ext);
|
||||
|
||||
// ALPN
|
||||
let mut alpn_list = Vec::new();
|
||||
for proto in profile_alpn(profile) {
|
||||
alpn_list.push(proto.len() as u8);
|
||||
alpn_list.extend_from_slice(proto);
|
||||
}
|
||||
if !alpn_list.is_empty() {
|
||||
let mut alpn_ext = Vec::with_capacity(2 + alpn_list.len());
|
||||
alpn_ext.extend_from_slice(&(alpn_list.len() as u16).to_be_bytes());
|
||||
alpn_ext.extend_from_slice(&alpn_list);
|
||||
push_extension(0x0010, &alpn_ext);
|
||||
}
|
||||
|
||||
if grease_enabled {
|
||||
let grease = grease_value(rng, deterministic, &format!("ext:{sni}"));
|
||||
push_extension(grease, &[]);
|
||||
}
|
||||
|
||||
// padding to reduce recognizability and keep length ~500 bytes
|
||||
let target_ext_len = profile_padding_target(profile);
|
||||
if exts.len() < target_ext_len {
|
||||
let remaining = target_ext_len - exts.len();
|
||||
if remaining > 4 {
|
||||
let pad_len = remaining - 4; // minus type+len
|
||||
exts.extend_from_slice(&0x0015u16.to_be_bytes()); // padding extension
|
||||
exts.extend_from_slice(&(pad_len as u16).to_be_bytes());
|
||||
exts.resize(exts.len() + pad_len, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Extensions length prefix
|
||||
body.extend_from_slice(&(exts.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&exts);
|
||||
|
||||
// === Handshake wrapper ===
|
||||
let mut handshake = Vec::new();
|
||||
handshake.push(0x01); // ClientHello
|
||||
let len_bytes = (body.len() as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&len_bytes[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
// === Record ===
|
||||
let mut record = Vec::new();
|
||||
record.push(TLS_RECORD_HANDSHAKE);
|
||||
record.extend_from_slice(&[0x03, 0x01]); // legacy record version
|
||||
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
|
||||
record.extend_from_slice(&handshake);
|
||||
|
||||
record
|
||||
}
|
||||
|
||||
pub(super) fn gen_key_share(rng: &SecureRandom, deterministic: bool, seed: &str) -> [u8; 32] {
|
||||
let mut scalar = [0u8; 32];
|
||||
if deterministic {
|
||||
scalar.copy_from_slice(&deterministic_bytes(seed, 32));
|
||||
} else {
|
||||
scalar.copy_from_slice(&rng.bytes(32));
|
||||
}
|
||||
x25519(scalar, X25519_BASEPOINT_BYTES)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn connect_with_dns_override(
|
||||
host: &str,
|
||||
port: u16,
|
||||
connect_timeout: Duration,
|
||||
) -> Result<TcpStream> {
|
||||
Ok(timeout(connect_timeout, TcpStream::connect((host, port))).await??)
|
||||
}
|
||||
|
||||
pub(super) async fn connect_tcp_with_upstream(
|
||||
host: &str,
|
||||
port: u16,
|
||||
connect_timeout: Duration,
|
||||
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
|
||||
scope: Option<&str>,
|
||||
strict_route: bool,
|
||||
) -> Result<UpstreamStream> {
|
||||
if let Some(manager) = upstream {
|
||||
let resolved = match manager.resolve_hostname(host, port).await {
|
||||
Ok(addr) => Some(addr),
|
||||
Err(e) => {
|
||||
if strict_route {
|
||||
return Err(anyhow!(
|
||||
"upstream route DNS resolution failed for {host}:{port}: {e}"
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
host = %host,
|
||||
port = port,
|
||||
scope = ?scope,
|
||||
error = %e,
|
||||
"Upstream DNS resolution failed, using direct connect"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(addr) = resolved {
|
||||
match manager.connect(addr, None, scope).await {
|
||||
Ok(stream) => return Ok(stream),
|
||||
Err(e) => {
|
||||
if strict_route {
|
||||
return Err(anyhow!(
|
||||
"upstream route connect failed for {host}:{port}: {e}"
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
host = %host,
|
||||
port = port,
|
||||
scope = ?scope,
|
||||
error = %e,
|
||||
"Upstream connect failed, using direct connect"
|
||||
);
|
||||
return Ok(UpstreamStream::Tcp(
|
||||
timeout(connect_timeout, TcpStream::connect(addr)).await??,
|
||||
));
|
||||
}
|
||||
}
|
||||
} else if strict_route {
|
||||
return Err(anyhow!(
|
||||
"upstream route resolution produced no usable address for {host}:{port}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(UpstreamStream::Tcp(
|
||||
connect_with_dns_override(host, port, connect_timeout).await?,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn socket_addrs_from_upstream_stream(
|
||||
stream: &UpstreamStream,
|
||||
) -> (Option<SocketAddr>, Option<SocketAddr>) {
|
||||
match stream {
|
||||
UpstreamStream::Tcp(tcp) => (tcp.local_addr().ok(), tcp.peer_addr().ok()),
|
||||
UpstreamStream::Shadowsocks(_) => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_tls_fetch_proxy_header(
|
||||
proxy_protocol: u8,
|
||||
src_addr: Option<SocketAddr>,
|
||||
dst_addr: Option<SocketAddr>,
|
||||
) -> Option<Vec<u8>> {
|
||||
match proxy_protocol {
|
||||
0 => None,
|
||||
2 => {
|
||||
let header = match (src_addr, dst_addr) {
|
||||
(Some(src @ SocketAddr::V4(_)), Some(dst @ SocketAddr::V4(_)))
|
||||
| (Some(src @ SocketAddr::V6(_)), Some(dst @ SocketAddr::V6(_))) => {
|
||||
ProxyProtocolV2Builder::new().with_addrs(src, dst).build()
|
||||
}
|
||||
_ => ProxyProtocolV2Builder::new().build(),
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
_ => {
|
||||
let header = match (src_addr, dst_addr) {
|
||||
(Some(SocketAddr::V4(src)), Some(SocketAddr::V4(dst))) => {
|
||||
ProxyProtocolV1Builder::new()
|
||||
.tcp4(src.into(), dst.into())
|
||||
.build()
|
||||
}
|
||||
(Some(SocketAddr::V6(src)), Some(SocketAddr::V6(dst))) => {
|
||||
ProxyProtocolV1Builder::new()
|
||||
.tcp6(src.into(), dst.into())
|
||||
.build()
|
||||
}
|
||||
_ => ProxyProtocolV1Builder::new().build(),
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn encode_tls13_certificate_message(cert_chain_der: &[Vec<u8>]) -> Option<Vec<u8>> {
|
||||
if cert_chain_der.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut certificate_list = Vec::new();
|
||||
for cert in cert_chain_der {
|
||||
if cert.is_empty() {
|
||||
return None;
|
||||
}
|
||||
certificate_list.extend_from_slice(&u24_bytes(cert.len())?);
|
||||
certificate_list.extend_from_slice(cert);
|
||||
certificate_list.extend_from_slice(&0u16.to_be_bytes()); // cert_entry extensions
|
||||
}
|
||||
|
||||
// Certificate = context_len(1) + certificate_list_len(3) + entries
|
||||
let body_len = 1usize.checked_add(3)?.checked_add(certificate_list.len())?;
|
||||
|
||||
let mut message = Vec::with_capacity(4 + body_len);
|
||||
message.push(0x0b); // HandshakeType::certificate
|
||||
message.extend_from_slice(&u24_bytes(body_len)?);
|
||||
message.push(0x00); // certificate_request_context length
|
||||
message.extend_from_slice(&u24_bytes(certificate_list.len())?);
|
||||
message.extend_from_slice(&certificate_list);
|
||||
Some(message)
|
||||
}
|
||||
|
||||
pub(super) async fn fetch_via_raw_tls_stream<S>(
|
||||
mut stream: S,
|
||||
sni: &str,
|
||||
connect_timeout: Duration,
|
||||
proxy_header: Option<Vec<u8>>,
|
||||
profile: TlsFetchProfile,
|
||||
grease_enabled: bool,
|
||||
deterministic: bool,
|
||||
) -> Result<TlsFetchResult>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin,
|
||||
{
|
||||
let rng = SecureRandom::new();
|
||||
let client_hello = build_client_hello(sni, &rng, profile, grease_enabled, deterministic);
|
||||
timeout(connect_timeout, async {
|
||||
if let Some(header) = proxy_header.as_ref() {
|
||||
stream.write_all(&header).await?;
|
||||
}
|
||||
stream.write_all(&client_hello).await?;
|
||||
stream.flush().await?;
|
||||
Ok::<(), std::io::Error>(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
let mut records = Vec::new();
|
||||
let mut app_records_seen = 0usize;
|
||||
// Read a bounded encrypted flight: ServerHello, CCS, certificate-like data,
|
||||
// and a small number of ticket-like tail records.
|
||||
for _ in 0..8 {
|
||||
match timeout(connect_timeout, read_tls_record(&mut stream)).await {
|
||||
Ok(Ok(rec)) => {
|
||||
if rec.0 == TLS_RECORD_APPLICATION {
|
||||
app_records_seen += 1;
|
||||
}
|
||||
records.push(rec);
|
||||
}
|
||||
Ok(Err(e)) => return Err(e),
|
||||
Err(_) => break,
|
||||
}
|
||||
if app_records_seen >= 4 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut server_hello = None;
|
||||
let mut server_hello_record_len = 0usize;
|
||||
for (t, body) in &records {
|
||||
if *t == TLS_RECORD_HANDSHAKE && server_hello.is_none() {
|
||||
server_hello = parse_server_hello(body);
|
||||
server_hello_record_len = body.len();
|
||||
}
|
||||
}
|
||||
|
||||
let parsed = server_hello.ok_or_else(|| anyhow!("ServerHello not received"))?;
|
||||
let mut behavior_profile = derive_behavior_profile(&records);
|
||||
behavior_profile.server_hello_record_len = server_hello_record_len;
|
||||
behavior_profile.refresh_server_hello_summary(&parsed);
|
||||
let mut app_sizes = behavior_profile.app_data_record_sizes.clone();
|
||||
app_sizes.extend_from_slice(&behavior_profile.ticket_record_sizes);
|
||||
let total_app_data_len = app_sizes.iter().sum::<usize>().max(1024);
|
||||
let app_data_records_sizes = if app_sizes.is_empty() {
|
||||
vec![total_app_data_len]
|
||||
} else {
|
||||
app_sizes
|
||||
};
|
||||
|
||||
Ok(TlsFetchResult {
|
||||
server_hello_parsed: parsed,
|
||||
app_data_records_sizes,
|
||||
total_app_data_len,
|
||||
behavior_profile,
|
||||
cert_info: None,
|
||||
cert_payload: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn fetch_via_raw_tls(
|
||||
host: &str,
|
||||
port: u16,
|
||||
sni: &str,
|
||||
connect_timeout: Duration,
|
||||
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
|
||||
scope: Option<&str>,
|
||||
proxy_protocol: u8,
|
||||
unix_sock: Option<&str>,
|
||||
strict_route: bool,
|
||||
profile: TlsFetchProfile,
|
||||
grease_enabled: bool,
|
||||
deterministic: bool,
|
||||
) -> Result<TlsFetchResult> {
|
||||
#[cfg(unix)]
|
||||
if let Some(sock_path) = unix_sock {
|
||||
match timeout(connect_timeout, UnixStream::connect(sock_path)).await {
|
||||
Ok(Ok(stream)) => {
|
||||
debug!(
|
||||
sni = %sni,
|
||||
sock = %sock_path,
|
||||
"Raw TLS fetch using mask unix socket"
|
||||
);
|
||||
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, None, None);
|
||||
return fetch_via_raw_tls_stream(
|
||||
stream,
|
||||
sni,
|
||||
connect_timeout,
|
||||
proxy_header,
|
||||
profile,
|
||||
grease_enabled,
|
||||
deterministic,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(
|
||||
sni = %sni,
|
||||
sock = %sock_path,
|
||||
error = %e,
|
||||
"Raw TLS unix socket connect failed, falling back to TCP"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
sni = %sni,
|
||||
sock = %sock_path,
|
||||
"Raw TLS unix socket connect timed out, falling back to TCP"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let _ = unix_sock;
|
||||
|
||||
let stream =
|
||||
connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope, strict_route)
|
||||
.await?;
|
||||
let (src_addr, dst_addr) = socket_addrs_from_upstream_stream(&stream);
|
||||
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, src_addr, dst_addr);
|
||||
fetch_via_raw_tls_stream(
|
||||
stream,
|
||||
sni,
|
||||
connect_timeout,
|
||||
proxy_header,
|
||||
profile,
|
||||
grease_enabled,
|
||||
deterministic,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user