mirror of
https://github.com/telemt/telemt.git
synced 2026-09-13 22:14:08 +03:00
Split oversized runtime modules + Async tests hardened
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
+17
-1551
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
/// Main dispatch after successful handshake.
|
||||
/// Two modes:
|
||||
/// - Direct: TCP relay to TG DC (existing behavior)
|
||||
/// - Middle Proxy: RPC multiplex through ME pool (supports CDN DCs)
|
||||
#[cfg(test)]
|
||||
pub(super) async fn handle_authenticated_static<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
local_addr: SocketAddr,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
None,
|
||||
route_runtime,
|
||||
local_addr,
|
||||
peer_addr,
|
||||
ip_tracker,
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_authenticated_static_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
local_addr: SocketAddr,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
run_authenticated(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
ClientRuntimeDeps {
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime,
|
||||
ip_tracker,
|
||||
shared,
|
||||
},
|
||||
local_addr,
|
||||
peer_addr,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn acquire_user_connection_reservation_static(
|
||||
user: &str,
|
||||
config: &ProxyConfig,
|
||||
stats: Arc<Stats>,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
) -> Result<UserConnectionReservation> {
|
||||
acquire_user_connection_reservation(user, config, stats, peer_addr, ip_tracker).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn check_user_limits_static(
|
||||
user: &str,
|
||||
config: &ProxyConfig,
|
||||
stats: &Stats,
|
||||
peer_addr: SocketAddr,
|
||||
ip_tracker: &UserIpTracker,
|
||||
) -> Result<()> {
|
||||
if let Some(expiration) = config.access.user_expirations.get(user)
|
||||
&& chrono::Utc::now() > *expiration
|
||||
{
|
||||
return Err(ProxyError::UserExpired {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(quota) = config.access.user_data_quota.get(user)
|
||||
&& stats.get_user_quota_used(user) >= *quota
|
||||
{
|
||||
return Err(ProxyError::DataQuotaExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let limit = config
|
||||
.access
|
||||
.user_max_tcp_conns
|
||||
.get(user)
|
||||
.copied()
|
||||
.filter(|limit| *limit > 0)
|
||||
.or((config.access.user_max_tcp_conns_global_each > 0)
|
||||
.then_some(config.access.user_max_tcp_conns_global_each))
|
||||
.map(|v| v as u64);
|
||||
if !stats.try_acquire_user_curr_connects(user, limit) {
|
||||
return Err(ProxyError::ConnectionLimitExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
match ip_tracker.check_and_add(user, peer_addr.ip()).await {
|
||||
Ok(()) => {
|
||||
ip_tracker.remove_ip(user, peer_addr.ip()).await;
|
||||
}
|
||||
Err(reason) => {
|
||||
stats.decrement_user_curr_connects(user);
|
||||
warn!(
|
||||
user = %user,
|
||||
ip = %peer_addr.ip(),
|
||||
reason = %reason,
|
||||
"IP limit exceeded"
|
||||
);
|
||||
return Err(ProxyError::ConnectionLimitExceeded {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
stats.decrement_user_curr_connects(user);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub(super) async fn handle_direct_client(
|
||||
mut self,
|
||||
first_bytes: [u8; 5],
|
||||
local_addr: SocketAddr,
|
||||
) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
|
||||
if !self.config.general.modes.classic && !self.config.general.modes.secure {
|
||||
debug!(peer = %peer, "Non-TLS modes disabled");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("direct_modes_disabled");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = [0u8; HANDSHAKE_LEN];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
self.stream.read_exact(&mut handshake[5..]).await?;
|
||||
|
||||
let config = self.config.clone();
|
||||
let replay_checker = self.replay_checker.clone();
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&handshake,
|
||||
read_half,
|
||||
write_half,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
false,
|
||||
None,
|
||||
self.shared.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
self.upstream_manager,
|
||||
self.stats,
|
||||
self.config,
|
||||
buffer_pool,
|
||||
self.rng,
|
||||
self.me_pool,
|
||||
self.me_pool_runtime,
|
||||
self.route_runtime.clone(),
|
||||
local_addr,
|
||||
peer,
|
||||
self.ip_tracker,
|
||||
self.shared,
|
||||
),
|
||||
)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn beobachten_ttl(config: &ProxyConfig) -> Duration {
|
||||
const BEOBACHTEN_TTL_MAX_MINUTES: u64 = 24 * 60;
|
||||
let minutes = config.general.beobachten_minutes;
|
||||
if minutes == 0 {
|
||||
static BEOBACHTEN_ZERO_MINUTES_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = BEOBACHTEN_ZERO_MINUTES_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
"general.beobachten_minutes=0 is insecure because entries expire immediately; forcing minimum TTL to 1 minute"
|
||||
);
|
||||
}
|
||||
return Duration::from_secs(60);
|
||||
}
|
||||
|
||||
if minutes > BEOBACHTEN_TTL_MAX_MINUTES {
|
||||
static BEOBACHTEN_OVERSIZED_MINUTES_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = BEOBACHTEN_OVERSIZED_MINUTES_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
configured_minutes = minutes,
|
||||
max_minutes = BEOBACHTEN_TTL_MAX_MINUTES,
|
||||
"general.beobachten_minutes is too large; clamping to secure maximum"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Duration::from_secs(minutes.min(BEOBACHTEN_TTL_MAX_MINUTES).saturating_mul(60))
|
||||
}
|
||||
|
||||
pub(super) fn wrap_tls_application_record(payload: &[u8]) -> Vec<u8> {
|
||||
let chunks = payload.len().div_ceil(u16::MAX as usize).max(1);
|
||||
let mut record = Vec::with_capacity(payload.len() + 5 * chunks);
|
||||
|
||||
if payload.is_empty() {
|
||||
record.push(TLS_RECORD_APPLICATION);
|
||||
record.extend_from_slice(&TLS_VERSION);
|
||||
record.extend_from_slice(&0u16.to_be_bytes());
|
||||
return record;
|
||||
}
|
||||
|
||||
for chunk in payload.chunks(u16::MAX as usize) {
|
||||
record.push(TLS_RECORD_APPLICATION);
|
||||
record.extend_from_slice(&TLS_VERSION);
|
||||
record.extend_from_slice(&(chunk.len() as u16).to_be_bytes());
|
||||
record.extend_from_slice(chunk);
|
||||
}
|
||||
|
||||
record
|
||||
}
|
||||
|
||||
pub(super) fn tls_clienthello_len_in_bounds(tls_len: usize) -> bool {
|
||||
(MIN_TLS_CLIENT_HELLO_SIZE..=MAX_TLS_PLAINTEXT_SIZE).contains(&tls_len)
|
||||
}
|
||||
|
||||
pub(super) async fn read_with_progress<R: AsyncRead + Unpin>(
|
||||
reader: &mut R,
|
||||
mut buf: &mut [u8],
|
||||
) -> std::io::Result<usize> {
|
||||
let mut total = 0usize;
|
||||
while !buf.is_empty() {
|
||||
match reader.read(buf).await {
|
||||
Ok(0) => return Ok(total),
|
||||
Ok(n) => {
|
||||
total += n;
|
||||
let (_, rest) = buf.split_at_mut(n);
|
||||
buf = rest;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
pub(super) async fn maybe_apply_mask_reject_delay(config: &ProxyConfig) {
|
||||
let min = config.censorship.server_hello_delay_min_ms;
|
||||
let max = config.censorship.server_hello_delay_max_ms;
|
||||
if max == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let delay_ms = if min >= max {
|
||||
max
|
||||
} else {
|
||||
rand::rng().random_range(min..=max)
|
||||
};
|
||||
|
||||
if delay_ms > 0 {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn handshake_timeout_with_mask_grace(config: &ProxyConfig) -> Duration {
|
||||
let base = Duration::from_secs(config.timeouts.client_handshake);
|
||||
if config.censorship.mask {
|
||||
base.saturating_add(Duration::from_millis(750))
|
||||
} else {
|
||||
base
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn effective_client_first_byte_idle_secs(
|
||||
config: &ProxyConfig,
|
||||
shared: &ProxySharedState,
|
||||
) -> u64 {
|
||||
let idle_secs = config.timeouts.client_first_byte_idle_secs;
|
||||
if idle_secs == 0 {
|
||||
return 0;
|
||||
}
|
||||
if shared.conntrack_pressure_active() {
|
||||
idle_secs.min(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.client_first_byte_idle_cap_secs(),
|
||||
)
|
||||
} else {
|
||||
idle_secs
|
||||
}
|
||||
}
|
||||
|
||||
const MASK_CLASSIFIER_PREFETCH_WINDOW: usize = 16;
|
||||
#[cfg(test)]
|
||||
pub(super) const MASK_CLASSIFIER_PREFETCH_TIMEOUT: Duration = Duration::from_millis(5);
|
||||
|
||||
pub(super) fn mask_classifier_prefetch_timeout(config: &ProxyConfig) -> Duration {
|
||||
Duration::from_millis(config.censorship.mask_classifier_prefetch_timeout_ms)
|
||||
}
|
||||
|
||||
pub(super) fn should_prefetch_mask_classifier_window(initial_data: &[u8]) -> bool {
|
||||
if initial_data.len() >= MASK_CLASSIFIER_PREFETCH_WINDOW {
|
||||
return false;
|
||||
}
|
||||
|
||||
if initial_data.is_empty() {
|
||||
// Empty initial_data means there is no client probe prefix to refine.
|
||||
// Prefetching in this case can consume fallback relay payload bytes and
|
||||
// accidentally route them through shaping heuristics.
|
||||
return false;
|
||||
}
|
||||
|
||||
if initial_data[0] == 0x16 || initial_data.starts_with(b"SSH-") {
|
||||
return false;
|
||||
}
|
||||
|
||||
initial_data
|
||||
.iter()
|
||||
.all(|b| b.is_ascii_alphabetic() || *b == b' ')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn extend_masking_initial_window<R>(reader: &mut R, initial_data: &mut Vec<u8>)
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
extend_masking_initial_window_with_timeout(
|
||||
reader,
|
||||
initial_data,
|
||||
MASK_CLASSIFIER_PREFETCH_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn extend_masking_initial_window_with_timeout<R>(
|
||||
reader: &mut R,
|
||||
initial_data: &mut Vec<u8>,
|
||||
prefetch_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
if !should_prefetch_mask_classifier_window(initial_data) {
|
||||
return;
|
||||
}
|
||||
|
||||
let need = MASK_CLASSIFIER_PREFETCH_WINDOW.saturating_sub(initial_data.len());
|
||||
if need == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut extra = [0u8; MASK_CLASSIFIER_PREFETCH_WINDOW];
|
||||
if let Ok(Ok(n)) = timeout(prefetch_timeout, reader.read(&mut extra[..need])).await
|
||||
&& n > 0
|
||||
{
|
||||
initial_data.extend_from_slice(&extra[..n]);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn masking_outcome<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: Vec<u8>,
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> HandshakeOutcome
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
HandshakeOutcome::NeedsMasking(Box::pin(async move {
|
||||
let mut reader = reader;
|
||||
let mut initial_data = initial_data;
|
||||
extend_masking_initial_window_with_timeout(
|
||||
&mut reader,
|
||||
&mut initial_data,
|
||||
mask_classifier_prefetch_timeout(&config),
|
||||
)
|
||||
.await;
|
||||
|
||||
crate::proxy::masking::handle_bad_client_with_shared_resolver(
|
||||
reader,
|
||||
writer,
|
||||
&initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
shared.as_ref(),
|
||||
Some(upstream_manager.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn record_beobachten_class(
|
||||
beobachten: &BeobachtenStore,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
class: &str,
|
||||
) {
|
||||
if !config.general.beobachten {
|
||||
return;
|
||||
}
|
||||
beobachten.record(class, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
|
||||
pub(super) fn tls_fingerprint_collection_enabled(config: &ProxyConfig) -> bool {
|
||||
config.general.beobachten || config.server.api.runtime_edge_enabled
|
||||
}
|
||||
|
||||
pub(super) fn observe_tls_client_fingerprint(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
handshake: &[u8],
|
||||
) -> Option<TlsClientFingerprint> {
|
||||
if !tls_fingerprint_collection_enabled(config) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match tls_fingerprint::fingerprint_client_hello(handshake) {
|
||||
Some(fingerprint) => {
|
||||
stats.record_tls_fingerprint_observed(&fingerprint, peer_ip, beobachten_ttl(config));
|
||||
Some(fingerprint)
|
||||
}
|
||||
None => {
|
||||
stats.increment_tls_fingerprint_parse_error();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_tls_fingerprint_auth_success(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
user: &str,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_auth_success(
|
||||
fingerprint,
|
||||
peer_ip,
|
||||
user,
|
||||
beobachten_ttl(config),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_tls_fingerprint_bad_or_probe(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_bad_or_probe(fingerprint, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn classify_expected_64_got_0(kind: std::io::ErrorKind) -> Option<&'static str> {
|
||||
match kind {
|
||||
std::io::ErrorKind::UnexpectedEof => Some("expected_64_got_0_unexpected_eof"),
|
||||
std::io::ErrorKind::ConnectionReset => Some("expected_64_got_0_connection_reset"),
|
||||
std::io::ErrorKind::ConnectionAborted => Some("expected_64_got_0_connection_aborted"),
|
||||
std::io::ErrorKind::BrokenPipe => Some("expected_64_got_0_broken_pipe"),
|
||||
std::io::ErrorKind::NotConnected => Some("expected_64_got_0_not_connected"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn classify_handshake_failure_class(error: &ProxyError) -> &'static str {
|
||||
match error {
|
||||
ProxyError::Io(err) => classify_expected_64_got_0(err.kind()).unwrap_or("other"),
|
||||
ProxyError::Stream(StreamError::UnexpectedEof) => "expected_64_got_0_unexpected_eof",
|
||||
ProxyError::Stream(StreamError::Io(err)) => {
|
||||
classify_expected_64_got_0(err.kind()).unwrap_or("other")
|
||||
}
|
||||
_ => "other",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_handshake_failure_class(
|
||||
beobachten: &BeobachtenStore,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
error: &ProxyError,
|
||||
) {
|
||||
// Keep beobachten buckets stable while detailed per-kind classification
|
||||
// is tracked in API counters.
|
||||
let class = match classify_handshake_failure_class(error) {
|
||||
value if value.starts_with("expected_64_got_0_") => "expected_64_got_0",
|
||||
_ => "other",
|
||||
};
|
||||
record_beobachten_class(beobachten, config, peer_ip, class);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn increment_bad_on_unknown_tls_sni(stats: &Stats, error: &ProxyError) {
|
||||
if matches!(error, ProxyError::UnknownTlsSni) {
|
||||
stats.increment_connects_bad_with_class("unknown_tls_sni");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_trusted_proxy_source(peer_ip: IpAddr, trusted: &[IpNetwork]) -> bool {
|
||||
if trusted.is_empty() {
|
||||
static EMPTY_PROXY_TRUST_WARNED: OnceLock<AtomicBool> = OnceLock::new();
|
||||
let warned = EMPTY_PROXY_TRUST_WARNED.get_or_init(|| AtomicBool::new(false));
|
||||
if !warned.swap(true, Ordering::Relaxed) {
|
||||
warn!(
|
||||
"PROXY protocol enabled but server.proxy_protocol_trusted_cidrs is empty; rejecting all PROXY headers"
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
trusted.iter().any(|cidr| cidr.contains(peer_ip))
|
||||
}
|
||||
|
||||
pub(super) fn synthetic_local_addr(port: u16) -> SocketAddr {
|
||||
SocketAddr::from(([0, 0, 0, 0], port))
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub async fn run(self) -> Result<()> {
|
||||
self.stats.increment_connects_all();
|
||||
let peer = self.peer;
|
||||
debug!(peer = %peer, "New connection");
|
||||
|
||||
if let Err(e) = configure_client_socket(
|
||||
&self.stream,
|
||||
self.config.timeouts.client_keepalive,
|
||||
self.config.timeouts.client_ack,
|
||||
) {
|
||||
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
let raw_fd = self.raw_fd;
|
||||
let rst_on_close = self.rst_on_close;
|
||||
|
||||
let outcome = match self.do_handshake().await? {
|
||||
Some(outcome) => outcome,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||
match outcome {
|
||||
HandshakeOutcome::NeedsRelay(fut) => {
|
||||
#[cfg(unix)]
|
||||
if matches!(rst_on_close, crate::config::RstOnCloseMode::Errors) {
|
||||
let _ = crate::transport::socket::clear_linger_fd(raw_fd);
|
||||
}
|
||||
fut.await
|
||||
}
|
||||
HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn do_handshake(mut self) -> Result<Option<HandshakeOutcome>> {
|
||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||
|
||||
if self.proxy_protocol_enabled {
|
||||
if !is_trusted_proxy_source(
|
||||
self.peer.ip(),
|
||||
&self.config.server.proxy_protocol_trusted_cidrs,
|
||||
) {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||
warn!(
|
||||
peer = %self.peer,
|
||||
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
|
||||
"Rejecting PROXY protocol header from untrusted source"
|
||||
);
|
||||
record_beobachten_class(&self.beobachten, &self.config, self.peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
|
||||
let proxy_header_timeout =
|
||||
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||
match timeout(
|
||||
proxy_header_timeout,
|
||||
parse_proxy_protocol(&mut self.stream, self.peer),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(info)) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
client = %info.src_addr,
|
||||
version = info.version,
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
self.peer = normalize_ip(info.src_addr);
|
||||
self.real_peer_from_proxy = Some(self.peer);
|
||||
if let Ok(mut slot) = self.real_peer_report.lock() {
|
||||
*slot = Some(self.peer);
|
||||
}
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_invalid_header");
|
||||
warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header");
|
||||
record_beobachten_class(
|
||||
&self.beobachten,
|
||||
&self.config,
|
||||
self.peer.ip(),
|
||||
"other",
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("proxy_protocol_header_timeout");
|
||||
warn!(
|
||||
peer = %self.peer,
|
||||
timeout_ms = proxy_header_timeout.as_millis(),
|
||||
"PROXY protocol header timeout"
|
||||
);
|
||||
record_beobachten_class(
|
||||
&self.beobachten,
|
||||
&self.config,
|
||||
self.peer.ip(),
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let first_byte_idle_secs =
|
||||
effective_client_first_byte_idle_secs(&self.config, self.shared.as_ref());
|
||||
let first_byte = if first_byte_idle_secs == 0 {
|
||||
None
|
||||
} else {
|
||||
let idle_timeout = Duration::from_secs(first_byte_idle_secs);
|
||||
let mut first_byte = [0u8; 1];
|
||||
match timeout(idle_timeout, self.stream.read(&mut first_byte)).await {
|
||||
Ok(Ok(0)) => {
|
||||
debug!(peer = %self.peer, "Connection closed before first client byte");
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(_)) => Some(first_byte[0]),
|
||||
Ok(Err(e))
|
||||
if matches!(
|
||||
e.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
| std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
) =>
|
||||
{
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
error = %e,
|
||||
"Connection closed before first client byte"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
error = %e,
|
||||
"Failed while waiting for first client byte"
|
||||
);
|
||||
return Err(ProxyError::Io(e));
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
peer = %self.peer,
|
||||
idle_secs = first_byte_idle_secs,
|
||||
"Closing idle pooled connection before first client byte"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let handshake_timeout = handshake_timeout_with_mask_grace(&self.config);
|
||||
let stats = self.stats.clone();
|
||||
let config_for_timeout = self.config.clone();
|
||||
let beobachten_for_timeout = self.beobachten.clone();
|
||||
let peer_for_timeout = self.peer.ip();
|
||||
let peer_for_log = self.peer;
|
||||
|
||||
let outcome = match timeout(handshake_timeout, async {
|
||||
let mut first_bytes = [0u8; 5];
|
||||
if let Some(first_byte) = first_byte {
|
||||
first_bytes[0] = first_byte;
|
||||
self.stream.read_exact(&mut first_bytes[1..]).await?;
|
||||
} else {
|
||||
self.stream.read_exact(&mut first_bytes).await?;
|
||||
}
|
||||
|
||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||
let peer = self.peer;
|
||||
|
||||
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
|
||||
|
||||
if is_tls {
|
||||
self.handle_tls_client(first_bytes, local_addr).await
|
||||
} else {
|
||||
self.handle_direct_client(first_bytes, local_addr).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(outcome)) => outcome,
|
||||
Ok(Err(e)) => {
|
||||
debug!(peer = %peer_for_log, error = %e, "Handshake failed");
|
||||
stats.increment_handshake_failure_class(classify_handshake_failure_class(&e));
|
||||
record_handshake_failure_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
&e,
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_handshake_timeouts();
|
||||
stats.increment_handshake_failure_class("timeout");
|
||||
debug!(peer = %peer_for_log, "Handshake timeout");
|
||||
record_beobachten_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::TgHandshakeTimeout);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Some(outcome))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn handle_client_stream<S>(
|
||||
stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_client_stream_with_shared(
|
||||
stream,
|
||||
peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
ProxySharedState::new(),
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(dead_code)]
|
||||
pub async fn handle_client_stream_with_shared<S>(
|
||||
stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_client_stream_with_shared_and_pool_runtime(
|
||||
stream,
|
||||
peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
None,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
shared,
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn handle_client_stream_with_shared_and_pool_runtime<S>(
|
||||
mut stream: S,
|
||||
peer: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
proxy_protocol_enabled: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
stats.increment_connects_all();
|
||||
let mut real_peer = normalize_ip(peer);
|
||||
|
||||
// For non-TCP streams, use a synthetic local address; may be overridden by PROXY protocol dst
|
||||
let mut local_addr = synthetic_local_addr(config.server.port);
|
||||
|
||||
if proxy_protocol_enabled {
|
||||
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs) {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||
warn!(
|
||||
peer = %peer,
|
||||
trusted = ?config.server.proxy_protocol_trusted_cidrs,
|
||||
"Rejecting PROXY protocol header from untrusted source"
|
||||
);
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
|
||||
let proxy_header_timeout =
|
||||
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||
match timeout(
|
||||
proxy_header_timeout,
|
||||
parse_proxy_protocol(&mut stream, peer),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(info)) => {
|
||||
debug!(
|
||||
peer = %peer,
|
||||
client = %info.src_addr,
|
||||
version = info.version,
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
real_peer = normalize_ip(info.src_addr);
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_invalid_header");
|
||||
warn!(peer = %peer, error = %e, "Invalid PROXY protocol header");
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_connects_bad_with_class("proxy_protocol_header_timeout");
|
||||
warn!(peer = %peer, timeout_ms = proxy_header_timeout.as_millis(), "PROXY protocol header timeout");
|
||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||
return Err(ProxyError::InvalidProxyProtocol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(peer = %real_peer, "New connection (generic stream)");
|
||||
|
||||
let first_byte_idle_secs = effective_client_first_byte_idle_secs(&config, shared.as_ref());
|
||||
let first_byte = if first_byte_idle_secs == 0 {
|
||||
None
|
||||
} else {
|
||||
let idle_timeout = Duration::from_secs(first_byte_idle_secs);
|
||||
let mut first_byte = [0u8; 1];
|
||||
match timeout(idle_timeout, stream.read(&mut first_byte)).await {
|
||||
Ok(Ok(0)) => {
|
||||
debug!(peer = %real_peer, "Connection closed before first client byte");
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Ok(_)) => Some(first_byte[0]),
|
||||
Ok(Err(e))
|
||||
if matches!(
|
||||
e.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
| std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
) =>
|
||||
{
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
error = %e,
|
||||
"Connection closed before first client byte"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
error = %e,
|
||||
"Failed while waiting for first client byte"
|
||||
);
|
||||
return Err(ProxyError::Io(e));
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
peer = %real_peer,
|
||||
idle_secs = first_byte_idle_secs,
|
||||
"Closing idle pooled connection before first client byte"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let handshake_timeout = handshake_timeout_with_mask_grace(&config);
|
||||
let stats_for_timeout = stats.clone();
|
||||
let config_for_timeout = config.clone();
|
||||
let beobachten_for_timeout = beobachten.clone();
|
||||
let peer_for_timeout = real_peer.ip();
|
||||
|
||||
// Phase 2: active handshake (with timeout after the first client byte)
|
||||
let outcome = match timeout(handshake_timeout, async {
|
||||
let mut first_bytes = [0u8; 5];
|
||||
if let Some(first_byte) = first_byte {
|
||||
first_bytes[0] = first_byte;
|
||||
stream.read_exact(&mut first_bytes[1..]).await?;
|
||||
} else {
|
||||
stream.read_exact(&mut first_bytes).await?;
|
||||
}
|
||||
|
||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||
debug!(peer = %real_peer, is_tls = is_tls, "Handshake type detected");
|
||||
|
||||
if is_tls {
|
||||
let tls_len = u16::from_be_bytes([first_bytes[3], first_bytes[4]]) as usize;
|
||||
|
||||
// RFC 8446 §5.1: TLS record payload MUST NOT exceed 2^14 (16_384) bytes.
|
||||
// Lower bound is a structural minimum for a valid TLS 1.3 ClientHello
|
||||
// (record header + handshake header + random + session_id + cipher_suites
|
||||
// + compression + at least one extension with SNI). The previous value of
|
||||
// 512 was implicitly coupled to TLS_REQUEST_LENGTH=517 from the official
|
||||
// Telegram MTProxy reference server, leaving only a 5-byte margin and
|
||||
// incorrectly rejecting compact but spec-compliant ClientHellos from
|
||||
// third-party clients or future Telegram versions.
|
||||
if !tls_clienthello_len_in_bounds(tls_len) {
|
||||
debug!(peer = %real_peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = vec![0u8; 5 + tls_len];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
let body_read = match read_with_progress(&mut stream, &mut handshake[5..]).await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
debug!(peer = %real_peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_read_error");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let initial_len = 5;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
if body_read < tls_len {
|
||||
debug!(peer = %real_peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
|
||||
stats.increment_connects_bad_with_class("tls_clienthello_truncated");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let initial_len = 5 + body_read;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint =
|
||||
observe_tls_client_fingerprint(stats.as_ref(), &config, real_peer.ip(), &handshake);
|
||||
|
||||
let (read_half, write_half) = tokio::io::split(stream);
|
||||
|
||||
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake_with_shared(
|
||||
&handshake, read_half, write_half, real_peer,
|
||||
&config, &replay_checker, &rng, tls_cache.clone(),
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.clone(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
let mtproto_handshake: [u8; HANDSHAKE_LEN] = mtproto_data[..].try_into()
|
||||
.map_err(|_| ProxyError::InvalidHandshake("Short MTProto handshake".into()))?;
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&mtproto_handshake, tls_reader, tls_writer, real_peer,
|
||||
&config, &replay_checker, true, Some(tls_user.as_str()),
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
// MTProto failed after TLS ServerHello was already sent.
|
||||
// Switch fallback relay back to raw transport so the mask
|
||||
// backend receives valid TLS records (not unwrapped payload).
|
||||
let (reader, pending_plaintext) = reader.into_inner_with_pending_plaintext();
|
||||
let writer = writer.into_inner();
|
||||
let pending_record = if pending_plaintext.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
wrap_tls_application_record(&pending_plaintext)
|
||||
};
|
||||
let reader = tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
|
||||
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
Vec::new(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
RunningClientHandler::handle_authenticated_static_with_shared(
|
||||
crypto_reader, crypto_writer, success,
|
||||
upstream_manager, stats, config, buffer_pool, rng, me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime.clone(),
|
||||
local_addr, real_peer, ip_tracker.clone(),
|
||||
shared.clone(),
|
||||
),
|
||||
)))
|
||||
} else {
|
||||
if !config.general.modes.classic && !config.general.modes.secure {
|
||||
debug!(peer = %real_peer, "Non-TLS modes disabled");
|
||||
stats.increment_connects_bad_with_class("direct_modes_disabled");
|
||||
maybe_apply_mask_reject_delay(&config).await;
|
||||
let (reader, writer) = tokio::io::split(stream);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = [0u8; HANDSHAKE_LEN];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
stream.read_exact(&mut handshake[5..]).await?;
|
||||
|
||||
let (read_half, write_half) = tokio::io::split(stream);
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&handshake, read_half, write_half, real_peer,
|
||||
&config, &replay_checker, false, None,
|
||||
shared.as_ref(),
|
||||
).await {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.to_vec(),
|
||||
real_peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
upstream_manager.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
RunningClientHandler::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime.clone(),
|
||||
local_addr,
|
||||
real_peer,
|
||||
ip_tracker.clone(),
|
||||
shared.clone(),
|
||||
)
|
||||
)))
|
||||
}
|
||||
}).await {
|
||||
Ok(Ok(outcome)) => outcome,
|
||||
Ok(Err(e)) => {
|
||||
debug!(peer = %peer, error = %e, "Handshake failed");
|
||||
stats_for_timeout.increment_handshake_failure_class(classify_handshake_failure_class(&e));
|
||||
record_handshake_failure_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
&e,
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Err(_) => {
|
||||
stats_for_timeout.increment_handshake_timeouts();
|
||||
stats_for_timeout.increment_handshake_failure_class("timeout");
|
||||
debug!(peer = %peer, "Handshake timeout");
|
||||
record_beobachten_class(
|
||||
&beobachten_for_timeout,
|
||||
&config_for_timeout,
|
||||
peer_for_timeout,
|
||||
"other",
|
||||
);
|
||||
return Err(ProxyError::TgHandshakeTimeout);
|
||||
}
|
||||
};
|
||||
|
||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||
match outcome {
|
||||
HandshakeOutcome::NeedsRelay(fut) | HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
use super::*;
|
||||
|
||||
impl RunningClientHandler {
|
||||
pub(super) async fn handle_tls_client(
|
||||
mut self,
|
||||
first_bytes: [u8; 5],
|
||||
local_addr: SocketAddr,
|
||||
) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
|
||||
let tls_len = u16::from_be_bytes([first_bytes[3], first_bytes[4]]) as usize;
|
||||
|
||||
debug!(peer = %peer, tls_len = tls_len, "Reading TLS handshake");
|
||||
|
||||
// RFC 8446 §5.1: TLS record payload MUST NOT exceed 2^14 (16_384) bytes.
|
||||
// Lower bound is a structural minimum for a valid TLS 1.3 ClientHello
|
||||
// (record header + handshake header + random + session_id + cipher_suites
|
||||
// + compression + at least one extension with SNI). The previous value of
|
||||
// 512 was implicitly coupled to TLS_REQUEST_LENGTH=517 from the official
|
||||
// Telegram MTProxy reference server, leaving only a 5-byte margin and
|
||||
// incorrectly rejecting compact but spec-compliant ClientHellos from
|
||||
// third-party clients or future Telegram versions.
|
||||
if !tls_clienthello_len_in_bounds(tls_len) {
|
||||
debug!(peer = %peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
first_bytes.to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut handshake = vec![0u8; 5 + tls_len];
|
||||
handshake[..5].copy_from_slice(&first_bytes);
|
||||
let body_read = match read_with_progress(&mut self.stream, &mut handshake[5..]).await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
debug!(peer = %peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_read_error");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..5].to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
if body_read < tls_len {
|
||||
debug!(peer = %peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
|
||||
self.stats
|
||||
.increment_connects_bad_with_class("tls_clienthello_truncated");
|
||||
maybe_apply_mask_reject_delay(&self.config).await;
|
||||
let initial_len = 5 + body_read;
|
||||
let (reader, writer) = self.stream.into_split();
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake[..initial_len].to_vec(),
|
||||
peer,
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint = observe_tls_client_fingerprint(
|
||||
self.stats.as_ref(),
|
||||
&self.config,
|
||||
peer.ip(),
|
||||
&handshake,
|
||||
);
|
||||
|
||||
let config = self.config.clone();
|
||||
let replay_checker = self.replay_checker.clone();
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let response_write_options =
|
||||
TlsResponseWriteOptions::tcp(self.raw_fd, self.tls_response_fragment_size);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let response_write_options = TlsResponseWriteOptions::default();
|
||||
|
||||
let (mut tls_reader, tls_writer, tls_user) =
|
||||
match handle_tls_handshake_with_shared_and_options(
|
||||
&handshake,
|
||||
read_half,
|
||||
write_half,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
&self.rng,
|
||||
self.tls_cache.clone(),
|
||||
self.shared.as_ref(),
|
||||
response_write_options,
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
handshake.clone(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
let mtproto_handshake: [u8; HANDSHAKE_LEN] = mtproto_data[..]
|
||||
.try_into()
|
||||
.map_err(|_| ProxyError::InvalidHandshake("Short MTProto handshake".into()))?;
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake_with_shared(
|
||||
&mtproto_handshake,
|
||||
tls_reader,
|
||||
tls_writer,
|
||||
peer,
|
||||
&config,
|
||||
&replay_checker,
|
||||
true,
|
||||
Some(tls_user.as_str()),
|
||||
self.shared.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
// MTProto failed after TLS ServerHello was already sent.
|
||||
// Switch fallback relay back to raw transport so the mask
|
||||
// backend receives valid TLS records (not unwrapped payload).
|
||||
let (reader, pending_plaintext) = reader.into_inner_with_pending_plaintext();
|
||||
let writer = writer.into_inner();
|
||||
let pending_record = if pending_plaintext.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
wrap_tls_application_record(&pending_plaintext)
|
||||
};
|
||||
let reader =
|
||||
tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
|
||||
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
Vec::new(),
|
||||
peer,
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.upstream_manager.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
};
|
||||
|
||||
Ok(HandshakeOutcome::NeedsRelay(Box::pin(
|
||||
Self::handle_authenticated_static_with_shared(
|
||||
crypto_reader,
|
||||
crypto_writer,
|
||||
success,
|
||||
self.upstream_manager,
|
||||
self.stats,
|
||||
self.config,
|
||||
buffer_pool,
|
||||
self.rng,
|
||||
self.me_pool,
|
||||
self.me_pool_runtime,
|
||||
self.route_runtime.clone(),
|
||||
local_addr,
|
||||
peer,
|
||||
self.ip_tracker,
|
||||
self.shared,
|
||||
),
|
||||
)))
|
||||
}
|
||||
}
|
||||
+12
-395
@@ -36,6 +36,15 @@ use nix::sys::stat::Mode;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
// Direct relay lifecycle and conntrack publication.
|
||||
mod relay;
|
||||
// Telegram DC resolution and upstream handshake.
|
||||
mod routing;
|
||||
|
||||
pub(crate) use relay::{
|
||||
handle_via_direct, handle_via_direct_with_shared, handle_via_direct_with_shared_and_conntrack,
|
||||
};
|
||||
use routing::*;
|
||||
const UNKNOWN_DC_LOG_DISTINCT_LIMIT: usize = 1024;
|
||||
static LOGGED_UNKNOWN_DCS: OnceLock<Mutex<HashSet<i16>>> = OnceLock::new();
|
||||
const MAX_SCOPE_HINT_LEN: usize = 64;
|
||||
@@ -224,401 +233,9 @@ fn clear_unknown_dc_log_cache_for_testing() {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn unknown_dc_test_lock() -> &'static Mutex<()> {
|
||||
static TEST_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
TEST_LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Runs Direct relay with standalone cancellation and shared-state defaults.
|
||||
pub(crate) async fn handle_via_direct<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config.clone(),
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
SocketAddr::from(([0, 0, 0, 0], config.server.port)),
|
||||
CancellationToken::new(),
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay for a kernel-backed TCP client tuple.
|
||||
pub(crate) async fn handle_via_direct_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared_and_conntrack(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel,
|
||||
shared,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay with explicit kernel-conntrack close publication policy.
|
||||
pub(crate) async fn handle_via_direct_with_shared_and_conntrack<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
mut route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
conntrack_close_policy: ConntrackClosePolicy,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let user = &success.user;
|
||||
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
||||
|
||||
debug!(
|
||||
user = %user,
|
||||
peer = %success.peer,
|
||||
dc = success.dc_idx,
|
||||
dc_addr = %dc_addr,
|
||||
proto = ?success.proto_tag,
|
||||
mode = "direct",
|
||||
"Connecting to Telegram DC"
|
||||
);
|
||||
|
||||
let scope_hint = validated_scope_hint(user);
|
||||
if user.starts_with("scope_") && scope_hint.is_none() {
|
||||
warn!(
|
||||
user = %user,
|
||||
"Ignoring invalid scope hint and falling back to default upstream selection"
|
||||
);
|
||||
}
|
||||
let tg_stream = tokio::select! {
|
||||
result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake");
|
||||
|
||||
let (tg_reader, tg_writer) = tokio::select! {
|
||||
result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, "TG handshake complete, starting relay");
|
||||
|
||||
stats.increment_user_connects(user);
|
||||
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
||||
let traffic_lease = shared
|
||||
.traffic_limiter
|
||||
.acquire_lease(user, success.peer.ip());
|
||||
|
||||
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
||||
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
||||
Duration::from_secs(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.direct_activity_timeout_secs(),
|
||||
)
|
||||
} else {
|
||||
Duration::from_secs(1800)
|
||||
};
|
||||
let relay_result = crate::proxy::relay::relay_direct_adaptive(
|
||||
client_reader,
|
||||
client_writer,
|
||||
tg_reader,
|
||||
tg_writer,
|
||||
config.general.direct_relay_copy_buf_c2s_bytes,
|
||||
config.general.direct_relay_copy_buf_s2c_bytes,
|
||||
config.server.max_connections,
|
||||
user,
|
||||
Arc::clone(&stats),
|
||||
config.access.user_data_quota.get(user).copied(),
|
||||
traffic_lease,
|
||||
relay_activity_timeout,
|
||||
session_cancel.clone(),
|
||||
Arc::clone(&shared.direct_buffer_budget),
|
||||
);
|
||||
tokio::pin!(relay_result);
|
||||
let relay_result = loop {
|
||||
if let Some(cutover) =
|
||||
affected_cutover_state(&route_rx, RelayRouteMode::Direct, route_snapshot.generation)
|
||||
{
|
||||
let delay = cutover_stagger_delay(session_id, cutover.generation);
|
||||
warn!(
|
||||
user = %user,
|
||||
target_mode = cutover.mode.as_str(),
|
||||
cutover_generation = cutover.generation,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Cutover affected direct session, closing client connection"
|
||||
);
|
||||
let _cutover_park_lease = stats.acquire_direct_cutover_park_lease();
|
||||
tokio::time::sleep(delay).await;
|
||||
break Err(ProxyError::RouteSwitched);
|
||||
}
|
||||
tokio::select! {
|
||||
result = &mut relay_result => {
|
||||
break result;
|
||||
}
|
||||
changed = route_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break relay_result.await;
|
||||
}
|
||||
}
|
||||
_ = session_cancel.cancelled() => {
|
||||
break Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &relay_result {
|
||||
Ok(()) => debug!(user = %user, "Direct relay completed"),
|
||||
Err(e) => debug!(user = %user, error = %e, "Direct relay ended with error"),
|
||||
}
|
||||
|
||||
let pool_snapshot = buffer_pool_trim.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
pool_snapshot.allocated,
|
||||
pool_snapshot.allocated.saturating_sub(pool_snapshot.pooled),
|
||||
);
|
||||
|
||||
if conntrack_close_policy == ConntrackClosePolicy::Publish {
|
||||
let close_reason = classify_conntrack_close_reason(&relay_result);
|
||||
let publish_result = shared.publish_conntrack_close_event(ConntrackCloseEvent {
|
||||
src: success.peer,
|
||||
dst: local_addr,
|
||||
reason: close_reason,
|
||||
});
|
||||
if !matches!(
|
||||
publish_result,
|
||||
ConntrackClosePublishResult::Sent | ConntrackClosePublishResult::Disabled
|
||||
) {
|
||||
stats.increment_conntrack_close_event_drop_total();
|
||||
}
|
||||
}
|
||||
|
||||
relay_result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(error.kind(), std::io::ErrorKind::TimedOut) =>
|
||||
{
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(crate::error::ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result<SocketAddr> {
|
||||
let prefer_v6 = config.network.prefer == 6 && config.network.ipv6.unwrap_or(true);
|
||||
let datacenters = if prefer_v6 {
|
||||
&*TG_DATACENTERS_V6
|
||||
} else {
|
||||
&*TG_DATACENTERS_V4
|
||||
};
|
||||
|
||||
let num_dcs = datacenters.len();
|
||||
|
||||
let dc_key = dc_idx.to_string();
|
||||
if let Some(addrs) = config.dc_overrides.get(&dc_key) {
|
||||
let mut parsed = Vec::new();
|
||||
for addr_str in addrs {
|
||||
match addr_str.parse::<SocketAddr>() {
|
||||
Ok(addr) => parsed.push(addr),
|
||||
Err(_) => {
|
||||
warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(addr) = parsed
|
||||
.iter()
|
||||
.find(|a| a.is_ipv6() == prefer_v6)
|
||||
.or_else(|| parsed.first())
|
||||
.copied()
|
||||
{
|
||||
debug!(dc_idx = dc_idx, addr = %addr, count = parsed.len(), "Using DC override from config");
|
||||
return Ok(addr);
|
||||
}
|
||||
}
|
||||
|
||||
let abs_dc = dc_idx.unsigned_abs() as usize;
|
||||
if abs_dc >= 1 && abs_dc <= num_dcs {
|
||||
return Ok(SocketAddr::new(datacenters[abs_dc - 1], TG_DATACENTER_PORT));
|
||||
}
|
||||
|
||||
// Unknown DC requested by client without override: log and fall back.
|
||||
if !config.dc_overrides.contains_key(&dc_key) {
|
||||
warn!(
|
||||
dc_idx = dc_idx,
|
||||
"Requested non-standard DC with no override; falling back to default cluster"
|
||||
);
|
||||
if config.general.unknown_dc_file_log_enabled
|
||||
&& let Some(path) = &config.general.unknown_dc_log_path
|
||||
&& let Ok(handle) = tokio::runtime::Handle::try_current()
|
||||
{
|
||||
if let Some(path) = sanitize_unknown_dc_log_path(path) {
|
||||
if should_log_unknown_dc(dc_idx) {
|
||||
handle.spawn_blocking(move || {
|
||||
if unknown_dc_log_path_is_still_safe(&path)
|
||||
&& let Ok(mut file) = open_unknown_dc_log_append_anchored(&path)
|
||||
{
|
||||
let _ = append_unknown_dc_line(&mut file, dc_idx);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
warn!(dc_idx = dc_idx, raw_path = %path, "Rejected unsafe unknown DC log path");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_dc = config.default_dc.unwrap_or(2) as usize;
|
||||
let fallback_idx = if default_dc >= 1 && default_dc <= num_dcs {
|
||||
default_dc - 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
info!(
|
||||
original_dc = dc_idx,
|
||||
fallback_dc = (fallback_idx + 1) as u16,
|
||||
fallback_addr = %datacenters[fallback_idx],
|
||||
"Special DC ---> default_cluster"
|
||||
);
|
||||
|
||||
Ok(SocketAddr::new(
|
||||
datacenters[fallback_idx],
|
||||
TG_DATACENTER_PORT,
|
||||
))
|
||||
}
|
||||
|
||||
async fn do_tg_handshake_static<S>(
|
||||
mut stream: S,
|
||||
success: &HandshakeSuccess,
|
||||
config: &ProxyConfig,
|
||||
rng: &SecureRandom,
|
||||
) -> Result<(CryptoReader<ReadHalf<S>>, CryptoWriter<WriteHalf<S>>)>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin,
|
||||
{
|
||||
let (nonce, _tg_enc_key, _tg_enc_iv, _tg_dec_key, _tg_dec_iv) = generate_tg_nonce(
|
||||
success.proto_tag,
|
||||
success.dc_idx,
|
||||
&success.enc_key,
|
||||
success.enc_iv,
|
||||
rng,
|
||||
config.general.fast_mode,
|
||||
);
|
||||
|
||||
let (encrypted_nonce, tg_encryptor, tg_decryptor) = encrypt_tg_nonce_with_ciphers(&nonce);
|
||||
|
||||
debug!(
|
||||
peer = %success.peer,
|
||||
nonce_head = %hex::encode(&nonce[..16]),
|
||||
"Sending nonce to Telegram"
|
||||
);
|
||||
|
||||
stream.write_all(&encrypted_nonce).await?;
|
||||
stream.flush().await?;
|
||||
|
||||
let (read_half, write_half) = split(stream);
|
||||
|
||||
let max_pending = config.general.crypto_pending_buffer;
|
||||
Ok((
|
||||
CryptoReader::new(read_half, tg_decryptor),
|
||||
CryptoWriter::new(write_half, tg_encryptor, max_pending),
|
||||
))
|
||||
fn unknown_dc_test_lock() -> &'static tokio::sync::Mutex<()> {
|
||||
static TEST_LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
|
||||
TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
use super::*;
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Runs Direct relay with standalone cancellation and shared-state defaults.
|
||||
pub(crate) async fn handle_via_direct<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config.clone(),
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
SocketAddr::from(([0, 0, 0, 0], config.server.port)),
|
||||
CancellationToken::new(),
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay for a kernel-backed TCP client tuple.
|
||||
pub(crate) async fn handle_via_direct_with_shared<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_via_direct_with_shared_and_conntrack(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_rx,
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel,
|
||||
shared,
|
||||
ConntrackClosePolicy::Publish,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Runs Direct relay with explicit kernel-conntrack close publication policy.
|
||||
pub(crate) async fn handle_via_direct_with_shared_and_conntrack<R, W>(
|
||||
client_reader: CryptoReader<R>,
|
||||
client_writer: CryptoWriter<W>,
|
||||
success: HandshakeSuccess,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
stats: Arc<Stats>,
|
||||
config: Arc<ProxyConfig>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
mut route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
conntrack_close_policy: ConntrackClosePolicy,
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let user = &success.user;
|
||||
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
||||
|
||||
debug!(
|
||||
user = %user,
|
||||
peer = %success.peer,
|
||||
dc = success.dc_idx,
|
||||
dc_addr = %dc_addr,
|
||||
proto = ?success.proto_tag,
|
||||
mode = "direct",
|
||||
"Connecting to Telegram DC"
|
||||
);
|
||||
|
||||
let scope_hint = validated_scope_hint(user);
|
||||
if user.starts_with("scope_") && scope_hint.is_none() {
|
||||
warn!(
|
||||
user = %user,
|
||||
"Ignoring invalid scope hint and falling back to default upstream selection"
|
||||
);
|
||||
}
|
||||
let tg_stream = tokio::select! {
|
||||
result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake");
|
||||
|
||||
let (tg_reader, tg_writer) = tokio::select! {
|
||||
result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, "TG handshake complete, starting relay");
|
||||
|
||||
stats.increment_user_connects(user);
|
||||
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
||||
let traffic_lease = shared
|
||||
.traffic_limiter
|
||||
.acquire_lease(user, success.peer.ip());
|
||||
|
||||
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
||||
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
||||
Duration::from_secs(
|
||||
config
|
||||
.server
|
||||
.conntrack_control
|
||||
.profile
|
||||
.direct_activity_timeout_secs(),
|
||||
)
|
||||
} else {
|
||||
Duration::from_secs(1800)
|
||||
};
|
||||
let relay_result = crate::proxy::relay::relay_direct_adaptive(
|
||||
client_reader,
|
||||
client_writer,
|
||||
tg_reader,
|
||||
tg_writer,
|
||||
config.general.direct_relay_copy_buf_c2s_bytes,
|
||||
config.general.direct_relay_copy_buf_s2c_bytes,
|
||||
config.server.max_connections,
|
||||
user,
|
||||
Arc::clone(&stats),
|
||||
config.access.user_data_quota.get(user).copied(),
|
||||
traffic_lease,
|
||||
relay_activity_timeout,
|
||||
session_cancel.clone(),
|
||||
Arc::clone(&shared.direct_buffer_budget),
|
||||
);
|
||||
tokio::pin!(relay_result);
|
||||
let relay_result = loop {
|
||||
if let Some(cutover) =
|
||||
affected_cutover_state(&route_rx, RelayRouteMode::Direct, route_snapshot.generation)
|
||||
{
|
||||
let delay = cutover_stagger_delay(session_id, cutover.generation);
|
||||
warn!(
|
||||
user = %user,
|
||||
target_mode = cutover.mode.as_str(),
|
||||
cutover_generation = cutover.generation,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"Cutover affected direct session, closing client connection"
|
||||
);
|
||||
let _cutover_park_lease = stats.acquire_direct_cutover_park_lease();
|
||||
tokio::time::sleep(delay).await;
|
||||
break Err(ProxyError::RouteSwitched);
|
||||
}
|
||||
tokio::select! {
|
||||
result = &mut relay_result => {
|
||||
break result;
|
||||
}
|
||||
changed = route_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break relay_result.await;
|
||||
}
|
||||
}
|
||||
_ = session_cancel.cancelled() => {
|
||||
break Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &relay_result {
|
||||
Ok(()) => debug!(user = %user, "Direct relay completed"),
|
||||
Err(e) => debug!(user = %user, error = %e, "Direct relay ended with error"),
|
||||
}
|
||||
|
||||
let pool_snapshot = buffer_pool_trim.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
pool_snapshot.allocated,
|
||||
pool_snapshot.allocated.saturating_sub(pool_snapshot.pooled),
|
||||
);
|
||||
|
||||
if conntrack_close_policy == ConntrackClosePolicy::Publish {
|
||||
let close_reason = classify_conntrack_close_reason(&relay_result);
|
||||
let publish_result = shared.publish_conntrack_close_event(ConntrackCloseEvent {
|
||||
src: success.peer,
|
||||
dst: local_addr,
|
||||
reason: close_reason,
|
||||
});
|
||||
if !matches!(
|
||||
publish_result,
|
||||
ConntrackClosePublishResult::Sent | ConntrackClosePublishResult::Disabled
|
||||
) {
|
||||
stats.increment_conntrack_close_event_drop_total();
|
||||
}
|
||||
}
|
||||
|
||||
relay_result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(error.kind(), std::io::ErrorKind::TimedOut) =>
|
||||
{
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(crate::error::ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(crate::error::ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result<SocketAddr> {
|
||||
let prefer_v6 = config.network.prefer == 6 && config.network.ipv6.unwrap_or(true);
|
||||
let datacenters = if prefer_v6 {
|
||||
&*TG_DATACENTERS_V6
|
||||
} else {
|
||||
&*TG_DATACENTERS_V4
|
||||
};
|
||||
|
||||
let num_dcs = datacenters.len();
|
||||
|
||||
let dc_key = dc_idx.to_string();
|
||||
if let Some(addrs) = config.dc_overrides.get(&dc_key) {
|
||||
let mut parsed = Vec::new();
|
||||
for addr_str in addrs {
|
||||
match addr_str.parse::<SocketAddr>() {
|
||||
Ok(addr) => parsed.push(addr),
|
||||
Err(_) => {
|
||||
warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(addr) = parsed
|
||||
.iter()
|
||||
.find(|a| a.is_ipv6() == prefer_v6)
|
||||
.or_else(|| parsed.first())
|
||||
.copied()
|
||||
{
|
||||
debug!(dc_idx = dc_idx, addr = %addr, count = parsed.len(), "Using DC override from config");
|
||||
return Ok(addr);
|
||||
}
|
||||
}
|
||||
|
||||
let abs_dc = dc_idx.unsigned_abs() as usize;
|
||||
if abs_dc >= 1 && abs_dc <= num_dcs {
|
||||
return Ok(SocketAddr::new(datacenters[abs_dc - 1], TG_DATACENTER_PORT));
|
||||
}
|
||||
|
||||
// Unknown DC requested by client without override: log and fall back.
|
||||
if !config.dc_overrides.contains_key(&dc_key) {
|
||||
warn!(
|
||||
dc_idx = dc_idx,
|
||||
"Requested non-standard DC with no override; falling back to default cluster"
|
||||
);
|
||||
if config.general.unknown_dc_file_log_enabled
|
||||
&& let Some(path) = &config.general.unknown_dc_log_path
|
||||
&& let Ok(handle) = tokio::runtime::Handle::try_current()
|
||||
{
|
||||
if let Some(path) = sanitize_unknown_dc_log_path(path) {
|
||||
if should_log_unknown_dc(dc_idx) {
|
||||
handle.spawn_blocking(move || {
|
||||
if unknown_dc_log_path_is_still_safe(&path)
|
||||
&& let Ok(mut file) = open_unknown_dc_log_append_anchored(&path)
|
||||
{
|
||||
let _ = append_unknown_dc_line(&mut file, dc_idx);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
warn!(dc_idx = dc_idx, raw_path = %path, "Rejected unsafe unknown DC log path");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_dc = config.default_dc.unwrap_or(2) as usize;
|
||||
let fallback_idx = if default_dc >= 1 && default_dc <= num_dcs {
|
||||
default_dc - 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
info!(
|
||||
original_dc = dc_idx,
|
||||
fallback_dc = (fallback_idx + 1) as u16,
|
||||
fallback_addr = %datacenters[fallback_idx],
|
||||
"Special DC ---> default_cluster"
|
||||
);
|
||||
|
||||
Ok(SocketAddr::new(
|
||||
datacenters[fallback_idx],
|
||||
TG_DATACENTER_PORT,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn do_tg_handshake_static<S>(
|
||||
mut stream: S,
|
||||
success: &HandshakeSuccess,
|
||||
config: &ProxyConfig,
|
||||
rng: &SecureRandom,
|
||||
) -> Result<(CryptoReader<ReadHalf<S>>, CryptoWriter<WriteHalf<S>>)>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin,
|
||||
{
|
||||
let (nonce, _tg_enc_key, _tg_enc_iv, _tg_dec_key, _tg_dec_iv) = generate_tg_nonce(
|
||||
success.proto_tag,
|
||||
success.dc_idx,
|
||||
&success.enc_key,
|
||||
success.enc_iv,
|
||||
rng,
|
||||
config.general.fast_mode,
|
||||
);
|
||||
|
||||
let (encrypted_nonce, tg_encryptor, tg_decryptor) = encrypt_tg_nonce_with_ciphers(&nonce);
|
||||
|
||||
debug!(
|
||||
peer = %success.peer,
|
||||
nonce_head = %hex::encode(&nonce[..16]),
|
||||
"Sending nonce to Telegram"
|
||||
);
|
||||
|
||||
stream.write_all(&encrypted_nonce).await?;
|
||||
stream.flush().await?;
|
||||
|
||||
let (read_half, write_half) = split(stream);
|
||||
|
||||
let max_pending = config.general.crypto_pending_buffer;
|
||||
Ok((
|
||||
CryptoReader::new(read_half, tg_decryptor),
|
||||
CryptoWriter::new(write_half, tg_encryptor, max_pending),
|
||||
))
|
||||
}
|
||||
@@ -98,7 +98,9 @@ pub(super) fn auth_probe_is_throttled_in(
|
||||
};
|
||||
if auth_probe_state_expired(&entry, now) {
|
||||
drop(entry);
|
||||
state.remove_if(&peer_ip, |_, current| auth_probe_state_expired(current, now));
|
||||
state.remove_if(&peer_ip, |_, current| {
|
||||
auth_probe_state_expired(current, now)
|
||||
});
|
||||
return false;
|
||||
}
|
||||
now < entry.blocked_until
|
||||
@@ -116,7 +118,9 @@ pub(super) fn auth_probe_saturation_grace_exhausted_in(
|
||||
};
|
||||
if auth_probe_state_expired(&entry, now) {
|
||||
drop(entry);
|
||||
state.remove_if(&peer_ip, |_, current| auth_probe_state_expired(current, now));
|
||||
state.remove_if(&peer_ip, |_, current| {
|
||||
auth_probe_state_expired(current, now)
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -264,7 +268,8 @@ pub(super) fn auth_probe_record_failure_with_state_in(
|
||||
}
|
||||
}
|
||||
|
||||
let Some((evict_key, evict_fail_streak, evict_last_seen)) = eviction_candidate else {
|
||||
let Some((evict_key, evict_fail_streak, evict_last_seen)) = eviction_candidate
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if state
|
||||
|
||||
@@ -266,8 +266,7 @@ where
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
}
|
||||
|
||||
let selected_tls_domain =
|
||||
matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
|
||||
let selected_tls_domain = matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
|
||||
let cached_entry = if config.censorship.tls_emulation {
|
||||
if let Some(cache) = tls_cache.as_ref() {
|
||||
let cached_entry = cache.get(selected_tls_domain).await;
|
||||
|
||||
+26
-1299
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn masking_beobachten_ttl(config: &ProxyConfig) -> Duration {
|
||||
let minutes = config.general.beobachten_minutes;
|
||||
let clamped = minutes.clamp(1, 24 * 60);
|
||||
Duration::from_secs(clamped.saturating_mul(60))
|
||||
}
|
||||
|
||||
pub(super) fn build_mask_proxy_header(
|
||||
version: u8,
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
) -> Option<Vec<u8>> {
|
||||
match version {
|
||||
0 => None,
|
||||
2 => Some(
|
||||
ProxyProtocolV2Builder::new()
|
||||
.with_addrs(peer, local_addr)
|
||||
.build(),
|
||||
),
|
||||
_ => {
|
||||
let header = match (peer, local_addr) {
|
||||
(SocketAddr::V4(src), SocketAddr::V4(dst)) => ProxyProtocolV1Builder::new()
|
||||
.tcp4(src.into(), dst.into())
|
||||
.build(),
|
||||
(SocketAddr::V6(src), SocketAddr::V6(dst)) => ProxyProtocolV1Builder::new()
|
||||
.tcp6(src.into(), dst.into())
|
||||
.build(),
|
||||
_ => ProxyProtocolV1Builder::new().build(),
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn configure_mask_backend_socket(stream: &TcpStream) {
|
||||
if let Err(e) = configure_tcp_socket(stream, false, Duration::from_secs(0)) {
|
||||
debug!(error = %e, "Failed to configure mask backend socket");
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles a bad client by forwarding it to the configured mask target.
|
||||
#[cfg(test)]
|
||||
pub async fn handle_bad_client<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let shared = ProxySharedState::new();
|
||||
handle_bad_client_with_shared(
|
||||
reader,
|
||||
writer,
|
||||
initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
config,
|
||||
beobachten,
|
||||
shared.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn mask_copy_read_len(total: usize, byte_cap: usize) -> usize {
|
||||
// Keep short scanner probes on the small baseline buffer and grow only
|
||||
// after the session has proven to be sustained masking relay traffic.
|
||||
let active_buffer_size = if total >= MASK_BUFFER_GROW_AFTER_BYTES {
|
||||
MASK_BUFFER_MAX_SIZE
|
||||
} else {
|
||||
MASK_BUFFER_SIZE
|
||||
};
|
||||
|
||||
if byte_cap == 0 {
|
||||
return active_buffer_size;
|
||||
}
|
||||
|
||||
let remaining_budget = byte_cap.saturating_sub(total);
|
||||
if remaining_budget == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
remaining_budget.min(active_buffer_size)
|
||||
}
|
||||
|
||||
pub(super) async fn copy_with_idle_timeout<R, W>(
|
||||
reader: &mut R,
|
||||
writer: &mut W,
|
||||
byte_cap: usize,
|
||||
shutdown_on_eof: bool,
|
||||
idle_timeout: Duration,
|
||||
) -> CopyOutcome
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
let mut ended_by_eof = false;
|
||||
|
||||
loop {
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let read_res = timeout(idle_timeout, reader.read(&mut buf[..read_len])).await;
|
||||
let n = match read_res {
|
||||
Ok(Ok(n)) => n,
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
};
|
||||
if n == 0 {
|
||||
ended_by_eof = true;
|
||||
if shutdown_on_eof {
|
||||
let _ = timeout(idle_timeout, writer.shutdown()).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
total = total.saturating_add(n);
|
||||
|
||||
let write_res = timeout(idle_timeout, writer.write_all(&buf[..n])).await;
|
||||
match write_res {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
}
|
||||
}
|
||||
CopyOutcome {
|
||||
total,
|
||||
ended_by_eof,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_http_probe(data: &[u8]) -> bool {
|
||||
// RFC 7540 section 3.5: HTTP/2 client preface starts with "PRI ".
|
||||
const HTTP_METHODS: [&[u8]; 10] = [
|
||||
b"GET ", b"POST", b"HEAD", b"PUT ", b"DELETE", b"OPTIONS", b"CONNECT", b"TRACE", b"PATCH",
|
||||
b"PRI ",
|
||||
];
|
||||
|
||||
if data.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let window = &data[..data.len().min(16)];
|
||||
for method in HTTP_METHODS {
|
||||
if data.len() >= method.len() && window.starts_with(method) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (2..=3).contains(&window.len()) && method.starts_with(window) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
pub(super) fn next_mask_shape_bucket(total: usize, floor: usize, cap: usize) -> usize {
|
||||
if total == 0 || floor == 0 || cap < floor {
|
||||
return total;
|
||||
}
|
||||
|
||||
if total >= cap {
|
||||
return total;
|
||||
}
|
||||
|
||||
let mut bucket = floor;
|
||||
while bucket < total {
|
||||
match bucket.checked_mul(2) {
|
||||
Some(next) => bucket = next,
|
||||
None => return total,
|
||||
}
|
||||
if bucket > cap {
|
||||
return cap;
|
||||
}
|
||||
}
|
||||
bucket
|
||||
}
|
||||
|
||||
pub(super) async fn maybe_write_shape_padding<W>(
|
||||
mask_write: &mut W,
|
||||
total_sent: usize,
|
||||
enabled: bool,
|
||||
floor: usize,
|
||||
cap: usize,
|
||||
above_cap_blur: bool,
|
||||
above_cap_blur_max_bytes: usize,
|
||||
aggressive_mode: bool,
|
||||
) where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
if !enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
let target_total = if total_sent >= cap && above_cap_blur && above_cap_blur_max_bytes > 0 {
|
||||
let mut rng = rand::rng();
|
||||
let extra = if aggressive_mode {
|
||||
rng.random_range(1..=above_cap_blur_max_bytes)
|
||||
} else {
|
||||
rng.random_range(0..=above_cap_blur_max_bytes)
|
||||
};
|
||||
total_sent.saturating_add(extra)
|
||||
} else {
|
||||
next_mask_shape_bucket(total_sent, floor, cap)
|
||||
};
|
||||
|
||||
if target_total <= total_sent {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut remaining = target_total - total_sent;
|
||||
let mut pad_chunk = [0u8; 1024];
|
||||
let deadline = Instant::now() + MASK_TIMEOUT;
|
||||
// Use a Send RNG so relay futures remain spawn-safe under Tokio.
|
||||
let mut rng = {
|
||||
let mut seed_source = rand::rng();
|
||||
StdRng::from_rng(&mut seed_source)
|
||||
};
|
||||
|
||||
while remaining > 0 {
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return;
|
||||
}
|
||||
|
||||
let write_len = remaining.min(pad_chunk.len());
|
||||
rng.fill_bytes(&mut pad_chunk[..write_len]);
|
||||
let write_budget = deadline.saturating_duration_since(now);
|
||||
match timeout(write_budget, mask_write.write_all(&pad_chunk[..write_len])).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) | Err(_) => return,
|
||||
}
|
||||
remaining -= write_len;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return;
|
||||
}
|
||||
let flush_budget = deadline.saturating_duration_since(now);
|
||||
let _ = timeout(flush_budget, mask_write.flush()).await;
|
||||
}
|
||||
|
||||
pub(super) async fn write_proxy_header_with_timeout<W>(mask_write: &mut W, header: &[u8]) -> bool
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
match timeout(MASK_TIMEOUT, mask_write.write_all(header)).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(_)) => false,
|
||||
Err(_) => {
|
||||
debug!("Timeout writing proxy protocol header to mask backend");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn consume_client_data_with_timeout_and_cap<R>(
|
||||
reader: R,
|
||||
byte_cap: usize,
|
||||
relay_timeout: Duration,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
consume_client_data(reader, byte_cap, idle_timeout),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Timed out while consuming client data on masking fallback path");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mask_failure_drain_cap(config: &ProxyConfig) -> usize {
|
||||
let configured_cap = config.censorship.mask_relay_max_bytes;
|
||||
if configured_cap == 0 {
|
||||
return MASK_BUFFER_SIZE;
|
||||
}
|
||||
|
||||
configured_cap.min(MASK_BUFFER_SIZE)
|
||||
}
|
||||
|
||||
pub(super) async fn consume_mask_failure_path<R>(
|
||||
reader: R,
|
||||
config: &ProxyConfig,
|
||||
relay_timeout: Duration,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
mask_failure_drain_cap(config),
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_connect_budget(started: Instant) {
|
||||
let elapsed = started.elapsed();
|
||||
if elapsed < MASK_TIMEOUT {
|
||||
tokio::time::sleep(MASK_TIMEOUT - elapsed).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Log-normal sample bounded to [floor, ceiling]. Median = sqrt(floor * ceiling).
|
||||
// Implements Box-Muller transform for standard normal sampling — no external
|
||||
// dependency on rand_distr (which is incompatible with rand 0.10).
|
||||
// sigma is chosen so ~99% of raw samples land inside [floor, ceiling] before clamp.
|
||||
// When floor > ceiling (misconfiguration), returns ceiling (the smaller value).
|
||||
// When floor == ceiling, returns that value. When both are 0, returns 0.
|
||||
@@ -0,0 +1,260 @@
|
||||
use super::*;
|
||||
|
||||
/// Handles a bad client with shared pre-auth fallback admission state.
|
||||
pub(crate) async fn handle_bad_client_with_shared<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
shared: &ProxySharedState,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
handle_bad_client_with_shared_resolver(
|
||||
reader,
|
||||
writer,
|
||||
initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
config,
|
||||
beobachten,
|
||||
shared,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(in crate::proxy) async fn handle_bad_client_with_shared_resolver<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
shared: &ProxySharedState,
|
||||
upstream_manager: Option<&crate::transport::UpstreamManager>,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let client_type = detect_client_type(initial_data);
|
||||
if config.general.beobachten {
|
||||
let ttl = masking_beobachten_ttl(config);
|
||||
beobachten.record(client_type, peer.ip(), ttl);
|
||||
}
|
||||
|
||||
let relay_timeout = Duration::from_millis(config.censorship.mask_relay_timeout_ms);
|
||||
let idle_timeout = Duration::from_millis(config.censorship.mask_relay_idle_timeout_ms);
|
||||
|
||||
if !config.censorship.mask {
|
||||
// Masking disabled, just consume data
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(_masking_permit) = shared.try_acquire_masking_fallback_permit() else {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
"Masking fallback concurrency limit reached"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
};
|
||||
|
||||
let client_sni = tls::extract_sni_from_client_hello(initial_data);
|
||||
let exclusive_tcp_target = client_sni
|
||||
.as_deref()
|
||||
.and_then(|sni| exclusive_mask_target_for_sni(config, sni));
|
||||
|
||||
// Connect via Unix socket or TCP
|
||||
#[cfg(unix)]
|
||||
if exclusive_tcp_target.is_none()
|
||||
&& let Some(ref sock_path) = config.censorship.mask_unix_sock
|
||||
{
|
||||
let outcome_started = Instant::now();
|
||||
let connect_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
sock = %sock_path,
|
||||
data_len = initial_data.len(),
|
||||
"Forwarding bad client to mask unix socket"
|
||||
);
|
||||
|
||||
let connect_result = timeout(MASK_TIMEOUT, UnixStream::connect(sock_path)).await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
let (mask_read, mut mask_write) = stream.into_split();
|
||||
let proxy_header = build_mask_proxy_header(
|
||||
config.censorship.mask_proxy_protocol,
|
||||
peer,
|
||||
local_addr,
|
||||
);
|
||||
if let Some(header) = proxy_header
|
||||
&& !write_proxy_header_with_timeout(&mut mask_write, &header).await
|
||||
{
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
relay_to_mask(
|
||||
reader,
|
||||
writer,
|
||||
mask_read,
|
||||
mask_write,
|
||||
initial_data,
|
||||
config.censorship.mask_shape_hardening,
|
||||
config.censorship.mask_shape_bucket_floor_bytes,
|
||||
config.censorship.mask_shape_bucket_cap_bytes,
|
||||
config.censorship.mask_shape_above_cap_blur,
|
||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
idle_timeout,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Mask relay timed out (unix socket)");
|
||||
}
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask unix socket");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask unix socket");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mask_target = exclusive_tcp_target.unwrap_or_else(|| {
|
||||
default_mask_tcp_target_for_initial_data(config, initial_data, client_sni.as_deref())
|
||||
});
|
||||
let mask_host = mask_target.host;
|
||||
let mask_port = mask_target.port;
|
||||
|
||||
let resolved_mask_addrs =
|
||||
match resolve_mask_target_addrs(mask_host, mask_port, upstream_manager).await {
|
||||
Ok(addrs) => addrs,
|
||||
Err(e) => {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
error = %e,
|
||||
"Failed to resolve mask target"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Fail closed when fallback points at our own listener endpoint.
|
||||
// Self-referential masking can create recursive proxy loops under
|
||||
// misconfiguration and leak distinguishable load spikes to adversaries.
|
||||
if is_mask_target_local_listener_async(mask_host, mask_port, local_addr, &resolved_mask_addrs)
|
||||
.await
|
||||
{
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
local = %local_addr,
|
||||
"Mask target resolves to local listener; refusing self-referential masking fallback"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let outcome_started = Instant::now();
|
||||
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
data_len = initial_data.len(),
|
||||
"Forwarding bad client to mask host"
|
||||
);
|
||||
|
||||
let connect_started = Instant::now();
|
||||
let connect_result = timeout(
|
||||
MASK_TIMEOUT,
|
||||
TcpStream::connect(resolved_mask_addrs.as_slice()),
|
||||
)
|
||||
.await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
configure_mask_backend_socket(&stream);
|
||||
let proxy_header =
|
||||
build_mask_proxy_header(config.censorship.mask_proxy_protocol, peer, local_addr);
|
||||
|
||||
let (mask_read, mut mask_write) = stream.into_split();
|
||||
if let Some(header) = proxy_header
|
||||
&& !write_proxy_header_with_timeout(&mut mask_write, &header).await
|
||||
{
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
if timeout(
|
||||
relay_timeout,
|
||||
relay_to_mask(
|
||||
reader,
|
||||
writer,
|
||||
mask_read,
|
||||
mask_write,
|
||||
initial_data,
|
||||
config.censorship.mask_shape_hardening,
|
||||
config.censorship.mask_shape_bucket_floor_bytes,
|
||||
config.censorship.mask_shape_bucket_cap_bytes,
|
||||
config.censorship.mask_shape_above_cap_blur,
|
||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
idle_timeout,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
debug!("Mask relay timed out");
|
||||
}
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask host");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask host");
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn canonical_ip(ip: IpAddr) -> IpAddr {
|
||||
match ip {
|
||||
IpAddr::V6(v6) => v6
|
||||
.to_ipv4_mapped()
|
||||
.map(IpAddr::V4)
|
||||
.unwrap_or(IpAddr::V6(v6)),
|
||||
IpAddr::V4(v4) => IpAddr::V4(v4),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) fn collect_local_interface_ips() -> Vec<IpAddr> {
|
||||
#[cfg(test)]
|
||||
LOCAL_INTERFACE_ENUMERATIONS.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let mut out = Vec::new();
|
||||
if let Ok(addrs) = getifaddrs() {
|
||||
for iface in addrs {
|
||||
if let Some(address) = iface.address {
|
||||
if let Some(v4) = address.as_sockaddr_in() {
|
||||
out.push(canonical_ip(IpAddr::V4(v4.ip())));
|
||||
} else if let Some(v6) = address.as_sockaddr_in6() {
|
||||
out.push(canonical_ip(IpAddr::V6(v6.ip())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) fn choose_interface_snapshot(
|
||||
previous: &[IpAddr],
|
||||
refreshed: Vec<IpAddr>,
|
||||
) -> Vec<IpAddr> {
|
||||
if refreshed.is_empty() && !previous.is_empty() {
|
||||
return previous.to_vec();
|
||||
}
|
||||
|
||||
refreshed
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[derive(Default)]
|
||||
struct LocalInterfaceCache {
|
||||
ips: Vec<IpAddr>,
|
||||
refreshed_at: Option<StdInstant>,
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
static LOCAL_INTERFACE_CACHE: OnceLock<Mutex<LocalInterfaceCache>> = OnceLock::new();
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) static LOCAL_INTERFACE_REFRESH_LOCK: OnceLock<AsyncMutex<()>> = OnceLock::new();
|
||||
|
||||
#[cfg(all(unix, test))]
|
||||
pub(super) fn local_interface_ips() -> Vec<IpAddr> {
|
||||
let cache = LOCAL_INTERFACE_CACHE.get_or_init(|| Mutex::new(LocalInterfaceCache::default()));
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if stale {
|
||||
let refreshed = collect_local_interface_ips();
|
||||
guard.ips = choose_interface_snapshot(&guard.ips, refreshed);
|
||||
guard.refreshed_at = Some(StdInstant::now());
|
||||
}
|
||||
|
||||
guard.ips.clone()
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(super) async fn local_interface_ips_async() -> Vec<IpAddr> {
|
||||
let cache = LOCAL_INTERFACE_CACHE.get_or_init(|| Mutex::new(LocalInterfaceCache::default()));
|
||||
|
||||
{
|
||||
let guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if !stale {
|
||||
return guard.ips.clone();
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_lock = LOCAL_INTERFACE_REFRESH_LOCK.get_or_init(|| AsyncMutex::new(()));
|
||||
let _refresh_guard = refresh_lock.lock().await;
|
||||
|
||||
{
|
||||
let guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if !stale {
|
||||
return guard.ips.clone();
|
||||
}
|
||||
}
|
||||
|
||||
let refreshed = tokio::task::spawn_blocking(collect_local_interface_ips)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
let stale = guard
|
||||
.refreshed_at
|
||||
.is_none_or(|at| at.elapsed() >= LOCAL_INTERFACE_CACHE_TTL);
|
||||
if stale {
|
||||
guard.ips = choose_interface_snapshot(&guard.ips, refreshed);
|
||||
guard.refreshed_at = Some(StdInstant::now());
|
||||
}
|
||||
|
||||
guard.ips.clone()
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), test))]
|
||||
pub(super) fn local_interface_ips() -> Vec<IpAddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub(super) async fn local_interface_ips_async() -> Vec<IpAddr> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static LOCAL_INTERFACE_ENUMERATIONS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn reset_local_interface_enumerations_for_tests() {
|
||||
LOCAL_INTERFACE_ENUMERATIONS.store(0, Ordering::Relaxed);
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(cache) = LOCAL_INTERFACE_CACHE.get() {
|
||||
let mut guard = cache.lock().unwrap_or_else(|poison| poison.into_inner());
|
||||
guard.ips.clear();
|
||||
guard.refreshed_at = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn local_interface_enumerations_for_tests() -> usize {
|
||||
LOCAL_INTERFACE_ENUMERATIONS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn interface_cache_test_lock() -> &'static tokio::sync::Mutex<()> {
|
||||
static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
|
||||
}
|
||||
|
||||
pub(super) fn is_mask_target_local_listener_with_interfaces(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
interface_ips: &[IpAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let local_ip = canonical_ip(local_addr.ip());
|
||||
let literal_mask_ip = parse_mask_host_ip_literal(mask_host).map(canonical_ip);
|
||||
|
||||
for addr in resolved_addrs {
|
||||
let resolved_ip = canonical_ip(addr.ip());
|
||||
if resolved_ip == local_ip {
|
||||
return true;
|
||||
}
|
||||
|
||||
if local_ip.is_unspecified()
|
||||
&& (resolved_ip.is_loopback()
|
||||
|| resolved_ip.is_unspecified()
|
||||
|| interface_ips.contains(&resolved_ip))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(mask_ip) = literal_mask_ip {
|
||||
if mask_ip == local_ip {
|
||||
return true;
|
||||
}
|
||||
|
||||
if local_ip.is_unspecified()
|
||||
&& (mask_ip.is_loopback()
|
||||
|| mask_ip.is_unspecified()
|
||||
|| interface_ips.contains(&mask_ip))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn is_mask_target_local_listener(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let interfaces = local_interface_ips();
|
||||
is_mask_target_local_listener_with_interfaces(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn is_mask_target_local_listener_async(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let interfaces = local_interface_ips_async().await;
|
||||
is_mask_target_local_listener_with_interfaces(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use super::*;
|
||||
|
||||
/// Relays traffic between the client and mask backend.
|
||||
pub(super) async fn relay_to_mask<R, W, MR, MW>(
|
||||
mut reader: R,
|
||||
mut writer: W,
|
||||
mut mask_read: MR,
|
||||
mut mask_write: MW,
|
||||
initial_data: &[u8],
|
||||
shape_hardening_enabled: bool,
|
||||
shape_bucket_floor_bytes: usize,
|
||||
shape_bucket_cap_bytes: usize,
|
||||
shape_above_cap_blur: bool,
|
||||
shape_above_cap_blur_max_bytes: usize,
|
||||
shape_hardening_aggressive_mode: bool,
|
||||
mask_relay_max_bytes: usize,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
MR: AsyncRead + Unpin + Send + 'static,
|
||||
MW: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
// Send initial data to mask host
|
||||
if mask_write.write_all(initial_data).await.is_err() {
|
||||
return;
|
||||
}
|
||||
if mask_write.flush().await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let (upstream_copy, downstream_copy) = tokio::join!(
|
||||
async {
|
||||
copy_with_idle_timeout(
|
||||
&mut reader,
|
||||
&mut mask_write,
|
||||
mask_relay_max_bytes,
|
||||
!shape_hardening_enabled,
|
||||
idle_timeout,
|
||||
)
|
||||
.await
|
||||
},
|
||||
async {
|
||||
copy_with_idle_timeout(
|
||||
&mut mask_read,
|
||||
&mut writer,
|
||||
mask_relay_max_bytes,
|
||||
true,
|
||||
idle_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
);
|
||||
|
||||
let total_sent = initial_data.len().saturating_add(upstream_copy.total);
|
||||
|
||||
let should_shape = shape_hardening_enabled
|
||||
&& !initial_data.is_empty()
|
||||
&& (upstream_copy.ended_by_eof
|
||||
|| (shape_hardening_aggressive_mode && downstream_copy.total == 0));
|
||||
|
||||
maybe_write_shape_padding(
|
||||
&mut mask_write,
|
||||
total_sent,
|
||||
should_shape,
|
||||
shape_bucket_floor_bytes,
|
||||
shape_bucket_cap_bytes,
|
||||
shape_above_cap_blur,
|
||||
shape_above_cap_blur_max_bytes,
|
||||
shape_hardening_aggressive_mode,
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = mask_write.shutdown().await;
|
||||
let _ = writer.shutdown().await;
|
||||
}
|
||||
|
||||
/// Just consume all data from client without responding.
|
||||
pub(super) async fn consume_client_data<R: AsyncRead + Unpin>(
|
||||
mut reader: R,
|
||||
byte_cap: usize,
|
||||
idle_timeout: Duration,
|
||||
) {
|
||||
// Keep drain path fail-closed under slow-loris stalls.
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
|
||||
loop {
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let n = match timeout(idle_timeout, reader.read(&mut buf[..read_len])).await {
|
||||
Ok(Ok(n)) => n,
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
};
|
||||
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
total = total.saturating_add(n);
|
||||
if byte_cap != 0 && total >= byte_cap {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use super::*;
|
||||
|
||||
/// Detect client type based on initial data.
|
||||
pub(super) fn detect_client_type(data: &[u8]) -> &'static str {
|
||||
// Check for HTTP request
|
||||
if is_http_probe(data) {
|
||||
return "HTTP";
|
||||
}
|
||||
|
||||
// Check for TLS ClientHello (0x16 = handshake, 0x03 0x01-0x03 = TLS version)
|
||||
if data.len() > 3 && data[0] == 0x16 && data[1] == 0x03 {
|
||||
return "TLS-scanner";
|
||||
}
|
||||
|
||||
// Check for SSH
|
||||
if data.starts_with(b"SSH-") {
|
||||
return "SSH";
|
||||
}
|
||||
|
||||
// Port scanner (very short data)
|
||||
if data.len() < 10 {
|
||||
return "port-scanner";
|
||||
}
|
||||
|
||||
"unknown"
|
||||
}
|
||||
|
||||
pub(super) fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
|
||||
if host.starts_with('[') && host.ends_with(']') {
|
||||
return host[1..host.len() - 1].parse::<IpAddr>().ok();
|
||||
}
|
||||
host.parse::<IpAddr>().ok()
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_mask_target_addrs(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
upstream_manager: Option<&crate::transport::UpstreamManager>,
|
||||
) -> std::io::Result<Vec<SocketAddr>> {
|
||||
if let Some(ip) = parse_mask_host_ip_literal(mask_host) {
|
||||
return Ok(vec![SocketAddr::new(ip, mask_port)]);
|
||||
}
|
||||
|
||||
if let Some(upstream_manager) = upstream_manager {
|
||||
return upstream_manager
|
||||
.resolve_all(mask_host, mask_port)
|
||||
.await
|
||||
.map_err(|error| IoError::new(ErrorKind::NotFound, error.to_string()));
|
||||
}
|
||||
|
||||
let addrs = timeout(MASK_TIMEOUT, lookup_host((mask_host, mask_port)))
|
||||
.await
|
||||
.map_err(|_| IoError::new(ErrorKind::TimedOut, "mask target DNS lookup timed out"))??;
|
||||
let addrs = addrs
|
||||
.take(MASK_DNS_RESULT_MAX_ADDRESSES)
|
||||
.collect::<Vec<_>>();
|
||||
if addrs.is_empty() {
|
||||
return Err(IoError::new(
|
||||
ErrorKind::NotFound,
|
||||
"mask target DNS lookup returned no addresses",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(addrs)
|
||||
}
|
||||
|
||||
pub(super) fn matching_tls_domain_for_sni<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
sni: &str,
|
||||
) -> Option<&'a str> {
|
||||
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
|
||||
return Some(config.censorship.tls_domain.as_str());
|
||||
}
|
||||
|
||||
for domain in &config.censorship.tls_domains {
|
||||
if domain.eq_ignore_ascii_case(sni) {
|
||||
return Some(domain.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn parse_exclusive_mask_target(target: &str) -> Option<MaskTcpTarget<'_>> {
|
||||
let target = target.trim();
|
||||
if target.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if target.starts_with('[') {
|
||||
let end = target.find(']')?;
|
||||
if target.get(end + 1..end + 2)? != ":" {
|
||||
return None;
|
||||
}
|
||||
let port = target[end + 2..].parse::<u16>().ok()?;
|
||||
return (port > 0).then_some(MaskTcpTarget {
|
||||
host: &target[..=end],
|
||||
port,
|
||||
});
|
||||
}
|
||||
|
||||
let (host, port) = target.rsplit_once(':')?;
|
||||
if host.is_empty() || host.contains(':') {
|
||||
return None;
|
||||
}
|
||||
let port = port.parse::<u16>().ok()?;
|
||||
(port > 0).then_some(MaskTcpTarget { host, port })
|
||||
}
|
||||
|
||||
pub(super) fn exclusive_mask_target_for_sni<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
sni: &str,
|
||||
) -> Option<MaskTcpTarget<'a>> {
|
||||
if let Some(target) = config.censorship.exclusive_mask_targets.get(sni) {
|
||||
return Some(MaskTcpTarget {
|
||||
host: target.host.as_str(),
|
||||
port: target.port,
|
||||
});
|
||||
}
|
||||
if let Some(target) = config.censorship.exclusive_mask.get(sni) {
|
||||
return parse_exclusive_mask_target(target);
|
||||
}
|
||||
|
||||
if sni.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||
let normalized_sni = sni.to_ascii_lowercase();
|
||||
if let Some(target) = config
|
||||
.censorship
|
||||
.exclusive_mask_targets
|
||||
.get(&normalized_sni)
|
||||
{
|
||||
return Some(MaskTcpTarget {
|
||||
host: target.host.as_str(),
|
||||
port: target.port,
|
||||
});
|
||||
}
|
||||
if let Some(target) = config.censorship.exclusive_mask.get(&normalized_sni) {
|
||||
return parse_exclusive_mask_target(target);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn mask_host_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
) -> &'a str {
|
||||
mask_tcp_target_for_initial_data(config, initial_data).host
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn mask_tcp_target_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
) -> MaskTcpTarget<'a> {
|
||||
let sni = tls::extract_sni_from_client_hello(initial_data);
|
||||
if let Some(target) = sni
|
||||
.as_deref()
|
||||
.and_then(|sni| exclusive_mask_target_for_sni(config, sni))
|
||||
{
|
||||
return target;
|
||||
}
|
||||
|
||||
default_mask_tcp_target_for_initial_data(config, initial_data, sni.as_deref())
|
||||
}
|
||||
|
||||
pub(super) fn default_mask_tcp_target_for_initial_data<'a>(
|
||||
config: &'a ProxyConfig,
|
||||
initial_data: &[u8],
|
||||
sni: Option<&str>,
|
||||
) -> MaskTcpTarget<'a> {
|
||||
let configured_mask_host = config
|
||||
.censorship
|
||||
.mask_host
|
||||
.as_deref()
|
||||
.unwrap_or(&config.censorship.tls_domain);
|
||||
|
||||
if config.censorship.mask_host.is_none() && config.censorship.mask_dynamic {
|
||||
let extracted_sni = if sni.is_none() {
|
||||
tls::extract_sni_from_client_hello(initial_data)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(host) = sni
|
||||
.or(extracted_sni.as_deref())
|
||||
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
|
||||
{
|
||||
return MaskTcpTarget {
|
||||
host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(mask_host) = config.censorship.mask_host.as_deref() {
|
||||
return MaskTcpTarget {
|
||||
host: mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
|
||||
MaskTcpTarget {
|
||||
host: configured_mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn sample_lognormal_percentile_bounded(
|
||||
floor: u64,
|
||||
ceiling: u64,
|
||||
rng: &mut impl Rng,
|
||||
) -> u64 {
|
||||
if ceiling == 0 && floor == 0 {
|
||||
return 0;
|
||||
}
|
||||
if floor > ceiling {
|
||||
return ceiling;
|
||||
}
|
||||
if floor == ceiling {
|
||||
return floor;
|
||||
}
|
||||
let floor_f = floor.max(1) as f64;
|
||||
let ceiling_f = ceiling.max(1) as f64;
|
||||
let mu = (floor_f.ln() + ceiling_f.ln()) / 2.0;
|
||||
// 4.65 ≈ 2 * 2.326 (double-sided z-score for 99th percentile)
|
||||
let sigma = ((ceiling_f / floor_f).ln() / 4.65).max(0.01);
|
||||
// Box-Muller transform: two uniform samples → one standard normal sample
|
||||
let u1: f64 = rng.random_range(f64::MIN_POSITIVE..1.0);
|
||||
let u2: f64 = rng.random_range(0.0_f64..std::f64::consts::TAU);
|
||||
let normal_sample = (-2.0_f64 * u1.ln()).sqrt() * u2.cos();
|
||||
let raw = (mu + sigma * normal_sample).exp();
|
||||
if raw.is_finite() {
|
||||
(raw as u64).clamp(floor, ceiling)
|
||||
} else {
|
||||
((floor_f * ceiling_f).sqrt()) as u64
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mask_outcome_target_budget(config: &ProxyConfig) -> Duration {
|
||||
if config.censorship.mask_timing_normalization_enabled {
|
||||
let floor = config.censorship.mask_timing_normalization_floor_ms;
|
||||
let ceiling = config.censorship.mask_timing_normalization_ceiling_ms;
|
||||
if floor == 0 {
|
||||
if ceiling == 0 {
|
||||
return Duration::from_millis(0);
|
||||
}
|
||||
// floor=0 stays uniform: log-normal cannot model distribution anchored at zero
|
||||
let mut rng = rand::rng();
|
||||
return Duration::from_millis(rng.random_range(0..=ceiling));
|
||||
}
|
||||
if ceiling > floor {
|
||||
let mut rng = rand::rng();
|
||||
return Duration::from_millis(sample_lognormal_percentile_bounded(
|
||||
floor, ceiling, &mut rng,
|
||||
));
|
||||
}
|
||||
// ceiling <= floor: use the larger value (fail-closed: preserve longer delay)
|
||||
return Duration::from_millis(floor.max(ceiling));
|
||||
}
|
||||
|
||||
MASK_TIMEOUT
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_connect_budget_if_needed(started: Instant, config: &ProxyConfig) {
|
||||
if config.censorship.mask_timing_normalization_enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
wait_mask_connect_budget(started).await;
|
||||
}
|
||||
|
||||
pub(super) async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) {
|
||||
let target = mask_outcome_target_budget(config);
|
||||
let elapsed = started.elapsed();
|
||||
if elapsed < target {
|
||||
tokio::time::sleep(target - elapsed).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tls_domain_mask_host_tests {
|
||||
use super::{
|
||||
mask_host_for_initial_data, mask_tcp_target_for_initial_data, matching_tls_domain_for_sni,
|
||||
};
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
fn client_hello_with_sni(sni_host: &str) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0x03, 0x03]);
|
||||
body.extend_from_slice(&[0u8; 32]);
|
||||
body.push(32);
|
||||
body.extend_from_slice(&[0x42u8; 32]);
|
||||
body.extend_from_slice(&2u16.to_be_bytes());
|
||||
body.extend_from_slice(&[0x13, 0x01]);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
|
||||
let host_bytes = sni_host.as_bytes();
|
||||
let mut sni_payload = Vec::new();
|
||||
sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes());
|
||||
sni_payload.push(0);
|
||||
sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes());
|
||||
sni_payload.extend_from_slice(host_bytes);
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
extensions.extend_from_slice(&0x0000u16.to_be_bytes());
|
||||
extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes());
|
||||
extensions.extend_from_slice(&sni_payload);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
|
||||
let mut handshake = Vec::new();
|
||||
handshake.push(0x01);
|
||||
let body_len = (body.len() as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&body_len[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
let mut record = Vec::new();
|
||||
record.push(0x16);
|
||||
record.extend_from_slice(&[0x03, 0x01]);
|
||||
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
|
||||
record.extend_from_slice(&handshake);
|
||||
record
|
||||
}
|
||||
|
||||
fn config_with_tls_domains() -> ProxyConfig {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_domain = "a.com".to_string();
|
||||
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
|
||||
config.censorship.mask_host = None;
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() {
|
||||
let config = config_with_tls_domains();
|
||||
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com"));
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com"));
|
||||
assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_preserves_explicit_non_primary_origin() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config.censorship.mask_host = Some("origin.example".to_string());
|
||||
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(
|
||||
mask_host_for_initial_data(&config, &initial_data),
|
||||
"origin.example"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() {
|
||||
let config = config_with_tls_domains();
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_uses_primary_domain_when_dynamic_masking_is_disabled() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config.censorship.mask_dynamic = false;
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "a.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusive_mask_target_overrides_only_matching_sni() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config
|
||||
.censorship
|
||||
.exclusive_mask
|
||||
.insert("b.com".to_string(), "origin-b.example:8443".to_string());
|
||||
let b_initial_data = client_hello_with_sni("B.COM");
|
||||
let c_initial_data = client_hello_with_sni("c.com");
|
||||
|
||||
let b_target = mask_tcp_target_for_initial_data(&config, &b_initial_data);
|
||||
let c_target = mask_tcp_target_for_initial_data(&config, &c_initial_data);
|
||||
|
||||
assert_eq!(b_target.host, "origin-b.example");
|
||||
assert_eq!(b_target.port, 8443);
|
||||
assert_eq!(c_target.host, "c.com");
|
||||
assert_eq!(c_target.port, config.censorship.mask_port);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,12 @@
|
||||
use super::*;
|
||||
|
||||
// Bounded C2ME sender and downstream writer tasks.
|
||||
mod tasks;
|
||||
// Conntrack close classification.
|
||||
mod close_reason;
|
||||
|
||||
use close_reason::classify_conntrack_close_reason;
|
||||
use tasks::{run_c2me_sender, run_me_writer};
|
||||
struct RelayConnLease {
|
||||
connection: Option<ConnLease>,
|
||||
conn_id: u64,
|
||||
@@ -174,47 +181,21 @@ where
|
||||
};
|
||||
let c2me_byte_budget = c2me_queued_permit_budget(c2me_channel_capacity, frame_limit);
|
||||
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
|
||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||
let (c2me_tx, c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||
let me_pool_c2me = me_pool.clone();
|
||||
let mut c2me_sender = tokio::spawn(async move {
|
||||
let mut sent_since_yield = 0usize;
|
||||
while let Some(cmd) = c2me_rx.recv().await {
|
||||
match cmd {
|
||||
C2MeCommand::Data {
|
||||
payload,
|
||||
flags,
|
||||
_permit,
|
||||
} => {
|
||||
me_pool_c2me
|
||||
.send_proxy_req_pooled(
|
||||
conn_id,
|
||||
success.dc_idx,
|
||||
peer,
|
||||
translated_local_addr,
|
||||
payload,
|
||||
_permit,
|
||||
flags,
|
||||
effective_tag_array,
|
||||
)
|
||||
.await?;
|
||||
sent_since_yield = sent_since_yield.saturating_add(1);
|
||||
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
|
||||
sent_since_yield = 0;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
C2MeCommand::Close => {
|
||||
let _ = me_pool_c2me.send_close(conn_id).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
let mut c2me_sender = tokio::spawn(run_c2me_sender(
|
||||
c2me_rx,
|
||||
me_pool_c2me,
|
||||
conn_id,
|
||||
success,
|
||||
peer,
|
||||
translated_local_addr,
|
||||
effective_tag_array,
|
||||
));
|
||||
|
||||
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
|
||||
let (stop_tx, stop_rx) = oneshot::channel::<()>();
|
||||
let flow_cancel = CancellationToken::new();
|
||||
let mut me_rx_task = me_rx;
|
||||
let me_rx_task = me_rx;
|
||||
let stats_clone = stats.clone();
|
||||
let rng_clone = rng.clone();
|
||||
let user_clone = user.clone();
|
||||
@@ -224,361 +205,24 @@ where
|
||||
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
||||
let bytes_me2c_clone = bytes_me2c.clone();
|
||||
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
||||
let mut me_writer = tokio::spawn(async move {
|
||||
let mut writer = crypto_writer;
|
||||
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
|
||||
fn shrink_session_vec(buf: &mut Vec<u8>, threshold: usize) {
|
||||
if buf.capacity() > threshold {
|
||||
buf.clear();
|
||||
buf.shrink_to(threshold);
|
||||
} else {
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
msg = me_rx_task.recv() => {
|
||||
let Some(first) = msg else {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
};
|
||||
|
||||
let mut batch_frames = 0usize;
|
||||
let mut batch_bytes = 0usize;
|
||||
let mut flush_immediately;
|
||||
let mut max_delay_fired = false;
|
||||
|
||||
let first_is_downstream_activity =
|
||||
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
first,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
false,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if first_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately = immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(next) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at =
|
||||
if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ =
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !flush_immediately
|
||||
&& !d2c_flush_policy.max_delay.is_zero()
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
stats_clone.increment_me_d2c_batch_timeout_armed_total();
|
||||
match tokio::time::timeout(d2c_flush_policy.max_delay, me_rx_task.recv()).await {
|
||||
Ok(Some(next)) => {
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(extra) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let extra_is_downstream_activity =
|
||||
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
extra,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if extra_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
}
|
||||
Err(_) => {
|
||||
max_delay_fired = true;
|
||||
stats_clone.increment_me_d2c_batch_timeout_fired_total();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let flush_reason = classify_me_d2c_flush_reason(
|
||||
flush_immediately,
|
||||
batch_frames,
|
||||
d2c_flush_policy.max_frames,
|
||||
batch_bytes,
|
||||
d2c_flush_policy.max_bytes,
|
||||
max_delay_fired,
|
||||
);
|
||||
let physical_flush =
|
||||
me_d2c_flush_reason_requires_client_flush(flush_reason);
|
||||
let flush_started_at = if physical_flush
|
||||
&& stats_clone.telemetry_policy().me_level.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if physical_flush {
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
}
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
flush_reason,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
let shrink_trigger = shrink_threshold
|
||||
.saturating_mul(ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR);
|
||||
if frame_buf.capacity() > shrink_trigger {
|
||||
let cap_before = frame_buf.capacity();
|
||||
frame_buf.shrink_to(shrink_threshold);
|
||||
let cap_after = frame_buf.capacity();
|
||||
let bytes_freed = cap_before.saturating_sub(cap_after) as u64;
|
||||
stats_clone.observe_me_d2c_frame_buf_shrink(bytes_freed);
|
||||
}
|
||||
}
|
||||
_ = &mut stop_rx => {
|
||||
debug!(conn_id, "ME writer stop signal");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let mut me_writer = tokio::spawn(run_me_writer(
|
||||
crypto_writer,
|
||||
me_rx_task,
|
||||
stats_clone,
|
||||
rng_clone,
|
||||
user_clone,
|
||||
quota_user_stats_me_writer,
|
||||
quota_limit,
|
||||
traffic_lease_me_writer,
|
||||
flow_cancel_me_writer,
|
||||
last_downstream_activity_ms_clone,
|
||||
bytes_me2c_clone,
|
||||
d2c_flush_policy,
|
||||
proto_tag,
|
||||
session_started_at,
|
||||
conn_id,
|
||||
stop_rx,
|
||||
));
|
||||
|
||||
let mut main_result: Result<()> = Ok(());
|
||||
let mut client_closed = false;
|
||||
@@ -875,30 +519,3 @@ where
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(ProxyError::Io(error)) if matches!(error.kind(), std::io::ErrorKind::TimedOut) => {
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn classify_conntrack_close_reason(result: &Result<()>) -> ConntrackCloseReason {
|
||||
match result {
|
||||
Ok(()) => ConntrackCloseReason::NormalEof,
|
||||
Err(ProxyError::Io(error)) if matches!(error.kind(), std::io::ErrorKind::TimedOut) => {
|
||||
ConntrackCloseReason::Timeout
|
||||
}
|
||||
Err(ProxyError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
| std::io::ErrorKind::UnexpectedEof
|
||||
) =>
|
||||
{
|
||||
ConntrackCloseReason::Reset
|
||||
}
|
||||
Err(ProxyError::Proxy(message))
|
||||
if message.contains("pressure") || message.contains("evicted") =>
|
||||
{
|
||||
ConntrackCloseReason::Pressure
|
||||
}
|
||||
Err(_) => ConntrackCloseReason::Other,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
use super::*;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_c2me_sender(
|
||||
mut c2me_rx: mpsc::Receiver<C2MeCommand>,
|
||||
me_pool_c2me: Arc<MePool>,
|
||||
conn_id: u64,
|
||||
success: HandshakeSuccess,
|
||||
peer: SocketAddr,
|
||||
translated_local_addr: SocketAddr,
|
||||
effective_tag_array: Option<[u8; 16]>,
|
||||
) -> Result<()> {
|
||||
let mut sent_since_yield = 0usize;
|
||||
while let Some(cmd) = c2me_rx.recv().await {
|
||||
match cmd {
|
||||
C2MeCommand::Data {
|
||||
payload,
|
||||
flags,
|
||||
_permit,
|
||||
} => {
|
||||
me_pool_c2me
|
||||
.send_proxy_req_pooled(
|
||||
conn_id,
|
||||
success.dc_idx,
|
||||
peer,
|
||||
translated_local_addr,
|
||||
payload,
|
||||
_permit,
|
||||
flags,
|
||||
effective_tag_array,
|
||||
)
|
||||
.await?;
|
||||
sent_since_yield = sent_since_yield.saturating_add(1);
|
||||
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
|
||||
sent_since_yield = 0;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
C2MeCommand::Close => {
|
||||
let _ = me_pool_c2me.send_close(conn_id).await;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_me_writer<W>(
|
||||
crypto_writer: CryptoWriter<W>,
|
||||
mut me_rx_task: mpsc::Receiver<MeResponse>,
|
||||
stats_clone: Arc<Stats>,
|
||||
rng_clone: Arc<SecureRandom>,
|
||||
user_clone: String,
|
||||
quota_user_stats_me_writer: Option<Arc<UserStats>>,
|
||||
quota_limit: Option<u64>,
|
||||
traffic_lease_me_writer: Option<Arc<TrafficLease>>,
|
||||
flow_cancel_me_writer: CancellationToken,
|
||||
last_downstream_activity_ms_clone: Arc<AtomicU64>,
|
||||
bytes_me2c_clone: Arc<AtomicU64>,
|
||||
d2c_flush_policy: MeD2cFlushPolicy,
|
||||
proto_tag: ProtoTag,
|
||||
session_started_at: Instant,
|
||||
conn_id: u64,
|
||||
mut stop_rx: oneshot::Receiver<()>,
|
||||
) -> Result<()>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let mut writer = crypto_writer;
|
||||
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
|
||||
fn shrink_session_vec(buf: &mut Vec<u8>, threshold: usize) {
|
||||
if buf.capacity() > threshold {
|
||||
buf.clear();
|
||||
buf.shrink_to(threshold);
|
||||
} else {
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
msg = me_rx_task.recv() => {
|
||||
let Some(first) = msg else {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
};
|
||||
|
||||
let mut batch_frames = 0usize;
|
||||
let mut batch_bytes = 0usize;
|
||||
let mut flush_immediately;
|
||||
let mut max_delay_fired = false;
|
||||
|
||||
let first_is_downstream_activity =
|
||||
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
first,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
false,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if first_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately = immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(next) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at =
|
||||
if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ =
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !flush_immediately
|
||||
&& !d2c_flush_policy.max_delay.is_zero()
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
stats_clone.increment_me_d2c_batch_timeout_armed_total();
|
||||
match tokio::time::timeout(d2c_flush_policy.max_delay, me_rx_task.recv()).await {
|
||||
Ok(Some(next)) => {
|
||||
let next_is_downstream_activity =
|
||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
next,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if next_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
while !flush_immediately
|
||||
&& batch_frames < d2c_flush_policy.max_frames
|
||||
&& batch_bytes < d2c_flush_policy.max_bytes
|
||||
{
|
||||
let Ok(extra) = me_rx_task.try_recv() else {
|
||||
break;
|
||||
};
|
||||
|
||||
let extra_is_downstream_activity =
|
||||
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||
match process_me_writer_response_with_traffic_lease(
|
||||
extra,
|
||||
&mut writer,
|
||||
proto_tag,
|
||||
rng_clone.as_ref(),
|
||||
&mut frame_buf,
|
||||
stats_clone.as_ref(),
|
||||
&user_clone,
|
||||
quota_user_stats_me_writer.as_deref(),
|
||||
quota_limit,
|
||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||
traffic_lease_me_writer.as_ref(),
|
||||
&flow_cancel_me_writer,
|
||||
bytes_me2c_clone.as_ref(),
|
||||
conn_id,
|
||||
d2c_flush_policy.ack_flush_immediate,
|
||||
true,
|
||||
).await? {
|
||||
MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => {
|
||||
if extra_is_downstream_activity {
|
||||
last_downstream_activity_ms_clone
|
||||
.store(session_started_at.elapsed().as_millis() as u64, Ordering::Relaxed);
|
||||
}
|
||||
batch_frames = batch_frames.saturating_add(frames);
|
||||
batch_bytes = batch_bytes.saturating_add(bytes);
|
||||
flush_immediately |= immediate;
|
||||
}
|
||||
MeWriterResponseOutcome::Close => {
|
||||
let flush_started_at = if stats_clone
|
||||
.telemetry_policy()
|
||||
.me_level
|
||||
.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _ = flush_client_or_cancel(
|
||||
&mut writer,
|
||||
&flow_cancel_me_writer,
|
||||
)
|
||||
.await;
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
MeD2cFlushReason::Close,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
debug!(conn_id, "ME channel closed");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Err(ProxyError::MiddleConnectionLost);
|
||||
}
|
||||
Err(_) => {
|
||||
max_delay_fired = true;
|
||||
stats_clone.increment_me_d2c_batch_timeout_fired_total();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let flush_reason = classify_me_d2c_flush_reason(
|
||||
flush_immediately,
|
||||
batch_frames,
|
||||
d2c_flush_policy.max_frames,
|
||||
batch_bytes,
|
||||
d2c_flush_policy.max_bytes,
|
||||
max_delay_fired,
|
||||
);
|
||||
let physical_flush =
|
||||
me_d2c_flush_reason_requires_client_flush(flush_reason);
|
||||
let flush_started_at = if physical_flush
|
||||
&& stats_clone.telemetry_policy().me_level.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if physical_flush {
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
}
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
.as_micros()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
});
|
||||
observe_me_d2c_flush_event(
|
||||
stats_clone.as_ref(),
|
||||
flush_reason,
|
||||
batch_frames,
|
||||
batch_bytes,
|
||||
flush_duration_us,
|
||||
);
|
||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||
let shrink_trigger = shrink_threshold
|
||||
.saturating_mul(ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR);
|
||||
if frame_buf.capacity() > shrink_trigger {
|
||||
let cap_before = frame_buf.capacity();
|
||||
frame_buf.shrink_to(shrink_threshold);
|
||||
let cap_after = frame_buf.capacity();
|
||||
let bytes_freed = cap_before.saturating_sub(cap_after) as u64;
|
||||
stats_clone.observe_me_d2c_frame_buf_shrink(bytes_freed);
|
||||
}
|
||||
}
|
||||
_ = &mut stop_rx => {
|
||||
debug!(conn_id, "ME writer stop signal");
|
||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,358 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-parent-swap-openat-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&base) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&base).expect("stale parent-swap-openat path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&base)
|
||||
.expect("stale parent-swap-openat directory must be removable");
|
||||
}
|
||||
}
|
||||
let moved = base.with_extension("bak");
|
||||
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&moved)
|
||||
.expect("stale parent-swap-openat backup path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&moved)
|
||||
.expect("stale parent-swap-openat backup directory must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-parent-swap-openat-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent swap");
|
||||
fs::write(&sanitized.resolved_path, "seed\n").expect("seed target file must be writable");
|
||||
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"precondition: target should initially pass revalidation"
|
||||
);
|
||||
|
||||
let outside_parent = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-parent-swap-openat-outside-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&outside_parent).expect("outside parent directory must be creatable");
|
||||
let outside_target = outside_parent.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&outside_target);
|
||||
|
||||
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
|
||||
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect_err("anchored open must fail when parent is swapped to symlink");
|
||||
let raw = err.raw_os_error();
|
||||
assert!(
|
||||
matches!(
|
||||
raw,
|
||||
Some(libc::ELOOP) | Some(libc::ENOTDIR) | Some(libc::ENOENT)
|
||||
),
|
||||
"anchored open must fail closed on parent swap race, got raw_os_error={raw:?}"
|
||||
);
|
||||
assert!(
|
||||
!outside_target.exists(),
|
||||
"anchored open must never create a log file in swapped outside parent"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_nix_path_writes_expected_lines() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-open-ok-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-open-ok base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-open-ok-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut first = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must create log file in allowed parent");
|
||||
append_unknown_dc_line(&mut first, 31_200).expect("first append must succeed");
|
||||
|
||||
let mut second = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored reopen must succeed for existing regular file");
|
||||
append_unknown_dc_line(&mut second, 31_201).expect("second append must succeed");
|
||||
|
||||
let content =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("anchored log file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 2, "expected one line per anchored append call");
|
||||
assert!(
|
||||
lines.contains(&"dc_idx=31200") && lines.contains(&"dc_idx=31201"),
|
||||
"anchored append output must contain both expected dc_idx lines"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_parallel_appends_preserve_line_integrity() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-open-parallel-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-open-parallel base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-open-parallel-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut workers = Vec::new();
|
||||
for idx in 0..64i16 {
|
||||
let sanitized = sanitized.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must succeed in worker");
|
||||
append_unknown_dc_line(&mut file, 32_000 + idx).expect("worker append must succeed");
|
||||
}));
|
||||
}
|
||||
|
||||
for worker in workers {
|
||||
worker.join().expect("worker must not panic");
|
||||
}
|
||||
|
||||
let content =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("parallel log file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lines.len(),
|
||||
64,
|
||||
"expected one complete line per worker append"
|
||||
);
|
||||
for line in lines {
|
||||
assert!(
|
||||
line.starts_with("dc_idx="),
|
||||
"line must keep dc_idx prefix and not be interleaved: {line}"
|
||||
);
|
||||
let value = line
|
||||
.strip_prefix("dc_idx=")
|
||||
.expect("prefix checked above")
|
||||
.parse::<i16>();
|
||||
assert!(
|
||||
value.is_ok(),
|
||||
"line payload must remain parseable i16 and not be corrupted: {line}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_creates_private_0600_file_permissions() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-perms-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-perms base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-perms-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must create file with restricted mode");
|
||||
append_unknown_dc_line(&mut file, 31_210).expect("initial append must succeed");
|
||||
drop(file);
|
||||
|
||||
let mode = fs::metadata(&sanitized.resolved_path)
|
||||
.expect("created log file metadata must be readable")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777;
|
||||
assert_eq!(
|
||||
mode, 0o600,
|
||||
"anchored open must create unknown-dc log file with owner-only rw permissions"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_rejects_existing_symlink_target() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-symlink-target-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-symlink-target base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-symlink-target-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-anchored-symlink-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside\n").expect("outside baseline file must be writable");
|
||||
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
symlink(&outside, &sanitized.resolved_path)
|
||||
.expect("target symlink for anchored-open rejection test must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect_err("anchored open must reject symlinked filename target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"anchored open should fail closed with ELOOP on symlinked target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn anchored_open_high_contention_multi_write_preserves_complete_lines() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-anchored-contention-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("anchored-contention base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-anchored-contention-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
let _ = fs::remove_file(&sanitized.resolved_path);
|
||||
|
||||
let workers = 24usize;
|
||||
let rounds = 40usize;
|
||||
let mut threads = Vec::new();
|
||||
|
||||
for worker in 0..workers {
|
||||
let sanitized = sanitized.clone();
|
||||
threads.push(std::thread::spawn(move || {
|
||||
for round in 0..rounds {
|
||||
let mut file = open_unknown_dc_log_append_anchored(&sanitized)
|
||||
.expect("anchored open must succeed under contention");
|
||||
let dc_idx = 20_000i16.wrapping_add((worker * rounds + round) as i16);
|
||||
append_unknown_dc_line(&mut file, dc_idx)
|
||||
.expect("each contention append must complete");
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for thread in threads {
|
||||
thread.join().expect("contention worker must not panic");
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(&sanitized.resolved_path)
|
||||
.expect("contention output file must be readable");
|
||||
let lines: Vec<&str> = content
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lines.len(),
|
||||
workers * rounds,
|
||||
"every contention append must produce exactly one line"
|
||||
);
|
||||
|
||||
let mut unique = std::collections::HashSet::new();
|
||||
for line in lines {
|
||||
assert!(
|
||||
line.starts_with("dc_idx="),
|
||||
"line must preserve expected prefix under heavy contention: {line}"
|
||||
);
|
||||
let value = line
|
||||
.strip_prefix("dc_idx=")
|
||||
.expect("prefix validated")
|
||||
.parse::<i16>()
|
||||
.expect("line payload must remain parseable i16 under contention");
|
||||
unique.insert(value);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
unique.len(),
|
||||
workers * rounds,
|
||||
"contention output must not lose or duplicate logical writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn append_unknown_dc_line_returns_error_for_read_only_descriptor() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-append-ro-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("append-ro base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-append-ro-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
fs::write(&sanitized.resolved_path, "seed\n").expect("seed file must be writable");
|
||||
|
||||
let mut readonly = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(&sanitized.resolved_path)
|
||||
.expect("readonly file open must succeed");
|
||||
|
||||
append_unknown_dc_line(&mut readonly, 31_222)
|
||||
.expect_err("append on readonly descriptor must fail closed");
|
||||
|
||||
let content_after =
|
||||
fs::read_to_string(&sanitized.resolved_path).expect("seed file must remain readable");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&content_after),
|
||||
1,
|
||||
"failed readonly append must not modify persisted unknown-dc log content"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_absolute_log_path_writes_one_entry() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_001;
|
||||
let file_path = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-abs-{}-{}.log",
|
||||
std::process::id(),
|
||||
dc_idx
|
||||
));
|
||||
let _ = fs::remove_file(&file_path);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(
|
||||
file_path
|
||||
.to_str()
|
||||
.expect("temp file path must be valid UTF-8")
|
||||
.to_string(),
|
||||
);
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..20 {
|
||||
if let Ok(text) = fs::read_to_string(&file_path) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("absolute unknown-DC log path must produce exactly one log write");
|
||||
assert!(
|
||||
text.contains(&format!("dc_idx={dc_idx}")),
|
||||
"absolute unknown-DC integration log must contain requested dc_idx"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_safe_relative_log_path_writes_one_entry() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_002;
|
||||
let rel_dir = format!("target/telemt-unknown-dc-int-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("integration test log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..20 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("safe relative path must produce exactly one log write");
|
||||
assert!(
|
||||
text.contains(&format!("dc_idx={dc_idx}")),
|
||||
"unknown-DC integration log must contain requested dc_idx"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_same_index_burst_writes_only_once() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_010;
|
||||
let rel_dir = format!("target/telemt-unknown-dc-same-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir().unwrap().join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("same-index log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
for _ in 0..64 {
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
}
|
||||
|
||||
let mut content = None;
|
||||
for _ in 0..30 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
content = Some(text);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let text = content.expect("same-index burst must produce at least one log write");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&text),
|
||||
1,
|
||||
"same unknown dc index must be deduplicated to one file line"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_distinct_burst_is_hard_capped_on_file_writes() {
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-unknown-dc-cap-{}", std::process::id());
|
||||
let rel_file = format!("{rel_dir}/unknown-dc.log");
|
||||
let abs_dir = std::env::current_dir().unwrap().join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("cap log directory must be creatable");
|
||||
let abs_file = abs_dir.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&abs_file);
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
for i in 0..(UNKNOWN_DC_LOG_DISTINCT_LIMIT + 128) {
|
||||
let dc_idx = 20_000i16.wrapping_add(i as i16);
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
}
|
||||
|
||||
let mut final_text = String::new();
|
||||
for _ in 0..80 {
|
||||
if let Ok(text) = fs::read_to_string(&abs_file) {
|
||||
final_text = text;
|
||||
if nonempty_line_count(&final_text) >= UNKNOWN_DC_LOG_DISTINCT_LIMIT {
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let line_count = nonempty_line_count(&final_text);
|
||||
assert!(
|
||||
line_count > 0,
|
||||
"distinct unknown-dc burst must write at least one line"
|
||||
);
|
||||
assert!(
|
||||
line_count <= UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"distinct unknown-dc writes must stay within dedup hard cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn unknown_dc_symlinked_target_escape_is_not_written_integration() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-no-write-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("integration symlink base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "baseline\n").expect("outside baseline file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("symlink target must be creatable");
|
||||
|
||||
let rel_file = format!(
|
||||
"target/telemt-unknown-dc-no-write-link-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let dc_idx: i16 = 31_050;
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some(rel_file);
|
||||
|
||||
let before = fs::read_to_string(&outside).expect("must read baseline outside file");
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
tokio::time::sleep(Duration::from_millis(80)).await;
|
||||
let after = fs::read_to_string(&outside).expect("must read outside file after attempt");
|
||||
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"symlink target escape must not be written by unknown-DC logging"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
use super::*;
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_open_append_rejects_symlink_target_with_nofollow() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-nofollow-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("nofollow base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-nofollow-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = fs::remove_file(&outside);
|
||||
fs::write(&outside, "outside\n").expect("outside file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("symlink target must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&linked_target)
|
||||
.expect_err("O_NOFOLLOW open must fail for symlink target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"symlink target must be rejected with ELOOP when O_NOFOLLOW is applied"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_open_append_rejects_broken_symlink_target_with_nofollow() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-broken-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("broken-link base must be creatable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(base.join("missing-target.log"), &linked_target)
|
||||
.expect("broken symlink target must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&linked_target)
|
||||
.expect_err("O_NOFOLLOW open must fail for broken symlink target");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"broken symlink target must be rejected with ELOOP when O_NOFOLLOW is applied"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_unknown_dc_open_append_symlink_flip_never_writes_outside_file() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-symlink-flip-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("symlink-flip base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-symlink-flip-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside-baseline\n").expect("outside baseline file must be writable");
|
||||
let outside_before = fs::read_to_string(&outside).expect("outside baseline must be readable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
for step in 0..1024usize {
|
||||
let _ = fs::remove_file(&target);
|
||||
if step % 2 == 0 {
|
||||
symlink(&outside, &target).expect("symlink creation in flip loop must succeed");
|
||||
}
|
||||
if let Ok(mut file) = open_unknown_dc_log_append(&target) {
|
||||
writeln!(file, "dc_idx={step}").expect("append on regular file must succeed");
|
||||
}
|
||||
}
|
||||
|
||||
let outside_after = fs::read_to_string(&outside).expect("outside file must remain readable");
|
||||
assert_eq!(
|
||||
outside_after, outside_before,
|
||||
"outside file must never be modified under symlink-flip adversarial churn"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_open_append_creates_regular_file() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-open-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("open test base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
{
|
||||
let mut file = open_unknown_dc_log_append(&target)
|
||||
.expect("regular target must be creatable with append open");
|
||||
writeln!(file, "dc_idx=1234").expect("append write must succeed");
|
||||
}
|
||||
|
||||
let meta = fs::symlink_metadata(&target).expect("created target metadata must be readable");
|
||||
assert!(meta.file_type().is_file(), "target must be a regular file");
|
||||
assert!(
|
||||
!meta.file_type().is_symlink(),
|
||||
"regular target open path must not produce symlink artifacts"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_unknown_dc_open_append_regular_file_preserves_line_integrity() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-open-stress-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("stress open base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&target);
|
||||
|
||||
let writes = 2048usize;
|
||||
for idx in 0..writes {
|
||||
let mut file = open_unknown_dc_log_append(&target)
|
||||
.expect("stress append open on regular file must succeed");
|
||||
writeln!(file, "dc_idx={idx}").expect("stress append write must succeed");
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(&target).expect("stress output file must be readable");
|
||||
assert_eq!(
|
||||
nonempty_line_count(&content),
|
||||
writes,
|
||||
"regular-file append stress must preserve one logical line per write"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_accepts_regular_existing_target() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-safe-target-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("safe target base must be creatable");
|
||||
|
||||
let target = base.join("unknown-dc.log");
|
||||
fs::write(&target, "seed\n").expect("safe target seed write must succeed");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-safe-target-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized =
|
||||
sanitize_unknown_dc_log_path(&rel_candidate).expect("safe candidate must sanitize");
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must allow safe existing regular files"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_deleted_parent_after_sanitize() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-vanish-parent-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("vanish-parent base must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-vanish-parent-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent deletion");
|
||||
|
||||
fs::remove_dir_all(&base).expect("test parent directory must be removable");
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must fail when sanitized parent disappears before write"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let parent = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-parent-swap-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&parent) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&parent).expect("stale parent-swap path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&parent).expect("stale parent-swap directory must be removable");
|
||||
}
|
||||
}
|
||||
let moved = parent.with_extension("bak");
|
||||
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&moved).expect("stale parent-swap backup path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&moved)
|
||||
.expect("stale parent-swap backup directory must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-parent-swap-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate must sanitize before parent swap");
|
||||
|
||||
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
|
||||
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
|
||||
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"revalidation must fail when canonical parent is swapped to a symlinked target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn adversarial_check_then_symlink_flip_is_blocked_by_nofollow_open() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let parent = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-check-open-race-{}",
|
||||
std::process::id()
|
||||
));
|
||||
if let Ok(meta) = fs::symlink_metadata(&parent) {
|
||||
if meta.file_type().is_symlink() || meta.is_file() {
|
||||
fs::remove_file(&parent).expect("stale check-open-race path must be removable");
|
||||
} else {
|
||||
fs::remove_dir_all(&parent).expect("stale check-open-race parent must be removable");
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(&parent).expect("check-open-race parent must be creatable");
|
||||
|
||||
let target = parent.join("unknown-dc.log");
|
||||
fs::write(&target, "seed\n").expect("seed target file must be writable");
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-check-open-race-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate).expect("candidate must sanitize");
|
||||
|
||||
assert!(
|
||||
unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"precondition: target should initially pass revalidation"
|
||||
);
|
||||
|
||||
let outside = std::env::temp_dir().join(format!(
|
||||
"telemt-unknown-dc-check-open-race-outside-{}.log",
|
||||
std::process::id()
|
||||
));
|
||||
fs::write(&outside, "outside\n").expect("outside file must be writable");
|
||||
fs::remove_file(&target).expect("target removal before flip must succeed");
|
||||
symlink(&outside, &target).expect("target symlink flip must be creatable");
|
||||
|
||||
let err = open_unknown_dc_log_append(&sanitized.resolved_path)
|
||||
.expect_err("nofollow open must fail after symlink flip between check and open");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(libc::ELOOP),
|
||||
"symlink flip in check/open window must be neutralized by O_NOFOLLOW"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_dc_never_panics_with_single_dc_list() {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.default_dc = Some(42);
|
||||
|
||||
let addr = get_dc_addr_static(999, &cfg).expect("fallback dc must resolve safely");
|
||||
let expected = SocketAddr::new(TG_DATACENTERS_V6[0], TG_DATACENTER_PORT);
|
||||
assert_eq!(addr, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_abort_midflight_releases_route_gauge() {
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
let _hold_stream = stream;
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "abort-direct-user".to_string(),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: "127.0.0.1:50000".parse().unwrap(),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let relay_task = tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats.clone(),
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xabad1dea,
|
||||
));
|
||||
|
||||
let started = tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
started.is_ok(),
|
||||
"direct relay must increment route gauge before abort"
|
||||
);
|
||||
|
||||
relay_task.abort();
|
||||
let joined = relay_task.await;
|
||||
assert!(
|
||||
joined.is_err(),
|
||||
"aborted direct relay task must return join error"
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"route gauge must be released when direct relay task is aborted mid-flight"
|
||||
);
|
||||
|
||||
drop(client_side);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_cutover_midflight_releases_route_gauge() {
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
let _hold_stream = stream;
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "cutover-direct-user".to_string(),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: "127.0.0.1:50002".parse().unwrap(),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let relay_task = tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats.clone(),
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xface_cafe,
|
||||
));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("direct relay must increment route gauge before cutover");
|
||||
|
||||
assert!(
|
||||
route_runtime.set_mode(RelayRouteMode::Middle).is_some(),
|
||||
"cutover must advance route generation"
|
||||
);
|
||||
|
||||
let relay_result = tokio::time::timeout(Duration::from_secs(6), relay_task)
|
||||
.await
|
||||
.expect("direct relay must terminate after cutover")
|
||||
.expect("direct relay task must not panic");
|
||||
assert!(
|
||||
relay_result.is_err(),
|
||||
"cutover should terminate direct relay session"
|
||||
);
|
||||
assert!(
|
||||
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||
"client-visible cutover error must stay generic and avoid route-internal metadata"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"route gauge must be released when direct relay exits on cutover"
|
||||
);
|
||||
|
||||
drop(client_side);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_releases_gauge() {
|
||||
let session_count = 6usize;
|
||||
let tg_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let tg_addr = tg_listener.local_addr().unwrap();
|
||||
|
||||
let tg_accept_task = tokio::spawn(async move {
|
||||
let mut held_streams = Vec::with_capacity(session_count);
|
||||
for _ in 0..session_count {
|
||||
let (stream, _) = tg_listener.accept().await.unwrap();
|
||||
held_streams.push(stream);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
drop(held_streams);
|
||||
});
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![tg_addr.to_string()]);
|
||||
let config = Arc::new(config);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
10,
|
||||
1,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(session_count);
|
||||
let mut client_sides = Vec::with_capacity(session_count);
|
||||
|
||||
for idx in 0..session_count {
|
||||
let (server_side, client_side) = duplex(64 * 1024);
|
||||
client_sides.push(client_side);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_side);
|
||||
let client_reader = make_crypto_reader(server_reader);
|
||||
let client_writer = make_crypto_writer(server_writer);
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: format!("cutover-storm-direct-user-{idx}"),
|
||||
dc_idx: 2,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
dec_key: [0u8; 32],
|
||||
dec_iv: 0,
|
||||
enc_key: [0u8; 32],
|
||||
enc_iv: 0,
|
||||
peer: SocketAddr::new(
|
||||
std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)),
|
||||
51000 + idx as u16,
|
||||
),
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
relay_tasks.push(tokio::spawn(handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager.clone(),
|
||||
stats.clone(),
|
||||
config.clone(),
|
||||
buffer_pool.clone(),
|
||||
rng.clone(),
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
0xA000_0000 + idx as u64,
|
||||
)));
|
||||
}
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(4), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == session_count as u64 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all direct sessions must become active before cutover storm");
|
||||
|
||||
let route_runtime_flipper = route_runtime.clone();
|
||||
let flipper = tokio::spawn(async move {
|
||||
for step in 0..64u32 {
|
||||
let mode = if (step & 1) == 0 {
|
||||
RelayRouteMode::Middle
|
||||
} else {
|
||||
RelayRouteMode::Direct
|
||||
};
|
||||
let _ = route_runtime_flipper.set_mode(mode);
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
}
|
||||
});
|
||||
|
||||
for relay_task in relay_tasks {
|
||||
let relay_result = tokio::time::timeout(Duration::from_secs(10), relay_task)
|
||||
.await
|
||||
.expect("direct relay task must finish under cutover storm")
|
||||
.expect("direct relay task must not panic");
|
||||
|
||||
assert!(
|
||||
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||
"storm-cutover termination must remain generic for all direct sessions"
|
||||
);
|
||||
}
|
||||
|
||||
flipper.abort();
|
||||
let _ = flipper.await;
|
||||
|
||||
assert_eq!(
|
||||
stats.get_current_connections_direct(),
|
||||
0,
|
||||
"direct route gauge must return to zero after cutover storm"
|
||||
);
|
||||
|
||||
drop(client_sides);
|
||||
tg_accept_task.abort();
|
||||
let _ = tg_accept_task.await;
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn prefer_v6_override_matrix_prefers_matching_family_then_degrades_safely() {
|
||||
let dc_idx: i16 = 2;
|
||||
|
||||
let mut cfg_a = ProxyConfig::default();
|
||||
cfg_a.network.prefer = 6;
|
||||
cfg_a.network.ipv6 = Some(true);
|
||||
cfg_a.dc_overrides.insert(
|
||||
dc_idx.to_string(),
|
||||
vec![
|
||||
"203.0.113.90:443".to_string(),
|
||||
"[2001:db8::90]:443".to_string(),
|
||||
],
|
||||
);
|
||||
let a = get_dc_addr_static(dc_idx, &cfg_a).expect("v6+v4 override set must resolve");
|
||||
assert!(
|
||||
a.is_ipv6(),
|
||||
"prefer_v6 should choose v6 override when present"
|
||||
);
|
||||
|
||||
let mut cfg_b = ProxyConfig::default();
|
||||
cfg_b.network.prefer = 6;
|
||||
cfg_b.network.ipv6 = Some(true);
|
||||
cfg_b
|
||||
.dc_overrides
|
||||
.insert(dc_idx.to_string(), vec!["203.0.113.91:443".to_string()]);
|
||||
let b = get_dc_addr_static(dc_idx, &cfg_b).expect("v4-only override must still resolve");
|
||||
assert!(
|
||||
b.is_ipv4(),
|
||||
"when no v6 override exists, v4 override must be used"
|
||||
);
|
||||
|
||||
let mut cfg_c = ProxyConfig::default();
|
||||
cfg_c.network.prefer = 6;
|
||||
cfg_c.network.ipv6 = Some(true);
|
||||
let c = get_dc_addr_static(dc_idx, &cfg_c).expect("table fallback must resolve");
|
||||
assert_eq!(
|
||||
c,
|
||||
SocketAddr::new(TG_DATACENTERS_V6[(dc_idx as usize) - 1], TG_DATACENTER_PORT),
|
||||
"without overrides, prefer_v6 path must resolve from static v6 datacenter table"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefer_v6_override_matrix_ignores_invalid_entries_and_keeps_fail_closed_fallback() {
|
||||
let dc_idx: i16 = 3;
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.dc_overrides.insert(
|
||||
dc_idx.to_string(),
|
||||
vec![
|
||||
"not-an-addr".to_string(),
|
||||
"also:bad".to_string(),
|
||||
"203.0.113.55:443".to_string(),
|
||||
],
|
||||
);
|
||||
|
||||
let addr = get_dc_addr_static(dc_idx, &cfg)
|
||||
.expect("at least one valid override must keep resolution alive");
|
||||
assert_eq!(addr, "203.0.113.55:443".parse::<SocketAddr>().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_prefer_v6_override_matrix_is_deterministic_under_mixed_inputs() {
|
||||
for idx in 1..=5i16 {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.network.prefer = 6;
|
||||
cfg.network.ipv6 = Some(true);
|
||||
cfg.dc_overrides.insert(
|
||||
idx.to_string(),
|
||||
vec![
|
||||
format!("203.0.113.{}:443", 100 + idx),
|
||||
format!("[2001:db8::{}]:443", 100 + idx),
|
||||
],
|
||||
);
|
||||
|
||||
let first = get_dc_addr_static(idx, &cfg).expect("first lookup must resolve");
|
||||
let second = get_dc_addr_static(idx, &cfg).expect("second lookup must resolve");
|
||||
assert_eq!(
|
||||
first, second,
|
||||
"override resolution must stay deterministic for dc {idx}"
|
||||
);
|
||||
assert!(first.is_ipv6(), "dc {idx}: v6 override should be preferred");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn negative_direct_relay_dc_connection_refused_fails_fast() {
|
||||
let (client_reader_side, _client_writer_side) = duplex(1024);
|
||||
let (_client_reader_relay, client_writer_side) = duplex(1024);
|
||||
|
||||
let key = [0u8; 32];
|
||||
let iv = 0u128;
|
||||
let client_reader = CryptoReader::new(client_reader_side, AesCtr::new(&key, iv));
|
||||
let client_writer = CryptoWriter::new(client_writer_side, AesCtr::new(&key, iv), 1024);
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(1024, 1));
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let route_runtime = RouteRuntimeController::new(RelayRouteMode::Direct);
|
||||
|
||||
// Reserve an ephemeral port and immediately release it to deterministically
|
||||
// exercise the direct-connect failure path without long-lived hangs.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let dc_addr = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
|
||||
let mut config_with_override = ProxyConfig::default();
|
||||
config_with_override
|
||||
.dc_overrides
|
||||
.insert("1".to_string(), vec![dc_addr.to_string()]);
|
||||
let config = Arc::new(config_with_override);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
enabled: true,
|
||||
weight: 1,
|
||||
scopes: String::new(),
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
100,
|
||||
5000,
|
||||
10,
|
||||
3,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "test-user".to_string(),
|
||||
peer: "127.0.0.1:12345".parse().unwrap(),
|
||||
dc_idx: 1,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
enc_key: key,
|
||||
enc_iv: iv,
|
||||
dec_key: key,
|
||||
dec_iv: iv,
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let result = timeout(
|
||||
TokioDuration::from_secs(2),
|
||||
handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
route_runtime.subscribe(),
|
||||
route_runtime.snapshot(),
|
||||
0xABCD_1234,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("direct relay must fail fast on connection-refused upstream");
|
||||
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"connection-refused upstream must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adversarial_direct_relay_cutover_integrity() {
|
||||
let (client_reader_side, _client_writer_side) = duplex(1024);
|
||||
let (_client_reader_relay, client_writer_side) = duplex(1024);
|
||||
|
||||
let key = [0u8; 32];
|
||||
let iv = 0u128;
|
||||
let client_reader = CryptoReader::new(client_reader_side, AesCtr::new(&key, iv));
|
||||
let client_writer = CryptoWriter::new(client_writer_side, AesCtr::new(&key, iv), 1024);
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(1024, 1));
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let route_runtime = RouteRuntimeController::new(RelayRouteMode::Direct);
|
||||
|
||||
// Mock upstream server.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let dc_addr = listener.local_addr().unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
// Read handshake nonce.
|
||||
let mut nonce = [0u8; 64];
|
||||
let _ = stream.read_exact(&mut nonce).await;
|
||||
// Keep connection open.
|
||||
tokio::time::sleep(TokioDuration::from_secs(5)).await;
|
||||
});
|
||||
|
||||
let mut config_with_override = ProxyConfig::default();
|
||||
config_with_override
|
||||
.dc_overrides
|
||||
.insert("1".to_string(), vec![dc_addr.to_string()]);
|
||||
let config = Arc::new(config_with_override);
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
vec![UpstreamConfig {
|
||||
enabled: true,
|
||||
weight: 1,
|
||||
scopes: String::new(),
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
100,
|
||||
5000,
|
||||
10,
|
||||
3,
|
||||
false,
|
||||
stats.clone(),
|
||||
));
|
||||
|
||||
let success = HandshakeSuccess {
|
||||
user: "test-user".to_string(),
|
||||
peer: "127.0.0.1:12345".parse().unwrap(),
|
||||
dc_idx: 1,
|
||||
proto_tag: ProtoTag::Intermediate,
|
||||
enc_key: key,
|
||||
enc_iv: iv,
|
||||
dec_key: key,
|
||||
dec_iv: iv,
|
||||
is_tls: false,
|
||||
};
|
||||
|
||||
let stats_for_task = stats.clone();
|
||||
let runtime_clone = route_runtime.clone();
|
||||
let session_task = tokio::spawn(async move {
|
||||
handle_via_direct(
|
||||
client_reader,
|
||||
client_writer,
|
||||
success,
|
||||
upstream_manager,
|
||||
stats_for_task,
|
||||
config,
|
||||
buffer_pool,
|
||||
rng,
|
||||
runtime_clone.subscribe(),
|
||||
runtime_clone.snapshot(),
|
||||
0xABCD_1234,
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
timeout(TokioDuration::from_secs(2), async {
|
||||
loop {
|
||||
if stats.get_current_connections_direct() == 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("direct relay session must start before cutover");
|
||||
|
||||
// Trigger cutover.
|
||||
route_runtime.set_mode(RelayRouteMode::Middle).unwrap();
|
||||
|
||||
// The session should terminate after the staggered delay (1000-2000ms).
|
||||
let result = timeout(TokioDuration::from_secs(5), session_task)
|
||||
.await
|
||||
.expect("Session must terminate after cutover")
|
||||
.expect("Session must not panic");
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::RouteSwitched)),
|
||||
"Session must terminate with route switch error on cutover"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_is_deduplicated_per_dc_idx() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
assert!(should_log_unknown_dc(777));
|
||||
assert!(
|
||||
!should_log_unknown_dc(777),
|
||||
"same unknown dc_idx must not be logged repeatedly"
|
||||
);
|
||||
assert!(
|
||||
should_log_unknown_dc(778),
|
||||
"different unknown dc_idx must still be loggable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_respects_distinct_limit() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
for dc in 1..=UNKNOWN_DC_LOG_DISTINCT_LIMIT {
|
||||
assert!(
|
||||
should_log_unknown_dc(dc as i16),
|
||||
"expected first-time unknown dc_idx to be loggable"
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
!should_log_unknown_dc(i16::MAX),
|
||||
"distinct unknown dc_idx entries above limit must not be logged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_fails_closed_when_dedup_lock_is_poisoned() {
|
||||
let poisoned = Arc::new(std::sync::Mutex::new(
|
||||
std::collections::HashSet::<i16>::new(),
|
||||
));
|
||||
let poisoned_for_thread = poisoned.clone();
|
||||
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poisoned_for_thread
|
||||
.lock()
|
||||
.expect("poison setup lock must be available");
|
||||
panic!("intentional poison for fail-closed regression");
|
||||
})
|
||||
.join();
|
||||
|
||||
assert!(
|
||||
!should_log_unknown_dc_with_set(poisoned.as_ref(), 4242),
|
||||
"poisoned unknown-DC dedup lock must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_unknown_dc_log_path_does_not_consume_dedup_slot() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let dc_idx: i16 = 31_123;
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.unknown_dc_file_log_enabled = true;
|
||||
cfg.general.unknown_dc_log_path = Some("../telemt-unknown-dc-unsafe.log".to_string());
|
||||
|
||||
let _ = get_dc_addr_static(dc_idx, &cfg).expect("fallback routing must still work");
|
||||
|
||||
assert!(
|
||||
should_log_unknown_dc(dc_idx),
|
||||
"rejected unsafe log path must not consume unknown-dc dedup entry"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_unknown_dc_log_concurrent_unique_churn_respects_cap() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let accepted = Arc::new(AtomicUsize::new(0));
|
||||
let mut workers = Vec::new();
|
||||
|
||||
// Adversarial model: many concurrent peers rotate dc_idx values rapidly.
|
||||
for worker in 0..16usize {
|
||||
let accepted = Arc::clone(&accepted);
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let base = (worker * 2048) as i32;
|
||||
for offset in 0..512i32 {
|
||||
let raw = base + offset;
|
||||
let dc = (raw % i16::MAX as i32) as i16;
|
||||
if should_log_unknown_dc(dc) {
|
||||
accepted.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for worker in workers {
|
||||
worker.join().expect("worker thread must not panic");
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
accepted.load(Ordering::Relaxed),
|
||||
UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"concurrent unique churn must never admit more than the configured distinct cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_unknown_dc_log_mixed_duplicates_never_exceeds_cap() {
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
// Deterministic xorshift sequence for reproducible mixed duplicate fuzzing.
|
||||
let mut s: u64 = 0xA5A5_5A5A_C3C3_3C3C;
|
||||
let mut admitted = 0usize;
|
||||
|
||||
for _ in 0..20_000 {
|
||||
s ^= s << 7;
|
||||
s ^= s >> 9;
|
||||
s ^= s << 8;
|
||||
|
||||
let dc = (s as i16).wrapping_sub(i16::MAX / 2);
|
||||
if should_log_unknown_dc(dc) {
|
||||
admitted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
admitted <= UNKNOWN_DC_LOG_DISTINCT_LIMIT,
|
||||
"mixed-duplicate fuzzed inputs must not admit more than cap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_hint_accepts_ascii_alnum_and_dash_within_limit() {
|
||||
assert_eq!(validated_scope_hint("scope_alpha-1"), Some("alpha-1"));
|
||||
assert_eq!(validated_scope_hint("scope_AZ09"), Some("AZ09"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_hint_rejects_invalid_or_oversized_values() {
|
||||
assert_eq!(validated_scope_hint("plain_user"), None);
|
||||
assert_eq!(validated_scope_hint("scope_"), None);
|
||||
assert_eq!(validated_scope_hint("scope_a/b"), None);
|
||||
assert_eq!(validated_scope_hint("scope_bad space"), None);
|
||||
assert_eq!(validated_scope_hint("scope_bad.dot"), None);
|
||||
|
||||
let oversized = format!("scope_{}", "a".repeat(MAX_SCOPE_HINT_LEN + 1));
|
||||
assert_eq!(validated_scope_hint(&oversized), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_parent_traversal_inputs() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("../unknown-dc.txt").is_none(),
|
||||
"parent traversal paths must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("logs/../unknown-dc.txt").is_none(),
|
||||
"embedded parent traversal must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("./../unknown-dc.txt").is_none(),
|
||||
"relative parent traversal must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_absolute_paths_with_existing_parent() {
|
||||
let absolute = std::env::temp_dir().join("unknown-dc.txt");
|
||||
let absolute_str = absolute
|
||||
.to_str()
|
||||
.expect("temp absolute path must be valid UTF-8");
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(absolute_str)
|
||||
.expect("absolute paths with existing parent must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, absolute);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_absolute_parent_traversal() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("/tmp/../etc/passwd").is_none(),
|
||||
"absolute parent traversal must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_safe_relative_path() {
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!("telemt-unknown-dc-log-{}", std::process::id()));
|
||||
fs::create_dir_all(&base).expect("temp test directory must be creatable");
|
||||
|
||||
let candidate = base.join("unknown-dc.txt");
|
||||
let candidate_relative = format!(
|
||||
"target/telemt-unknown-dc-log-{}/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&candidate_relative)
|
||||
.expect("safe relative path with existing parent must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, candidate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_empty_or_dot_only_inputs() {
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path("").is_none(),
|
||||
"empty path must be rejected"
|
||||
);
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(".").is_none(),
|
||||
"dot-only path without filename must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_directory_only_as_filename_projection() {
|
||||
let sanitized = sanitize_unknown_dc_log_path("target/")
|
||||
.expect("directory-only input is interpreted as filename projection in current sanitizer");
|
||||
assert!(
|
||||
sanitized.resolved_path.ends_with("target"),
|
||||
"directory-only input should resolve to canonical parent plus filename projection"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_dot_prefixed_relative_path() {
|
||||
let rel_dir = format!("target/telemt-unknown-dc-dot-{}", std::process::id());
|
||||
let abs_dir = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join(&rel_dir);
|
||||
fs::create_dir_all(&abs_dir).expect("dot-prefixed test directory must be creatable");
|
||||
|
||||
let rel_candidate = format!("./{rel_dir}/unknown-dc.log");
|
||||
let expected = abs_dir.join("unknown-dc.log");
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("dot-prefixed safe path must be accepted");
|
||||
assert_eq!(sanitized.resolved_path, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_unknown_dc_path_parentdir_inputs_always_rejected() {
|
||||
let mut s: u64 = 0xD00D_BAAD_1234_5678;
|
||||
for _ in 0..4096 {
|
||||
s ^= s << 7;
|
||||
s ^= s >> 9;
|
||||
s ^= s << 8;
|
||||
let a = (s as usize) % 32;
|
||||
let b = ((s >> 8) as usize) % 32;
|
||||
let candidate = format!("target/{a}/../{b}/unknown-dc.log");
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(&candidate).is_none(),
|
||||
"parent-dir candidate must be rejected: {candidate}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_rejects_nonexistent_parent_directory() {
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-missing-{}/nested/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
assert!(
|
||||
sanitize_unknown_dc_log_path(&rel_candidate).is_none(),
|
||||
"path with missing parent must be rejected to avoid implicit directory creation"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_symlinked_parent_inside_workspace() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-log-symlink-internal-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let real_parent = base.join("real_parent");
|
||||
fs::create_dir_all(&real_parent).expect("real parent dir must be creatable");
|
||||
|
||||
let symlink_parent = base.join("internal_link");
|
||||
let _ = fs::remove_file(&symlink_parent);
|
||||
symlink(&real_parent, &symlink_parent).expect("internal symlink must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-log-symlink-internal-{}/internal_link/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("symlinked parent that resolves inside workspace must be accepted");
|
||||
assert!(
|
||||
sanitized.resolved_path.starts_with(&real_parent),
|
||||
"sanitized path must resolve to canonical internal parent"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_sanitizer_accepts_symlink_parent_escape_as_canonical_path() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-log-symlink-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("symlink test directory must be creatable");
|
||||
|
||||
let symlink_parent = base.join("escape_link");
|
||||
let _ = fs::remove_file(&symlink_parent);
|
||||
symlink("/tmp", &symlink_parent).expect("symlink parent must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-log-symlink-{}/escape_link/unknown-dc.txt",
|
||||
std::process::id()
|
||||
);
|
||||
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("symlinked parent must canonicalize to target path");
|
||||
assert!(
|
||||
sanitized.resolved_path.starts_with(Path::new("/tmp")),
|
||||
"sanitized path must resolve to canonical symlink target"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn unknown_dc_log_path_revalidation_rejects_symlinked_target_escape() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let base = std::env::current_dir()
|
||||
.expect("cwd must be available")
|
||||
.join("target")
|
||||
.join(format!(
|
||||
"telemt-unknown-dc-target-link-{}",
|
||||
std::process::id()
|
||||
));
|
||||
fs::create_dir_all(&base).expect("target-link base must be creatable");
|
||||
|
||||
let outside = std::env::temp_dir().join(format!("telemt-outside-{}", std::process::id()));
|
||||
let _ = fs::remove_file(&outside);
|
||||
fs::write(&outside, "outside").expect("outside file must be writable");
|
||||
|
||||
let linked_target = base.join("unknown-dc.log");
|
||||
let _ = fs::remove_file(&linked_target);
|
||||
symlink(&outside, &linked_target).expect("target symlink must be creatable");
|
||||
|
||||
let rel_candidate = format!(
|
||||
"target/telemt-unknown-dc-target-link-{}/unknown-dc.log",
|
||||
std::process::id()
|
||||
);
|
||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||
.expect("candidate should sanitize before final revalidation");
|
||||
|
||||
assert!(
|
||||
!unknown_dc_log_path_is_still_safe(&sanitized),
|
||||
"final revalidation must reject symlinked target escape"
|
||||
);
|
||||
}
|
||||
@@ -8,9 +8,7 @@ fn nonempty_line_count(text: &str) -> usize {
|
||||
|
||||
#[test]
|
||||
fn subtle_stress_single_unknown_dc_under_concurrency_logs_once() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().blocking_lock();
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let winners = Arc::new(AtomicUsize::new(0));
|
||||
@@ -103,9 +101,7 @@ fn subtle_light_fuzz_dc_resolution_never_panics_and_preserves_port() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn subtle_integration_parallel_same_dc_logs_one_line() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-direct-relay-same-{}", std::process::id());
|
||||
@@ -148,9 +144,7 @@ async fn subtle_integration_parallel_same_dc_logs_one_line() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn subtle_integration_parallel_unique_dcs_log_unique_lines() {
|
||||
let _guard = unknown_dc_test_lock()
|
||||
.lock()
|
||||
.expect("unknown dc test lock must be available");
|
||||
let _guard = unknown_dc_test_lock().lock().await;
|
||||
clear_unknown_dc_log_cache_for_testing();
|
||||
|
||||
let rel_dir = format!("target/telemt-direct-relay-unique-{}", std::process::id());
|
||||
|
||||
@@ -13,795 +13,9 @@ use tokio::time::{Duration, Instant};
|
||||
// Probing Indistinguishability (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_probes_indistinguishable_timing() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 80; // Should timeout/refuse
|
||||
|
||||
let peer: SocketAddr = "192.0.2.10:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
// Test different probe types
|
||||
let probes = vec![
|
||||
(b"GET / HTTP/1.1\r\nHost: x\r\n\r\n".to_vec(), "HTTP"),
|
||||
(b"SSH-2.0-probe".to_vec(), "SSH"),
|
||||
(
|
||||
vec![0x16, 0x03, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x01, 0x00],
|
||||
"TLS-scanner",
|
||||
),
|
||||
(vec![0x42; 5], "port-scanner"),
|
||||
];
|
||||
|
||||
for (probe, type_name) in probes {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// We expect any outcome to take roughly MASK_TIMEOUT (50ms in tests)
|
||||
// to mask whether the backend was reachable or refused.
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Probe {type_name} finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Masking Budget Stress Tests (OWASP ASVS 5.1.6)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_budget_stress_under_load() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let peer: SocketAddr = "192.0.2.20:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..50 {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
let config = config.clone();
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
start.elapsed()
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
let elapsed = task.await.unwrap();
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Stress probe finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// detect_client_type Fingerprint Check
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_detect_client_type_boundary_cases() {
|
||||
// 9 bytes = port-scanner
|
||||
assert_eq!(detect_client_type(&[0x42; 9]), "port-scanner");
|
||||
// 10 bytes = unknown
|
||||
assert_eq!(detect_client_type(&[0x42; 10]), "unknown");
|
||||
|
||||
// HTTP verbs without trailing space
|
||||
assert_eq!(detect_client_type(b"GET/"), "port-scanner"); // because len < 10
|
||||
assert_eq!(detect_client_type(b"GET /path"), "HTTP");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Slowloris and Slow Read Attacks (OWASP ASVS 5.1.5)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_slowloris_client_idle_timeout_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let initial = b"GET / HTTP/1.1\r\nHost: front.example\r\n\r\n".to_vec();
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let initial = initial.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; initial.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(observed, initial);
|
||||
|
||||
let mut drip = [0u8; 1];
|
||||
let drip_read =
|
||||
tokio::time::timeout(Duration::from_millis(220), stream.read_exact(&mut drip))
|
||||
.await;
|
||||
assert!(
|
||||
drip_read.is_err() || drip_read.unwrap().is_err(),
|
||||
"backend must not receive post-timeout slowloris drip bytes"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.10:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(1024);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&initial,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(160)).await;
|
||||
let _ = client_writer.write_all(b"X").await;
|
||||
|
||||
handle.await.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Fallback Server Down / Fingerprinting (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_fallback_down_mimics_timeout() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let (server_reader, server_writer) = duplex(1024);
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.12:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
server_reader,
|
||||
server_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
// It should wait for MASK_TIMEOUT (50ms in tests) even if connection was refused immediately
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(40),
|
||||
"Must respect connect budget even on failure: {:?}",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: SSRF Prevention (OWASP ASVS 5.1.2)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_ssrf_resolve_internal_ranges_blocked() {
|
||||
use crate::network::dns_overrides::DnsOverrides;
|
||||
|
||||
let blocked_ips = [
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"10.0.0.1",
|
||||
"192.168.1.1",
|
||||
"0.0.0.0",
|
||||
];
|
||||
let resolver = DnsOverrides::default();
|
||||
|
||||
for ip in blocked_ips {
|
||||
assert!(
|
||||
resolver.resolve_socket_addr(ip, 80).is_none(),
|
||||
"runtime DNS overrides must not resolve unconfigured literal host targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_unknown_proxy_protocol_version_falls_back_to_v1_unknown_header() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
let mut header = [0u8; 15];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(&header, b"PROXY UNKNOWN\r\n");
|
||||
|
||||
let mut payload = [0u8; 5];
|
||||
stream.read_exact(&mut payload).await.unwrap();
|
||||
assert_eq!(&payload, b"probe");
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 255;
|
||||
|
||||
let peer: SocketAddr = "198.51.100.77:50001".parse().unwrap();
|
||||
let local_addr: SocketAddr = "[2001:db8::10]:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_zero_length_initial_data_does_not_hang_or_panic() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut one = [0u8; 1];
|
||||
let n = tokio::time::timeout(Duration::from_millis(150), stream.read(&mut one))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
n, 0,
|
||||
"backend must observe clean EOF for empty initial payload"
|
||||
);
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.70:50002".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (client_reader, client_writer) = duplex(64);
|
||||
drop(client_writer);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_oversized_initial_payload_is_forwarded_verbatim() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let payload = vec![0xA5u8; 32 * 1024];
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let payload = payload.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; payload.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(
|
||||
observed, payload,
|
||||
"large initial payload must stay byte-for-byte"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.71:50003".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(64);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&payload,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_refused_backend_keeps_constantish_timing_floor_under_burst() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.72:50004".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
for _ in 0..16 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(30),
|
||||
"refused-backend path must keep timing floor to reduce fingerprinting"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_backend_half_close_then_client_half_close_completes_without_hang() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut pre = [0u8; 4];
|
||||
stream.read_exact(&mut pre).await.unwrap();
|
||||
assert_eq!(&pre, b"PING");
|
||||
stream.write_all(b"PONG").await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.73:50005".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(256);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"PING",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
client_writer.shutdown().await.unwrap();
|
||||
|
||||
let mut got = [0u8; 4];
|
||||
client_visible_reader.read_exact(&mut got).await.unwrap();
|
||||
assert_eq!(&got, b"PONG");
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking task must terminate after bilateral half-close")
|
||||
.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_concurrently() {
|
||||
const MASKING_SESSIONS: usize = 48;
|
||||
const RELAY_SESSIONS: usize = 48;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..MASKING_SESSIONS {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"masking backend must receive storm reconnect probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.200:55555".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
let mut masking_tasks = Vec::with_capacity(MASKING_SESSIONS);
|
||||
for i in 0..MASKING_SESSIONS {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{i:04} HTTP/1.1\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking reconnect task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(RELAY_SESSIONS);
|
||||
for i in 0..RELAY_SESSIONS {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-storm-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(i as u8).wrapping_add(1); 64];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(i as u8).wrapping_add(17); 96];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(2), relay_task)
|
||||
.await
|
||||
.expect("relay reconnect task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("masking storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("relay storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(3), backend_task)
|
||||
.await
|
||||
.expect("masking backend accept loop must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn read_env_usize_or_default(name: &str, default: usize) -> usize {
|
||||
match std::env::var(name) {
|
||||
Ok(raw) => match raw.parse::<usize>() {
|
||||
Ok(parsed) if parsed > 0 => parsed,
|
||||
_ => default,
|
||||
},
|
||||
Err(_) => default,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_multiwave_soak() {
|
||||
let waves = read_env_usize_or_default("CHAOS_WAVES", 4);
|
||||
let masking_per_wave = read_env_usize_or_default("CHAOS_MASKING_PER_WAVE", 160);
|
||||
let relay_per_wave = read_env_usize_or_default("CHAOS_RELAY_PER_WAVE", 160);
|
||||
let total_masking = waves * masking_per_wave;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..total_masking {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"mask backend must only receive storm probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.201:56565".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
for wave in 0..waves {
|
||||
let mut masking_tasks = Vec::with_capacity(masking_per_wave);
|
||||
for i in 0..masking_per_wave {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{wave:02}-{i:03}\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(3), handle)
|
||||
.await
|
||||
.expect("masking storm task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(relay_per_wave);
|
||||
for i in 0..relay_per_wave {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-multiwave-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(1); 32];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(17); 48];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(3), relay_task)
|
||||
.await
|
||||
.expect("relay storm task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("masking wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("relay wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(8), backend_task)
|
||||
.await
|
||||
.expect("mask backend must complete all accepted storm sessions")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn masking_timing_bucket_soak_refused_backend_stays_within_narrow_band() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.74:50006".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let mut samples = Vec::with_capacity(128);
|
||||
for _ in 0..128 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
samples.push(started.elapsed().as_millis());
|
||||
}
|
||||
|
||||
samples.sort_unstable();
|
||||
let p10 = samples[samples.len() / 10];
|
||||
let p90 = samples[(samples.len() * 9) / 10];
|
||||
assert!(
|
||||
p90.saturating_sub(p10) <= 40,
|
||||
"timing spread too wide for refused-backend masking path: p10={p10}ms p90={p90}ms"
|
||||
);
|
||||
}
|
||||
// Masking timing, fallback, and relay boundary cases.
|
||||
#[path = "masking_adversarial_tests/boundaries.rs"]
|
||||
mod boundaries;
|
||||
// Concurrent reconnect storms and manual soak cases.
|
||||
#[path = "masking_adversarial_tests/chaos.rs"]
|
||||
mod chaos;
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_probes_indistinguishable_timing() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 80; // Should timeout/refuse
|
||||
|
||||
let peer: SocketAddr = "192.0.2.10:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
// Test different probe types
|
||||
let probes = vec![
|
||||
(b"GET / HTTP/1.1\r\nHost: x\r\n\r\n".to_vec(), "HTTP"),
|
||||
(b"SSH-2.0-probe".to_vec(), "SSH"),
|
||||
(
|
||||
vec![0x16, 0x03, 0x03, 0x00, 0x05, 0x01, 0x00, 0x00, 0x01, 0x00],
|
||||
"TLS-scanner",
|
||||
),
|
||||
(vec![0x42; 5], "port-scanner"),
|
||||
];
|
||||
|
||||
for (probe, type_name) in probes {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// We expect any outcome to take roughly MASK_TIMEOUT (50ms in tests)
|
||||
// to mask whether the backend was reachable or refused.
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Probe {type_name} finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Masking Budget Stress Tests (OWASP ASVS 5.1.6)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_budget_stress_under_load() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let peer: SocketAddr = "192.0.2.20:443".parse().unwrap();
|
||||
let local_addr: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..50 {
|
||||
let (client_reader, _client_writer) = duplex(256);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(256);
|
||||
let config = config.clone();
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
start.elapsed()
|
||||
}));
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
let elapsed = task.await.unwrap();
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(30),
|
||||
"Stress probe finished too fast: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// detect_client_type Fingerprint Check
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_detect_client_type_boundary_cases() {
|
||||
// 9 bytes = port-scanner
|
||||
assert_eq!(detect_client_type(&[0x42; 9]), "port-scanner");
|
||||
// 10 bytes = unknown
|
||||
assert_eq!(detect_client_type(&[0x42; 10]), "unknown");
|
||||
|
||||
// HTTP verbs without trailing space
|
||||
assert_eq!(detect_client_type(b"GET/"), "port-scanner"); // because len < 10
|
||||
assert_eq!(detect_client_type(b"GET /path"), "HTTP");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Slowloris and Slow Read Attacks (OWASP ASVS 5.1.5)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_slowloris_client_idle_timeout_rejected() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let initial = b"GET / HTTP/1.1\r\nHost: front.example\r\n\r\n".to_vec();
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let initial = initial.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; initial.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(observed, initial);
|
||||
|
||||
let mut drip = [0u8; 1];
|
||||
let drip_read =
|
||||
tokio::time::timeout(Duration::from_millis(220), stream.read_exact(&mut drip))
|
||||
.await;
|
||||
assert!(
|
||||
drip_read.is_err() || drip_read.unwrap().is_err(),
|
||||
"backend must not receive post-timeout slowloris drip bytes"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.10:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(1024);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&initial,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(160)).await;
|
||||
let _ = client_writer.write_all(b"X").await;
|
||||
|
||||
handle.await.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: Fallback Server Down / Fingerprinting (OWASP ASVS 5.1.7)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_fallback_down_mimics_timeout() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1; // Unlikely port
|
||||
|
||||
let (server_reader, server_writer) = duplex(1024);
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let peer: SocketAddr = "192.0.2.12:12345".parse().unwrap();
|
||||
let local: SocketAddr = "192.0.2.1:443".parse().unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
handle_bad_client(
|
||||
server_reader,
|
||||
server_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
// It should wait for MASK_TIMEOUT (50ms in tests) even if connection was refused immediately
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(40),
|
||||
"Must respect connect budget even on failure: {:?}",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Priority 2: SSRF Prevention (OWASP ASVS 5.1.2)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_ssrf_resolve_internal_ranges_blocked() {
|
||||
use crate::network::dns_overrides::DnsOverrides;
|
||||
|
||||
let blocked_ips = [
|
||||
"127.0.0.1",
|
||||
"169.254.169.254",
|
||||
"10.0.0.1",
|
||||
"192.168.1.1",
|
||||
"0.0.0.0",
|
||||
];
|
||||
let resolver = DnsOverrides::default();
|
||||
|
||||
for ip in blocked_ips {
|
||||
assert!(
|
||||
resolver.resolve_socket_addr(ip, 80).is_none(),
|
||||
"runtime DNS overrides must not resolve unconfigured literal host targets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_unknown_proxy_protocol_version_falls_back_to_v1_unknown_header() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
|
||||
let mut header = [0u8; 15];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(&header, b"PROXY UNKNOWN\r\n");
|
||||
|
||||
let mut payload = [0u8; 5];
|
||||
stream.read_exact(&mut payload).await.unwrap();
|
||||
assert_eq!(&payload, b"probe");
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 255;
|
||||
|
||||
let peer: SocketAddr = "198.51.100.77:50001".parse().unwrap();
|
||||
let local_addr: SocketAddr = "[2001:db8::10]:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"probe",
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_zero_length_initial_data_does_not_hang_or_panic() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut one = [0u8; 1];
|
||||
let n = tokio::time::timeout(Duration::from_millis(150), stream.read(&mut one))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
n, 0,
|
||||
"backend must observe clean EOF for empty initial payload"
|
||||
);
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.70:50002".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (client_reader, client_writer) = duplex(64);
|
||||
drop(client_writer);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_oversized_initial_payload_is_forwarded_verbatim() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let payload = vec![0xA5u8; 32 * 1024];
|
||||
|
||||
let accept_task = tokio::spawn({
|
||||
let payload = payload.clone();
|
||||
async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut observed = vec![0u8; payload.len()];
|
||||
stream.read_exact(&mut observed).await.unwrap();
|
||||
assert_eq!(
|
||||
observed, payload,
|
||||
"large initial payload must stay byte-for-byte"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.71:50003".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
let (client_reader, _client_writer) = duplex(64);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(64);
|
||||
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&payload,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_refused_backend_keeps_constantish_timing_floor_under_burst() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.72:50004".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
for _ in 0..16 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(30),
|
||||
"refused-backend path must keep timing floor to reduce fingerprinting"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn masking_backend_half_close_then_client_half_close_completes_without_hang() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut pre = [0u8; 4];
|
||||
stream.read_exact(&mut pre).await.unwrap();
|
||||
assert_eq!(&pre, b"PING");
|
||||
stream.write_all(b"PONG").await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
|
||||
let peer: SocketAddr = "203.0.113.73:50005".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let (mut client_writer, client_reader) = duplex(256);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(256);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"PING",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
client_writer.shutdown().await.unwrap();
|
||||
|
||||
let mut got = [0u8; 4];
|
||||
client_visible_reader.read_exact(&mut got).await.unwrap();
|
||||
assert_eq!(&got, b"PONG");
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking task must terminate after bilateral half-close")
|
||||
.unwrap();
|
||||
accept_task.await.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_concurrently() {
|
||||
const MASKING_SESSIONS: usize = 48;
|
||||
const RELAY_SESSIONS: usize = 48;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..MASKING_SESSIONS {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"masking backend must receive storm reconnect probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.200:55555".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
let mut masking_tasks = Vec::with_capacity(MASKING_SESSIONS);
|
||||
for i in 0..MASKING_SESSIONS {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{i:04} HTTP/1.1\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(2), handle)
|
||||
.await
|
||||
.expect("masking reconnect task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(RELAY_SESSIONS);
|
||||
for i in 0..RELAY_SESSIONS {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-storm-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(i as u8).wrapping_add(1); 64];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(i as u8).wrapping_add(17); 96];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(2), relay_task)
|
||||
.await
|
||||
.expect("relay reconnect task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("masking storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("relay storm join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(3), backend_task)
|
||||
.await
|
||||
.expect("masking backend accept loop must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn read_env_usize_or_default(name: &str, default: usize) -> usize {
|
||||
match std::env::var(name) {
|
||||
Ok(raw) => match raw.parse::<usize>() {
|
||||
Ok(parsed) if parsed > 0 => parsed,
|
||||
_ => default,
|
||||
},
|
||||
Err(_) => default,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn chaos_burst_reconnect_storm_for_masking_and_relay_multiwave_soak() {
|
||||
let waves = read_env_usize_or_default("CHAOS_WAVES", 4);
|
||||
let masking_per_wave = read_env_usize_or_default("CHAOS_MASKING_PER_WAVE", 160);
|
||||
let relay_per_wave = read_env_usize_or_default("CHAOS_RELAY_PER_WAVE", 160);
|
||||
let total_masking = waves * masking_per_wave;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let backend_addr = listener.local_addr().unwrap();
|
||||
let backend_reply = b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n".to_vec();
|
||||
|
||||
let backend_task = tokio::spawn({
|
||||
let backend_reply = backend_reply.clone();
|
||||
async move {
|
||||
for _ in 0..total_masking {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut req = [0u8; 32];
|
||||
stream.read_exact(&mut req).await.unwrap();
|
||||
assert!(
|
||||
req.starts_with(b"GET /storm/"),
|
||||
"mask backend must only receive storm probes"
|
||||
);
|
||||
stream.write_all(&backend_reply).await.unwrap();
|
||||
stream.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = backend_addr.port();
|
||||
config.censorship.mask_proxy_protocol = 0;
|
||||
|
||||
let config = Arc::new(config);
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let peer: SocketAddr = "198.51.100.201:56565".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
|
||||
for wave in 0..waves {
|
||||
let mut masking_tasks = Vec::with_capacity(masking_per_wave);
|
||||
for i in 0..masking_per_wave {
|
||||
let config = Arc::clone(&config);
|
||||
let beobachten = Arc::clone(&beobachten);
|
||||
let expected_reply = backend_reply.clone();
|
||||
masking_tasks.push(tokio::spawn(async move {
|
||||
let mut probe = [0u8; 32];
|
||||
let template = format!("GET /storm/{wave:02}-{i:03}\r\n\r\n");
|
||||
let bytes = template.as_bytes();
|
||||
probe[..bytes.len()].copy_from_slice(bytes);
|
||||
|
||||
let (client_reader, client_writer) = duplex(256);
|
||||
drop(client_writer);
|
||||
let (mut client_visible_reader, client_visible_writer) = duplex(1024);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
&probe,
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let mut observed = vec![0u8; expected_reply.len()];
|
||||
client_visible_reader
|
||||
.read_exact(&mut observed)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observed, expected_reply);
|
||||
|
||||
timeout(Duration::from_secs(3), handle)
|
||||
.await
|
||||
.expect("masking storm task must complete")
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
let mut relay_tasks = Vec::with_capacity(relay_per_wave);
|
||||
for i in 0..relay_per_wave {
|
||||
relay_tasks.push(tokio::spawn(async move {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let (mut client_peer, relay_client) = duplex(4096);
|
||||
let (relay_server, mut server_peer) = duplex(4096);
|
||||
|
||||
let (client_reader, client_writer) = tokio::io::split(relay_client);
|
||||
let (server_reader, server_writer) = tokio::io::split(relay_server);
|
||||
|
||||
let relay_task = tokio::spawn(relay_bidirectional(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
1024,
|
||||
1024,
|
||||
"chaos-multiwave-relay",
|
||||
stats,
|
||||
None,
|
||||
Arc::new(BufferPool::new()),
|
||||
));
|
||||
|
||||
let c2s = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(1); 32];
|
||||
client_peer.write_all(&c2s).await.unwrap();
|
||||
let mut c2s_seen = vec![0u8; c2s.len()];
|
||||
server_peer.read_exact(&mut c2s_seen).await.unwrap();
|
||||
assert_eq!(c2s_seen, c2s);
|
||||
|
||||
let s2c = vec![(wave as u8).wrapping_add(i as u8).wrapping_add(17); 48];
|
||||
server_peer.write_all(&s2c).await.unwrap();
|
||||
let mut s2c_seen = vec![0u8; s2c.len()];
|
||||
client_peer.read_exact(&mut s2c_seen).await.unwrap();
|
||||
assert_eq!(s2c_seen, s2c);
|
||||
|
||||
drop(client_peer);
|
||||
drop(server_peer);
|
||||
timeout(Duration::from_secs(3), relay_task)
|
||||
.await
|
||||
.expect("relay storm task must complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}));
|
||||
}
|
||||
|
||||
for task in masking_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("masking wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for task in relay_tasks {
|
||||
timeout(Duration::from_secs(6), task)
|
||||
.await
|
||||
.expect("relay wave task join must complete")
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(8), backend_task)
|
||||
.await
|
||||
.expect("mask backend must complete all accepted storm sessions")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "heavy soak; run manually"]
|
||||
async fn masking_timing_bucket_soak_refused_backend_stays_within_narrow_band() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.mask = true;
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
|
||||
let peer: SocketAddr = "203.0.113.74:50006".parse().unwrap();
|
||||
let local: SocketAddr = "127.0.0.1:443".parse().unwrap();
|
||||
let beobachten = BeobachtenStore::new();
|
||||
|
||||
let mut samples = Vec::with_capacity(128);
|
||||
for _ in 0..128 {
|
||||
let (client_reader, _client_writer) = duplex(128);
|
||||
let (_client_visible_reader, client_visible_writer) = duplex(128);
|
||||
let started = Instant::now();
|
||||
handle_bad_client(
|
||||
client_reader,
|
||||
client_visible_writer,
|
||||
b"GET / HTTP/1.1\r\n",
|
||||
peer,
|
||||
local,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
.await;
|
||||
samples.push(started.elapsed().as_millis());
|
||||
}
|
||||
|
||||
samples.sort_unstable();
|
||||
let p10 = samples[samples.len() / 10];
|
||||
let p90 = samples[(samples.len() * 9) / 10];
|
||||
assert!(
|
||||
p90.saturating_sub(p10) <= 40,
|
||||
"timing spread too wide for refused-backend masking path: p10={p10}ms p90={p90}ms"
|
||||
);
|
||||
}
|
||||
@@ -86,15 +86,14 @@ async fn connect_failure_refusal_close_behavior_matrix() {
|
||||
let peer: SocketAddr = format!("203.0.113.210:{}", 54100 + idx as u16)
|
||||
.parse()
|
||||
.unwrap();
|
||||
let elapsed =
|
||||
run_connect_failure_case(
|
||||
"127.0.0.1",
|
||||
unused_port,
|
||||
timing_normalization_enabled,
|
||||
peer,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
let elapsed = run_connect_failure_case(
|
||||
"127.0.0.1",
|
||||
unused_port,
|
||||
timing_normalization_enabled,
|
||||
peer,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
if timing_normalization_enabled {
|
||||
assert!(
|
||||
|
||||
@@ -1,19 +1,11 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::*;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
fn interface_cache_test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn adversarial_parallel_cold_miss_performs_single_interface_refresh() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
#![cfg(unix)]
|
||||
|
||||
use super::*;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
fn interface_cache_test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tdd_repeated_local_listener_checks_do_not_repeat_interface_enumeration_within_window() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
@@ -29,9 +21,7 @@ async fn tdd_repeated_local_listener_checks_do_not_repeat_interface_enumeration_
|
||||
|
||||
#[tokio::test]
|
||||
async fn tdd_non_local_port_short_circuit_does_not_enumerate_interfaces() {
|
||||
let _guard = interface_cache_test_lock()
|
||||
.lock()
|
||||
.unwrap_or_else(|poison| poison.into_inner());
|
||||
let _guard = interface_cache_test_lock().lock().await;
|
||||
reset_local_interface_enumerations_for_tests();
|
||||
|
||||
let local_addr: SocketAddr = "0.0.0.0:443".parse().expect("valid local addr");
|
||||
|
||||
@@ -95,710 +95,12 @@ fn simulate_tiny_debt_pattern(pattern: &[bool], max_steps: usize) -> (Option<usi
|
||||
(None, debt, reals)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_constants_match_security_budget_expectations() {
|
||||
assert_eq!(TINY_FRAME_DEBT_PER_TINY, 8);
|
||||
assert_eq!(TINY_FRAME_DEBT_LIMIT, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relay_client_idle_state_initial_debt_is_zero() {
|
||||
let state = RelayClientIdleState::new(Instant::now());
|
||||
assert_eq!(state.tiny_frame_debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_client_frame_does_not_reset_tiny_frame_debt() {
|
||||
let now = Instant::now();
|
||||
let mut state = RelayClientIdleState::new(now);
|
||||
state.tiny_frame_debt = 77;
|
||||
state.on_client_frame(now);
|
||||
assert_eq!(state.tiny_frame_debt, 77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_increment_is_saturating() {
|
||||
let mut debt = u32::MAX - 1;
|
||||
debt = debt.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
||||
assert_eq!(debt, u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_decrement_is_saturating() {
|
||||
let mut debt = 0u32;
|
||||
debt = debt.saturating_sub(1);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_tiny_frames_close_exactly_at_threshold() {
|
||||
let max_tiny_without_close = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize;
|
||||
let pattern = vec![true; max_tiny_without_close];
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, Some(max_tiny_without_close));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_less_than_threshold_tiny_frames_do_not_close() {
|
||||
let tiny_count = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize - 1;
|
||||
let pattern = vec![true; tiny_count];
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_one_closes_with_bounded_real_frame_count() {
|
||||
let mut pattern = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
pattern.push(true);
|
||||
pattern.push(false);
|
||||
}
|
||||
let (closed_at, _, reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(closed_at.is_some());
|
||||
assert!(
|
||||
reals <= 80,
|
||||
"expected bounded real frames before close, got {reals}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_eight_is_stable_for_long_runs() {
|
||||
let mut pattern = Vec::with_capacity(9 * 5000);
|
||||
for _ in 0..5000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..8 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt <= TINY_FRAME_DEBT_PER_TINY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_seven_eventually_closes() {
|
||||
let mut pattern = Vec::with_capacity(8 * 2000);
|
||||
for _ in 0..2000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..7 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(
|
||||
closed_at.is_some(),
|
||||
"1:7 tiny-to-real must eventually close"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_tiny_one_real_closes_faster_than_one_to_one() {
|
||||
let mut one_to_one = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
one_to_one.push(true);
|
||||
one_to_one.push(false);
|
||||
}
|
||||
|
||||
let mut two_to_one = Vec::with_capacity(768);
|
||||
for _ in 0..256 {
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(false);
|
||||
}
|
||||
|
||||
let (a_close, _, _) = simulate_tiny_debt_pattern(&one_to_one, one_to_one.len());
|
||||
let (b_close, _, _) = simulate_tiny_debt_pattern(&two_to_one, two_to_one.len());
|
||||
assert!(a_close.is_some() && b_close.is_some());
|
||||
assert!(b_close.unwrap_or(usize::MAX) < a_close.unwrap_or(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn burst_then_drain_can_recover_without_close() {
|
||||
let burst_tiny = ((TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) / 2) as usize;
|
||||
let mut pattern = Vec::with_capacity(burst_tiny + 600);
|
||||
for _ in 0..burst_tiny {
|
||||
pattern.push(true);
|
||||
}
|
||||
pattern.extend(std::iter::repeat_n(false, 600));
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_tiny_frame_debt_model_stays_within_bounds() {
|
||||
let mut seed = 0xA5A5_91C3_2026_0322u64;
|
||||
for _case in 0..128 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let len = 512 + ((seed as usize) & 0x3ff);
|
||||
let mut pattern = Vec::with_capacity(len);
|
||||
let mut local_seed = seed;
|
||||
for _ in 0..len {
|
||||
local_seed ^= local_seed << 7;
|
||||
local_seed ^= local_seed >> 9;
|
||||
local_seed ^= local_seed << 8;
|
||||
pattern.push((local_seed & 1) == 0);
|
||||
}
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
if closed_at.is_none() {
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
assert!(debt <= u32::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_many_independent_simulations_keep_isolated_debt_state() {
|
||||
for idx in 0..2048usize {
|
||||
let mut pattern = Vec::with_capacity(64);
|
||||
for j in 0..64usize {
|
||||
pattern.push(((idx ^ j) & 3) == 0);
|
||||
}
|
||||
let (_closed_at, debt, _reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_intermediate_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(11, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_secure_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(12, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Secure,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intermediate_alternating_zero_and_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(13, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(3000);
|
||||
for idx in 0..160u8 {
|
||||
plaintext.extend_from_slice(&0u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&4u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x11, idx ^ 0x22, idx ^ 0x33]);
|
||||
}
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..220 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some(_)) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(other) => panic!("unexpected error while probing alternating close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(closed, "intermediate alternating attack must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn small_tiny_burst_followed_by_real_frame_does_not_spuriously_close() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(14, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(64);
|
||||
for _ in 0..8 {
|
||||
plaintext.push(0x00);
|
||||
}
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[1, 2, 3, 4]);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
|
||||
let first = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match first {
|
||||
Ok(Some((payload, _))) => assert_eq!(payload.as_ref(), &[1, 2, 3, 4]),
|
||||
Err(e) => panic!("unexpected close after small tiny burst: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before real frame"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(1, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 1024];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer
|
||||
.write_all(&flood_encrypted)
|
||||
.await
|
||||
.expect("zero-length flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"idle policy enabled must fail closed for pure zero-length flood"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_alternating_tiny_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(2, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(256 * 6);
|
||||
for idx in 0..=255u8 {
|
||||
plaintext.push(0x00);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x55, idx ^ 0xAA, 0x11]);
|
||||
}
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("alternating flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let mut saw_proxy_close = false;
|
||||
for _ in 0..300 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some((_payload, _quickack))) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
saw_proxy_close = true;
|
||||
break;
|
||||
}
|
||||
Err(ProxyError::Io(e)) => panic!("unexpected IO error before close: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before debt-based closure"),
|
||||
Err(other) => panic!("unexpected error before close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_proxy_close,
|
||||
"alternating tiny/real sequence must eventually fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_idle_policy_valid_nonzero_frame_still_passes() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(3, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let payload = [7u8, 8, 9, 10];
|
||||
let mut plaintext = Vec::with_capacity(1 + payload.len());
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&payload);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("nonzero frame must be writable");
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
.expect("valid frame should decode")
|
||||
.expect("valid frame should return payload");
|
||||
|
||||
assert_eq!(result.0.as_ref(), &payload);
|
||||
assert!(!result.1);
|
||||
assert_eq!(frame_counter, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_quickack_tiny_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(21, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0x80u8; 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"quickack-marked zero-length flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_extended_zero_len_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(22, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut flood_plaintext = Vec::with_capacity(4 * 256);
|
||||
for _ in 0..256 {
|
||||
flood_plaintext.extend_from_slice(&[0x7f, 0x00, 0x00, 0x00]);
|
||||
}
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"extended zero-length abridged flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_to_eight_abridged_wire_pattern_survives_without_false_positive_close() {
|
||||
let mut plaintext = Vec::with_capacity(9 * 300);
|
||||
for idx in 0..300usize {
|
||||
plaintext.push(0x00);
|
||||
for _ in 0..8 {
|
||||
let b = idx as u8;
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x11, b ^ 0x22, b ^ 0x33]);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the test single-task and deterministic: make duplex capacity larger than the
|
||||
// generated ciphertext so write_all cannot block waiting for a concurrent reader.
|
||||
let duplex_capacity = plaintext.len().saturating_add(1024);
|
||||
let (reader, mut writer) = duplex(duplex_capacity);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(23, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..3000 {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected error in 1:8 wire test: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!closed,
|
||||
"wire-level 1:8 tiny-to-real pattern should not trigger debt close"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deterministic_light_fuzz_abridged_wire_behavior_matches_model() {
|
||||
let mut seed = 0xD1CE_BAAD_2026_0322u64;
|
||||
|
||||
for case_idx in 0..32u64 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let events = 300 + ((seed as usize) & 0xff);
|
||||
let mut pattern = Vec::with_capacity(events);
|
||||
let mut local = seed;
|
||||
for _ in 0..events {
|
||||
local ^= local << 7;
|
||||
local ^= local >> 9;
|
||||
local ^= local << 8;
|
||||
pattern.push((local & 0x03) == 0);
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::with_capacity(events * 6);
|
||||
for (idx, tiny) in pattern.iter().copied().enumerate() {
|
||||
if tiny {
|
||||
plaintext.push(0x00);
|
||||
} else {
|
||||
let b = (idx as u8) ^ (case_idx as u8);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x1F, b ^ 0x7A, b ^ 0xC3]);
|
||||
}
|
||||
}
|
||||
|
||||
let (reader, mut writer) = duplex(16 * 1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(500 + case_idx, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
writer
|
||||
.write_all(&encrypt_for_reader(&plaintext))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let (expected_close, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
let mut observed_close = false;
|
||||
|
||||
for _ in 0..(events + 8) {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
observed_close = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected fuzz error: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
observed_close,
|
||||
expected_close.is_some(),
|
||||
"wire parser behavior must match debt model for case {case_idx}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Pure tiny-frame debt model invariants.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/model.rs"]
|
||||
mod model;
|
||||
// Intermediate and secure transport debt behavior.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/transport.rs"]
|
||||
mod transport;
|
||||
// Abridged framing debt behavior.
|
||||
#[path = "middle_relay_tiny_frame_debt_security_tests/abridged.rs"]
|
||||
mod abridged;
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_quickack_tiny_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(21, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0x80u8; 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"quickack-marked zero-length flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abridged_extended_zero_len_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(22, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut flood_plaintext = Vec::with_capacity(4 * 256);
|
||||
for _ in 0..256 {
|
||||
flood_plaintext.extend_from_slice(&[0x7f, 0x00, 0x00, 0x00]);
|
||||
}
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"extended zero-length abridged flood must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_to_eight_abridged_wire_pattern_survives_without_false_positive_close() {
|
||||
let mut plaintext = Vec::with_capacity(9 * 300);
|
||||
for idx in 0..300usize {
|
||||
plaintext.push(0x00);
|
||||
for _ in 0..8 {
|
||||
let b = idx as u8;
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x11, b ^ 0x22, b ^ 0x33]);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the test single-task and deterministic: make duplex capacity larger than the
|
||||
// generated ciphertext so write_all cannot block waiting for a concurrent reader.
|
||||
let duplex_capacity = plaintext.len().saturating_add(1024);
|
||||
let (reader, mut writer) = duplex(duplex_capacity);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(23, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..3000 {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected error in 1:8 wire test: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!closed,
|
||||
"wire-level 1:8 tiny-to-real pattern should not trigger debt close"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deterministic_light_fuzz_abridged_wire_behavior_matches_model() {
|
||||
let mut seed = 0xD1CE_BAAD_2026_0322u64;
|
||||
|
||||
for case_idx in 0..32u64 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let events = 300 + ((seed as usize) & 0xff);
|
||||
let mut pattern = Vec::with_capacity(events);
|
||||
let mut local = seed;
|
||||
for _ in 0..events {
|
||||
local ^= local << 7;
|
||||
local ^= local >> 9;
|
||||
local ^= local << 8;
|
||||
pattern.push((local & 0x03) == 0);
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::with_capacity(events * 6);
|
||||
for (idx, tiny) in pattern.iter().copied().enumerate() {
|
||||
if tiny {
|
||||
plaintext.push(0x00);
|
||||
} else {
|
||||
let b = (idx as u8) ^ (case_idx as u8);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[b, b ^ 0x1F, b ^ 0x7A, b ^ 0xC3]);
|
||||
}
|
||||
}
|
||||
|
||||
let (reader, mut writer) = duplex(16 * 1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(500 + case_idx, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
writer
|
||||
.write_all(&encrypt_for_reader(&plaintext))
|
||||
.await
|
||||
.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let (expected_close, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
let mut observed_close = false;
|
||||
|
||||
for _ in 0..(events + 8) {
|
||||
match read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => break,
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
observed_close = true;
|
||||
break;
|
||||
}
|
||||
Err(other) => panic!("unexpected fuzz error: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
observed_close,
|
||||
expected_close.is_some(),
|
||||
"wire parser behavior must match debt model for case {case_idx}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_constants_match_security_budget_expectations() {
|
||||
assert_eq!(TINY_FRAME_DEBT_PER_TINY, 8);
|
||||
assert_eq!(TINY_FRAME_DEBT_LIMIT, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relay_client_idle_state_initial_debt_is_zero() {
|
||||
let state = RelayClientIdleState::new(Instant::now());
|
||||
assert_eq!(state.tiny_frame_debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_client_frame_does_not_reset_tiny_frame_debt() {
|
||||
let now = Instant::now();
|
||||
let mut state = RelayClientIdleState::new(now);
|
||||
state.tiny_frame_debt = 77;
|
||||
state.on_client_frame(now);
|
||||
assert_eq!(state.tiny_frame_debt, 77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_increment_is_saturating() {
|
||||
let mut debt = u32::MAX - 1;
|
||||
debt = debt.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
||||
assert_eq!(debt, u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_frame_debt_decrement_is_saturating() {
|
||||
let mut debt = 0u32;
|
||||
debt = debt.saturating_sub(1);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_tiny_frames_close_exactly_at_threshold() {
|
||||
let max_tiny_without_close = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize;
|
||||
let pattern = vec![true; max_tiny_without_close];
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, Some(max_tiny_without_close));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_less_than_threshold_tiny_frames_do_not_close() {
|
||||
let tiny_count = (TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) as usize - 1;
|
||||
let pattern = vec![true; tiny_count];
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_one_closes_with_bounded_real_frame_count() {
|
||||
let mut pattern = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
pattern.push(true);
|
||||
pattern.push(false);
|
||||
}
|
||||
let (closed_at, _, reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(closed_at.is_some());
|
||||
assert!(
|
||||
reals <= 80,
|
||||
"expected bounded real frames before close, got {reals}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_eight_is_stable_for_long_runs() {
|
||||
let mut pattern = Vec::with_capacity(9 * 5000);
|
||||
for _ in 0..5000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..8 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert!(debt <= TINY_FRAME_DEBT_PER_TINY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alternating_one_to_seven_eventually_closes() {
|
||||
let mut pattern = Vec::with_capacity(8 * 2000);
|
||||
for _ in 0..2000 {
|
||||
pattern.push(true);
|
||||
for _ in 0..7 {
|
||||
pattern.push(false);
|
||||
}
|
||||
}
|
||||
let (closed_at, _, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(
|
||||
closed_at.is_some(),
|
||||
"1:7 tiny-to-real must eventually close"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_tiny_one_real_closes_faster_than_one_to_one() {
|
||||
let mut one_to_one = Vec::with_capacity(512);
|
||||
for _ in 0..256 {
|
||||
one_to_one.push(true);
|
||||
one_to_one.push(false);
|
||||
}
|
||||
|
||||
let mut two_to_one = Vec::with_capacity(768);
|
||||
for _ in 0..256 {
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(true);
|
||||
two_to_one.push(false);
|
||||
}
|
||||
|
||||
let (a_close, _, _) = simulate_tiny_debt_pattern(&one_to_one, one_to_one.len());
|
||||
let (b_close, _, _) = simulate_tiny_debt_pattern(&two_to_one, two_to_one.len());
|
||||
assert!(a_close.is_some() && b_close.is_some());
|
||||
assert!(b_close.unwrap_or(usize::MAX) < a_close.unwrap_or(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn burst_then_drain_can_recover_without_close() {
|
||||
let burst_tiny = ((TINY_FRAME_DEBT_LIMIT / TINY_FRAME_DEBT_PER_TINY) / 2) as usize;
|
||||
let mut pattern = Vec::with_capacity(burst_tiny + 600);
|
||||
for _ in 0..burst_tiny {
|
||||
pattern.push(true);
|
||||
}
|
||||
pattern.extend(std::iter::repeat_n(false, 600));
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert_eq!(closed_at, None);
|
||||
assert_eq!(debt, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_fuzz_tiny_frame_debt_model_stays_within_bounds() {
|
||||
let mut seed = 0xA5A5_91C3_2026_0322u64;
|
||||
for _case in 0..128 {
|
||||
seed ^= seed << 7;
|
||||
seed ^= seed >> 9;
|
||||
seed ^= seed << 8;
|
||||
|
||||
let len = 512 + ((seed as usize) & 0x3ff);
|
||||
let mut pattern = Vec::with_capacity(len);
|
||||
let mut local_seed = seed;
|
||||
for _ in 0..len {
|
||||
local_seed ^= local_seed << 7;
|
||||
local_seed ^= local_seed >> 9;
|
||||
local_seed ^= local_seed << 8;
|
||||
pattern.push((local_seed & 1) == 0);
|
||||
}
|
||||
|
||||
let (closed_at, debt, _) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
if closed_at.is_none() {
|
||||
assert!(debt < TINY_FRAME_DEBT_LIMIT);
|
||||
}
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stress_many_independent_simulations_keep_isolated_debt_state() {
|
||||
for idx in 0..2048usize {
|
||||
let mut pattern = Vec::with_capacity(64);
|
||||
for j in 0..64usize {
|
||||
pattern.push(((idx ^ j) & 3) == 0);
|
||||
}
|
||||
let (_closed_at, debt, _reals) = simulate_tiny_debt_pattern(&pattern, pattern.len());
|
||||
assert!(debt <= TINY_FRAME_DEBT_LIMIT.saturating_add(TINY_FRAME_DEBT_PER_TINY));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_intermediate_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(11, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_secure_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(12, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 4 * 256];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer.write_all(&flood_encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Secure,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProxyError::Proxy(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn intermediate_alternating_zero_and_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(13, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(3000);
|
||||
for idx in 0..160u8 {
|
||||
plaintext.extend_from_slice(&0u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&4u32.to_le_bytes());
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x11, idx ^ 0x22, idx ^ 0x33]);
|
||||
}
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
drop(writer);
|
||||
|
||||
let mut closed = false;
|
||||
for _ in 0..220 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Intermediate,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some(_)) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(other) => panic!("unexpected error while probing alternating close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(closed, "intermediate alternating attack must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn small_tiny_burst_followed_by_real_frame_does_not_spuriously_close() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(14, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(64);
|
||||
for _ in 0..8 {
|
||||
plaintext.push(0x00);
|
||||
}
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[1, 2, 3, 4]);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer.write_all(&encrypted).await.unwrap();
|
||||
|
||||
let first = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match first {
|
||||
Ok(Some((payload, _))) => assert_eq!(payload.as_ref(), &[1, 2, 3, 4]),
|
||||
Err(e) => panic!("unexpected close after small tiny burst: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before real frame"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_zero_length_flood_is_fail_closed() {
|
||||
let (reader, mut writer) = duplex(4096);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(1, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let flood_plaintext = vec![0u8; 1024];
|
||||
let flood_encrypted = encrypt_for_reader(&flood_plaintext);
|
||||
writer
|
||||
.write_all(&flood_encrypted)
|
||||
.await
|
||||
.expect("zero-length flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(ProxyError::Proxy(_))),
|
||||
"idle policy enabled must fail closed for pure zero-length flood"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn idle_policy_enabled_alternating_tiny_real_eventually_closes() {
|
||||
let (reader, mut writer) = duplex(8192);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(2, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let mut plaintext = Vec::with_capacity(256 * 6);
|
||||
for idx in 0..=255u8 {
|
||||
plaintext.push(0x00);
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&[idx, idx ^ 0x55, idx ^ 0xAA, 0x11]);
|
||||
}
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("alternating flood bytes must be writable");
|
||||
drop(writer);
|
||||
|
||||
let mut saw_proxy_close = false;
|
||||
for _ in 0..300 {
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some((_payload, _quickack))) => {}
|
||||
Err(ProxyError::Proxy(_)) => {
|
||||
saw_proxy_close = true;
|
||||
break;
|
||||
}
|
||||
Err(ProxyError::Io(e)) => panic!("unexpected IO error before close: {e}"),
|
||||
Ok(None) => panic!("unexpected EOF before debt-based closure"),
|
||||
Err(other) => panic!("unexpected error before close: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_proxy_close,
|
||||
"alternating tiny/real sequence must eventually fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_idle_policy_valid_nonzero_frame_still_passes() {
|
||||
let (reader, mut writer) = duplex(1024);
|
||||
let mut crypto_reader = make_crypto_reader(reader);
|
||||
let buffer_pool = Arc::new(BufferPool::new());
|
||||
let stats = Stats::new();
|
||||
let session_started_at = Instant::now();
|
||||
let forensics = make_forensics(3, session_started_at);
|
||||
let mut frame_counter = 0u64;
|
||||
let mut idle_state = RelayClientIdleState::new(session_started_at);
|
||||
let idle_policy = make_enabled_idle_policy();
|
||||
let last_downstream_activity_ms = AtomicU64::new(0);
|
||||
|
||||
let payload = [7u8, 8, 9, 10];
|
||||
let mut plaintext = Vec::with_capacity(1 + payload.len());
|
||||
plaintext.push(0x01);
|
||||
plaintext.extend_from_slice(&payload);
|
||||
|
||||
let encrypted = encrypt_for_reader(&plaintext);
|
||||
writer
|
||||
.write_all(&encrypted)
|
||||
.await
|
||||
.expect("nonzero frame must be writable");
|
||||
|
||||
let result = read_bounded(
|
||||
&mut crypto_reader,
|
||||
ProtoTag::Abridged,
|
||||
&buffer_pool,
|
||||
&forensics,
|
||||
&mut frame_counter,
|
||||
&stats,
|
||||
&idle_policy,
|
||||
&mut idle_state,
|
||||
&last_downstream_activity_ms,
|
||||
session_started_at,
|
||||
)
|
||||
.await
|
||||
.expect("valid frame should decode")
|
||||
.expect("valid frame should return payload");
|
||||
|
||||
assert_eq!(result.0.as_ref(), &payload);
|
||||
assert!(!result.1);
|
||||
assert_eq!(frame_counter, 1);
|
||||
}
|
||||
+20
-849
@@ -1,17 +1,33 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use dashmap::DashMap;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::{CidrRateLimitKey, RateLimitBps};
|
||||
use crate::config::RateLimitBps;
|
||||
|
||||
// Atomic per-user and per-CIDR accounting.
|
||||
mod buckets;
|
||||
// Immutable policy matching and sharded registries.
|
||||
mod policy;
|
||||
// Traffic lease accounting and cleanup.
|
||||
mod lease;
|
||||
// Runtime policy application and admission.
|
||||
mod limiter;
|
||||
// Epoch and arithmetic helpers.
|
||||
mod helpers;
|
||||
|
||||
pub use helpers::next_refill_delay;
|
||||
use helpers::{
|
||||
auto_cidr_bucket_key, bytes_per_epoch, current_epoch, decrement_atomic_saturating,
|
||||
now_epoch_secs,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
const REGISTRY_SHARDS: usize = 64;
|
||||
const FAIR_EPOCH_MS: u64 = 20;
|
||||
const MAX_BORROW_CHUNK_BYTES: u64 = 32 * 1024;
|
||||
@@ -56,114 +72,18 @@ struct ScopeMetrics {
|
||||
policy_entries: AtomicU64,
|
||||
}
|
||||
|
||||
impl ScopeMetrics {
|
||||
fn throttle(&self, direction: RateDirection) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.throttle_up_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.throttle_down_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_ms(&self, direction: RateDirection, wait_ms: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.wait_up_ms_total.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.wait_down_ms_total
|
||||
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AtomicRatePair {
|
||||
up_bps: AtomicU64,
|
||||
down_bps: AtomicU64,
|
||||
}
|
||||
|
||||
impl AtomicRatePair {
|
||||
fn set(&self, limits: RateLimitBps) {
|
||||
self.up_bps.store(limits.up_bps, Ordering::Relaxed);
|
||||
self.down_bps.store(limits.down_bps, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn get(&self, direction: RateDirection) -> u64 {
|
||||
match direction {
|
||||
RateDirection::Up => self.up_bps.load(Ordering::Relaxed),
|
||||
RateDirection::Down => self.down_bps.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DirectionBucket {
|
||||
epoch: AtomicU64,
|
||||
used: AtomicU64,
|
||||
}
|
||||
|
||||
impl DirectionBucket {
|
||||
fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn try_consume(&self, cap_bps: u64, requested: u64) -> u64 {
|
||||
if requested == 0 {
|
||||
return 0;
|
||||
}
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
|
||||
loop {
|
||||
let used = self.used.load(Ordering::Relaxed);
|
||||
if used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let remaining = cap_epoch.saturating_sub(used);
|
||||
let grant = requested.min(remaining);
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
let next = used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(used, next, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
struct UserBucket {
|
||||
rates: AtomicRatePair,
|
||||
up: DirectionBucket,
|
||||
@@ -171,38 +91,6 @@ struct UserBucket {
|
||||
active_leases: AtomicU64,
|
||||
}
|
||||
|
||||
impl UserBucket {
|
||||
fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: DirectionBucket::default(),
|
||||
down: DirectionBucket::default(),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
fn try_consume(&self, direction: RateDirection, requested: u64) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(cap_bps, requested),
|
||||
RateDirection::Down => self.down.try_consume(cap_bps, requested),
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => self.up.refund(bytes),
|
||||
RateDirection::Down => self.down.refund(bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CidrDirectionBucket {
|
||||
epoch: AtomicU64,
|
||||
@@ -210,125 +98,18 @@ struct CidrDirectionBucket {
|
||||
active_users: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrDirectionBucket {
|
||||
fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
self.active_users.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn try_consume(
|
||||
&self,
|
||||
user_state: &CidrUserDirectionState,
|
||||
cap_epoch: u64,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
if requested == 0 || cap_epoch == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
user_state.sync_epoch_and_mark_active(epoch, &self.active_users);
|
||||
let active_users = self.active_users.load(Ordering::Relaxed).max(1);
|
||||
let fair_share = cap_epoch.saturating_div(active_users).max(1);
|
||||
|
||||
loop {
|
||||
let total_used = self.used.load(Ordering::Relaxed);
|
||||
if total_used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let total_remaining = cap_epoch.saturating_sub(total_used);
|
||||
let user_used = user_state.used.load(Ordering::Relaxed);
|
||||
let guaranteed_remaining = fair_share.saturating_sub(user_used);
|
||||
|
||||
let grant = if guaranteed_remaining > 0 {
|
||||
requested.min(guaranteed_remaining).min(total_remaining)
|
||||
} else {
|
||||
requested.min(total_remaining).min(MAX_BORROW_CHUNK_BYTES)
|
||||
};
|
||||
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let next_total = total_used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(total_used, next_total, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
user_state.used.fetch_add(grant, Ordering::Relaxed);
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CidrUserDirectionState {
|
||||
epoch: AtomicU64,
|
||||
used: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrUserDirectionState {
|
||||
fn sync_epoch_and_mark_active(&self, epoch: u64, active_users: &AtomicU64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
active_users.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
struct CidrUserShare {
|
||||
active_conns: AtomicU64,
|
||||
up: CidrUserDirectionState,
|
||||
down: CidrUserDirectionState,
|
||||
}
|
||||
|
||||
impl CidrUserShare {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
active_conns: AtomicU64::new(0),
|
||||
up: CidrUserDirectionState::default(),
|
||||
down: CidrUserDirectionState::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CidrBucket {
|
||||
rates: AtomicRatePair,
|
||||
up: CidrDirectionBucket,
|
||||
@@ -337,75 +118,6 @@ struct CidrBucket {
|
||||
active_leases: AtomicU64,
|
||||
}
|
||||
|
||||
impl CidrBucket {
|
||||
fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: CidrDirectionBucket::default(),
|
||||
down: CidrDirectionBucket::default(),
|
||||
users: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
fn acquire_user_share(&self, user: &str) -> Arc<CidrUserShare> {
|
||||
self.users
|
||||
.get_or_insert_with(user, CidrUserShare::new, |share| {
|
||||
share.active_conns.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
fn release_user_share(&self, user: &str, share: &Arc<CidrUserShare>) {
|
||||
decrement_atomic_saturating(&share.active_conns, 1);
|
||||
let share_for_remove = Arc::clone(share);
|
||||
let _ = self.users.remove_if(user, |candidate| {
|
||||
Arc::ptr_eq(candidate, &share_for_remove)
|
||||
&& candidate.active_conns.load(Ordering::Relaxed) == 0
|
||||
});
|
||||
}
|
||||
|
||||
fn try_consume_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(&share.up, cap_epoch, requested),
|
||||
RateDirection::Down => self.down.try_consume(&share.down, cap_epoch, requested),
|
||||
}
|
||||
}
|
||||
|
||||
fn refund_for_user(&self, direction: RateDirection, share: &CidrUserShare, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.up.refund(bytes);
|
||||
share.up.refund(bytes);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.down.refund(bytes);
|
||||
share.down.refund(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_idle_users(&self) {
|
||||
self.users
|
||||
.retain(|_, share| share.active_conns.load(Ordering::Relaxed) > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CidrRule {
|
||||
key: String,
|
||||
@@ -435,97 +147,11 @@ struct PolicySnapshot {
|
||||
cidr_rule_keys: HashSet<String>,
|
||||
}
|
||||
|
||||
impl PolicySnapshot {
|
||||
fn match_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
match ip {
|
||||
IpAddr::V4(_) => self
|
||||
.cidr_rules_v4
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
IpAddr::V6(_) => self
|
||||
.cidr_rules_v6
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
}
|
||||
.map(CidrPolicyMatch::Explicit)
|
||||
.or_else(|| self.match_auto_cidr(ip))
|
||||
}
|
||||
|
||||
fn match_auto_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
let rule = match ip {
|
||||
IpAddr::V4(_) => self.cidr_auto_rules_v4.first()?,
|
||||
IpAddr::V6(_) => self.cidr_auto_rules_v6.first()?,
|
||||
};
|
||||
let key = auto_cidr_bucket_key(ip, rule.prefix_len)?;
|
||||
Some(CidrPolicyMatch::Auto {
|
||||
key,
|
||||
limits: rule.limits,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ShardedRegistry<T> {
|
||||
shards: Box<[DashMap<String, Arc<T>>]>,
|
||||
mask: usize,
|
||||
}
|
||||
|
||||
impl<T> ShardedRegistry<T> {
|
||||
fn new(shards: usize) -> Self {
|
||||
let shard_count = shards.max(1).next_power_of_two();
|
||||
let mut items = Vec::with_capacity(shard_count);
|
||||
for _ in 0..shard_count {
|
||||
items.push(DashMap::<String, Arc<T>>::new());
|
||||
}
|
||||
Self {
|
||||
shards: items.into_boxed_slice(),
|
||||
mask: shard_count.saturating_sub(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn shard_index(&self, key: &str) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
key.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.mask
|
||||
}
|
||||
|
||||
fn get_or_insert_with<F, A>(&self, key: &str, make: F, activate: A) -> Arc<T>
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
A: FnOnce(&Arc<T>),
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
match shard.entry(key.to_string()) {
|
||||
dashmap::mapref::entry::Entry::Occupied(entry) => {
|
||||
activate(entry.get());
|
||||
Arc::clone(entry.get())
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Vacant(slot) => {
|
||||
let value = Arc::new(make());
|
||||
activate(&value);
|
||||
slot.insert(Arc::clone(&value));
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn retain<F>(&self, predicate: F)
|
||||
where
|
||||
F: Fn(&String, &Arc<T>) -> bool + Copy,
|
||||
{
|
||||
for shard in &*self.shards {
|
||||
shard.retain(|key, value| predicate(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_if<F>(&self, key: &str, predicate: F) -> bool
|
||||
where
|
||||
F: Fn(&Arc<T>) -> bool,
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
shard.remove_if(key, |_, value| predicate(value)).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TrafficLease {
|
||||
limiter: Arc<TrafficLimiter>,
|
||||
user_bucket: Option<Arc<UserBucket>>,
|
||||
@@ -534,107 +160,6 @@ pub struct TrafficLease {
|
||||
cidr_user_share: Option<Arc<CidrUserShare>>,
|
||||
}
|
||||
|
||||
impl TrafficLease {
|
||||
pub fn try_consume(&self, direction: RateDirection, requested: u64) -> TrafficConsumeResult {
|
||||
if requested == 0 {
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
|
||||
let mut granted = requested;
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
let user_granted = user_bucket.try_consume(direction, granted);
|
||||
if user_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, true, false);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: true,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
granted = user_granted;
|
||||
}
|
||||
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
let cidr_granted =
|
||||
cidr_bucket.try_consume_for_user(direction, cidr_user_share, granted);
|
||||
if cidr_granted < granted
|
||||
&& let Some(user_bucket) = self.user_bucket.as_ref()
|
||||
{
|
||||
user_bucket.refund(direction, granted.saturating_sub(cidr_granted));
|
||||
}
|
||||
if cidr_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, false, true);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: true,
|
||||
};
|
||||
}
|
||||
granted = cidr_granted;
|
||||
}
|
||||
|
||||
TrafficConsumeResult {
|
||||
granted,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
user_bucket.refund(direction, bytes);
|
||||
}
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
cidr_bucket.refund_for_user(direction, cidr_user_share, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe_wait_ms(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if wait_ms == 0 {
|
||||
return;
|
||||
}
|
||||
self.limiter
|
||||
.observe_wait(direction, blocked_user, blocked_cidr, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TrafficLease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(bucket) = self.user_bucket.as_ref() {
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.user_scope.active_leases, 1);
|
||||
}
|
||||
|
||||
if let Some(bucket) = self.cidr_bucket.as_ref() {
|
||||
if let (Some(user_key), Some(share)) =
|
||||
(self.cidr_user_key.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
bucket.release_user_share(user_key, share);
|
||||
}
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.cidr_scope.active_leases, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TrafficLimiter {
|
||||
policy: ArcSwap<PolicySnapshot>,
|
||||
user_buckets: ShardedRegistry<UserBucket>,
|
||||
@@ -643,357 +168,3 @@ pub struct TrafficLimiter {
|
||||
cidr_scope: ScopeMetrics,
|
||||
last_cleanup_epoch_secs: AtomicU64,
|
||||
}
|
||||
|
||||
impl TrafficLimiter {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
policy: ArcSwap::from_pointee(PolicySnapshot::default()),
|
||||
user_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
cidr_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
user_scope: ScopeMetrics::default(),
|
||||
cidr_scope: ScopeMetrics::default(),
|
||||
last_cleanup_epoch_secs: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_policy(
|
||||
&self,
|
||||
user_limits: HashMap<String, RateLimitBps>,
|
||||
cidr_limits: HashMap<CidrRateLimitKey, RateLimitBps>,
|
||||
) {
|
||||
let filtered_users = user_limits
|
||||
.into_iter()
|
||||
.filter(|(_, limit)| limit.up_bps > 0 || limit.down_bps > 0)
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut cidr_rules_v4 = Vec::new();
|
||||
let mut cidr_rules_v6 = Vec::new();
|
||||
let mut cidr_auto_rules_v4 = Vec::new();
|
||||
let mut cidr_auto_rules_v6 = Vec::new();
|
||||
let mut cidr_rule_keys = HashSet::new();
|
||||
for (key, limits) in cidr_limits {
|
||||
if limits.up_bps == 0 && limits.down_bps == 0 {
|
||||
continue;
|
||||
}
|
||||
match key {
|
||||
CidrRateLimitKey::Network(cidr) => {
|
||||
let key = cidr.to_string();
|
||||
let rule = CidrRule {
|
||||
key: key.clone(),
|
||||
cidr,
|
||||
limits,
|
||||
prefix_len: cidr.prefix(),
|
||||
};
|
||||
cidr_rule_keys.insert(key);
|
||||
match rule.cidr {
|
||||
IpNetwork::V4(_) => cidr_rules_v4.push(rule),
|
||||
IpNetwork::V6(_) => cidr_rules_v6.push(rule),
|
||||
}
|
||||
}
|
||||
CidrRateLimitKey::AutoV4(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoV6(prefix_len) => {
|
||||
cidr_auto_rules_v6.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoDual(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
cidr_auto_rules_v6.push(CidrAutoRule {
|
||||
prefix_len: prefix_len.saturating_mul(4),
|
||||
limits,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cidr_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
let cidr_policy_entries =
|
||||
cidr_rule_keys.len() + cidr_auto_rules_v4.len() + cidr_auto_rules_v6.len();
|
||||
|
||||
self.user_scope
|
||||
.policy_entries
|
||||
.store(filtered_users.len() as u64, Ordering::Relaxed);
|
||||
self.cidr_scope
|
||||
.policy_entries
|
||||
.store(cidr_policy_entries as u64, Ordering::Relaxed);
|
||||
|
||||
self.policy.store(Arc::new(PolicySnapshot {
|
||||
user_limits: filtered_users,
|
||||
cidr_rules_v4,
|
||||
cidr_rules_v6,
|
||||
cidr_auto_rules_v4,
|
||||
cidr_auto_rules_v6,
|
||||
cidr_rule_keys,
|
||||
}));
|
||||
|
||||
self.maybe_cleanup();
|
||||
}
|
||||
|
||||
pub fn acquire_lease(
|
||||
self: &Arc<Self>,
|
||||
user: &str,
|
||||
client_ip: IpAddr,
|
||||
) -> Option<Arc<TrafficLease>> {
|
||||
let policy = self.policy.load_full();
|
||||
let mut user_bucket = None;
|
||||
if let Some(limit) = policy.user_limits.get(user).copied() {
|
||||
let bucket = self
|
||||
.user_buckets
|
||||
.get_or_insert_with(user, || UserBucket::new(limit), |bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
bucket.set_rates(limit);
|
||||
self.user_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
user_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
let mut cidr_bucket = None;
|
||||
let mut cidr_user_key = None;
|
||||
let mut cidr_user_share = None;
|
||||
if let Some(rule_match) = policy.match_cidr(client_ip) {
|
||||
let (key, limits) = match &rule_match {
|
||||
CidrPolicyMatch::Explicit(rule) => (rule.key.as_str(), rule.limits),
|
||||
CidrPolicyMatch::Auto { key, limits } => (key.as_str(), *limits),
|
||||
};
|
||||
let bucket = self
|
||||
.cidr_buckets
|
||||
.get_or_insert_with(key, || CidrBucket::new(limits), |bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
bucket.set_rates(limits);
|
||||
self.cidr_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
let share = bucket.acquire_user_share(user);
|
||||
cidr_user_key = Some(user.to_string());
|
||||
cidr_user_share = Some(share);
|
||||
cidr_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
if user_bucket.is_none() && cidr_bucket.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.maybe_cleanup();
|
||||
Some(Arc::new(TrafficLease {
|
||||
limiter: Arc::clone(self),
|
||||
user_bucket,
|
||||
cidr_bucket,
|
||||
cidr_user_key,
|
||||
cidr_user_share,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn metrics_snapshot(&self) -> TrafficLimiterMetricsSnapshot {
|
||||
TrafficLimiterMetricsSnapshot {
|
||||
user_throttle_up_total: self.user_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
user_throttle_down_total: self.user_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_up_total: self.cidr_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_down_total: self.cidr_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
user_wait_up_ms_total: self.user_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
user_wait_down_ms_total: self.user_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_up_ms_total: self.cidr_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_down_ms_total: self.cidr_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
user_active_leases: self.user_scope.active_leases.load(Ordering::Relaxed),
|
||||
cidr_active_leases: self.cidr_scope.active_leases.load(Ordering::Relaxed),
|
||||
user_policy_entries: self.user_scope.policy_entries.load(Ordering::Relaxed),
|
||||
cidr_policy_entries: self.cidr_scope.policy_entries.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_throttle(&self, direction: RateDirection, blocked_user: bool, blocked_cidr: bool) {
|
||||
if blocked_user {
|
||||
self.user_scope.throttle(direction);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.throttle(direction);
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_wait(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_cleanup(&self) {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed);
|
||||
if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS {
|
||||
return;
|
||||
}
|
||||
if self
|
||||
.last_cleanup_epoch_secs
|
||||
.compare_exchange(last, now_epoch_secs, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let policy = self.policy.load_full();
|
||||
self.user_buckets.retain(|user, bucket| {
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.user_limits.contains_key(user)
|
||||
});
|
||||
self.cidr_buckets.retain(|cidr_key, bucket| {
|
||||
bucket.cleanup_idle_users();
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.cidr_rule_keys.contains(cidr_key)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_refill_delay() -> Duration {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
let epoch_pos = elapsed_ms % FAIR_EPOCH_MS;
|
||||
let wait_ms = FAIR_EPOCH_MS.saturating_sub(epoch_pos).max(1);
|
||||
Duration::from_millis(wait_ms)
|
||||
}
|
||||
|
||||
fn decrement_atomic_saturating(counter: &AtomicU64, by: u64) {
|
||||
if by == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current == 0 {
|
||||
return;
|
||||
}
|
||||
let next = current.saturating_sub(by);
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn bytes_per_epoch(bps: u64) -> u64 {
|
||||
if bps == 0 {
|
||||
return 0;
|
||||
}
|
||||
let numerator = bps.saturating_mul(FAIR_EPOCH_MS);
|
||||
let bytes = numerator.saturating_div(8_000);
|
||||
bytes.max(1)
|
||||
}
|
||||
|
||||
fn auto_cidr_bucket_key(ip: IpAddr, prefix_len: u8) -> Option<String> {
|
||||
let cidr = IpNetwork::new(ip, prefix_len).ok()?;
|
||||
let network = IpNetwork::new(cidr.network(), prefix_len).ok()?;
|
||||
let family = match network {
|
||||
IpNetwork::V4(_) => "4",
|
||||
IpNetwork::V6(_) => "6",
|
||||
};
|
||||
Some(format!("auto:{family}:{network}"))
|
||||
}
|
||||
|
||||
fn current_epoch() -> u64 {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
elapsed_ms / FAIR_EPOCH_MS
|
||||
}
|
||||
|
||||
fn limiter_epoch_start() -> &'static Instant {
|
||||
static START: OnceLock<Instant> = OnceLock::new();
|
||||
START.get_or_init(Instant::now)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rate(up_bps: u64, down_bps: u64) -> RateLimitBps {
|
||||
RateLimitBps { up_bps, down_bps }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_cidr_rule_wins_over_auto_template() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(
|
||||
CidrRateLimitKey::Network("203.0.113.7/32".parse().unwrap()),
|
||||
rate(2_000, 0),
|
||||
);
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.7".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Explicit(rule) => assert_eq!(rule.key.as_str(), "203.0.113.7/32"),
|
||||
CidrPolicyMatch::Auto { .. } => panic!("explicit CIDR must have priority"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_template_uses_longest_prefix() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(32), rate(2_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.129".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, limits } => {
|
||||
assert_eq!(key, "auto:4:203.0.113.129/32");
|
||||
assert_eq!(limits.up_bps, 2_000);
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dual_auto_template_maps_v6_prefix_by_four() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoDual(32), rate(1_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("2001:db8::1".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, .. } => {
|
||||
assert_eq!(key, "auto:6:2001:db8::1/128");
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_cidr_bucket_key_canonicalizes_network_address() {
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("203.0.113.129".parse().unwrap(), 24).unwrap(),
|
||||
"auto:4:203.0.113.0/24"
|
||||
);
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("2001:db8::abcd".parse().unwrap(), 64).unwrap(),
|
||||
"auto:6:2001:db8::/64"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
use super::*;
|
||||
|
||||
impl ScopeMetrics {
|
||||
pub(super) fn throttle(&self, direction: RateDirection) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.throttle_up_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.throttle_down_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn wait_ms(&self, direction: RateDirection, wait_ms: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.wait_up_ms_total.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.wait_down_ms_total
|
||||
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AtomicRatePair {
|
||||
pub(super) fn set(&self, limits: RateLimitBps) {
|
||||
self.up_bps.store(limits.up_bps, Ordering::Relaxed);
|
||||
self.down_bps.store(limits.down_bps, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(super) fn get(&self, direction: RateDirection) -> u64 {
|
||||
match direction {
|
||||
RateDirection::Up => self.up_bps.load(Ordering::Relaxed),
|
||||
RateDirection::Down => self.down_bps.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DirectionBucket {
|
||||
pub(super) fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(&self, cap_bps: u64, requested: u64) -> u64 {
|
||||
if requested == 0 {
|
||||
return 0;
|
||||
}
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
|
||||
loop {
|
||||
let used = self.used.load(Ordering::Relaxed);
|
||||
if used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let remaining = cap_epoch.saturating_sub(used);
|
||||
let grant = requested.min(remaining);
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
let next = used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(used, next, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl UserBucket {
|
||||
pub(super) fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: DirectionBucket::default(),
|
||||
down: DirectionBucket::default(),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(&self, direction: RateDirection, requested: u64) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(cap_bps, requested),
|
||||
RateDirection::Down => self.down.try_consume(cap_bps, requested),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
match direction {
|
||||
RateDirection::Up => self.up.refund(bytes),
|
||||
RateDirection::Down => self.down.refund(bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrDirectionBucket {
|
||||
pub(super) fn sync_epoch(&self, epoch: u64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
self.active_users.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_consume(
|
||||
&self,
|
||||
user_state: &CidrUserDirectionState,
|
||||
cap_epoch: u64,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
if requested == 0 || cap_epoch == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let epoch = current_epoch();
|
||||
self.sync_epoch(epoch);
|
||||
user_state.sync_epoch_and_mark_active(epoch, &self.active_users);
|
||||
let active_users = self.active_users.load(Ordering::Relaxed).max(1);
|
||||
let fair_share = cap_epoch.saturating_div(active_users).max(1);
|
||||
|
||||
loop {
|
||||
let total_used = self.used.load(Ordering::Relaxed);
|
||||
if total_used >= cap_epoch {
|
||||
return 0;
|
||||
}
|
||||
let total_remaining = cap_epoch.saturating_sub(total_used);
|
||||
let user_used = user_state.used.load(Ordering::Relaxed);
|
||||
let guaranteed_remaining = fair_share.saturating_sub(user_used);
|
||||
|
||||
let grant = if guaranteed_remaining > 0 {
|
||||
requested.min(guaranteed_remaining).min(total_remaining)
|
||||
} else {
|
||||
requested.min(total_remaining).min(MAX_BORROW_CHUNK_BYTES)
|
||||
};
|
||||
|
||||
if grant == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let next_total = total_used.saturating_add(grant);
|
||||
if self
|
||||
.used
|
||||
.compare_exchange_weak(total_used, next_total, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
user_state.used.fetch_add(grant, Ordering::Relaxed);
|
||||
return grant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrUserDirectionState {
|
||||
pub(super) fn sync_epoch_and_mark_active(&self, epoch: u64, active_users: &AtomicU64) {
|
||||
let current = self.epoch.load(Ordering::Relaxed);
|
||||
if current == epoch {
|
||||
return;
|
||||
}
|
||||
if current < epoch
|
||||
&& self
|
||||
.epoch
|
||||
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.used.store(0, Ordering::Relaxed);
|
||||
active_users.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund(&self, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
decrement_atomic_saturating(&self.used, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrUserShare {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
active_conns: AtomicU64::new(0),
|
||||
up: CidrUserDirectionState::default(),
|
||||
down: CidrUserDirectionState::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrBucket {
|
||||
pub(super) fn new(limits: RateLimitBps) -> Self {
|
||||
let rates = AtomicRatePair::default();
|
||||
rates.set(limits);
|
||||
Self {
|
||||
rates,
|
||||
up: CidrDirectionBucket::default(),
|
||||
down: CidrDirectionBucket::default(),
|
||||
users: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
active_leases: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn set_rates(&self, limits: RateLimitBps) {
|
||||
self.rates.set(limits);
|
||||
}
|
||||
|
||||
pub(super) fn acquire_user_share(&self, user: &str) -> Arc<CidrUserShare> {
|
||||
self.users
|
||||
.get_or_insert_with(user, CidrUserShare::new, |share| {
|
||||
share.active_conns.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn release_user_share(&self, user: &str, share: &Arc<CidrUserShare>) {
|
||||
decrement_atomic_saturating(&share.active_conns, 1);
|
||||
let share_for_remove = Arc::clone(share);
|
||||
let _ = self.users.remove_if(user, |candidate| {
|
||||
Arc::ptr_eq(candidate, &share_for_remove)
|
||||
&& candidate.active_conns.load(Ordering::Relaxed) == 0
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn try_consume_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
requested: u64,
|
||||
) -> u64 {
|
||||
let cap_bps = self.rates.get(direction);
|
||||
if cap_bps == 0 {
|
||||
return requested;
|
||||
}
|
||||
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||
match direction {
|
||||
RateDirection::Up => self.up.try_consume(&share.up, cap_epoch, requested),
|
||||
RateDirection::Down => self.down.try_consume(&share.down, cap_epoch, requested),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn refund_for_user(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
share: &CidrUserShare,
|
||||
bytes: u64,
|
||||
) {
|
||||
match direction {
|
||||
RateDirection::Up => {
|
||||
self.up.refund(bytes);
|
||||
share.up.refund(bytes);
|
||||
}
|
||||
RateDirection::Down => {
|
||||
self.down.refund(bytes);
|
||||
share.down.refund(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cleanup_idle_users(&self) {
|
||||
self.users
|
||||
.retain(|_, share| share.active_conns.load(Ordering::Relaxed) > 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::*;
|
||||
pub fn next_refill_delay() -> Duration {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
let epoch_pos = elapsed_ms % FAIR_EPOCH_MS;
|
||||
let wait_ms = FAIR_EPOCH_MS.saturating_sub(epoch_pos).max(1);
|
||||
Duration::from_millis(wait_ms)
|
||||
}
|
||||
|
||||
pub(super) fn decrement_atomic_saturating(counter: &AtomicU64, by: u64) {
|
||||
if by == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current == 0 {
|
||||
return;
|
||||
}
|
||||
let next = current.saturating_sub(by);
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
pub(super) fn bytes_per_epoch(bps: u64) -> u64 {
|
||||
if bps == 0 {
|
||||
return 0;
|
||||
}
|
||||
let numerator = bps.saturating_mul(FAIR_EPOCH_MS);
|
||||
let bytes = numerator.saturating_div(8_000);
|
||||
bytes.max(1)
|
||||
}
|
||||
|
||||
pub(super) fn auto_cidr_bucket_key(ip: IpAddr, prefix_len: u8) -> Option<String> {
|
||||
let cidr = IpNetwork::new(ip, prefix_len).ok()?;
|
||||
let network = IpNetwork::new(cidr.network(), prefix_len).ok()?;
|
||||
let family = match network {
|
||||
IpNetwork::V4(_) => "4",
|
||||
IpNetwork::V6(_) => "6",
|
||||
};
|
||||
Some(format!("auto:{family}:{network}"))
|
||||
}
|
||||
|
||||
pub(super) fn current_epoch() -> u64 {
|
||||
let start = limiter_epoch_start();
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
elapsed_ms / FAIR_EPOCH_MS
|
||||
}
|
||||
|
||||
pub(super) fn limiter_epoch_start() -> &'static Instant {
|
||||
static START: OnceLock<Instant> = OnceLock::new();
|
||||
START.get_or_init(Instant::now)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use super::*;
|
||||
|
||||
impl TrafficLease {
|
||||
pub fn try_consume(&self, direction: RateDirection, requested: u64) -> TrafficConsumeResult {
|
||||
if requested == 0 {
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
|
||||
let mut granted = requested;
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
let user_granted = user_bucket.try_consume(direction, granted);
|
||||
if user_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, true, false);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: true,
|
||||
blocked_cidr: false,
|
||||
};
|
||||
}
|
||||
granted = user_granted;
|
||||
}
|
||||
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
let cidr_granted =
|
||||
cidr_bucket.try_consume_for_user(direction, cidr_user_share, granted);
|
||||
if cidr_granted < granted
|
||||
&& let Some(user_bucket) = self.user_bucket.as_ref()
|
||||
{
|
||||
user_bucket.refund(direction, granted.saturating_sub(cidr_granted));
|
||||
}
|
||||
if cidr_granted == 0 {
|
||||
self.limiter.observe_throttle(direction, false, true);
|
||||
return TrafficConsumeResult {
|
||||
granted: 0,
|
||||
blocked_user: false,
|
||||
blocked_cidr: true,
|
||||
};
|
||||
}
|
||||
granted = cidr_granted;
|
||||
}
|
||||
|
||||
TrafficConsumeResult {
|
||||
granted,
|
||||
blocked_user: false,
|
||||
blocked_cidr: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||
user_bucket.refund(direction, bytes);
|
||||
}
|
||||
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
cidr_bucket.refund_for_user(direction, cidr_user_share, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe_wait_ms(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if wait_ms == 0 {
|
||||
return;
|
||||
}
|
||||
self.limiter
|
||||
.observe_wait(direction, blocked_user, blocked_cidr, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TrafficLease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(bucket) = self.user_bucket.as_ref() {
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.user_scope.active_leases, 1);
|
||||
}
|
||||
|
||||
if let Some(bucket) = self.cidr_bucket.as_ref() {
|
||||
if let (Some(user_key), Some(share)) =
|
||||
(self.cidr_user_key.as_ref(), self.cidr_user_share.as_ref())
|
||||
{
|
||||
bucket.release_user_share(user_key, share);
|
||||
}
|
||||
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||
decrement_atomic_saturating(&self.limiter.cidr_scope.active_leases, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
use crate::config::CidrRateLimitKey;
|
||||
|
||||
use super::*;
|
||||
impl TrafficLimiter {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
policy: ArcSwap::from_pointee(PolicySnapshot::default()),
|
||||
user_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
cidr_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||
user_scope: ScopeMetrics::default(),
|
||||
cidr_scope: ScopeMetrics::default(),
|
||||
last_cleanup_epoch_secs: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn apply_policy(
|
||||
&self,
|
||||
user_limits: HashMap<String, RateLimitBps>,
|
||||
cidr_limits: HashMap<CidrRateLimitKey, RateLimitBps>,
|
||||
) {
|
||||
let filtered_users = user_limits
|
||||
.into_iter()
|
||||
.filter(|(_, limit)| limit.up_bps > 0 || limit.down_bps > 0)
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut cidr_rules_v4 = Vec::new();
|
||||
let mut cidr_rules_v6 = Vec::new();
|
||||
let mut cidr_auto_rules_v4 = Vec::new();
|
||||
let mut cidr_auto_rules_v6 = Vec::new();
|
||||
let mut cidr_rule_keys = HashSet::new();
|
||||
for (key, limits) in cidr_limits {
|
||||
if limits.up_bps == 0 && limits.down_bps == 0 {
|
||||
continue;
|
||||
}
|
||||
match key {
|
||||
CidrRateLimitKey::Network(cidr) => {
|
||||
let key = cidr.to_string();
|
||||
let rule = CidrRule {
|
||||
key: key.clone(),
|
||||
cidr,
|
||||
limits,
|
||||
prefix_len: cidr.prefix(),
|
||||
};
|
||||
cidr_rule_keys.insert(key);
|
||||
match rule.cidr {
|
||||
IpNetwork::V4(_) => cidr_rules_v4.push(rule),
|
||||
IpNetwork::V6(_) => cidr_rules_v6.push(rule),
|
||||
}
|
||||
}
|
||||
CidrRateLimitKey::AutoV4(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoV6(prefix_len) => {
|
||||
cidr_auto_rules_v6.push(CidrAutoRule { prefix_len, limits });
|
||||
}
|
||||
CidrRateLimitKey::AutoDual(prefix_len) => {
|
||||
cidr_auto_rules_v4.push(CidrAutoRule { prefix_len, limits });
|
||||
cidr_auto_rules_v6.push(CidrAutoRule {
|
||||
prefix_len: prefix_len.saturating_mul(4),
|
||||
limits,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cidr_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
cidr_auto_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||
let cidr_policy_entries =
|
||||
cidr_rule_keys.len() + cidr_auto_rules_v4.len() + cidr_auto_rules_v6.len();
|
||||
|
||||
self.user_scope
|
||||
.policy_entries
|
||||
.store(filtered_users.len() as u64, Ordering::Relaxed);
|
||||
self.cidr_scope
|
||||
.policy_entries
|
||||
.store(cidr_policy_entries as u64, Ordering::Relaxed);
|
||||
|
||||
self.policy.store(Arc::new(PolicySnapshot {
|
||||
user_limits: filtered_users,
|
||||
cidr_rules_v4,
|
||||
cidr_rules_v6,
|
||||
cidr_auto_rules_v4,
|
||||
cidr_auto_rules_v6,
|
||||
cidr_rule_keys,
|
||||
}));
|
||||
|
||||
self.maybe_cleanup();
|
||||
}
|
||||
|
||||
pub fn acquire_lease(
|
||||
self: &Arc<Self>,
|
||||
user: &str,
|
||||
client_ip: IpAddr,
|
||||
) -> Option<Arc<TrafficLease>> {
|
||||
let policy = self.policy.load_full();
|
||||
let mut user_bucket = None;
|
||||
if let Some(limit) = policy.user_limits.get(user).copied() {
|
||||
let bucket = self.user_buckets.get_or_insert_with(
|
||||
user,
|
||||
|| UserBucket::new(limit),
|
||||
|bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
},
|
||||
);
|
||||
bucket.set_rates(limit);
|
||||
self.user_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
user_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
let mut cidr_bucket = None;
|
||||
let mut cidr_user_key = None;
|
||||
let mut cidr_user_share = None;
|
||||
if let Some(rule_match) = policy.match_cidr(client_ip) {
|
||||
let (key, limits) = match &rule_match {
|
||||
CidrPolicyMatch::Explicit(rule) => (rule.key.as_str(), rule.limits),
|
||||
CidrPolicyMatch::Auto { key, limits } => (key.as_str(), *limits),
|
||||
};
|
||||
let bucket = self.cidr_buckets.get_or_insert_with(
|
||||
key,
|
||||
|| CidrBucket::new(limits),
|
||||
|bucket| {
|
||||
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||
},
|
||||
);
|
||||
bucket.set_rates(limits);
|
||||
self.cidr_scope
|
||||
.active_leases
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
let share = bucket.acquire_user_share(user);
|
||||
cidr_user_key = Some(user.to_string());
|
||||
cidr_user_share = Some(share);
|
||||
cidr_bucket = Some(bucket);
|
||||
}
|
||||
|
||||
if user_bucket.is_none() && cidr_bucket.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
self.maybe_cleanup();
|
||||
Some(Arc::new(TrafficLease {
|
||||
limiter: Arc::clone(self),
|
||||
user_bucket,
|
||||
cidr_bucket,
|
||||
cidr_user_key,
|
||||
cidr_user_share,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn metrics_snapshot(&self) -> TrafficLimiterMetricsSnapshot {
|
||||
TrafficLimiterMetricsSnapshot {
|
||||
user_throttle_up_total: self.user_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
user_throttle_down_total: self.user_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_up_total: self.cidr_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||
cidr_throttle_down_total: self.cidr_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||
user_wait_up_ms_total: self.user_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
user_wait_down_ms_total: self.user_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_up_ms_total: self.cidr_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||
cidr_wait_down_ms_total: self.cidr_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||
user_active_leases: self.user_scope.active_leases.load(Ordering::Relaxed),
|
||||
cidr_active_leases: self.cidr_scope.active_leases.load(Ordering::Relaxed),
|
||||
user_policy_entries: self.user_scope.policy_entries.load(Ordering::Relaxed),
|
||||
cidr_policy_entries: self.cidr_scope.policy_entries.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn observe_throttle(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.throttle(direction);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.throttle(direction);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn observe_wait(
|
||||
&self,
|
||||
direction: RateDirection,
|
||||
blocked_user: bool,
|
||||
blocked_cidr: bool,
|
||||
wait_ms: u64,
|
||||
) {
|
||||
if blocked_user {
|
||||
self.user_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
if blocked_cidr {
|
||||
self.cidr_scope.wait_ms(direction, wait_ms);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn maybe_cleanup(&self) {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed);
|
||||
if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS {
|
||||
return;
|
||||
}
|
||||
if self
|
||||
.last_cleanup_epoch_secs
|
||||
.compare_exchange(last, now_epoch_secs, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let policy = self.policy.load_full();
|
||||
self.user_buckets.retain(|user, bucket| {
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.user_limits.contains_key(user)
|
||||
});
|
||||
self.cidr_buckets.retain(|cidr_key, bucket| {
|
||||
bucket.cleanup_idle_users();
|
||||
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||
|| policy.cidr_rule_keys.contains(cidr_key)
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
use super::*;
|
||||
impl PolicySnapshot {
|
||||
pub(super) fn match_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
match ip {
|
||||
IpAddr::V4(_) => self
|
||||
.cidr_rules_v4
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
IpAddr::V6(_) => self
|
||||
.cidr_rules_v6
|
||||
.iter()
|
||||
.find(|rule| rule.cidr.contains(ip)),
|
||||
}
|
||||
.map(CidrPolicyMatch::Explicit)
|
||||
.or_else(|| self.match_auto_cidr(ip))
|
||||
}
|
||||
|
||||
pub(super) fn match_auto_cidr(&self, ip: IpAddr) -> Option<CidrPolicyMatch<'_>> {
|
||||
let rule = match ip {
|
||||
IpAddr::V4(_) => self.cidr_auto_rules_v4.first()?,
|
||||
IpAddr::V6(_) => self.cidr_auto_rules_v6.first()?,
|
||||
};
|
||||
let key = auto_cidr_bucket_key(ip, rule.prefix_len)?;
|
||||
Some(CidrPolicyMatch::Auto {
|
||||
key,
|
||||
limits: rule.limits,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ShardedRegistry<T> {
|
||||
pub(super) fn new(shards: usize) -> Self {
|
||||
let shard_count = shards.max(1).next_power_of_two();
|
||||
let mut items = Vec::with_capacity(shard_count);
|
||||
for _ in 0..shard_count {
|
||||
items.push(DashMap::<String, Arc<T>>::new());
|
||||
}
|
||||
Self {
|
||||
shards: items.into_boxed_slice(),
|
||||
mask: shard_count.saturating_sub(1),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn shard_index(&self, key: &str) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
key.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.mask
|
||||
}
|
||||
|
||||
pub(super) fn get_or_insert_with<F, A>(&self, key: &str, make: F, activate: A) -> Arc<T>
|
||||
where
|
||||
F: FnOnce() -> T,
|
||||
A: FnOnce(&Arc<T>),
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
match shard.entry(key.to_string()) {
|
||||
dashmap::mapref::entry::Entry::Occupied(entry) => {
|
||||
activate(entry.get());
|
||||
Arc::clone(entry.get())
|
||||
}
|
||||
dashmap::mapref::entry::Entry::Vacant(slot) => {
|
||||
let value = Arc::new(make());
|
||||
activate(&value);
|
||||
slot.insert(Arc::clone(&value));
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn retain<F>(&self, predicate: F)
|
||||
where
|
||||
F: Fn(&String, &Arc<T>) -> bool + Copy,
|
||||
{
|
||||
for shard in &*self.shards {
|
||||
shard.retain(|key, value| predicate(key, value));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remove_if<F>(&self, key: &str, predicate: F) -> bool
|
||||
where
|
||||
F: Fn(&Arc<T>) -> bool,
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
shard.remove_if(key, |_, value| predicate(value)).is_some()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use super::*;
|
||||
use crate::config::CidrRateLimitKey;
|
||||
|
||||
fn rate(up_bps: u64, down_bps: u64) -> RateLimitBps {
|
||||
RateLimitBps { up_bps, down_bps }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_cidr_rule_wins_over_auto_template() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(
|
||||
CidrRateLimitKey::Network("203.0.113.7/32".parse().unwrap()),
|
||||
rate(2_000, 0),
|
||||
);
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.7".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Explicit(rule) => assert_eq!(rule.key.as_str(), "203.0.113.7/32"),
|
||||
CidrPolicyMatch::Auto { .. } => panic!("explicit CIDR must have priority"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_template_uses_longest_prefix() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(24), rate(1_000, 0));
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoV4(32), rate(2_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("203.0.113.129".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, limits } => {
|
||||
assert_eq!(key, "auto:4:203.0.113.129/32");
|
||||
assert_eq!(limits.up_bps, 2_000);
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dual_auto_template_maps_v6_prefix_by_four() {
|
||||
let limiter = TrafficLimiter::new();
|
||||
let mut cidr_limits = HashMap::new();
|
||||
cidr_limits.insert(CidrRateLimitKey::AutoDual(32), rate(1_000, 0));
|
||||
|
||||
limiter.apply_policy(HashMap::new(), cidr_limits);
|
||||
let policy = limiter.policy.load_full();
|
||||
let matched = policy.match_cidr("2001:db8::1".parse().unwrap()).unwrap();
|
||||
|
||||
match matched {
|
||||
CidrPolicyMatch::Auto { key, .. } => {
|
||||
assert_eq!(key, "auto:6:2001:db8::1/128");
|
||||
}
|
||||
CidrPolicyMatch::Explicit(_) => panic!("auto-template match expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_cidr_bucket_key_canonicalizes_network_address() {
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("203.0.113.129".parse().unwrap(), 24).unwrap(),
|
||||
"auto:4:203.0.113.0/24"
|
||||
);
|
||||
assert_eq!(
|
||||
auto_cidr_bucket_key("2001:db8::abcd".parse().unwrap(), 64).unwrap(),
|
||||
"auto:6:2001:db8::/64"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user