Compare commits

..

14 Commits

Author SHA1 Message Date
Alexey bc691539a1 Bump 2026-04-07 19:28:05 +03:00
Alexey 2162a63e3e Memory Hard-bounds + Handshake Budget in Metrics + No mutable in hotpath ConnRegistry + Build-info in Metrics + TLS Fronting fixes + Round-bounded Retries + Bounded Retry-Round Constant + QueueFall Bounded Retry on Data-route: merge pull request #655 from telemt/flow
Memory Hard-bounds + Handshake Budget in Metrics + No mutable in hotpath ConnRegistry + Build-info in Metrics + TLS Fronting fixes + Round-bounded Retries + Bounded Retry-Round Constant + QueueFall Bounded Retry on Data-route
2026-04-07 19:26:07 +03:00
Alexey 4a77335ba9 Round-bounded Retries + Bounded Retry-Round Constant
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-07 19:19:40 +03:00
Alexey ba29b66c4c Merge branch 'flow' of https://github.com/telemt/telemt into flow 2026-04-07 18:42:10 +03:00
Alexey e8cf97095f QueueFall Bounded Retry on Data-route
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-07 18:41:59 +03:00
Alexey ee4264af50 Merge pull request #624 from mammuthus/feature/metrics-build-info
metrics: export CARGO_PKG_VERSION as telemt_build_info version metric
2026-04-07 18:35:06 +03:00
Alexey 89d6be267d Merge pull request #652 from groozchique/flow
[docs] Hotfix for link's obtaining command
2026-04-07 18:23:34 +03:00
Nick Parfyonov 3af7673342 [docs] add classic/secure links to the output
After further testing I discovered that the current command only returns TLS links, ignoring classic/secure links if they are present
2026-04-07 13:53:12 +03:00
Alexey 5cbcfb2a91 Merge pull request #643 from Dimasssss/patch-2
Update install.sh - fix "Permission denied (os error 13)"
2026-04-07 13:20:46 +03:00
Alexey aec2c23a0c Merge pull request #650 from pavlozt/fix/zabbix-storage
Zabbix template: disable intermediate data storage
2026-04-07 13:19:35 +03:00
PavelZ 12f99eebab Zabbix template: disable intermediate data storage 2026-04-07 11:55:51 +03:00
Dimasssss f829439e8f Update install.sh - fix "Permission denied (os error 13)" 2026-04-06 14:33:02 +03:00
Alexey 6996d6e597 Update LICENSE 2026-04-06 13:21:16 +03:00
Alexey b3f11624c9 Update LICENSE 2026-04-06 13:12:06 +03:00
14 changed files with 268 additions and 63 deletions
Generated
+1 -1
View File
@@ -2780,7 +2780,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "telemt"
version = "3.3.38"
version = "3.3.39"
dependencies = [
"aes",
"anyhow",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "telemt"
version = "3.3.38"
version = "3.3.39"
edition = "2024"
[features]
+10 -6
View File
@@ -1,4 +1,4 @@
###### TELEMT Public License 3 ######
######## TELEMT LICENSE 3.3 #########
##### Copyright (c) 2026 Telemt #####
Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -14,11 +14,15 @@ are preserved and complied with.
The canonical version of this License is the English version.
Official translations are provided for informational purposes only
and for convenience, and do not have legal force. In case of any
discrepancy, the English version of this License shall prevail.
Available versions:
- English in Markdown: docs/LICENSE/LICENSE.md
- German: docs/LICENSE/LICENSE.de.md
- Russian: docs/LICENSE/LICENSE.ru.md
discrepancy, the English version of this License shall prevail
/----------------------------------------------------------\
| Language | Location |
|-------------|--------------------------------------------|
| English | docs/LICENSE/TELEMT-LICENSE.en.md |
| German | docs/LICENSE/TELEMT-LICENSE.de.md |
| Russian | docs/LICENSE/TELEMT-LICENSE.ru.md |
\----------------------------------------------------------/
### License Versioning Policy
+1 -1
View File
@@ -150,7 +150,7 @@ systemctl daemon-reload
**7.** To get the link(s), enter:
```bash
curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | "User: \(.username)\n\(.links.tls[0] // empty)"'
curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | "[\(.username)]", (.links.classic[]? | "classic: \(.)"), (.links.secure[]? | "secure: \(.)"), (.links.tls[]? | "tls: \(.)"), ""'
```
> Any number of people can use one link.
+1 -1
View File
@@ -150,7 +150,7 @@ systemctl daemon-reload
**7.** Для получения ссылки/ссылок введите
```bash
curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | "User: \(.username)\n\(.links.tls[0] // empty)"'
curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | "[\(.username)]", (.links.classic[]? | "classic: \(.)"), (.links.secure[]? | "secure: \(.)"), (.links.tls[]? | "tls: \(.)"), ""'
```
> Одной ссылкой может пользоваться сколько угодно человек.
+1 -1
View File
@@ -315,7 +315,7 @@ setup_dirs() {
$SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" "$CONFIG_PARENT_DIR" || die "Failed to create directories"
$SUDO chown telemt:telemt "$WORK_DIR" && $SUDO chmod 750 "$WORK_DIR"
$SUDO chown root:telemt "$CONFIG_DIR" && $SUDO chmod 750 "$CONFIG_DIR"
$SUDO chown telemt:telemt "$CONFIG_DIR" && $SUDO chmod 750 "$CONFIG_DIR"
if [ "$CONFIG_PARENT_DIR" != "$CONFIG_DIR" ] && [ "$CONFIG_PARENT_DIR" != "." ] && [ "$CONFIG_PARENT_DIR" != "/" ]; then
$SUDO chown root:telemt "$CONFIG_PARENT_DIR" && $SUDO chmod 750 "$CONFIG_PARENT_DIR"
@@ -26,7 +26,8 @@ me_writer_cmd_channel_capacity = 16385
"#,
);
let err = ProxyConfig::load(&path).expect_err("writer command capacity above hard cap must fail");
let err =
ProxyConfig::load(&path).expect_err("writer command capacity above hard cap must fail");
let msg = err.to_string();
assert!(
msg.contains("general.me_writer_cmd_channel_capacity must be within [1, 16384]"),
@@ -45,7 +46,8 @@ me_route_channel_capacity = 8193
"#,
);
let err = ProxyConfig::load(&path).expect_err("route channel capacity above hard cap must fail");
let err =
ProxyConfig::load(&path).expect_err("route channel capacity above hard cap must fail");
let msg = err.to_string();
assert!(
msg.contains("general.me_route_channel_capacity must be within [1, 8192]"),
+23 -8
View File
@@ -196,7 +196,15 @@ async fn serve_listener(
let ip_tracker = ip_tracker.clone();
let config = config_rx_conn.borrow().clone();
async move {
handle(req, &stats, &beobachten, &shared_state, &ip_tracker, &config).await
handle(
req,
&stats,
&beobachten,
&shared_state,
&ip_tracker,
&config,
)
.await
}
});
if let Err(e) = http1::Builder::new()
@@ -3145,9 +3153,16 @@ mod tests {
stats.increment_connects_all();
let req = Request::builder().uri("/metrics").body(()).unwrap();
let resp = handle(req, &stats, &beobachten, shared_state.as_ref(), &tracker, &config)
.await
.unwrap();
let resp = handle(
req,
&stats,
&beobachten,
shared_state.as_ref(),
&tracker,
&config,
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = resp.into_body().collect().await.unwrap().to_bytes();
assert!(
@@ -3180,8 +3195,8 @@ mod tests {
&tracker,
&config,
)
.await
.unwrap();
.await
.unwrap();
assert_eq!(resp_beob.status(), StatusCode::OK);
let body_beob = resp_beob.into_body().collect().await.unwrap().to_bytes();
let beob_text = std::str::from_utf8(body_beob.as_ref()).unwrap();
@@ -3197,8 +3212,8 @@ mod tests {
&tracker,
&config,
)
.await
.unwrap();
.await
.unwrap();
assert_eq!(resp404.status(), StatusCode::NOT_FOUND);
}
}
+7
View File
@@ -56,6 +56,8 @@ const ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN: usize = 4096;
const ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR: usize = 2;
const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
const QUOTA_RESERVE_BACKOFF_MIN_MS: u64 = 1;
const QUOTA_RESERVE_BACKOFF_MAX_MS: u64 = 16;
#[derive(Default)]
pub(crate) struct DesyncDedupRotationState {
@@ -573,6 +575,7 @@ async fn reserve_user_quota_with_yield(
bytes: u64,
limit: u64,
) -> std::result::Result<u64, QuotaReserveError> {
let mut backoff_ms = QUOTA_RESERVE_BACKOFF_MIN_MS;
loop {
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match user_stats.quota_try_reserve(bytes, limit) {
@@ -585,6 +588,10 @@ async fn reserve_user_quota_with_yield(
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
backoff_ms = backoff_ms
.saturating_mul(2)
.min(QUOTA_RESERVE_BACKOFF_MAX_MS);
}
}
+24 -11
View File
@@ -271,6 +271,7 @@ const QUOTA_LARGE_CHARGE_BYTES: u64 = 16 * 1024;
const QUOTA_ADAPTIVE_INTERVAL_MIN_BYTES: u64 = 4 * 1024;
const QUOTA_ADAPTIVE_INTERVAL_MAX_BYTES: u64 = 64 * 1024;
const QUOTA_RESERVE_SPIN_RETRIES: usize = 64;
const QUOTA_RESERVE_MAX_ROUNDS: usize = 8;
#[inline]
fn quota_adaptive_interval_bytes(remaining_before: u64) -> u64 {
@@ -319,6 +320,7 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
let mut reserved_total = None;
let mut reserve_rounds = 0usize;
while reserved_total.is_none() {
let mut saw_contention = false;
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(n_to_charge, limit) {
Ok(total) => {
@@ -331,15 +333,20 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
return Poll::Ready(Err(quota_io_error()));
}
Err(crate::stats::QuotaReserveError::Contended) => {
std::hint::spin_loop();
saw_contention = true;
}
}
}
reserve_rounds = reserve_rounds.saturating_add(1);
if reserved_total.is_none() && reserve_rounds >= 8 {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
if reserved_total.is_none() {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
}
}
}
@@ -407,6 +414,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
remaining_before = Some(remaining);
let desired = remaining.min(buf.len() as u64);
let mut saw_contention = false;
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(desired, limit) {
Ok(_) => {
@@ -418,15 +426,20 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
break;
}
Err(crate::stats::QuotaReserveError::Contended) => {
std::hint::spin_loop();
saw_contention = true;
}
}
}
reserve_rounds = reserve_rounds.saturating_add(1);
if reserved_bytes == 0 && reserve_rounds >= 8 {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
if reserved_bytes == 0 {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
}
}
}
} else {
+3 -2
View File
@@ -1,6 +1,5 @@
#![allow(clippy::too_many_arguments)]
use crc32fast::Hasher;
use crate::crypto::{SecureRandom, sha256_hmac};
use crate::protocol::constants::{
MAX_TLS_CIPHERTEXT_SIZE, TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER,
@@ -8,6 +7,7 @@ use crate::protocol::constants::{
};
use crate::protocol::tls::{TLS_DIGEST_LEN, TLS_DIGEST_POS, gen_fake_x25519_key};
use crate::tls_front::types::{CachedTlsData, ParsedCertificateInfo, TlsProfileSource};
use crc32fast::Hasher;
const MIN_APP_DATA: usize = 64;
const MAX_APP_DATA: usize = MAX_TLS_CIPHERTEXT_SIZE;
@@ -343,7 +343,8 @@ mod tests {
};
use super::{
build_compact_cert_info_payload, build_emulated_server_hello, hash_compact_cert_info_payload,
build_compact_cert_info_payload, build_emulated_server_hello,
hash_compact_cert_info_payload,
};
use crate::crypto::SecureRandom;
use crate::protocol::constants::{
+178 -20
View File
@@ -23,6 +23,60 @@ use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
use super::registry::RouteResult;
use super::{ConnRegistry, MeResponse};
const DATA_ROUTE_MAX_ATTEMPTS: usize = 3;
const DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD: u8 = 3;
fn should_close_on_route_result_for_data(result: RouteResult) -> bool {
matches!(result, RouteResult::NoConn | RouteResult::ChannelClosed)
}
fn should_close_on_route_result_for_ack(result: RouteResult) -> bool {
matches!(result, RouteResult::NoConn | RouteResult::ChannelClosed)
}
fn is_data_route_queue_full(result: RouteResult) -> bool {
matches!(
result,
RouteResult::QueueFullBase | RouteResult::QueueFullHigh
)
}
fn should_close_on_queue_full_streak(streak: u8) -> bool {
streak >= DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD
}
async fn route_data_with_retry(
reg: &ConnRegistry,
conn_id: u64,
flags: u32,
data: Bytes,
timeout_ms: u64,
) -> RouteResult {
let mut attempt = 0usize;
loop {
let routed = reg
.route_with_timeout(
conn_id,
MeResponse::Data {
flags,
data: data.clone(),
},
timeout_ms,
)
.await;
match routed {
RouteResult::QueueFullBase | RouteResult::QueueFullHigh => {
attempt = attempt.saturating_add(1);
if attempt >= DATA_ROUTE_MAX_ATTEMPTS {
return routed;
}
tokio::task::yield_now().await;
}
_ => return routed,
}
}
}
pub(crate) async fn reader_loop(
mut rd: tokio::io::ReadHalf<TcpStream>,
dk: [u8; 32],
@@ -43,6 +97,7 @@ pub(crate) async fn reader_loop(
) -> Result<()> {
let mut raw = enc_leftover;
let mut expected_seq: i32 = 0;
let mut data_route_queue_full_streak = HashMap::<u64, u8>::new();
loop {
let mut tmp = [0u8; 65_536];
@@ -127,27 +182,39 @@ pub(crate) async fn reader_loop(
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
let routed = reg
.route_with_timeout(cid, MeResponse::Data { flags, data }, route_wait_ms)
.await;
if !matches!(routed, RouteResult::Routed) {
match routed {
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
RouteResult::ChannelClosed => {
stats.increment_me_route_drop_channel_closed()
}
RouteResult::QueueFullBase => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_base();
}
RouteResult::QueueFullHigh => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
}
RouteResult::Routed => {}
let routed =
route_data_with_retry(reg.as_ref(), cid, flags, data, route_wait_ms).await;
if matches!(routed, RouteResult::Routed) {
data_route_queue_full_streak.remove(&cid);
continue;
}
match routed {
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
RouteResult::ChannelClosed => stats.increment_me_route_drop_channel_closed(),
RouteResult::QueueFullBase => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_base();
}
RouteResult::QueueFullHigh => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
}
RouteResult::Routed => {}
}
if should_close_on_route_result_for_data(routed) {
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
continue;
}
if is_data_route_queue_full(routed) {
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak(*streak) {
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
}
}
} else if pt == RPC_SIMPLE_ACK_U32 && body.len() >= 12 {
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
@@ -171,19 +238,23 @@ pub(crate) async fn reader_loop(
}
RouteResult::Routed => {}
}
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
if should_close_on_route_result_for_ack(routed) {
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
}
}
} else if pt == RPC_CLOSE_EXT_U32 && body.len() >= 8 {
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
debug!(cid, "RPC_CLOSE_EXT from ME");
let _ = reg.route_nowait(cid, MeResponse::Close).await;
reg.unregister(cid).await;
data_route_queue_full_streak.remove(&cid);
} else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 {
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
debug!(cid, "RPC_CLOSE_CONN from ME");
let _ = reg.route_nowait(cid, MeResponse::Close).await;
reg.unregister(cid).await;
data_route_queue_full_streak.remove(&cid);
} else if pt == RPC_PING_U32 && body.len() >= 8 {
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
trace!(ping_id, "RPC_PING -> RPC_PONG");
@@ -243,6 +314,93 @@ pub(crate) async fn reader_loop(
}
}
#[cfg(test)]
mod tests {
use bytes::Bytes;
use crate::transport::middle_proxy::ConnRegistry;
use super::{
MeResponse, RouteResult, is_data_route_queue_full, route_data_with_retry,
should_close_on_queue_full_streak, should_close_on_route_result_for_ack,
should_close_on_route_result_for_data,
};
#[test]
fn data_route_only_fatal_results_close_immediately() {
assert!(!should_close_on_route_result_for_data(RouteResult::Routed));
assert!(!should_close_on_route_result_for_data(
RouteResult::QueueFullBase
));
assert!(!should_close_on_route_result_for_data(
RouteResult::QueueFullHigh
));
assert!(should_close_on_route_result_for_data(RouteResult::NoConn));
assert!(should_close_on_route_result_for_data(
RouteResult::ChannelClosed
));
}
#[test]
fn data_route_queue_full_uses_starvation_threshold() {
assert!(is_data_route_queue_full(RouteResult::QueueFullBase));
assert!(is_data_route_queue_full(RouteResult::QueueFullHigh));
assert!(!is_data_route_queue_full(RouteResult::NoConn));
assert!(!should_close_on_queue_full_streak(1));
assert!(!should_close_on_queue_full_streak(2));
assert!(should_close_on_queue_full_streak(3));
assert!(should_close_on_queue_full_streak(u8::MAX));
}
#[test]
fn ack_queue_full_is_soft_dropped_without_forced_close() {
assert!(!should_close_on_route_result_for_ack(RouteResult::Routed));
assert!(!should_close_on_route_result_for_ack(
RouteResult::QueueFullBase
));
assert!(!should_close_on_route_result_for_ack(
RouteResult::QueueFullHigh
));
assert!(should_close_on_route_result_for_ack(RouteResult::NoConn));
assert!(should_close_on_route_result_for_ack(
RouteResult::ChannelClosed
));
}
#[tokio::test]
async fn route_data_with_retry_returns_routed_when_channel_has_capacity() {
let reg = ConnRegistry::with_route_channel_capacity(1);
let (conn_id, mut rx) = reg.register().await;
let routed = route_data_with_retry(&reg, conn_id, 0, Bytes::from_static(b"a"), 20).await;
assert!(matches!(routed, RouteResult::Routed));
match rx.recv().await {
Some(MeResponse::Data { flags, data }) => {
assert_eq!(flags, 0);
assert_eq!(data, Bytes::from_static(b"a"));
}
other => panic!("expected routed data response, got {other:?}"),
}
}
#[tokio::test]
async fn route_data_with_retry_stops_after_bounded_attempts() {
let reg = ConnRegistry::with_route_channel_capacity(1);
let (conn_id, _rx) = reg.register().await;
assert!(matches!(
reg.route_nowait(conn_id, MeResponse::Ack(1)).await,
RouteResult::Routed
));
let routed = route_data_with_retry(&reg, conn_id, 0, Bytes::from_static(b"a"), 0).await;
assert!(matches!(
routed,
RouteResult::QueueFullBase | RouteResult::QueueFullHigh
));
}
}
async fn send_close_conn(tx: &mpsc::Sender<WriterCommand>, conn_id: u64) {
let mut p = Vec::with_capacity(12);
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
+13 -9
View File
@@ -356,13 +356,9 @@ impl ConnRegistry {
.entry(writer_id)
.or_insert_with(HashSet::new)
.insert(conn_id);
self.hot_binding.map.insert(
conn_id,
HotConnBinding {
writer_id,
meta,
},
);
self.hot_binding
.map
.insert(conn_id, HotConnBinding { writer_id, meta });
true
}
@@ -427,8 +423,16 @@ impl ConnRegistry {
return None;
}
let writer_id = self.hot_binding.map.get(&conn_id).map(|entry| entry.writer_id)?;
let writer = self.writers.map.get(&writer_id).map(|entry| entry.value().clone())?;
let writer_id = self
.hot_binding
.map
.get(&conn_id)
.map(|entry| entry.writer_id)?;
let writer = self
.writers
.map
.get(&writer_id)
.map(|entry| entry.value().clone())?;
Some(ConnWriter {
writer_id,
tx: writer,
+1
View File
@@ -842,6 +842,7 @@ zabbix_export:
name: 'Prometheus metrics'
type: HTTP_AGENT
key: telemt.prom_metrics
history: '0'
value_type: TEXT
trends: '0'
url: '{$TELEMT_URL}'