Compare commits

..

69 Commits
3.1.2 ... 3.1.4

Author SHA1 Message Date
Alexey
4300720d35 Merge pull request #296 from telemt/bump
Update Cargo.toml
2026-03-02 21:36:12 +03:00
Alexey
b7a8e759eb Update Cargo.toml 2026-03-02 21:36:00 +03:00
Alexey
1a68dc1c2d ME Dual-Trio Pool + ME Pool Shadow Writers: merge pull request #295 from telemt/flow-drift
ME Dual-Trio Pool + ME Pool Shadow Writers
2026-03-02 21:10:55 +03:00
Alexey
a6d22e8a57 ME Pool Shadow Writers
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-02 21:04:06 +03:00
Alexey
9477103f89 Update pool.rs
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-02 20:45:43 +03:00
Alexey
e589891706 ME Dual-Trio Pool Drafts 2026-03-02 20:41:51 +03:00
Alexey
fad4b652c4 Merge pull request #292 from telemt/flow-mep
ME Hardswap Generation stability + Dead-code deletion
2026-03-02 01:23:39 +03:00
Alexey
96bfc223fe Merge pull request #293 from telemt/l7-router
Create XRAY-SINGBOX-ROUTING.ru.md
2026-03-02 01:23:20 +03:00
Alexey
265b9a5f11 Create XRAY-SINGBOX-ROUTING.ru.md 2026-03-02 01:23:09 +03:00
Alexey
74ad9037de Dead-code deletion: has_proxy_tag 2026-03-02 00:54:02 +03:00
Alexey
49f4a7bb22 ME Hardswap Generation stability
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-02 00:39:18 +03:00
Alexey
ac453638b8 Adtag + ME Pool improvements: merge pull request #291 from telemt/flow-adtag
Adtag + ME Pool improvements
2026-03-02 00:22:45 +03:00
Alexey
e7773b2bda Merge branch 'main' into flow-adtag 2026-03-02 00:18:47 +03:00
Alexey
6f1980dfd7 ME Pool improvements
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-02 00:17:58 +03:00
Alexey
427fbef50f Merge pull request #289 from telemt/docs-me-kdf
Docs me kdf
2026-03-01 23:41:09 +03:00
Alexey
08609f4b6d Create MIDDLE-END-KDF.ru.md 2026-03-01 23:40:46 +03:00
Alexey
501d802b8d Create MIDDLE-END-KDF.de.md 2026-03-01 23:39:42 +03:00
Alexey
e8ff39d2ae Merge pull request #288 from telemt/docs-me-kdf
Create MIDDLE-END-KDF.en.md
2026-03-01 23:38:04 +03:00
Alexey
6c1b837d5b Create MIDDLE-END-KDF.en.md 2026-03-01 23:37:49 +03:00
Alexey
b112908c86 Merge pull request #286 from Dimasssss/patch-4
Update QUICK_START_GUIDE.ru.md
2026-03-01 22:32:29 +03:00
Dimasssss
1e400d4cc2 Update QUICK_START_GUIDE.ru.md 2026-03-01 19:05:53 +03:00
Alexey
a11c8b659b Merge pull request #285 from xaosproxy/adtag_per_user
Add per-user ad_tag with global fallback and hot-reload
2026-03-01 16:36:25 +03:00
sintanial
bc432f06e2 Add per-user ad_tag with global fallback and hot-reload
- Per-user ad_tag in [access.user_ad_tags], global fallback in general.ad_tag
- User tag overrides global; if no user tag, general.ad_tag is used
- Both general.ad_tag and user_ad_tags support hot-reload (no restart)
2026-03-01 16:28:55 +03:00
Alexey
338636ede6 Merge pull request #283 from Dimasssss/patch-3
Fix typos and update save instructions in documentation
2026-03-01 15:12:14 +03:00
Dimasssss
c05779208e Update QUICK_START_GUIDE.en.md 2026-03-01 15:05:39 +03:00
Dimasssss
7ba21ec5a8 Update save instructions in QUICK_START_GUIDE.ru.md 2026-03-01 15:05:25 +03:00
Dimasssss
d997c0b216 Fix typos and update save instructions in FAQ.ru.md 2026-03-01 15:03:44 +03:00
Alexey
62cf4f0a1c Merge pull request #278 from Dimasssss/patch-1
Update config.full.toml
2026-03-01 14:48:49 +03:00
Alexey
e710fefed2 Merge pull request #279 from Dimasssss/patch-2
Create FAQ.ru.md
2026-03-01 14:48:36 +03:00
Dimasssss
edef06edb5 Update FAQ.ru.md 2026-03-01 14:45:33 +03:00
Dimasssss
7a0b015e65 Create FAQ.ru.md 2026-03-01 14:04:18 +03:00
Dimasssss
8b2ec35c46 Update config.full.toml 2026-03-01 13:38:50 +03:00
Alexey
d324d84ec7 Merge pull request #276 from telemt/flow-mep
UpstreamManager Health-check for ME Pool over SOCKS
2026-03-01 04:02:59 +03:00
Alexey
47b12f9489 UpstreamManager Health-check for ME Pool over SOCKS
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-01 04:02:32 +03:00
Alexey
a5967d0ca3 Merge pull request #275 from telemt/flow-mep
ME Pool improvements
2026-03-01 03:38:53 +03:00
Alexey
44cdfd4b23 ME Pool improvements
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-03-01 03:36:00 +03:00
Alexey
25ffcf6081 Merge pull request #273 from ivulit/fix/proxy-protocol-unix-sock
fix: send PROXY protocol header to mask unix socket
2026-03-01 03:19:52 +03:00
Alexey
60322807b6 Merge pull request #271 from An0nX/patch-1
Rewrite configuration as a self-contained deployment guide with hardened anti-censorship defaults
2026-03-01 03:14:13 +03:00
ivulit
ed93b0a030 fix: send PROXY protocol header to mask unix socket
When mask_unix_sock is configured, mask_proxy_protocol was silently
ignored and no PROXY protocol header was sent to the backend. Apply
the same header-building logic as the TCP path in both masking relay
and TLS fetcher (raw and rustls).
2026-03-01 00:14:55 +03:00
Alexey
2370c8d5e4 Merge pull request #268 from radjah/patch-1
Update install.sh
2026-02-28 23:56:20 +03:00
Alexey
a3197b0fe1 Merge pull request #270 from ivulit/fix/proxy-protocol-dst-addr
fix: pass correct dst address to outgoing PROXY protocol header
2026-02-28 23:56:04 +03:00
ivulit
e27ef04c3d fix: pass correct dst address to outgoing PROXY protocol header
Previously handle_bad_client used stream.local_addr() (the ephemeral
socket to the mask backend) as the dst in the outgoing PROXY protocol
header. This is wrong: the dst should be the address telemt is listening
on, or the dst from the incoming PROXY protocol header if one was present.

- handle_bad_client now receives local_addr from the caller
- handle_client_stream resolves local_addr from PROXY protocol info.dst_addr
  or falls back to a synthetic address based on config.server.port
- RunningClientHandler.do_handshake resolves local_addr from stream.local_addr()
  overridden by PROXY protocol info.dst_addr when present, and passes it
  down to handle_tls_client / handle_direct_client
- masking.rs uses the caller-supplied local_addr directly, eliminating the
  stream.local_addr() call
2026-02-28 22:47:24 +03:00
An0nX
cf7e2ebf4b refactor: rewrite telemt config as self-documenting deployment reference
- Reorganize all sections with clear visual block separators
- Move inline comments to dedicated lines above each parameter
- Add Quick Start guide in the file header explaining 7-step deployment
- Add Modes of Operation explanation (Direct vs Middle-Proxy)
- Group related parameters under labeled subsections with separators
- Expand every comment to full plain-English explanation
- Remove all inline comments to prevent TOML parser edge cases
- Tune anti-censorship defaults for maximum DPI resistance:
  fast_mode_min_tls_record=1400, server_hello_delay=50-150ms,
  tls_new_session_tickets=2, tls_full_cert_ttl_secs=0,
  tls_emulation=true, desync_all_full=true, beobachten_minutes=30,
  me_reinit_every_secs=600
2026-02-28 21:36:56 +03:00
Pavel Frolov
685bfafe74 Update install.sh
Попытался привести к единообразию текст.
2026-02-28 19:02:00 +03:00
Alexey
0f6fcf49a7 Merge pull request #267 from Dimasssss/main
QUICK_START_GUIDE.en.md
2026-02-28 17:47:30 +03:00
Dimasssss
036f0e1569 Add files via upload 2026-02-28 17:46:11 +03:00
Dimasssss
291c22583f Update QUICK_START_GUIDE.ru.md 2026-02-28 17:39:12 +03:00
Alexey
ee5b01bb31 Merge pull request #266 from Dimasssss/main
Create QUICK_START_GUIDE.ru.md
2026-02-28 17:21:29 +03:00
Dimasssss
ccacf78890 Create QUICK_START_GUIDE.ru.md 2026-02-28 17:17:50 +03:00
Alexey
42db1191a8 Merge pull request #265 from Dimasssss/main
install.sh
2026-02-28 17:08:15 +03:00
Dimasssss
9ce26d16cb Add files via upload 2026-02-28 17:04:06 +03:00
Alexey
12e68f805f Update Cargo.toml 2026-02-28 15:51:15 +03:00
Alexey
62bf31fc73 Merge pull request #264 from telemt/flow-net
DNS-Overrides + STUN fixes + Bind_addr prio + Fetch for unix-socket + ME/DC Method Detection + Metrics impovements
2026-02-28 14:59:44 +03:00
Alexey
29d4636249 Merge branch 'main' into flow-net 2026-02-28 14:55:04 +03:00
Alexey
9afaa28add UpstreamManager: Backoff Retries 2026-02-28 14:21:09 +03:00
Alexey
6c12af2b94 ME Connectivity: socks-url
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-02-28 13:38:30 +03:00
Alexey
8b39a4ef6d Statistics on ME + Dynamic backpressure + KDF with SOCKS
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-02-28 13:18:31 +03:00
Alexey
fa2423dadf ME/DC Method Detection fixes
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-02-28 03:21:22 +03:00
Alexey
449a87d2e3 Merge branch 'flow-net' of https://github.com/telemt/telemt into flow-net 2026-02-28 02:55:23 +03:00
Alexey
a61882af6e TLS Fetch on unix-socket 2026-02-28 02:55:21 +03:00
Alexey
bf11ebbaa3 Update TUNING.ru.md 2026-02-28 02:23:34 +03:00
Alexey
e0d5561095 TUNING.md 2026-02-28 02:19:19 +03:00
Alexey
6b8aa7270e Bind_addresses prio over interfaces 2026-02-28 01:54:29 +03:00
Alexey
372f477927 Merge pull request #263 from Dimasssss/main
Update README.md
2026-02-28 01:27:42 +03:00
Dimasssss
05edbab06c Update README.md
Нашелся тот, кто не смог найти ссылку.
2026-02-28 01:20:49 +03:00
Alexey
3d9660f83e Upstreams for ME + Egress-data from UM + ME-over-SOCKS + Bind-aware STUN 2026-02-28 01:20:17 +03:00
Alexey
ac064fe773 STUN switch + Ad-tag fixes + DNS-overrides 2026-02-27 15:59:27 +03:00
Alexey
eba158ff8b Merge pull request #261 from nimbo78/nimbo78-patch-docker-compose-yml
Update docker-compose.yml
2026-02-27 02:46:12 +03:00
nimbo78
54ee6ff810 Update docker-compose.yml
docker pull image first, if fail - build
2026-02-27 01:53:22 +03:00
53 changed files with 7352 additions and 746 deletions

2
Cargo.lock generated
View File

@@ -2087,7 +2087,7 @@ dependencies = [
[[package]]
name = "telemt"
version = "3.0.13"
version = "3.1.3"
dependencies = [
"aes",
"anyhow",

View File

@@ -1,6 +1,6 @@
[package]
name = "telemt"
version = "3.1.2"
version = "3.1.4"
edition = "2024"
[dependencies]

View File

@@ -191,6 +191,8 @@ then Ctrl+X -> Y -> Enter to save
**5.** In Shell type `systemctl enable telemt` - then telemt will start with system startup, after the network is up
**6.** In Shell type `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` - get the connection links
## Configuration
### Minimal Configuration for First Start
```toml
@@ -213,10 +215,12 @@ hello = "00000000000000000000000000000000"
```
### Advanced
#### Adtag
To use channel advertising and usage statistics from Telegram, get Adtag from [@mtproxybot](https://t.me/mtproxybot), add this parameter to section `[General]`
#### Adtag (per-user)
To use channel advertising and usage statistics from Telegram, get an Adtag from [@mtproxybot](https://t.me/mtproxybot). Set it per user in `[access.user_ad_tags]` (32 hex chars):
```toml
ad_tag = "00000000000000000000000000000000" # Replace zeros to your adtag from @mtproxybot
[access.user_ad_tags]
username1 = "11111111111111111111111111111111" # Replace with your tag from @mtproxybot
username2 = "22222222222222222222222222222222"
```
#### Listening and Announce IPs
To specify listening address and/or address in links, add to section `[[server.listeners]]` of config.toml:

View File

@@ -1,176 +1,669 @@
# Telemt full config with default values.
# Examples are kept in comments after '#'.
# ==============================================================================
#
# TELEMT — Advanced Rust-based Telegram MTProto Proxy
# Full Configuration Reference
#
# This file is both a working config and a complete documentation.
# Every parameter is explained. Read it top to bottom before deploying.
#
# Quick Start:
# 1. Set [server].port to your desired port (443 recommended)
# 2. Generate a secret: openssl rand -hex 16
# 3. Put it in [access.users] under a name you choose
# 4. Set [censorship].tls_domain to a popular unblocked HTTPS site
# 5. Set your public IP in [general].middle_proxy_nat_ip
# and [general.links].public_host
# 6. Set announce IP in [[server.listeners]]
# 7. Run Telemt. It prints a tg:// link. Send it to your users.
#
# Modes of Operation:
# Direct Mode (use_middle_proxy = false)
# Connects straight to Telegram DCs via TCP. Simple, fast, low overhead.
# No ad_tag support. No CDN DC support (203, etc).
#
# Middle-Proxy Mode (use_middle_proxy = true)
# Connects to Telegram Middle-End servers via RPC protocol.
# Required for ad_tag monetization and CDN support.
# Requires proxy_secret_path and a valid public IP.
#
# ==============================================================================
# Top-level legacy field.
show_link = [] # example: "*" or ["alice", "bob"]
# default_dc = 2 # example: default DC for unmapped non-standard DCs
# ==============================================================================
# LEGACY TOP-LEVEL FIELDS
# ==============================================================================
# Deprecated. Use [general.links].show instead.
# Accepts "*" for all users, or an array like ["alice", "bob"].
show_link = ["0"]
# Fallback Datacenter index (1-5) when a client requests an unknown DC ID.
# DC 2 is Amsterdam (Europe), closest for most CIS users.
# default_dc = 2
# ==============================================================================
# GENERAL SETTINGS
# ==============================================================================
[general]
# ------------------------------------------------------------------------------
# Core Protocol
# ------------------------------------------------------------------------------
# Coalesce the MTProto handshake and first data payload into a single TCP packet.
# Significantly reduces connection latency. No reason to disable.
fast_mode = true
use_middle_proxy = false
# ad_tag = "00000000000000000000000000000000" # example
# proxy_secret_path = "proxy-secret" # example custom path
# middle_proxy_nat_ip = "203.0.113.10" # example public NAT IP override
# How the proxy connects to Telegram servers.
# false = Direct TCP to Telegram DCs (simple, low overhead)
# true = Middle-End RPC protocol (required for ad_tag and CDN DCs)
use_middle_proxy = true
# 32-char hex Ad-Tag from @MTProxybot for sponsored channel injection.
# Only works when use_middle_proxy = true.
# Obtain yours: message @MTProxybot on Telegram, register your proxy.
# ad_tag = "00000000000000000000000000000000"
# ------------------------------------------------------------------------------
# Middle-End Authentication
# ------------------------------------------------------------------------------
# Path to the Telegram infrastructure AES key file.
# Auto-downloaded from https://core.telegram.org/getProxySecret on first run.
# This key authenticates your proxy with Middle-End servers.
proxy_secret_path = "proxy-secret"
# ------------------------------------------------------------------------------
# Public IP Configuration (Critical for Middle-Proxy Mode)
# ------------------------------------------------------------------------------
# Your server's PUBLIC IPv4 address.
# Middle-End servers need this for the cryptographic Key Derivation Function.
# If your server has a direct public IP, set it here.
# If behind NAT (AWS, Docker, etc.), this MUST be your external IP.
# If omitted, Telemt uses STUN to auto-detect (see middle_proxy_nat_probe).
# middle_proxy_nat_ip = "203.0.113.10"
# Auto-detect public IP via STUN servers defined in [network].
# Set to false if you hardcoded middle_proxy_nat_ip above.
# Set to true if you want automatic detection.
middle_proxy_nat_probe = true
# middle_proxy_nat_stun = "stun.l.google.com:19302" # example
# middle_proxy_nat_stun_servers = [] # example: ["stun1.l.google.com:19302", "stun2.l.google.com:19302"]
# ------------------------------------------------------------------------------
# Middle-End Connection Pool
# ------------------------------------------------------------------------------
# Number of persistent multiplexed RPC connections to ME servers.
# All client traffic is routed through these "fat pipes".
# 8 handles thousands of concurrent users comfortably.
middle_proxy_pool_size = 8
# Legacy field. Connections kept initialized but idle as warm standby.
middle_proxy_warm_standby = 16
# ------------------------------------------------------------------------------
# Middle-End Keepalive
# Telegram ME servers aggressively kill idle TCP connections.
# These settings send periodic RPC_PING frames to keep pipes alive.
# ------------------------------------------------------------------------------
me_keepalive_enabled = true
# Base interval between pings in seconds.
me_keepalive_interval_secs = 25
# Random jitter added to interval to prevent all connections pinging simultaneously.
me_keepalive_jitter_secs = 5
# Randomize ping payload bytes to prevent DPI from fingerprinting ping patterns.
me_keepalive_payload_random = true
# ------------------------------------------------------------------------------
# Client-Side Limits
# ------------------------------------------------------------------------------
# Max buffered ciphertext per client (bytes) when upstream is slow.
# Acts as backpressure to prevent memory exhaustion. 256KB is safe.
crypto_pending_buffer = 262144
# Maximum single MTProto frame size from client. 16MB is protocol standard.
max_client_frame = 16777216
desync_all_full = false
# ------------------------------------------------------------------------------
# Crypto Desynchronization Logging
# Desync errors usually mean DPI/GFW is tampering with connections.
# ------------------------------------------------------------------------------
# true = full forensics (trace ID, IP hash, hex dumps) for EVERY desync event
# false = deduplicated logging, one entry per time window (prevents log spam)
# Set true if you are actively debugging DPI interference.
desync_all_full = true
# ------------------------------------------------------------------------------
# Beobachten — Built-in Honeypot / Active Probe Tracker
# Tracks IPs that fail handshakes or behave like TLS scanners.
# Output file can be fed into fail2ban or iptables for auto-blocking.
# ------------------------------------------------------------------------------
beobachten = true
beobachten_minutes = 10
# How long (minutes) to remember a suspicious IP before expiring it.
beobachten_minutes = 30
# How often (seconds) to flush tracker state to disk.
beobachten_flush_secs = 15
# File path for the tracker output.
beobachten_file = "cache/beobachten.txt"
# ------------------------------------------------------------------------------
# Hardswap — Zero-Downtime ME Pool Rotation
# When Telegram updates ME server IPs, Hardswap creates a completely new pool,
# waits until it is fully ready, migrates traffic, then kills the old pool.
# Users experience zero interruption.
# ------------------------------------------------------------------------------
hardswap = true
# ------------------------------------------------------------------------------
# ME Pool Warmup Staggering
# When creating a new pool, connections are opened one by one with delays
# to avoid a burst of SYN packets that could trigger ISP flood protection.
# ------------------------------------------------------------------------------
me_warmup_stagger_enabled = true
# Delay between each connection creation (milliseconds).
me_warmup_step_delay_ms = 500
# Random jitter added to the delay (milliseconds).
me_warmup_step_jitter_ms = 300
# ------------------------------------------------------------------------------
# ME Reconnect Backoff
# If an ME server drops the connection, Telemt retries with this strategy.
# ------------------------------------------------------------------------------
# Max simultaneous reconnect attempts per DC.
me_reconnect_max_concurrent_per_dc = 8
# Exponential backoff base (milliseconds).
me_reconnect_backoff_base_ms = 500
# Backoff ceiling (milliseconds). Will never wait longer than this.
me_reconnect_backoff_cap_ms = 30000
# Number of instant retries before switching to exponential backoff.
me_reconnect_fast_retry_count = 12
# ------------------------------------------------------------------------------
# NAT Mismatch Behavior
# If STUN-detected IP differs from local interface IP (you are behind NAT).
# false = abort ME mode (safe default)
# true = force ME mode anyway (use if you know your NAT setup is correct)
# ------------------------------------------------------------------------------
stun_iface_mismatch_ignore = false
unknown_dc_log_path = "unknown-dc.txt" # to disable: set to null
log_level = "normal" # debug | verbose | normal | silent
# ------------------------------------------------------------------------------
# Logging
# ------------------------------------------------------------------------------
# File to log unknown DC requests (DC IDs outside standard 1-5).
unknown_dc_log_path = "unknown-dc.txt"
# Verbosity: "debug" | "verbose" | "normal" | "silent"
log_level = "normal"
# Disable ANSI color codes in log output (useful for file logging).
disable_colors = false
fast_mode_min_tls_record = 0
# ------------------------------------------------------------------------------
# FakeTLS Record Sizing
# Buffer small MTProto packets into larger TLS records to mimic real HTTPS.
# Real HTTPS servers send records close to MTU size (~1400 bytes).
# A stream of tiny TLS records is a strong DPI signal.
# Set to 0 to disable. Set to 1400 for realistic HTTPS emulation.
# ------------------------------------------------------------------------------
fast_mode_min_tls_record = 1400
# ------------------------------------------------------------------------------
# Periodic Updates
# ------------------------------------------------------------------------------
# How often (seconds) to re-fetch ME server lists and proxy secrets
# from core.telegram.org. Keeps your proxy in sync with Telegram infrastructure.
update_every = 300
me_reinit_every_secs = 900
# How often (seconds) to force a Hardswap even if the ME map is unchanged.
# Shorter intervals mean shorter-lived TCP flows, harder for DPI to profile.
me_reinit_every_secs = 600
# ------------------------------------------------------------------------------
# Hardswap Warmup Tuning
# Fine-grained control over how the new pool is warmed up before traffic switch.
# ------------------------------------------------------------------------------
me_hardswap_warmup_delay_min_ms = 1000
me_hardswap_warmup_delay_max_ms = 2000
me_hardswap_warmup_extra_passes = 3
me_hardswap_warmup_pass_backoff_base_ms = 500
# ------------------------------------------------------------------------------
# Config Update Debouncing
# Telegram sometimes pushes transient/broken configs. Debouncing requires
# N consecutive identical fetches before applying a change.
# ------------------------------------------------------------------------------
# ME server list must be identical for this many fetches before applying.
me_config_stable_snapshots = 2
# Minimum seconds between config applications.
me_config_apply_cooldown_secs = 300
# Proxy secret must be identical for this many fetches before applying.
proxy_secret_stable_snapshots = 2
# ------------------------------------------------------------------------------
# Proxy Secret Rotation
# ------------------------------------------------------------------------------
# Apply newly downloaded secrets at runtime without restart.
proxy_secret_rotate_runtime = true
# Maximum acceptable secret length (bytes). Rejects abnormally large secrets.
proxy_secret_len_max = 256
# ------------------------------------------------------------------------------
# Hardswap Drain Settings
# Controls graceful shutdown of old ME connections during pool rotation.
# ------------------------------------------------------------------------------
# Seconds to keep old connections alive for in-flight data before force-closing.
me_pool_drain_ttl_secs = 90
# Minimum ratio of healthy connections in new pool before draining old pool.
# 0.8 = at least 80% of new pool must be ready.
me_pool_min_fresh_ratio = 0.8
# Maximum seconds to wait for drain to complete before force-killing.
me_reinit_drain_timeout_secs = 120
# Legacy compatibility fields used when update_every is omitted.
proxy_secret_auto_reload_secs = 3600
proxy_config_auto_reload_secs = 3600
# ------------------------------------------------------------------------------
# NTP Clock Check
# MTProto uses timestamps. Clock drift > 30 seconds breaks handshakes.
# Telemt checks on startup and warns if out of sync.
# ------------------------------------------------------------------------------
ntp_check = true
ntp_servers = ["pool.ntp.org"] # example: ["pool.ntp.org", "time.cloudflare.com"]
ntp_servers = ["pool.ntp.org"]
# ------------------------------------------------------------------------------
# Auto-Degradation
# If ME servers become completely unreachable (ISP blocking),
# automatically fall back to Direct Mode so users stay connected.
# ------------------------------------------------------------------------------
auto_degradation_enabled = true
# Number of DC groups that must be unreachable before triggering fallback.
degradation_min_unavailable_dc_groups = 2
# ==============================================================================
# ALLOWED CLIENT PROTOCOLS
# Only enable what you need. In censored regions, TLS-only is safest.
# ==============================================================================
[general.modes]
# Classic MTProto. Unobfuscated length prefixes. Trivially detected by DPI.
# No reason to enable unless you have ancient clients.
classic = false
# Obfuscated MTProto with randomized padding. Better than classic, but
# still detectable by statistical analysis of packet sizes.
secure = false
# FakeTLS (ee-secrets). Wraps MTProto in TLS 1.3 framing.
# To DPI, it looks like a normal HTTPS connection.
# This should be the ONLY enabled mode in censored environments.
tls = true
# ==============================================================================
# STARTUP LINK GENERATION
# Controls what tg:// invite links are printed to console on startup.
# ==============================================================================
[general.links]
show ="*" # example: "*" or ["alice", "bob"]
# public_host = "proxy.example.com" # example explicit host/IP for tg:// links
# public_port = 443 # example explicit port for tg:// links
# Which users to generate links for.
# "*" = all users, or an array like ["alice", "bob"].
show = "*"
# IP or domain to embed in the tg:// link.
# If omitted, Telemt uses STUN to auto-detect.
# Set this to your server's public IP or domain for reliable links.
# public_host = "proxy.example.com"
# Port to embed in the tg:// link.
# If omitted, uses [server].port.
# public_port = 443
# ==============================================================================
# NETWORK & IP RESOLUTION
# ==============================================================================
[network]
# Enable IPv4 for outbound connections to Telegram.
ipv4 = true
ipv6 = false # set true to enable IPv6
prefer = 4 # 4 or 6
# Enable IPv6 for outbound connections to Telegram.
ipv6 = false
# Prefer IPv4 (4) or IPv6 (6) when both are available.
prefer = 4
# Experimental: use both IPv4 and IPv6 ME servers simultaneously.
# May improve reliability but doubles connection count.
multipath = false
# STUN servers for external IP discovery.
# Used for Middle-Proxy KDF (if nat_probe=true) and link generation.
stun_servers = [
"stun.l.google.com:5349",
"stun1.l.google.com:3478",
"stun.gmx.net:3478",
"stun.l.google.com:19302",
"stun.1und1.de:3478",
"stun1.l.google.com:19302",
"stun2.l.google.com:19302",
"stun3.l.google.com:19302",
"stun4.l.google.com:19302",
"stun.services.mozilla.com:3478",
"stun.stunprotocol.org:3478",
"stun.nextcloud.com:3478",
"stun.voip.eutelia.it:3478",
"stun.l.google.com:5349",
"stun1.l.google.com:3478",
"stun.gmx.net:3478",
"stun.l.google.com:19302"
]
# If UDP STUN is blocked, attempt TCP-based STUN as fallback.
stun_tcp_fallback = true
http_ip_detect_urls = ["https://ifconfig.me/ip", "https://api.ipify.org"]
# If all STUN fails, use HTTP APIs to discover public IP.
http_ip_detect_urls = [
"https://ifconfig.me/ip",
"https://api.ipify.org"
]
# Cache discovered public IP to this file to survive restarts.
cache_public_ip_path = "cache/public_ip.txt"
# ==============================================================================
# SERVER BINDING & METRICS
# ==============================================================================
[server]
# TCP port to listen on.
# 443 is recommended (looks like normal HTTPS traffic).
port = 443
# IPv4 bind address. "0.0.0.0" = all interfaces.
listen_addr_ipv4 = "0.0.0.0"
# IPv6 bind address. "::" = all interfaces.
listen_addr_ipv6 = "::"
# listen_unix_sock = "/var/run/telemt.sock" # example
# listen_unix_sock_perm = "0660" # example unix socket mode
# listen_tcp = true # example explicit override (auto-detected when omitted)
# Unix socket listener (for reverse proxy setups with Nginx/HAProxy).
# listen_unix_sock = "/var/run/telemt.sock"
# listen_unix_sock_perm = "0660"
# Enable PROXY protocol header parsing.
# Set true ONLY if Telemt is behind HAProxy/Nginx that injects PROXY headers.
# If enabled without a proxy in front, clients will fail to connect.
proxy_protocol = false
# metrics_port = 9090 # example
metrics_whitelist = ["127.0.0.1/32", "::1/128"]
# Example explicit listeners (default: omitted, auto-generated from listen_addr_*):
# Prometheus metrics HTTP endpoint port.
# Uncomment to enable. Access at http://your-server:9090/metrics
# metrics_port = 9090
# IP ranges allowed to access the metrics endpoint.
metrics_whitelist = [
"127.0.0.1/32",
"::1/128"
]
# ------------------------------------------------------------------------------
# Listener Overrides
# Define explicit listeners with specific bind IPs and announce IPs.
# The announce IP is what gets embedded in tg:// links and sent to ME servers.
# You MUST set announce to your server's public IP for ME mode to work.
# ------------------------------------------------------------------------------
# [[server.listeners]]
# ip = "0.0.0.0"
# announce = "proxy-v4.example.com"
# # announce_ip = "203.0.113.10" # deprecated alias
# proxy_protocol = false
# reuse_allow = false
#
# [[server.listeners]]
# ip = "::"
# announce = "proxy-v6.example.com"
# proxy_protocol = false
# announce = "203.0.113.10"
# reuse_allow = false
# ==============================================================================
# TIMEOUTS (seconds unless noted)
# ==============================================================================
[timeouts]
# Maximum time for client to complete FakeTLS + MTProto handshake.
client_handshake = 15
# Maximum time to establish TCP connection to upstream Telegram DC.
tg_connect = 10
# TCP keepalive interval for client connections.
client_keepalive = 60
# Maximum client inactivity before dropping the connection.
client_ack = 300
# Instant retry count for a single ME endpoint before giving up on it.
me_one_retry = 3
# Timeout (milliseconds) for a single ME endpoint connection attempt.
me_one_timeout_ms = 1500
# ==============================================================================
# ANTI-CENSORSHIP / FAKETLS / MASKING
# This is where Telemt becomes invisible to Deep Packet Inspection.
# ==============================================================================
[censorship]
tls_domain = "petrovich.ru"
# tls_domains = ["example.com", "cdn.example.net"] # Additional domains for EE links
# ------------------------------------------------------------------------------
# TLS Domain Fronting
# The SNI (Server Name Indication) your proxy presents to connecting clients.
# Must be a popular, unblocked HTTPS website in your target country.
# DPI sees traffic to this domain. Choose carefully.
# Good choices: major CDNs, banks, government sites, search engines.
# Bad choices: obscure sites, already-blocked domains.
# ------------------------------------------------------------------------------
tls_domain = "www.google.com"
# ------------------------------------------------------------------------------
# Active Probe Masking
# When someone connects but fails the MTProto handshake (wrong secret),
# they might be an ISP active prober testing if this is a proxy.
#
# mask = false: drop the connection (prober knows something is here)
# mask = true: transparently proxy them to mask_host (prober sees a real website)
#
# With mask enabled, your server is indistinguishable from a real web server
# to anyone who doesn't have the correct secret.
# ------------------------------------------------------------------------------
mask = true
# mask_host = "www.google.com" # example, defaults to tls_domain when both mask_host/mask_unix_sock are unset
# mask_unix_sock = "/var/run/nginx.sock" # example, mutually exclusive with mask_host
# The real web server to forward failed handshakes to.
# If omitted, defaults to tls_domain.
# mask_host = "www.google.com"
# Port on the mask host to connect to.
mask_port = 443
# mask_proxy_protocol = 0 # Send PROXY protocol header to mask_host: 0 = off, 1 = v1 (text), 2 = v2 (binary)
fake_cert_len = 2048 # if tls_emulation=false and default value is used, loader may randomize this value at runtime
# Inject PROXY protocol header when forwarding to mask host.
# 0 = disabled, 1 = v1, 2 = v2. Leave disabled unless mask_host expects it.
# mask_proxy_protocol = 0
# ------------------------------------------------------------------------------
# TLS Certificate Emulation
# ------------------------------------------------------------------------------
# Size (bytes) of the locally generated fake TLS certificate.
# Only used when tls_emulation is disabled.
fake_cert_len = 2048
# KILLER FEATURE: Real-Time TLS Emulation.
# Telemt connects to tls_domain, fetches its actual TLS 1.3 certificate chain,
# and exactly replicates the byte sizes of ServerHello and Certificate records.
# Defeats DPI that uses TLS record length heuristics to detect proxies.
# Strongly recommended in censored environments.
tls_emulation = true
# Directory to cache fetched TLS certificates.
tls_front_dir = "tlsfront"
server_hello_delay_min_ms = 0
server_hello_delay_max_ms = 0
tls_new_session_tickets = 0
tls_full_cert_ttl_secs = 90
# ------------------------------------------------------------------------------
# ServerHello Timing
# Real web servers take 30-150ms to respond to ClientHello due to network
# latency and crypto processing. A proxy responding in <1ms is suspicious.
# These settings add realistic delay to mimic genuine server behavior.
# ------------------------------------------------------------------------------
# Minimum delay before sending ServerHello (milliseconds).
server_hello_delay_min_ms = 50
# Maximum delay before sending ServerHello (milliseconds).
server_hello_delay_max_ms = 150
# ------------------------------------------------------------------------------
# TLS Session Tickets
# Real TLS 1.3 servers send 1-2 NewSessionTicket messages after handshake.
# A server that sends zero tickets is anomalous and may trigger DPI flags.
# Set this to match your tls_domain's behavior (usually 2).
# ------------------------------------------------------------------------------
# tls_new_session_tickets = 0
# ------------------------------------------------------------------------------
# Full Certificate Frequency
# When tls_emulation is enabled, this controls how often (per client IP)
# to send the complete emulated certificate chain.
#
# > 0: Subsequent connections within TTL seconds get a smaller cached version.
# Saves bandwidth but creates a detectable size difference between
# first and repeat connections.
#
# = 0: Every connection gets the full certificate. More bandwidth but
# perfectly consistent behavior, no anomalies for DPI to detect.
# ------------------------------------------------------------------------------
tls_full_cert_ttl_secs = 0
# ------------------------------------------------------------------------------
# ALPN Enforcement
# Ensure ServerHello responds with the exact ALPN protocol the client requested.
# Mismatched ALPN (e.g., client asks h2, server says http/1.1) is a DPI red flag.
# ------------------------------------------------------------------------------
alpn_enforce = true
# ==============================================================================
# ACCESS CONTROL & USERS
# ==============================================================================
[access]
# ------------------------------------------------------------------------------
# Replay Attack Protection
# DPI can record a legitimate user's handshake and replay it later to probe
# whether the server is a proxy. Telemt remembers recent handshake nonces
# and rejects duplicates.
# ------------------------------------------------------------------------------
# Number of nonce slots in the replay detection buffer.
replay_check_len = 65536
# How long (seconds) to remember nonces before expiring them.
replay_window_secs = 1800
# Allow clients with incorrect system clocks to connect.
# false = reject clients with significant time skew (more secure)
# true = accept anyone regardless of clock (more permissive)
ignore_time_skew = false
# ------------------------------------------------------------------------------
# User Secrets
# Each user needs a unique 32-character hex string as their secret.
# Generate with: openssl rand -hex 16
#
# This secret is embedded in the tg:// link. Anyone with it can connect.
# Format: username = "hex_secret"
# ------------------------------------------------------------------------------
[access.users]
# format: "username" = "32_hex_chars_secret"
hello = "00000000000000000000000000000000"
# alice = "11111111111111111111111111111111" # example
# alice = "0123456789abcdef0123456789abcdef"
# bob = "fedcba9876543210fedcba9876543210"
# ------------------------------------------------------------------------------
# Per-User Connection Limits
# Limits concurrent TCP connections per user to prevent secret sharing.
# Uncomment and set for each user as needed.
# ------------------------------------------------------------------------------
[access.user_max_tcp_conns]
# alice = 100 # example
# alice = 100
# bob = 50
# ------------------------------------------------------------------------------
# Per-User Expiration Dates
# Automatically revoke access after the specified date (ISO 8601 format).
# ------------------------------------------------------------------------------
[access.user_expirations]
# alice = "2078-01-01T00:00:00Z" # example
# alice = "2025-12-31T23:59:59Z"
# bob = "2026-06-15T00:00:00Z"
# ------------------------------------------------------------------------------
# Per-User Data Quotas
# Maximum total bytes transferred per user. Connection refused after limit.
# ------------------------------------------------------------------------------
[access.user_data_quota]
# hello = 10737418240 # example bytes
# alice = 10737418240 # example bytes
# alice = 107374182400
# bob = 53687091200
# ------------------------------------------------------------------------------
# Per-User Unique IP Limits
# Maximum number of different IP addresses that can use this secret
# at the same time. Highly effective against secret leaking/sharing.
# Set to 1 for single-device, 2-3 for phone+desktop, etc.
# ------------------------------------------------------------------------------
[access.user_max_unique_ips]
# hello = 10 # example
# alice = 100 # example
# alice = 3
# bob = 2
# ==============================================================================
# UPSTREAM ROUTING
# Controls how Telemt connects to Telegram servers (or ME servers).
# If omitted entirely, uses the OS default route.
# ==============================================================================
# ------------------------------------------------------------------------------
# Direct upstream: use the server's own network interface.
# You can optionally bind to a specific interface or local IP.
# ------------------------------------------------------------------------------
# Default behavior if [[upstreams]] is omitted: loader injects one direct upstream.
# Example explicit upstreams:
# [[upstreams]]
# type = "direct"
# interface = "eth0"
@@ -178,28 +671,27 @@ hello = "00000000000000000000000000000000"
# weight = 1
# enabled = true
# scopes = "*"
#
# [[upstreams]]
# type = "socks4"
# address = "198.51.100.20:1080"
# interface = "eth0"
# user_id = "telemt"
# weight = 1
# enabled = true
# scopes = "*"
#
# ------------------------------------------------------------------------------
# SOCKS5 upstream: route Telegram traffic through a SOCKS5 proxy.
# Useful if your server's IP is blocked from reaching Telegram DCs.
# ------------------------------------------------------------------------------
# [[upstreams]]
# type = "socks5"
# address = "198.51.100.30:1080"
# interface = "eth0"
# username = "proxy-user"
# password = "proxy-pass"
# weight = 1
# enabled = true
# scopes = "*"
# === DC Address Overrides ===
# ==============================================================================
# DATACENTER OVERRIDES
# Force specific DC IDs to route to specific IP:Port combinations.
# DC 203 (CDN) is auto-injected by Telemt if not specified here.
# ==============================================================================
# [dc_overrides]
# "201" = "149.154.175.50:443" # example
# "202" = ["149.154.167.51:443", "149.154.175.100:443"] # example
# "203" = "91.105.192.100:443" # loader auto-adds this one when omitted
# "201" = "149.154.175.50:443"
# "202" = ["149.154.167.51:443", "149.154.175.100:443"]

View File

@@ -5,7 +5,9 @@
# === General Settings ===
[general]
use_middle_proxy = false
# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags]
# ad_tag = "00000000000000000000000000000000"
# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot)
# === Log Level ===
# Log level: debug | verbose | normal | silent

View File

@@ -1,5 +1,6 @@
services:
telemt:
image: ghcr.io/telemt/telemt:latest
build: .
container_name: telemt
restart: unless-stopped

64
docs/FAQ.ru.md Normal file
View File

@@ -0,0 +1,64 @@
## Как настроить канал "спонсор прокси"
1. Зайти в бота @MTProxybot.
2. Ввести команду `/newproxy`
3. Отправить IP и порт сервера. Например: 1.2.3.4:443
4. Открыть конфиг `nano /etc/telemt.toml`.
5. Скопировать и отправить боту секрет пользователя из раздела [access.users].
6. Скопировать полученный tag у бота. Например 1234567890abcdef1234567890abcdef.
7. Раскомментировать параметр ad_tag и вписать tag, полученный у бота.
8. Раскомментировать/добавить параметр use_middle_proxy = true.
Пример конфига:
```toml
[general]
ad_tag = "1234567890abcdef1234567890abcdef"
use_middle_proxy = true
```
9. Сохранить конфиг. Ctrl+S -> Ctrl+X.
10. Перезапустить telemt `systemctl restart telemt`.
11. В боте отправить команду /myproxies и выбрать добавленный сервер.
12. Нажать кнопку "Set promotion".
13. Отправить **публичную ссылку** на канал. Приватный канал добавить нельзя!
14. Подождать примерно 1 час, пока информация обновится на серверах Telegram.
> [!WARNING]
> У вас не будет отображаться "спонсор прокси" если вы уже подписаны на канал.
## Сколько человек может пользоваться 1 ссылкой
По умолчанию 1 ссылкой может пользоваться сколько угодно человек.
Вы можете ограничить число IP, использующих прокси.
```toml
[access.user_max_unique_ips]
hello = 1
```
Этот параметр ограничивает, сколько уникальных IP может использовать 1 ссылку одновременно. Если один пользователь отключится, второй сможет подключиться. Также с одного IP может сидеть несколько пользователей.
## Как сделать несколько разных ссылок
1. Сгенерируйте нужное число секретов `openssl rand -hex 16`
2. Открыть конфиг `nano /etc/telemt.toml`
3. Добавить новых пользователей.
```toml
[access.users]
user1 = "00000000000000000000000000000001"
user2 = "00000000000000000000000000000002"
user3 = "00000000000000000000000000000003"
```
4. Сохранить конфиг. Ctrl+S -> Ctrl+X. Перезапускать telemt не нужно.
5. Получить ссылки через `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
## Как посмотреть метрики
1. Открыть конфиг `nano /etc/telemt.toml`
2. Добавить следующие параметры
```toml
[server]
metrics_port = 9090
metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"]
```
3. Сохранить конфиг. Ctrl+S -> Ctrl+X.
4. Метрики доступны по адресу SERVER_IP:9090/metrics.
> [!WARNING]
> "0.0.0.0/0" в metrics_whitelist открывает доступ с любого IP. Замените на свой ip. Например "1.2.3.4"

40
docs/MIDDLE-END-KDF.de.md Normal file
View File

@@ -0,0 +1,40 @@
# Middle-End Proxy
## KDF-Adressierung — Implementierungs-FAQ
### Benötigt die C-Referenzimplementierung sowohl externe IP-Adresse als auch Port für die KDF?
Ja.
In der C-Referenzimplementierung werden **sowohl IP-Adresse als auch Port in die KDF einbezogen** — auf beiden Seiten der Verbindung.
In `aes_create_keys()` enthält der KDF-Input:
- `server_ip + client_port`
- `client_ip + server_port`
- sowie Secret / Nonces
Für IPv6:
- IPv4-Felder werden auf 0 gesetzt
- IPv6-Adressen werden ergänzt
Die **Ports bleiben weiterhin Bestandteil der KDF**.
> Wenn sich externe IP oder Port (z. B. durch NAT, SOCKS oder Proxy) von den erwarteten Werten unterscheiden, entstehen unterschiedliche Schlüssel — der Handshake schlägt fehl.
---
### Kann der Port aus der KDF ausgeschlossen werden (z. B. durch Port = 0)?
**Nein!**
Die C-Referenzimplementierung enthält **keine Möglichkeit, den Port zu ignorieren**:
- `client_port` und `server_port` sind fester Bestandteil der KDF
- Es werden immer reale Socket-Ports übergeben:
- `c->our_port`
- `c->remote_port`
Falls ein Port den Wert `0` hat, wird er dennoch als `0` in die KDF übernommen.
Eine „Port-Ignore“-Logik existiert nicht.

41
docs/MIDDLE-END-KDF.en.md Normal file
View File

@@ -0,0 +1,41 @@
# Middle-End Proxy
## KDF Addressing — Implementation FAQ
### Does the C-implementation require both external IP address and port for the KDF?
**Yes!**
In the C reference implementation, **both IP address and port are included in the KDF input** from both sides of the connection.
Inside `aes_create_keys()`, the KDF input explicitly contains:
- `server_ip + client_port`
- `client_ip + server_port`
- followed by shared secret / nonces
For IPv6:
- IPv4 fields are zeroed
- IPv6 addresses are inserted
However, **client_port and server_port remain part of the KDF regardless of IP version**.
> If externally observed IP or port (e.g. due to NAT, SOCKS, or proxy traversal) differs from what the peer expects, the derived keys will not match and the handshake will fail.
---
### Can port be excluded from KDF (e.g. by using port = 0)?
**No!**
The C-implementation provides **no mechanism to ignore the port**:
- `client_port` and `server_port` are explicitly included in the KDF input
- Real socket ports are always passed:
- `c->our_port`
- `c->remote_port`
If a port is `0`, it is still incorporated into the KDF as `0`.
There is **no conditional logic to exclude ports**

41
docs/MIDDLE-END-KDF.ru.md Normal file
View File

@@ -0,0 +1,41 @@
# Middle-End Proxy
## KDF Addressing — FAQ по реализации
### Требует ли C-референсная реализация KDF внешний IP и порт?
**Да**
В C-референсе **в KDF участвуют и IP-адрес, и порт**с обеих сторон соединения.
В `aes_create_keys()` в строку KDF входят:
- `server_ip + client_port`
- `client_ip + server_port`
- далее secret / nonces
Для IPv6:
- IPv4-поля заполняются нулями
- добавляются IPv6-адреса
Однако **порты client_port и server_port всё равно участвуют в KDF**.
> Если внешний IP или порт (например, из-за NAT, SOCKS или прокси) не совпадает с ожидаемым другой стороной — ключи расходятся и handshake ломается.
---
### Можно ли исключить порт из KDF (например, установив порт = 0)?
**Нет.**
В C-референсе **нет механики отключения порта**.
- `client_port` и `server_port` явно включены в KDF
- Передаются реальные порты сокета:
- `c->our_port`
- `c->remote_port`
Если порт равен `0`, он всё равно попадёт в KDF как `0`.
Отдельной логики «игнорировать порт» не предусмотрено.

View File

@@ -0,0 +1,152 @@
# Telemt via Systemd
## Installation
This software is designed for Debian-based OS: in addition to Debian, these are Ubuntu, Mint, Kali, MX and many other Linux
**1. Download**
```bash
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
```
**2. Move to the Bin folder**
```bash
mv telemt /bin
```
**3. Make the file executable**
```bash
chmod +x /bin/telemt
```
## How to use?
**This guide "assumes" that you:**
- logged in as root or executed `su -` / `sudo su`
- Already have the "telemt" executable file in the /bin folder. Read the **[Installation](#Installation)** section.
---
**0. Check port and generate secrets**
The port you have selected for use should be MISSING from the list, when:
```bash
netstat -lnp
```
Generate 16 bytes/32 characters HEX with OpenSSL or another way:
```bash
openssl rand -hex 16
```
OR
```bash
xxd -l 16 -p /dev/urandom
```
OR
```bash
python3 -c 'import os; print(os.urandom(16).hex())'
```
Save the obtained result somewhere. You will need it later!
---
**1. Place your config to /etc/telemt.toml**
Open nano
```bash
nano /etc/telemt.toml
```
paste your config
```toml
# === General Settings ===
[general]
# ad_tag = "00000000000000000000000000000000"
[general.modes]
classic = false
secure = false
tls = true
# === Anti-Censorship & Masking ===
[censorship]
tls_domain = "petrovich.ru"
[access.users]
# format: "username" = "32_hex_chars_secret"
hello = "00000000000000000000000000000000"
```
then Ctrl+S -> Ctrl+X to save
> [!WARNING]
> Replace the value of the hello parameter with the value you obtained in step 0.
> Replace the value of the tls_domain parameter with another website.
---
**2. Create service on /etc/systemd/system/telemt.service**
Open nano
```bash
nano /etc/systemd/system/telemt.service
```
paste this Systemd Module
```bash
[Unit]
Description=Telemt
After=network.target
[Service]
Type=simple
WorkingDirectory=/bin
ExecStart=/bin/telemt /etc/telemt.toml
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
```
then Ctrl+S -> Ctrl+X to save
**3.** To start it, enter the command `systemctl start telemt`
**4.** To get status information, enter `systemctl status telemt`
**5.** For automatic startup at system boot, enter `systemctl enable telemt`
**6.** To get the links, enter `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
---
# Telemt via Docker Compose
**1. Edit `config.toml` in repo root (at least: port, users secrets, tls_domain)**
**2. Start container:**
```bash
docker compose up -d --build
```
**3. Check logs:**
```bash
docker compose logs -f telemt
```
**4. Stop:**
```bash
docker compose down
```
> [!NOTE]
> - `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only)
> - By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added)
> - If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host`
**Run without Compose**
```bash
docker build -t telemt:local .
docker run --name telemt --restart unless-stopped \
-p 443:443 \
-e RUST_LOG=info \
-v "$PWD/config.toml:/app/config.toml:ro" \
--read-only \
--cap-drop ALL --cap-add NET_BIND_SERVICE \
--ulimit nofile=65536:65536 \
telemt:local
```

View File

@@ -0,0 +1,152 @@
# Telemt через Systemd
## Установка
Это программное обеспечение разработано для ОС на базе Debian: помимо Debian, это Ubuntu, Mint, Kali, MX и многие другие Linux
**1. Скачать**
```bash
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
```
**2. Переместить в папку Bin**
```bash
mv telemt /bin
```
**3. Сделать файл исполняемым**
```bash
chmod +x /bin/telemt
```
## Как правильно использовать?
**Эта инструкция "предполагает", что вы:**
- Авторизовались как пользователь root или выполнил `su -` / `sudo su`
- У вас уже есть исполняемый файл "telemt" в папке /bin. Читайте раздел **[Установка](#установка)**
---
**0. Проверьте порт и сгенерируйте секреты**
Порт, который вы выбрали для использования, должен отсутствовать в списке:
```bash
netstat -lnp
```
Сгенерируйте 16 bytes/32 символа в шестнадцатеричном формате с помощью OpenSSL или другим способом:
```bash
openssl rand -hex 16
```
ИЛИ
```bash
xxd -l 16 -p /dev/urandom
```
ИЛИ
```bash
python3 -c 'import os; print(os.urandom(16).hex())'
```
Полученный результат сохраняем где-нибудь. Он понадобиться вам дальше!
---
**1. Поместите свою конфигурацию в файл /etc/telemt.toml**
Открываем nano
```bash
nano /etc/telemt.toml
```
Вставьте свою конфигурацию
```toml
# === General Settings ===
[general]
# ad_tag = "00000000000000000000000000000000"
[general.modes]
classic = false
secure = false
tls = true
# === Anti-Censorship & Masking ===
[censorship]
tls_domain = "petrovich.ru"
[access.users]
# format: "username" = "32_hex_chars_secret"
hello = "00000000000000000000000000000000"
```
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
> [!WARNING]
> Замените значение параметра hello на значение, которое вы получили в пункте 0.
> Так же замените значение параметра tls_domain на другой сайт.
---
**2. Создайте службу в /etc/systemd/system/telemt.service**
Открываем nano
```bash
nano /etc/systemd/system/telemt.service
```
Вставьте этот модуль Systemd
```bash
[Unit]
Description=Telemt
After=network.target
[Service]
Type=simple
WorkingDirectory=/bin
ExecStart=/bin/telemt /etc/telemt.toml
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
```
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
**3.** Для запуска введите команду `systemctl start telemt`
**4.** Для получения информации о статусе введите `systemctl status telemt`
**5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt`
**6.** Для получения ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
---
# Telemt через Docker Compose
**1. Отредактируйте `config.toml` в корневом каталоге репозитория (как минимум: порт, пользовательские секреты, tls_domain)**
**2. Запустите контейнер:**
```bash
docker compose up -d --build
```
**3. Проверьте логи:**
```bash
docker compose logs -f telemt
```
**4. Остановите контейнер:**
```bash
docker compose down
```
> [!NOTE]
> - В `docker-compose.yml` файл `./config.toml` монтируется в `/app/config.toml` (доступно только для чтения)
> - По умолчанию публикуются порты 443:443, а контейнер запускается со сброшенными привилегиями (добавлена только `NET_BIND_SERVICE`)
> - Если вам действительно нужна сеть хоста (обычно это требуется только для некоторых конфигураций IPv6), раскомментируйте `network_mode: host`
**Запуск в Docker Compose**
```bash
docker build -t telemt:local .
docker run --name telemt --restart unless-stopped \
-p 443:443 \
-e RUST_LOG=info \
-v "$PWD/config.toml:/app/config.toml:ro" \
--read-only \
--cap-drop ALL --cap-add NET_BIND_SERVICE \
--ulimit nofile=65536:65536 \
telemt:local
```

219
docs/TUNING.de.md Normal file
View File

@@ -0,0 +1,219 @@
# Telemt Tuning-Leitfaden: Middle-End und Upstreams
Dieses Dokument beschreibt das aktuelle Laufzeitverhalten für Middle-End (ME) und Upstream-Routing basierend auf:
- `src/config/types.rs`
- `src/config/defaults.rs`
- `src/config/load.rs`
- `src/transport/upstream.rs`
Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüssel), nicht zwingend die Werte aus `config.full.toml`.
## Middle-End-Parameter
### 1) ME-Grundmodus, NAT und STUN
| Parameter | Typ | Default | Einschränkungen / Validierung | Laufzeiteffekt | Beispiel |
|---|---|---:|---|---|---|
| `general.use_middle_proxy` | `bool` | `true` | keine | Aktiviert den ME-Transportmodus. Bei `false` wird Direct-Modus verwendet. | `use_middle_proxy = true` |
| `general.proxy_secret_path` | `Option<String>` | `"proxy-secret"` | Pfad kann `null` sein | Pfad zur Telegram-Infrastrukturdatei `proxy-secret`. | `proxy_secret_path = "proxy-secret"` |
| `general.middle_proxy_nat_ip` | `Option<IpAddr>` | `null` | gültige IP bei gesetztem Wert | Manueller Override der öffentlichen NAT-IP für ME-Adressmaterial. | `middle_proxy_nat_ip = "203.0.113.10"` |
| `general.middle_proxy_nat_probe` | `bool` | `true` | wird auf `true` erzwungen, wenn `use_middle_proxy=true` | Aktiviert NAT-Probing für ME. | `middle_proxy_nat_probe = true` |
| `general.stun_nat_probe_concurrency` | `usize` | `8` | muss `> 0` sein | Maximale parallele STUN-Probes während NAT-Erkennung. | `stun_nat_probe_concurrency = 16` |
| `network.stun_use` | `bool` | `true` | keine | Globaler STUN-Schalter. Bei `false` wird STUN deaktiviert. | `stun_use = true` |
| `network.stun_servers` | `Vec<String>` | integrierter öffentlicher Pool | Duplikate/leer werden entfernt | Primäre STUN-Serverliste für NAT/Public-Endpoint-Erkennung. | `stun_servers = ["stun1.l.google.com:19302"]` |
| `network.stun_tcp_fallback` | `bool` | `true` | keine | Aktiviert TCP-Fallback, wenn UDP-STUN blockiert ist. | `stun_tcp_fallback = true` |
| `network.http_ip_detect_urls` | `Vec<String>` | `ifconfig.me` + `api.ipify.org` | keine | HTTP-Fallback zur öffentlichen IPv4-Erkennung, falls STUN ausfällt. | `http_ip_detect_urls = ["https://api.ipify.org"]` |
| `general.stun_iface_mismatch_ignore` | `bool` | `false` | keine | Reserviertes Feld in der aktuellen Revision (derzeit kein aktiver Runtime-Verbrauch). | `stun_iface_mismatch_ignore = false` |
| `timeouts.me_one_retry` | `u8` | `12` | keine | Anzahl schneller Reconnect-Versuche bei Single-Endpoint-DC-Fällen. | `me_one_retry = 6` |
| `timeouts.me_one_timeout_ms` | `u64` | `1200` | keine | Timeout pro schnellem Einzelversuch (ms). | `me_one_timeout_ms = 1500` |
### 2) Poolgröße, Keepalive und Reconnect-Policy
| Parameter | Typ | Default | Einschränkungen / Validierung | Laufzeiteffekt | Beispiel |
|---|---|---:|---|---|---|
| `general.middle_proxy_pool_size` | `usize` | `8` | keine | Zielgröße des aktiven ME-Writer-Pools. | `middle_proxy_pool_size = 12` |
| `general.middle_proxy_warm_standby` | `usize` | `16` | keine | Reserviertes Kompatibilitätsfeld in der aktuellen Revision (kein aktiver Runtime-Consumer). | `middle_proxy_warm_standby = 16` |
| `general.me_keepalive_enabled` | `bool` | `true` | keine | Aktiviert periodischen ME-Keepalive/Ping-Traffic. | `me_keepalive_enabled = true` |
| `general.me_keepalive_interval_secs` | `u64` | `25` | keine | Basisintervall für Keepalive (Sekunden). | `me_keepalive_interval_secs = 20` |
| `general.me_keepalive_jitter_secs` | `u64` | `5` | keine | Keepalive-Jitter zur Vermeidung synchroner Peaks. | `me_keepalive_jitter_secs = 3` |
| `general.me_keepalive_payload_random` | `bool` | `true` | keine | Randomisiert Keepalive-Payload-Bytes. | `me_keepalive_payload_random = true` |
| `general.me_warmup_stagger_enabled` | `bool` | `true` | keine | Aktiviert gestaffeltes Warmup zusätzlicher ME-Verbindungen. | `me_warmup_stagger_enabled = true` |
| `general.me_warmup_step_delay_ms` | `u64` | `500` | keine | Basisverzögerung zwischen Warmup-Schritten (ms). | `me_warmup_step_delay_ms = 300` |
| `general.me_warmup_step_jitter_ms` | `u64` | `300` | keine | Zusätzlicher zufälliger Warmup-Jitter (ms). | `me_warmup_step_jitter_ms = 200` |
| `general.me_reconnect_max_concurrent_per_dc` | `u32` | `8` | keine | Begrenzung paralleler Reconnect-Worker pro DC. | `me_reconnect_max_concurrent_per_dc = 12` |
| `general.me_reconnect_backoff_base_ms` | `u64` | `500` | keine | Initiales Reconnect-Backoff (ms). | `me_reconnect_backoff_base_ms = 250` |
| `general.me_reconnect_backoff_cap_ms` | `u64` | `30000` | keine | Maximales Reconnect-Backoff (ms). | `me_reconnect_backoff_cap_ms = 10000` |
| `general.me_reconnect_fast_retry_count` | `u32` | `16` | keine | Budget für Sofort-Retries vor längerem Backoff. | `me_reconnect_fast_retry_count = 8` |
### 3) Reinit/Hardswap, Secret-Rotation und Degradation
| Parameter | Typ | Default | Einschränkungen / Validierung | Laufzeiteffekt | Beispiel |
|---|---|---:|---|---|---|
| `general.hardswap` | `bool` | `true` | keine | Aktiviert generation-basierte Hardswap-Strategie für den ME-Pool. | `hardswap = true` |
| `general.me_reinit_every_secs` | `u64` | `900` | muss `> 0` sein | Intervall für periodische ME-Reinitialisierung. | `me_reinit_every_secs = 600` |
| `general.me_hardswap_warmup_delay_min_ms` | `u64` | `1000` | muss `<= me_hardswap_warmup_delay_max_ms` sein | Untere Grenze für Warmup-Dial-Abstände. | `me_hardswap_warmup_delay_min_ms = 500` |
| `general.me_hardswap_warmup_delay_max_ms` | `u64` | `2000` | muss `> 0` sein | Obere Grenze für Warmup-Dial-Abstände. | `me_hardswap_warmup_delay_max_ms = 1200` |
| `general.me_hardswap_warmup_extra_passes` | `u8` | `3` | Bereich `[0,10]` | Zusätzliche Warmup-Pässe nach dem Basispass. | `me_hardswap_warmup_extra_passes = 2` |
| `general.me_hardswap_warmup_pass_backoff_base_ms` | `u64` | `500` | muss `> 0` sein | Basis-Backoff zwischen zusätzlichen Warmup-Pässen. | `me_hardswap_warmup_pass_backoff_base_ms = 400` |
| `general.me_config_stable_snapshots` | `u8` | `2` | muss `> 0` sein | Anzahl identischer ME-Config-Snapshots vor Apply. | `me_config_stable_snapshots = 3` |
| `general.me_config_apply_cooldown_secs` | `u64` | `300` | keine | Cooldown zwischen angewendeten ME-Map-Updates. | `me_config_apply_cooldown_secs = 120` |
| `general.proxy_secret_stable_snapshots` | `u8` | `2` | muss `> 0` sein | Anzahl identischer Secret-Snapshots vor Rotation. | `proxy_secret_stable_snapshots = 3` |
| `general.proxy_secret_rotate_runtime` | `bool` | `true` | keine | Aktiviert Runtime-Rotation des Proxy-Secrets. | `proxy_secret_rotate_runtime = true` |
| `general.proxy_secret_len_max` | `usize` | `256` | Bereich `[32,4096]` | Obergrenze für akzeptierte Secret-Länge. | `proxy_secret_len_max = 512` |
| `general.update_every` | `Option<u64>` | `300` | wenn gesetzt: `> 0`; bei `null`: Legacy-Min-Fallback | Einheitliches Refresh-Intervall für ME-Config + Secret-Updater. | `update_every = 300` |
| `general.me_pool_drain_ttl_secs` | `u64` | `90` | keine | Zeitraum, in dem stale Writer noch als Fallback zulässig sind. | `me_pool_drain_ttl_secs = 120` |
| `general.me_pool_min_fresh_ratio` | `f32` | `0.8` | Bereich `[0.0,1.0]` | Coverage-Schwelle vor Drain der alten Generation. | `me_pool_min_fresh_ratio = 0.9` |
| `general.me_reinit_drain_timeout_secs` | `u64` | `120` | `0` = kein Force-Close; wenn `>0 && < TTL`, dann auf TTL angehoben | Force-Close-Timeout für draining stale Writer. | `me_reinit_drain_timeout_secs = 0` |
| `general.auto_degradation_enabled` | `bool` | `true` | keine | Reserviertes Kompatibilitätsfeld in aktueller Revision (kein aktiver Runtime-Consumer). | `auto_degradation_enabled = true` |
| `general.degradation_min_unavailable_dc_groups` | `u8` | `2` | keine | Reservierter Kompatibilitäts-Schwellenwert in aktueller Revision (kein aktiver Runtime-Consumer). | `degradation_min_unavailable_dc_groups = 2` |
## Deprecated / Legacy Parameter
| Parameter | Status | Ersatz | Aktuelles Verhalten | Migrationshinweis |
|---|---|---|---|---|
| `general.middle_proxy_nat_stun` | Deprecated | `network.stun_servers` | Wird nur dann in `network.stun_servers` gemerged, wenn `network.stun_servers` nicht explizit gesetzt ist. | Wert nach `network.stun_servers` verschieben, Legacy-Key entfernen. |
| `general.middle_proxy_nat_stun_servers` | Deprecated | `network.stun_servers` | Wird nur dann in `network.stun_servers` gemerged, wenn `network.stun_servers` nicht explizit gesetzt ist. | Werte nach `network.stun_servers` verschieben, Legacy-Key entfernen. |
| `general.proxy_secret_auto_reload_secs` | Deprecated | `general.update_every` | Nur aktiv, wenn `update_every = null` (Legacy-Fallback). | `general.update_every` explizit setzen, Legacy-Key entfernen. |
| `general.proxy_config_auto_reload_secs` | Deprecated | `general.update_every` | Nur aktiv, wenn `update_every = null` (Legacy-Fallback). | `general.update_every` explizit setzen, Legacy-Key entfernen. |
## Wie Upstreams konfiguriert werden
### Upstream-Schema
| Feld | Gilt für | Typ | Pflicht | Default | Bedeutung |
|---|---|---|---|---|---|
| `[[upstreams]].type` | alle Upstreams | `"direct" \| "socks4" \| "socks5"` | ja | n/a | Upstream-Transporttyp. |
| `[[upstreams]].weight` | alle Upstreams | `u16` | nein | `1` | Basisgewicht für weighted-random Auswahl. |
| `[[upstreams]].enabled` | alle Upstreams | `bool` | nein | `true` | Deaktivierte Einträge werden beim Start ignoriert. |
| `[[upstreams]].scopes` | alle Upstreams | `String` | nein | `""` | Komma-separierte Scope-Tags für Request-Routing. |
| `interface` | `direct` | `Option<String>` | nein | `null` | Interface-Name (z. B. `eth0`) oder lokale Literal-IP. |
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | nein | `null` | Explizite Source-IP-Kandidaten (strikter Vorrang vor `interface`). |
| `address` | `socks4` | `String` | ja | n/a | SOCKS4-Server (`ip:port` oder `host:port`). |
| `interface` | `socks4` | `Option<String>` | nein | `null` | Wird nur genutzt, wenn `address` als `ip:port` angegeben ist. |
| `user_id` | `socks4` | `Option<String>` | nein | `null` | SOCKS4 User-ID für CONNECT. |
| `address` | `socks5` | `String` | ja | n/a | SOCKS5-Server (`ip:port` oder `host:port`). |
| `interface` | `socks5` | `Option<String>` | nein | `null` | Wird nur genutzt, wenn `address` als `ip:port` angegeben ist. |
| `username` | `socks5` | `Option<String>` | nein | `null` | SOCKS5 Benutzername. |
| `password` | `socks5` | `Option<String>` | nein | `null` | SOCKS5 Passwort. |
### Runtime-Regeln (wichtig)
1. Wenn `[[upstreams]]` fehlt, injiziert der Loader einen Default-`direct`-Upstream.
2. Scope-Filterung basiert auf exaktem Token-Match:
- mit Request-Scope -> nur Einträge, deren `scopes` genau dieses Token enthält;
- ohne Request-Scope -> nur Einträge mit leerem `scopes`.
3. Unter healthy Upstreams erfolgt die Auswahl per weighted random: `weight * latency_factor`.
4. Gibt es im gefilterten Set keinen healthy Upstream, wird zufällig aus dem gefilterten Set gewählt.
5. `direct`-Bind-Auflösung:
- zuerst `bind_addresses` (nur gleiche IP-Familie wie Target);
- bei `interface` (Name) + `bind_addresses` wird jede Candidate-IP gegen Interface-Adressen validiert;
- ungültige Kandidaten werden mit `WARN` verworfen;
- bleiben keine gültigen Kandidaten übrig, erfolgt unbound direct connect (`bind_ip=None`);
- wenn `bind_addresses` nicht passt, wird `interface` verwendet (Literal-IP oder Interface-Primäradresse).
6. Für `socks4/socks5` mit Hostname-`address` ist Interface-Binding nicht unterstützt und wird mit Warnung ignoriert.
7. Runtime DNS Overrides werden für Hostname-Auflösung bei Upstream-Verbindungen genutzt.
8. Im ME-Modus wird der gewählte Upstream auch für den ME-TCP-Dial-Pfad verwendet.
9. Im ME-Modus ist bei `direct` mit bind/interface die STUN-Reflection bind-aware für KDF-Adressmaterial.
10. Im ME-Modus werden bei SOCKS-Upstream `BND.ADDR/BND.PORT` für KDF verwendet, wenn gültig/öffentlich und gleiche IP-Familie.
## Upstream-Konfigurationsbeispiele
### Beispiel 1: Minimaler direct Upstream
```toml
[[upstreams]]
type = "direct"
weight = 1
enabled = true
```
### Beispiel 2: direct mit Interface + expliziten bind IPs
```toml
[[upstreams]]
type = "direct"
interface = "eth0"
bind_addresses = ["192.168.1.100", "192.168.1.101"]
weight = 3
enabled = true
```
### Beispiel 3: SOCKS5 Upstream mit Authentifizierung
```toml
[[upstreams]]
type = "socks5"
address = "198.51.100.30:1080"
username = "proxy-user"
password = "proxy-pass"
weight = 2
enabled = true
```
### Beispiel 4: Gemischte Upstreams mit Scopes
```toml
[[upstreams]]
type = "direct"
weight = 5
enabled = true
scopes = ""
[[upstreams]]
type = "socks5"
address = "203.0.113.40:1080"
username = "edge"
password = "edgepass"
weight = 3
enabled = true
scopes = "premium,me"
```
### Beispiel 5: ME-orientiertes Tuning-Profil
```toml
[general]
use_middle_proxy = true
proxy_secret_path = "proxy-secret"
middle_proxy_nat_probe = true
stun_nat_probe_concurrency = 16
middle_proxy_pool_size = 12
me_keepalive_enabled = true
me_keepalive_interval_secs = 20
me_keepalive_jitter_secs = 4
me_reconnect_max_concurrent_per_dc = 12
me_reconnect_backoff_base_ms = 300
me_reconnect_backoff_cap_ms = 10000
me_reconnect_fast_retry_count = 10
hardswap = true
me_reinit_every_secs = 600
me_hardswap_warmup_delay_min_ms = 500
me_hardswap_warmup_delay_max_ms = 1200
me_hardswap_warmup_extra_passes = 2
me_hardswap_warmup_pass_backoff_base_ms = 400
me_config_stable_snapshots = 3
me_config_apply_cooldown_secs = 120
proxy_secret_stable_snapshots = 3
proxy_secret_rotate_runtime = true
proxy_secret_len_max = 512
update_every = 300
me_pool_drain_ttl_secs = 120
me_pool_min_fresh_ratio = 0.9
me_reinit_drain_timeout_secs = 180
[timeouts]
me_one_retry = 8
me_one_timeout_ms = 1200
[network]
stun_use = true
stun_tcp_fallback = true
stun_servers = [
"stun1.l.google.com:19302",
"stun2.l.google.com:19302"
]
http_ip_detect_urls = [
"https://api.ipify.org",
"https://ifconfig.me/ip"
]
```

219
docs/TUNING.en.md Normal file
View File

@@ -0,0 +1,219 @@
# Telemt Tuning Guide: Middle-End and Upstreams
This document describes the current runtime behavior for Middle-End (ME) and upstream routing based on:
- `src/config/types.rs`
- `src/config/defaults.rs`
- `src/config/load.rs`
- `src/transport/upstream.rs`
Defaults below are code defaults (used when a key is omitted), not necessarily values from `config.full.toml` examples.
## Middle-End Parameters
### 1) Core ME mode, NAT, and STUN
| Parameter | Type | Default | Constraints / validation | Runtime effect | Example |
|---|---|---:|---|---|---|
| `general.use_middle_proxy` | `bool` | `true` | none | Enables ME transport mode. If `false`, Direct mode is used. | `use_middle_proxy = true` |
| `general.proxy_secret_path` | `Option<String>` | `"proxy-secret"` | path may be `null` | Path to Telegram infrastructure proxy-secret file. | `proxy_secret_path = "proxy-secret"` |
| `general.middle_proxy_nat_ip` | `Option<IpAddr>` | `null` | valid IP when set | Manual public NAT IP override for ME address material. | `middle_proxy_nat_ip = "203.0.113.10"` |
| `general.middle_proxy_nat_probe` | `bool` | `true` | auto-forced to `true` when `use_middle_proxy=true` | Enables ME NAT probing. | `middle_proxy_nat_probe = true` |
| `general.stun_nat_probe_concurrency` | `usize` | `8` | must be `> 0` | Max parallel STUN probes during NAT discovery. | `stun_nat_probe_concurrency = 16` |
| `network.stun_use` | `bool` | `true` | none | Global STUN switch. If `false`, STUN probing is disabled. | `stun_use = true` |
| `network.stun_servers` | `Vec<String>` | built-in public pool | deduplicated + empty values removed | Primary STUN server list for NAT/public endpoint discovery. | `stun_servers = ["stun1.l.google.com:19302"]` |
| `network.stun_tcp_fallback` | `bool` | `true` | none | Enables TCP fallback path when UDP STUN is blocked. | `stun_tcp_fallback = true` |
| `network.http_ip_detect_urls` | `Vec<String>` | `ifconfig.me` + `api.ipify.org` | none | HTTP fallback for public IPv4 detection if STUN is unavailable. | `http_ip_detect_urls = ["https://api.ipify.org"]` |
| `general.stun_iface_mismatch_ignore` | `bool` | `false` | none | Reserved flag in current revision (not consumed by runtime path). | `stun_iface_mismatch_ignore = false` |
| `timeouts.me_one_retry` | `u8` | `12` | none | Fast reconnect attempts for single-endpoint DC cases. | `me_one_retry = 6` |
| `timeouts.me_one_timeout_ms` | `u64` | `1200` | none | Timeout per quick single-endpoint attempt (ms). | `me_one_timeout_ms = 1500` |
### 2) Pool size, keepalive, and reconnect policy
| Parameter | Type | Default | Constraints / validation | Runtime effect | Example |
|---|---|---:|---|---|---|
| `general.middle_proxy_pool_size` | `usize` | `8` | none | Target active ME writer pool size. | `middle_proxy_pool_size = 12` |
| `general.middle_proxy_warm_standby` | `usize` | `16` | none | Reserved compatibility field in current revision (no active runtime consumer). | `middle_proxy_warm_standby = 16` |
| `general.me_keepalive_enabled` | `bool` | `true` | none | Enables periodic ME keepalive/ping traffic. | `me_keepalive_enabled = true` |
| `general.me_keepalive_interval_secs` | `u64` | `25` | none | Base keepalive interval (seconds). | `me_keepalive_interval_secs = 20` |
| `general.me_keepalive_jitter_secs` | `u64` | `5` | none | Keepalive jitter to avoid synchronization bursts. | `me_keepalive_jitter_secs = 3` |
| `general.me_keepalive_payload_random` | `bool` | `true` | none | Randomizes keepalive payload bytes. | `me_keepalive_payload_random = true` |
| `general.me_warmup_stagger_enabled` | `bool` | `true` | none | Staggers extra ME warmup dials to avoid spikes. | `me_warmup_stagger_enabled = true` |
| `general.me_warmup_step_delay_ms` | `u64` | `500` | none | Base delay between warmup dial steps (ms). | `me_warmup_step_delay_ms = 300` |
| `general.me_warmup_step_jitter_ms` | `u64` | `300` | none | Additional random delay for warmup steps (ms). | `me_warmup_step_jitter_ms = 200` |
| `general.me_reconnect_max_concurrent_per_dc` | `u32` | `8` | none | Limits concurrent reconnect workers per DC in health recovery. | `me_reconnect_max_concurrent_per_dc = 12` |
| `general.me_reconnect_backoff_base_ms` | `u64` | `500` | none | Initial reconnect backoff (ms). | `me_reconnect_backoff_base_ms = 250` |
| `general.me_reconnect_backoff_cap_ms` | `u64` | `30000` | none | Maximum reconnect backoff (ms). | `me_reconnect_backoff_cap_ms = 10000` |
| `general.me_reconnect_fast_retry_count` | `u32` | `16` | none | Immediate retry budget before long backoff behavior. | `me_reconnect_fast_retry_count = 8` |
### 3) Reinit/hardswap, secret rotation, and degradation
| Parameter | Type | Default | Constraints / validation | Runtime effect | Example |
|---|---|---:|---|---|---|
| `general.hardswap` | `bool` | `true` | none | Enables generation-based ME hardswap strategy. | `hardswap = true` |
| `general.me_reinit_every_secs` | `u64` | `900` | must be `> 0` | Periodic ME reinit interval. | `me_reinit_every_secs = 600` |
| `general.me_hardswap_warmup_delay_min_ms` | `u64` | `1000` | must be `<= me_hardswap_warmup_delay_max_ms` | Lower bound for hardswap warmup dial spacing. | `me_hardswap_warmup_delay_min_ms = 500` |
| `general.me_hardswap_warmup_delay_max_ms` | `u64` | `2000` | must be `> 0` | Upper bound for hardswap warmup dial spacing. | `me_hardswap_warmup_delay_max_ms = 1200` |
| `general.me_hardswap_warmup_extra_passes` | `u8` | `3` | must be within `[0,10]` | Additional warmup passes after base pass. | `me_hardswap_warmup_extra_passes = 2` |
| `general.me_hardswap_warmup_pass_backoff_base_ms` | `u64` | `500` | must be `> 0` | Base backoff between extra warmup passes. | `me_hardswap_warmup_pass_backoff_base_ms = 400` |
| `general.me_config_stable_snapshots` | `u8` | `2` | must be `> 0` | Number of identical ME config snapshots required before apply. | `me_config_stable_snapshots = 3` |
| `general.me_config_apply_cooldown_secs` | `u64` | `300` | none | Cooldown between applied ME map updates. | `me_config_apply_cooldown_secs = 120` |
| `general.proxy_secret_stable_snapshots` | `u8` | `2` | must be `> 0` | Number of identical proxy-secret snapshots required before rotation. | `proxy_secret_stable_snapshots = 3` |
| `general.proxy_secret_rotate_runtime` | `bool` | `true` | none | Enables runtime proxy-secret rotation. | `proxy_secret_rotate_runtime = true` |
| `general.proxy_secret_len_max` | `usize` | `256` | must be within `[32,4096]` | Upper limit for accepted proxy-secret length. | `proxy_secret_len_max = 512` |
| `general.update_every` | `Option<u64>` | `300` | if set: must be `> 0`; if `null`: legacy min fallback | Unified refresh interval for ME config + secret updater. | `update_every = 300` |
| `general.me_pool_drain_ttl_secs` | `u64` | `90` | none | Time window where stale writers remain fallback-eligible. | `me_pool_drain_ttl_secs = 120` |
| `general.me_pool_min_fresh_ratio` | `f32` | `0.8` | must be within `[0.0,1.0]` | Coverage threshold before stale generation can be drained. | `me_pool_min_fresh_ratio = 0.9` |
| `general.me_reinit_drain_timeout_secs` | `u64` | `120` | `0` means no force-close; if `>0 && < TTL` it is bumped to TTL | Force-close timeout for draining stale writers. | `me_reinit_drain_timeout_secs = 0` |
| `general.auto_degradation_enabled` | `bool` | `true` | none | Reserved compatibility flag in current revision (no active runtime consumer). | `auto_degradation_enabled = true` |
| `general.degradation_min_unavailable_dc_groups` | `u8` | `2` | none | Reserved compatibility threshold in current revision (no active runtime consumer). | `degradation_min_unavailable_dc_groups = 2` |
## Deprecated / Legacy Parameters
| Parameter | Status | Replacement | Current behavior | Migration recommendation |
|---|---|---|---|---|
| `general.middle_proxy_nat_stun` | Deprecated | `network.stun_servers` | Merged into `network.stun_servers` only when `network.stun_servers` is not explicitly set. | Move value into `network.stun_servers` and remove legacy key. |
| `general.middle_proxy_nat_stun_servers` | Deprecated | `network.stun_servers` | Merged into `network.stun_servers` only when `network.stun_servers` is not explicitly set. | Move values into `network.stun_servers` and remove legacy key. |
| `general.proxy_secret_auto_reload_secs` | Deprecated | `general.update_every` | Used only when `update_every = null` (legacy fallback path). | Set `general.update_every` explicitly and remove legacy key. |
| `general.proxy_config_auto_reload_secs` | Deprecated | `general.update_every` | Used only when `update_every = null` (legacy fallback path). | Set `general.update_every` explicitly and remove legacy key. |
## How Upstreams Are Configured
### Upstream schema
| Field | Applies to | Type | Required | Default | Meaning |
|---|---|---|---|---|---|
| `[[upstreams]].type` | all upstreams | `"direct" \| "socks4" \| "socks5"` | yes | n/a | Upstream transport type. |
| `[[upstreams]].weight` | all upstreams | `u16` | no | `1` | Base weight for weighted-random selection. |
| `[[upstreams]].enabled` | all upstreams | `bool` | no | `true` | Disabled entries are ignored at startup. |
| `[[upstreams]].scopes` | all upstreams | `String` | no | `""` | Comma-separated scope tags for request-level routing. |
| `interface` | `direct` | `Option<String>` | no | `null` | Interface name (e.g. `eth0`) or literal local IP for bind selection. |
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | no | `null` | Explicit local source IP candidates (strict priority over `interface`). |
| `address` | `socks4` | `String` | yes | n/a | SOCKS4 server endpoint (`ip:port` or `host:port`). |
| `interface` | `socks4` | `Option<String>` | no | `null` | Used only for SOCKS server `ip:port` dial path. |
| `user_id` | `socks4` | `Option<String>` | no | `null` | SOCKS4 user ID for CONNECT request. |
| `address` | `socks5` | `String` | yes | n/a | SOCKS5 server endpoint (`ip:port` or `host:port`). |
| `interface` | `socks5` | `Option<String>` | no | `null` | Used only for SOCKS server `ip:port` dial path. |
| `username` | `socks5` | `Option<String>` | no | `null` | SOCKS5 username auth. |
| `password` | `socks5` | `Option<String>` | no | `null` | SOCKS5 password auth. |
### Runtime rules (important)
1. If `[[upstreams]]` is omitted, loader injects one default `direct` upstream.
2. Scope filtering is exact-token based:
- when request scope is set -> only entries whose `scopes` contains that exact token;
- when request scope is not set -> only entries with empty `scopes`.
3. Healthy upstreams are selected by weighted random using: `weight * latency_factor`.
4. If no healthy upstream exists in filtered set, random selection is used among filtered entries.
5. `direct` bind resolution order:
- `bind_addresses` candidates (same IP family as target) first;
- if `interface` is an interface name and `bind_addresses` is set, each candidate IP is validated against addresses currently assigned to that interface;
- invalid candidates are dropped with `WARN`;
- if no valid candidate remains, connection falls back to unbound direct connect (`bind_ip=None`);
- if no `bind_addresses` candidate, `interface` is used (literal IP or resolved interface primary IP).
6. For `socks4/socks5` with `address` as hostname, interface binding is not supported and is ignored with warning.
7. Runtime DNS overrides are used for upstream hostname resolution.
8. In ME mode, the selected upstream is also used for ME TCP dial path.
9. In ME mode for `direct` upstream with bind/interface, STUN reflection logic is bind-aware for KDF source material.
10. In ME mode for SOCKS upstream, SOCKS `BND.ADDR/BND.PORT` is used for KDF when it is valid/public for the same family.
## Upstream Configuration Examples
### Example 1: Minimal direct upstream
```toml
[[upstreams]]
type = "direct"
weight = 1
enabled = true
```
### Example 2: Direct with interface + explicit bind addresses
```toml
[[upstreams]]
type = "direct"
interface = "eth0"
bind_addresses = ["192.168.1.100", "192.168.1.101"]
weight = 3
enabled = true
```
### Example 3: SOCKS5 upstream with authentication
```toml
[[upstreams]]
type = "socks5"
address = "198.51.100.30:1080"
username = "proxy-user"
password = "proxy-pass"
weight = 2
enabled = true
```
### Example 4: Mixed upstreams with scopes
```toml
[[upstreams]]
type = "direct"
weight = 5
enabled = true
scopes = ""
[[upstreams]]
type = "socks5"
address = "203.0.113.40:1080"
username = "edge"
password = "edgepass"
weight = 3
enabled = true
scopes = "premium,me"
```
### Example 5: ME-focused tuning profile
```toml
[general]
use_middle_proxy = true
proxy_secret_path = "proxy-secret"
middle_proxy_nat_probe = true
stun_nat_probe_concurrency = 16
middle_proxy_pool_size = 12
me_keepalive_enabled = true
me_keepalive_interval_secs = 20
me_keepalive_jitter_secs = 4
me_reconnect_max_concurrent_per_dc = 12
me_reconnect_backoff_base_ms = 300
me_reconnect_backoff_cap_ms = 10000
me_reconnect_fast_retry_count = 10
hardswap = true
me_reinit_every_secs = 600
me_hardswap_warmup_delay_min_ms = 500
me_hardswap_warmup_delay_max_ms = 1200
me_hardswap_warmup_extra_passes = 2
me_hardswap_warmup_pass_backoff_base_ms = 400
me_config_stable_snapshots = 3
me_config_apply_cooldown_secs = 120
proxy_secret_stable_snapshots = 3
proxy_secret_rotate_runtime = true
proxy_secret_len_max = 512
update_every = 300
me_pool_drain_ttl_secs = 120
me_pool_min_fresh_ratio = 0.9
me_reinit_drain_timeout_secs = 180
[timeouts]
me_one_retry = 8
me_one_timeout_ms = 1200
[network]
stun_use = true
stun_tcp_fallback = true
stun_servers = [
"stun1.l.google.com:19302",
"stun2.l.google.com:19302"
]
http_ip_detect_urls = [
"https://api.ipify.org",
"https://ifconfig.me/ip"
]
```

219
docs/TUNING.ru.md Normal file
View File

@@ -0,0 +1,219 @@
# Руководство по тюнингу Telemt: Middle-End и Upstreams
Документ описывает актуальное поведение Middle-End (ME) и маршрутизации через upstream на основе:
- `src/config/types.rs`
- `src/config/defaults.rs`
- `src/config/load.rs`
- `src/transport/upstream.rs`
Значения `Default` ниже — это значения из кода при отсутствии ключа в конфиге, а не обязательно значения из примеров `config.full.toml`.
## Параметры Middle-End
### 1) Базовый режим ME, NAT и STUN
| Параметр | Тип | Default | Ограничения / валидация | Влияние на runtime | Пример |
|---|---|---:|---|---|---|
| `general.use_middle_proxy` | `bool` | `true` | нет | Включает транспорт ME. При `false` используется Direct-режим. | `use_middle_proxy = true` |
| `general.proxy_secret_path` | `Option<String>` | `"proxy-secret"` | путь может быть `null` | Путь к инфраструктурному proxy-secret Telegram. | `proxy_secret_path = "proxy-secret"` |
| `general.middle_proxy_nat_ip` | `Option<IpAddr>` | `null` | валидный IP при задании | Ручной override публичного NAT IP для адресного материала ME. | `middle_proxy_nat_ip = "203.0.113.10"` |
| `general.middle_proxy_nat_probe` | `bool` | `true` | авто-принудительно `true`, если `use_middle_proxy=true` | Включает NAT probing для ME. | `middle_proxy_nat_probe = true` |
| `general.stun_nat_probe_concurrency` | `usize` | `8` | должно быть `> 0` | Максимум параллельных STUN-проб при NAT-детекте. | `stun_nat_probe_concurrency = 16` |
| `network.stun_use` | `bool` | `true` | нет | Глобальный переключатель STUN. При `false` STUN отключен. | `stun_use = true` |
| `network.stun_servers` | `Vec<String>` | встроенный публичный пул | удаляются дубликаты и пустые значения | Основной список STUN-серверов для NAT/public endpoint discovery. | `stun_servers = ["stun1.l.google.com:19302"]` |
| `network.stun_tcp_fallback` | `bool` | `true` | нет | Включает TCP fallback, если UDP STUN недоступен. | `stun_tcp_fallback = true` |
| `network.http_ip_detect_urls` | `Vec<String>` | `ifconfig.me` + `api.ipify.org` | нет | HTTP fallback для определения публичного IPv4 при недоступности STUN. | `http_ip_detect_urls = ["https://api.ipify.org"]` |
| `general.stun_iface_mismatch_ignore` | `bool` | `false` | нет | Зарезервированный флаг в текущей ревизии (runtime его не использует). | `stun_iface_mismatch_ignore = false` |
| `timeouts.me_one_retry` | `u8` | `12` | нет | Количество быстрых reconnect-попыток для DC с одним endpoint. | `me_one_retry = 6` |
| `timeouts.me_one_timeout_ms` | `u64` | `1200` | нет | Таймаут одной быстрой попытки (мс). | `me_one_timeout_ms = 1500` |
### 2) Размер пула, keepalive и reconnect-политика
| Параметр | Тип | Default | Ограничения / валидация | Влияние на runtime | Пример |
|---|---|---:|---|---|---|
| `general.middle_proxy_pool_size` | `usize` | `8` | нет | Целевой размер активного пула ME-writer соединений. | `middle_proxy_pool_size = 12` |
| `general.middle_proxy_warm_standby` | `usize` | `16` | нет | Зарезервированное поле совместимости в текущей ревизии (активного runtime-consumer нет). | `middle_proxy_warm_standby = 16` |
| `general.me_keepalive_enabled` | `bool` | `true` | нет | Включает периодические keepalive/ping кадры ME. | `me_keepalive_enabled = true` |
| `general.me_keepalive_interval_secs` | `u64` | `25` | нет | Базовый интервал keepalive (сек). | `me_keepalive_interval_secs = 20` |
| `general.me_keepalive_jitter_secs` | `u64` | `5` | нет | Джиттер keepalive для предотвращения синхронных всплесков. | `me_keepalive_jitter_secs = 3` |
| `general.me_keepalive_payload_random` | `bool` | `true` | нет | Рандомизирует payload keepalive-кадров. | `me_keepalive_payload_random = true` |
| `general.me_warmup_stagger_enabled` | `bool` | `true` | нет | Включает staggered warmup дополнительных ME-коннектов. | `me_warmup_stagger_enabled = true` |
| `general.me_warmup_step_delay_ms` | `u64` | `500` | нет | Базовая задержка между шагами warmup (мс). | `me_warmup_step_delay_ms = 300` |
| `general.me_warmup_step_jitter_ms` | `u64` | `300` | нет | Дополнительный случайный warmup-джиттер (мс). | `me_warmup_step_jitter_ms = 200` |
| `general.me_reconnect_max_concurrent_per_dc` | `u32` | `8` | нет | Ограничивает параллельные reconnect worker'ы на один DC. | `me_reconnect_max_concurrent_per_dc = 12` |
| `general.me_reconnect_backoff_base_ms` | `u64` | `500` | нет | Начальный backoff reconnect (мс). | `me_reconnect_backoff_base_ms = 250` |
| `general.me_reconnect_backoff_cap_ms` | `u64` | `30000` | нет | Верхняя граница backoff reconnect (мс). | `me_reconnect_backoff_cap_ms = 10000` |
| `general.me_reconnect_fast_retry_count` | `u32` | `16` | нет | Бюджет быстрых retry до длинного backoff. | `me_reconnect_fast_retry_count = 8` |
### 3) Reinit/hardswap, ротация секрета и деградация
| Параметр | Тип | Default | Ограничения / валидация | Влияние на runtime | Пример |
|---|---|---:|---|---|---|
| `general.hardswap` | `bool` | `true` | нет | Включает generation-based стратегию hardswap для ME-пула. | `hardswap = true` |
| `general.me_reinit_every_secs` | `u64` | `900` | должно быть `> 0` | Интервал периодического reinit ME-пула. | `me_reinit_every_secs = 600` |
| `general.me_hardswap_warmup_delay_min_ms` | `u64` | `1000` | должно быть `<= me_hardswap_warmup_delay_max_ms` | Нижняя граница пауз между warmup dial попытками. | `me_hardswap_warmup_delay_min_ms = 500` |
| `general.me_hardswap_warmup_delay_max_ms` | `u64` | `2000` | должно быть `> 0` | Верхняя граница пауз между warmup dial попытками. | `me_hardswap_warmup_delay_max_ms = 1200` |
| `general.me_hardswap_warmup_extra_passes` | `u8` | `3` | диапазон `[0,10]` | Дополнительные warmup-проходы после базового. | `me_hardswap_warmup_extra_passes = 2` |
| `general.me_hardswap_warmup_pass_backoff_base_ms` | `u64` | `500` | должно быть `> 0` | Базовый backoff между extra-pass в warmup. | `me_hardswap_warmup_pass_backoff_base_ms = 400` |
| `general.me_config_stable_snapshots` | `u8` | `2` | должно быть `> 0` | Количество одинаковых snapshot перед применением ME map update. | `me_config_stable_snapshots = 3` |
| `general.me_config_apply_cooldown_secs` | `u64` | `300` | нет | Cooldown между применёнными обновлениями ME map. | `me_config_apply_cooldown_secs = 120` |
| `general.proxy_secret_stable_snapshots` | `u8` | `2` | должно быть `> 0` | Количество одинаковых snapshot перед runtime-rotation proxy-secret. | `proxy_secret_stable_snapshots = 3` |
| `general.proxy_secret_rotate_runtime` | `bool` | `true` | нет | Включает runtime-ротацию proxy-secret. | `proxy_secret_rotate_runtime = true` |
| `general.proxy_secret_len_max` | `usize` | `256` | диапазон `[32,4096]` | Верхний лимит длины принимаемого proxy-secret. | `proxy_secret_len_max = 512` |
| `general.update_every` | `Option<u64>` | `300` | если задано: `> 0`; если `null`: fallback на legacy минимум | Единый интервал refresh для ME config + secret updater. | `update_every = 300` |
| `general.me_pool_drain_ttl_secs` | `u64` | `90` | нет | Время, когда stale writer ещё может использоваться как fallback. | `me_pool_drain_ttl_secs = 120` |
| `general.me_pool_min_fresh_ratio` | `f32` | `0.8` | диапазон `[0.0,1.0]` | Порог покрытия fresh-поколения перед drain старого поколения. | `me_pool_min_fresh_ratio = 0.9` |
| `general.me_reinit_drain_timeout_secs` | `u64` | `120` | `0` = без force-close; если `>0 && < TTL`, поднимается до TTL | Таймаут force-close для draining stale writer. | `me_reinit_drain_timeout_secs = 0` |
| `general.auto_degradation_enabled` | `bool` | `true` | нет | Зарезервированный флаг совместимости в текущей ревизии (активного runtime-consumer нет). | `auto_degradation_enabled = true` |
| `general.degradation_min_unavailable_dc_groups` | `u8` | `2` | нет | Зарезервированный порог совместимости в текущей ревизии (активного runtime-consumer нет). | `degradation_min_unavailable_dc_groups = 2` |
## Устаревшие / legacy параметры
| Параметр | Статус | Замена | Текущее поведение | Рекомендация миграции |
|---|---|---|---|---|
| `general.middle_proxy_nat_stun` | Deprecated | `network.stun_servers` | Добавляется в `network.stun_servers`, только если `network.stun_servers` не задан явно. | Перенести значение в `network.stun_servers`, legacy-ключ удалить. |
| `general.middle_proxy_nat_stun_servers` | Deprecated | `network.stun_servers` | Добавляется в `network.stun_servers`, только если `network.stun_servers` не задан явно. | Перенести значения в `network.stun_servers`, legacy-ключ удалить. |
| `general.proxy_secret_auto_reload_secs` | Deprecated | `general.update_every` | Используется только если `update_every = null` (legacy fallback). | Явно задать `general.update_every`, legacy-ключ удалить. |
| `general.proxy_config_auto_reload_secs` | Deprecated | `general.update_every` | Используется только если `update_every = null` (legacy fallback). | Явно задать `general.update_every`, legacy-ключ удалить. |
## Как конфигурируются Upstreams
### Схема upstream
| Поле | Применимость | Тип | Обязательно | Default | Назначение |
|---|---|---|---|---|---|
| `[[upstreams]].type` | все upstream | `"direct" \| "socks4" \| "socks5"` | да | n/a | Тип upstream транспорта. |
| `[[upstreams]].weight` | все upstream | `u16` | нет | `1` | Базовый вес в weighted-random выборе. |
| `[[upstreams]].enabled` | все upstream | `bool` | нет | `true` | Выключенные записи игнорируются на старте. |
| `[[upstreams]].scopes` | все upstream | `String` | нет | `""` | Список scope-токенов через запятую для маршрутизации. |
| `interface` | `direct` | `Option<String>` | нет | `null` | Имя интерфейса (например `eth0`) или literal локальный IP. |
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | нет | `null` | Явные кандидаты source IP (имеют приоритет над `interface`). |
| `address` | `socks4` | `String` | да | n/a | Адрес SOCKS4 сервера (`ip:port` или `host:port`). |
| `interface` | `socks4` | `Option<String>` | нет | `null` | Используется только если `address` задан как `ip:port`. |
| `user_id` | `socks4` | `Option<String>` | нет | `null` | SOCKS4 user ID в CONNECT-запросе. |
| `address` | `socks5` | `String` | да | n/a | Адрес SOCKS5 сервера (`ip:port` или `host:port`). |
| `interface` | `socks5` | `Option<String>` | нет | `null` | Используется только если `address` задан как `ip:port`. |
| `username` | `socks5` | `Option<String>` | нет | `null` | Логин SOCKS5 auth. |
| `password` | `socks5` | `Option<String>` | нет | `null` | Пароль SOCKS5 auth. |
### Runtime-правила
1. Если `[[upstreams]]` отсутствует, loader добавляет один upstream `direct` по умолчанию.
2. Scope-фильтрация — по точному совпадению токена:
- если scope запроса задан -> используются только записи, где `scopes` содержит такой же токен;
- если scope запроса не задан -> используются только записи с пустым `scopes`.
3. Среди healthy upstream используется weighted-random выбор: `weight * latency_factor`.
4. Если в отфильтрованном наборе нет healthy upstream, выбирается случайный из отфильтрованных.
5. Порядок выбора bind для `direct`:
- сначала `bind_addresses` (только IP нужного семейства);
- если одновременно заданы `interface` (имя) и `bind_addresses`, каждый IP проверяется на принадлежность интерфейсу;
- несовпадающие IP отбрасываются с `WARN`;
- если валидных IP не осталось, используется unbound direct connect (`bind_ip=None`);
- если `bind_addresses` не подходит, применяется `interface` (literal IP или адрес интерфейса).
6. Для `socks4/socks5` с `address` в виде hostname интерфейсный bind не поддерживается и игнорируется с предупреждением.
7. Runtime DNS overrides применяются к резолвингу hostname в upstream-подключениях.
8. В ME-режиме выбранный upstream также используется для ME TCP dial path.
9. В ME-режиме для `direct` upstream с bind/interface STUN-рефлексия выполняется bind-aware для KDF материала.
10. В ME-режиме для SOCKS upstream используются `BND.ADDR/BND.PORT` для KDF, если адрес валиден/публичен и соответствует IP family.
## Примеры конфигурации Upstreams
### Пример 1: минимальный direct upstream
```toml
[[upstreams]]
type = "direct"
weight = 1
enabled = true
```
### Пример 2: direct с interface + явными bind IP
```toml
[[upstreams]]
type = "direct"
interface = "eth0"
bind_addresses = ["192.168.1.100", "192.168.1.101"]
weight = 3
enabled = true
```
### Пример 3: SOCKS5 upstream с аутентификацией
```toml
[[upstreams]]
type = "socks5"
address = "198.51.100.30:1080"
username = "proxy-user"
password = "proxy-pass"
weight = 2
enabled = true
```
### Пример 4: смешанные upstream с scopes
```toml
[[upstreams]]
type = "direct"
weight = 5
enabled = true
scopes = ""
[[upstreams]]
type = "socks5"
address = "203.0.113.40:1080"
username = "edge"
password = "edgepass"
weight = 3
enabled = true
scopes = "premium,me"
```
### Пример 5: профиль тюнинга под ME
```toml
[general]
use_middle_proxy = true
proxy_secret_path = "proxy-secret"
middle_proxy_nat_probe = true
stun_nat_probe_concurrency = 16
middle_proxy_pool_size = 12
me_keepalive_enabled = true
me_keepalive_interval_secs = 20
me_keepalive_jitter_secs = 4
me_reconnect_max_concurrent_per_dc = 12
me_reconnect_backoff_base_ms = 300
me_reconnect_backoff_cap_ms = 10000
me_reconnect_fast_retry_count = 10
hardswap = true
me_reinit_every_secs = 600
me_hardswap_warmup_delay_min_ms = 500
me_hardswap_warmup_delay_max_ms = 1200
me_hardswap_warmup_extra_passes = 2
me_hardswap_warmup_pass_backoff_base_ms = 400
me_config_stable_snapshots = 3
me_config_apply_cooldown_secs = 120
proxy_secret_stable_snapshots = 3
proxy_secret_rotate_runtime = true
proxy_secret_len_max = 512
update_every = 300
me_pool_drain_ttl_secs = 120
me_pool_min_fresh_ratio = 0.9
me_reinit_drain_timeout_secs = 180
[timeouts]
me_one_retry = 8
me_one_timeout_ms = 1200
[network]
stun_use = true
stun_tcp_fallback = true
stun_servers = [
"stun1.l.google.com:19302",
"stun2.l.google.com:19302"
]
http_ip_detect_urls = [
"https://api.ipify.org",
"https://ifconfig.me/ip"
]
```

View File

@@ -0,0 +1,321 @@
# SNI-маршрутизация в xray-core / sing-box + TLS-fronting
## Термины (в контексте этого кейса)
- **TLS-fronting домен** — домен, который фигурирует в TLS ClientHello как **SNI** (например, `petrovich.ru`): он используется как "маска" на L7 и как ключ маршрутизации в прокси-роутере.
- **xray-core / sing-box** — локальный или удалённый L7/TLS-роутер (прокси), который:
1) принимает входящее TCP/TLS-соединение,
2) читает TLS ClientHello,
3) извлекает SNI,
4) по SNI выбирает outbound/апстрим,
5) устанавливает новое TCP-соединение к целевому хосту уже **от себя**.
- **SNI (Server Name Indication)** — поле в TLS ClientHello, где клиент Telegram сообщает доменное имя для "маскировки"
- **DNS-resolve на стороне L7-роутера** — если выходной адрес задан доменом (или роутер решил "всё равно идти по SNI"), то DNS резолвится **на стороне xray/sing-box**, а не на стороне Telegram-клиента
---
## Ключевая идея: куда на самом деле идёт соединение решает не то, что вы указали клиенту, а то как L7-роутер трактует SNI
Механика:
1) Telegram-клиенту вы можете указать **IP/домен telemt**,как "сервер".
2) Между клиентом и telemt стоит xray-core/sing-box, который принимает TCP, читает TLS ClientHello и видит **SNI=petrovich.ru**
3) Дальше роутер говорит: "Вижу SNI - направить на апстрим/маршрут N"
4) И устанавливает исходящее соединение не "по тому IP, который пользователь подразумевал", а **по домену из SNI** (или по сопоставлению SNI→outbound), используя для определния его IP собственный DNS-кеш или резолвер
5) `petrovich.ru` по A-записи указывает **не на IP telemt**, а значит при L7-маршрутизации трафик уйдёт на "оригинальный" сайт за этим доменом, а не в telemt: Telegram-клиент, естественно, не сможет получить ожидаемое поведение, потому что ответить с handshake на той стороне некому
---
## Схема №1 "Как это НЕ работает"
```text
Telegram Client
|
| (указан IP/домен telemt)
v
telemt instance
````
Ожидание: "я указал telemt -> значит трафик попадёт в telemt" - **нет!**
---
## Схема №2. "Как это реально работает с TLS/L7-роутером и SNI"
```text
Telegram Client
|
| 1) TCP/TLS connection:
| - ClientHello:
| - SNI=petrovich.ru
v
xray-core / sing-box / любой L7 router
|
| 2) читает ClientHello -> вытаскивает SNI
| 3) выбирает маршрут по SNI
| 4) делает DNS для petrovich.ru
| 5) подключается к полученному IP по TLS с этим SNI
v
"Оригинальный" сайт, A-запись которого не на telemt
|
X не telemt -> Telegram-клиент не коннектится как ожидалось
```
---
## Почему указанный в клиенте IP/домен telemt "не спасает"
Потому что в таком режиме xray/sing-box выступает как **точка терминации TCP/TLS**, можно сказать - TLS-инспектор на уровне ClientHello, это означает:
* TCP-сессия от Telegram-клиента заканчивается на xray/sing-box
* Дальше создаётся **новая** TCP-сессия "от имени" xray/sing-box к апстриму
* Выбор апстрима делается правилами роутинга, а в TLS-сценариях самый удобный и распространённый ключ — **SNI**
То есть, "куда идти дальше" определяется логикой L7-роутера:
* либо правилами вида `if SNI == petrovich.ru -> outbound X`,
* либо более "автоматическим" поведением: `подключаться к тому хосту, который указан в SNI`,
* плюс кэш DNS и собственные резолверы роутера
---
## Что именно извлекается из TLS ClientHello и почему этого достаточно
TLS ClientHello отправляется **в начале** TLS-сессии и, в классическом TLS без ECH, содержит SNI в открытом виде.
Упрощённо:
```text
ClientHello:
- supported_versions
- cipher_suites
- extensions:
- server_name: petrovich.ru <-- SNI
- alpn: h2/http1.1/...
- ...
```
Роутеру не нужно расшифровывать трафик и завершать TLS "как сервер" — часто достаточно просто прочитать первые пакеты и распарсить ClientHello, чтобы получить SNI и принять решение
---
## Типовой алгоритм SNI-роутинга
1. Принять входящий TCP.
2. Подождать первые байты.
3. Определить протокол:
* если видим TLS ClientHello → парсим SNI/ALPN
4. Применить route rules:
* match по `server_name` / `domain` / `tls.sni`
5. Выбрать outbound:
* direct / proxy / specific upstream / detour
6. Установить исходящее соединение:
* либо на фиксированный IP:порт,
* либо на домен через DNS-resolve на стороне роутера
7. Начать проксирование данных между входом и выходом
---
## Почему "A-запись фронтинг-домена не на telemt" ломает кейс
### Ситуация
* В ClientHello: `SNI = petrovich.ru`
* DNS: `petrovich.ru -> 203.0.113.77` - "оригинальный" сайт
* telemt живёт на: `198.51.100.10`
### Что делает роутер
* Видит SNI `petrovich.ru`
* Либо:
* (а) напрямую коннектится к `petrovich.ru:443`, резолвя A-запись в `203.0.113.77`,
* либо:
* (б) выбирает outbound, который указывает на `petrovich.ru` как destination,
* либо:
* (в) делает sniffing/override destination по SNI
В итоге исходящий коннект идёт на `203.0.113.77:443`, а не на telemt!
Другой сервер, другой протокол, другая логика, где telemt не участвует
---
## "Где именно происходит подмена destination на SNI"
Это зависит от конфигурации, но типовые варианты:
### Вариант A: outbound задан доменом (и он совпадает с SNI)
Правило по SNI выбирает outbound, у которого destination задан доменом фронтинга,
тогда DNS резолвится на стороне роутера и вы уходите на "оригинальный" хост
### Вариант B: destination override / sniffing
Роутер "снифает" SNI и **перезаписывает** destination на домен из SNI (даже если вход изначально был на IP telemt),
это особенно коварно: пользователь видит "я подключаюсь к IP telemt", но роутер после sniffing решает иначе
### Вариант C: split DNS / кеш / независимый резолвер
Даже если клиент "где-то" резолвит иначе, это не важно: конечный DNS для исходящего коннекта — на стороне xray/sing-box,
который может иметь:
* свой DoH/DoT,
* свой кеш,
* свои правила fake-ip / system resolver,
* и, как следствие, своя "карта" **домен/SNI -> IP**
---
## Признаки того, что трафик "утёк на оригинал", а не попал в telemt
* На стороне telemt отсутствуют входящие соединения/логи
* На стороне роутера видно, что destination — домен фронтинга, а IP соответствует публичному сайту
* TLS-метрики/сертификат на выходе соответствует "оригинальному" сайту в записах трафика
* Telegram-клиент получает неожиданный тип ответов/ошибку handshaking/timeout в debug-режиме
---
## Best-practice решение для этого кейса: свой домен фронтинга + заглушка на telemt + Let's Encrypt
### Цель
Сделать так, чтобы:
* SNI (фронтинг-домен) **резолвился в IP telemt**,
* на IP telemt реально был TLS-сервис с валидным сертификатом под этот домен,
* даже если кто-то "попробует открыть домен как сайт", он увидит нормальную заглушку, а не "пустоту"
### Что это даёт
* xray/sing-box, маршрутизируя по SNI, будет неизбежно приходить на telemt, потому что DNS(SNI-домен) → IP telemt
* Внешний вид будет правдоподобным: обычный домен с обычным сертификатом
* Устойчивость: меньше сюрпризов от DNS-кеша/перерезолва/"умных" правил роутера
---
## Рекомендуемая схема (целевое состояние)
```text
Telegram Client
|
| TLS ClientHello: SNI = hello.example.com
v
xray-core / sing-box
|
| Route by SNI -> outbound -> connect to hello.example.com:443
| DNS(hello.example.com) = IP telemt
v
telemt instance (IP telemt)
|
| TLS cert for hello.example.com (Let's Encrypt)
| + сайт-заглушка / health endpoint
v
OK
```
---
## Практический чеклист (минимальный)
1. Купить/иметь домен: `hello.example.com`
2. В DNS:
* `A hello.example.com -> <IP telemt>`
* (опционально) AAAA, если используете IPv6 и он стабилен
3. На telemt-хосте:
* поднять TLS endpoint на 443 с валидным сертификатом LE под `hello.example.com`
* отдать "заглушку" (например, статический сайт), чтобы домен выглядел как обычный веб-сервис
4. В xray/sing-box правилах:
* маршрутизировать нужный трафик по SNI = `hello.example.com` в "правильный" outbound (к telemt)
* избегать конфигураций, где destination override уводит на чужой домен
5. Важно:
* если вы используете кеш DNS на роутере — сбросить/обновить его после смены A-записи
---
## Пояснение про сайт-заглушку
Для эмуляции TLS, telemt имеет подсистему TLS-F в `src/tls_front`:
- её модуль - fetcher, собирает TLS-профили, чтоб максимально поведенчески корректно повторять TLS конкретно указанного сайта
Когда вы указываете сайт, который не отвечает по TLS:
- fetcher не может собрать TLS-профиль и происходит fallback на `fake_cert_len` - примитивный алгоритм,
- он забивает служебную информацию TLS рандомными байтами,
- простые системы DPI не распознают это
- однако, продвинутые системы, такие как nEdge или Fraud Control в сетях мобильной связи легко заблокируют или замедлят такой трафик
Создав сайт-заглушку с Let's Encrypt сертификатом, вы даёте TLS-F возможность получить данные сертификата и корректно его "повторять" в дальнейшем
---
## Вариант конфиг-подхода: "SNI строго привязываем к telemt - фиксированный IP"
Чтобы полностью исключить зависимость от DNS если вам это нужно, можно сделать outbound, который ходит на **фиксированный IP telemt**, но при этом выставляет SNI/Host как `hello.example.com`.
Идея:
* destination: `IP:443`
* SNI: `hello.example.com`
* сертификат на telemt именно под `hello.example.com`
Так вы получаете:
* TLS выглядит корректно, ведь SNI совпадает с сертификатом,
* а routing никогда не уйдёт на "оригинал", потому что A-запись указывает на telemt и контроллируется вами!
Но в вашем описании проблема как раз в том, что роутер "сам решает по SNI и резолвит домен", поэтому самый универсальный вариант — сделать так, чтобы DNS всегда приводил в telemt
---
## Пример логики правил на псевдоконфиге L7-роутера
```text
if inbound is TLS and sni == "hello.example.com":
route -> outbound "telemt"
else:
route -> outbound "default"
```
Outbound `telemt`:
* destination: `hello.example.com:443`
* TLS enabled
* SNI: `hello.example.com`
---
## Отдельно: что может неожиданно сломать даже "правильный" DNS
* **Кеширование DNS** на xray/sing-box или на системном резолвере, особенно при смене A-записи
* **Split-horizon DNS**: разные ответы внутри/снаружи, попытки подмены/терминирования в других точках
* **IPv6**: если есть AAAA и он указывает не туда, роутер может предпочесть IPv6: помните, что поддержка v6 нестабильна и не рекомендуется в prod
* **DoH/DoT** на роутере: он может резолвить не тем резолвером, которым вы проверяли
Минимальная гигиена:
* контролировать A/AAAA,
* держать TTL разумным,
* проверять, каким резолвером пользуется именно роутер,
* при необходимости отключить/ограничить destination override
---
## Итог
В режиме TLS-fronting с xray-core/sing-box как L7/TLS-роутером **SNI становится приоритетным "source-of-truth" для маршрутизации**
Если фронтинг-домен по DNS указывает не на IP telemt, роутер честно уводит трафик на "оригинальный" сайт, потому что он строит исходящее соединение "по SNI"
Надёжное решение для этого кейса:
* использовать **свой домен** для фронтинга,
* направить его **A/AAAA** на IP telemt,
* поднять на telemt **TLS-сервис с Lets Encrypt сертификатом** под этот домен,
* (желательно) держать **сайт-заглушку**, чтобы 443 выглядел как обычный HTTPS

73
install.sh Normal file
View File

@@ -0,0 +1,73 @@
sudo bash -c '
set -e
# --- Проверка на существующую установку ---
if systemctl list-unit-files | grep -q telemt.service; then
# --- РЕЖИМ ОБНОВЛЕНИЯ ---
echo "--- Обнаружена существующая установка Telemt. Запускаю обновление... ---"
echo "[*] Остановка службы telemt..."
systemctl stop telemt || true # Игнорируем ошибку, если служба уже остановлена
echo "[1/2] Скачивание последней версии Telemt..."
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
echo "[1/2] Замена исполняемого файла в /usr/local/bin..."
mv telemt /usr/local/bin/telemt
chmod +x /usr/local/bin/telemt
echo "[2/2] Запуск службы..."
systemctl start telemt
echo "--- Обновление Telemt успешно завершено! ---"
echo
echo "Для проверки статуса службы выполните:"
echo " systemctl status telemt"
else
# --- РЕЖИМ НОВОЙ УСТАНОВКИ ---
echo "--- Начало автоматической установки Telemt ---"
# Шаг 1: Скачивание и установка бинарного файла
echo "[1/5] Скачивание последней версии Telemt..."
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
echo "[1/5] Перемещение исполняемого файла в /usr/local/bin и установка прав..."
mv telemt /usr/local/bin/telemt
chmod +x /usr/local/bin/telemt
# Шаг 2: Генерация секрета
echo "[2/5] Генерация секретного ключа..."
SECRET=$(openssl rand -hex 16)
# Шаг 3: Создание файла конфигурации
echo "[3/5] Создание файла конфигурации /etc/telemt.toml..."
printf "# === General Settings ===\n[general]\n[general.modes]\nclassic = false\nsecure = false\ntls = true\n\n# === Anti-Censorship & Masking ===\n[censorship]\n# !!! ВАЖНО: Замените на ваш домен или домен, который вы хотите использовать для маскировки !!!\ntls_domain = \"petrovich.ru\"\n\n[access.users]\nhello = \"%s\"\n" "$SECRET" > /etc/telemt.toml
# Шаг 4: Создание службы Systemd
echo "[4/5] Создание службы systemd..."
printf "[Unit]\nDescription=Telemt Proxy\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/telemt /etc/telemt.toml\nRestart=on-failure\nRestartSec=5\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/telemt.service
# Шаг 5: Запуск службы
echo "[5/5] Перезагрузка systemd, запуск и включение службы telemt..."
systemctl daemon-reload
systemctl start telemt
systemctl enable telemt
echo "--- Установка и запуск Telemt успешно завершены! ---"
echo
echo "ВАЖНАЯ ИНФОРМАЦИЯ:"
echo "==================="
echo "1. Вам НЕОБХОДИМО отредактировать файл /etc/telemt.toml и заменить '\''petrovich.ru'\'' на другой домен"
echo " с помощью команды:"
echo " nano /etc/telemt.toml"
echo " После редактирования файла перезапустите службу командой:"
echo " sudo systemctl restart telemt"
echo
echo "2. Для проверки статуса службы выполните команду:"
echo " systemctl status telemt"
echo
echo "3. Для получения ссылок на подключение выполните команду:"
echo " journalctl -u telemt -n -g '\''links'\'' --no-pager -o cat | tac"
fi
'

View File

@@ -8,6 +8,9 @@ const DEFAULT_STUN_TCP_FALLBACK: bool = true;
const DEFAULT_MIDDLE_PROXY_WARM_STANDBY: usize = 16;
const DEFAULT_ME_RECONNECT_MAX_CONCURRENT_PER_DC: u32 = 8;
const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16;
const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2;
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3;
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4;
const DEFAULT_LISTEN_ADDR_IPV6: &str = "::";
const DEFAULT_ACCESS_USER: &str = "default";
const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000";
@@ -158,6 +161,42 @@ pub(crate) fn default_me_reconnect_fast_retry_count() -> u32 {
DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT
}
pub(crate) fn default_me_single_endpoint_shadow_writers() -> u8 {
DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS
}
pub(crate) fn default_me_single_endpoint_outage_mode_enabled() -> bool {
true
}
pub(crate) fn default_me_single_endpoint_outage_disable_quarantine() -> bool {
true
}
pub(crate) fn default_me_single_endpoint_outage_backoff_min_ms() -> u64 {
250
}
pub(crate) fn default_me_single_endpoint_outage_backoff_max_ms() -> u64 {
3000
}
pub(crate) fn default_me_single_endpoint_shadow_rotate_every_secs() -> u64 {
900
}
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
}
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
250
}
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
}
pub(crate) fn default_crypto_pending_buffer() -> usize {
256 * 1024
}
@@ -170,6 +209,18 @@ pub(crate) fn default_desync_all_full() -> bool {
false
}
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
25
}
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
120
}
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
80
}
pub(crate) fn default_beobachten_minutes() -> u64 {
10
}
@@ -251,6 +302,18 @@ pub(crate) fn default_me_reinit_every_secs() -> u64 {
15 * 60
}
pub(crate) fn default_me_reinit_singleflight() -> bool {
true
}
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
64
}
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
200
}
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
1000
}
@@ -275,6 +338,18 @@ pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
300
}
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
true
}
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
true
}
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
1
}
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
2
}
@@ -283,6 +358,10 @@ pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
true
}
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
true
}
pub(crate) fn default_proxy_secret_len_max() -> usize {
256
}
@@ -295,10 +374,18 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
90
}
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
default_me_pool_drain_ttl_secs()
}
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
0.8
}
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
true
}
pub(crate) fn default_hardswap() -> bool {
true
}

View File

@@ -4,19 +4,22 @@
//!
//! # What can be reloaded without restart
//!
//! | Section | Field | Effect |
//! |-----------|-------------------------------|-----------------------------------|
//! | `general` | `log_level` | Filter updated via `log_level_tx` |
//! | `general` | `ad_tag` | Passed on next connection |
//! | `general` | `middle_proxy_pool_size` | Passed on next connection |
//! | `general` | `me_keepalive_*` | Passed on next connection |
//! | `general` | `desync_all_full` | Applied immediately |
//! | `general` | `update_every` | Applied to ME updater immediately |
//! | `general` | `hardswap` | Applied on next ME map update |
//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update |
//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update |
//! | `general` | `me_reinit_drain_timeout_secs`| Applied on next ME map update |
//! | `access` | All user/quota fields | Effective immediately |
//! | Section | Field | Effect |
//! |-----------|--------------------------------|------------------------------------------------|
//! | `general` | `log_level` | Filter updated via `log_level_tx` |
//! | `access` | `user_ad_tags` | Passed on next connection |
//! | `general` | `ad_tag` | Passed on next connection (fallback per-user) |
//! | `general` | `middle_proxy_pool_size` | Passed on next connection |
//! | `general` | `me_keepalive_*` | Passed on next connection |
//! | `general` | `desync_all_full` | Applied immediately |
//! | `general` | `update_every` | Applied to ME updater immediately |
//! | `general` | `hardswap` | Applied on next ME map update |
//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update |
//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update |
//! | `general` | `me_reinit_drain_timeout_secs` | Applied on next ME map update |
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
//! | `network` | `dns_overrides` | Applied immediately |
//! | `access` | All user/quota fields | Effective immediately |
//!
//! Fields that require re-binding sockets (`server.port`, `censorship.*`,
//! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted.
@@ -29,7 +32,7 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher};
use tokio::sync::{mpsc, watch};
use tracing::{error, info, warn};
use crate::config::LogLevel;
use crate::config::{LogLevel, MeSocksKdfPolicy, MeTelemetryLevel};
use super::load::ProxyConfig;
// ── Hot fields ────────────────────────────────────────────────────────────────
@@ -39,6 +42,7 @@ use super::load::ProxyConfig;
pub struct HotFields {
pub log_level: LogLevel,
pub ad_tag: Option<String>,
pub dns_overrides: Vec<String>,
pub middle_proxy_pool_size: usize,
pub desync_all_full: bool,
pub update_every_secs: u64,
@@ -50,6 +54,13 @@ pub struct HotFields {
pub me_keepalive_interval_secs: u64,
pub me_keepalive_jitter_secs: u64,
pub me_keepalive_payload_random: bool,
pub telemetry_core_enabled: bool,
pub telemetry_user_enabled: bool,
pub telemetry_me_level: MeTelemetryLevel,
pub me_socks_kdf_policy: MeSocksKdfPolicy,
pub me_route_backpressure_base_timeout_ms: u64,
pub me_route_backpressure_high_timeout_ms: u64,
pub me_route_backpressure_high_watermark_pct: u8,
pub access: crate::config::AccessConfig,
}
@@ -58,6 +69,7 @@ impl HotFields {
Self {
log_level: cfg.general.log_level.clone(),
ad_tag: cfg.general.ad_tag.clone(),
dns_overrides: cfg.network.dns_overrides.clone(),
middle_proxy_pool_size: cfg.general.middle_proxy_pool_size,
desync_all_full: cfg.general.desync_all_full,
update_every_secs: cfg.general.effective_update_every_secs(),
@@ -69,6 +81,13 @@ impl HotFields {
me_keepalive_interval_secs: cfg.general.me_keepalive_interval_secs,
me_keepalive_jitter_secs: cfg.general.me_keepalive_jitter_secs,
me_keepalive_payload_random: cfg.general.me_keepalive_payload_random,
telemetry_core_enabled: cfg.general.telemetry.core_enabled,
telemetry_user_enabled: cfg.general.telemetry.user_enabled,
telemetry_me_level: cfg.general.telemetry.me_level,
me_socks_kdf_policy: cfg.general.me_socks_kdf_policy,
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
access: cfg.access.clone(),
}
}
@@ -99,6 +118,14 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) {
if old.general.stun_nat_probe_concurrency != new.general.stun_nat_probe_concurrency {
warn!("config reload: general.stun_nat_probe_concurrency changed; restart required");
}
if old.general.upstream_connect_retry_attempts != new.general.upstream_connect_retry_attempts
|| old.general.upstream_connect_retry_backoff_ms
!= new.general.upstream_connect_retry_backoff_ms
|| old.general.upstream_unhealthy_fail_threshold
!= new.general.upstream_unhealthy_fail_threshold
{
warn!("config reload: general.upstream_* changed; restart required");
}
}
/// Resolve the public host for link generation — mirrors the logic in main.rs.
@@ -181,11 +208,21 @@ fn log_changes(
log_tx.send(new_hot.log_level.clone()).ok();
}
if old_hot.ad_tag != new_hot.ad_tag {
if old_hot.access.user_ad_tags != new_hot.access.user_ad_tags {
info!(
"config reload: ad_tag: {} → {}",
old_hot.ad_tag.as_deref().unwrap_or("none"),
new_hot.ad_tag.as_deref().unwrap_or("none"),
"config reload: user_ad_tags updated ({} entries)",
new_hot.access.user_ad_tags.len(),
);
}
if old_hot.ad_tag != new_hot.ad_tag {
info!("config reload: general.ad_tag updated (applied on next connection)");
}
if old_hot.dns_overrides != new_hot.dns_overrides {
info!(
"config reload: network.dns_overrides updated ({} entries)",
new_hot.dns_overrides.len()
);
}
@@ -252,6 +289,41 @@ fn log_changes(
);
}
if old_hot.telemetry_core_enabled != new_hot.telemetry_core_enabled
|| old_hot.telemetry_user_enabled != new_hot.telemetry_user_enabled
|| old_hot.telemetry_me_level != new_hot.telemetry_me_level
{
info!(
"config reload: telemetry: core_enabled={} user_enabled={} me_level={}",
new_hot.telemetry_core_enabled,
new_hot.telemetry_user_enabled,
new_hot.telemetry_me_level,
);
}
if old_hot.me_socks_kdf_policy != new_hot.me_socks_kdf_policy {
info!(
"config reload: me_socks_kdf_policy: {:?} → {:?}",
old_hot.me_socks_kdf_policy,
new_hot.me_socks_kdf_policy,
);
}
if old_hot.me_route_backpressure_base_timeout_ms
!= new_hot.me_route_backpressure_base_timeout_ms
|| old_hot.me_route_backpressure_high_timeout_ms
!= new_hot.me_route_backpressure_high_timeout_ms
|| old_hot.me_route_backpressure_high_watermark_pct
!= new_hot.me_route_backpressure_high_watermark_pct
{
info!(
"config reload: me_route_backpressure: base={}ms high={}ms watermark={}%",
new_hot.me_route_backpressure_base_timeout_ms,
new_hot.me_route_backpressure_high_timeout_ms,
new_hot.me_route_backpressure_high_watermark_pct,
);
}
if old_hot.access.users != new_hot.access.users {
let mut added: Vec<&String> = new_hot.access.users.keys()
.filter(|u| !old_hot.access.users.contains_key(*u))
@@ -354,6 +426,16 @@ fn reload_config(
return;
}
if old_hot.dns_overrides != new_hot.dns_overrides
&& let Err(e) = crate::network::dns_overrides::install_entries(&new_hot.dns_overrides)
{
error!(
"config reload: invalid network.dns_overrides: {}; keeping old config",
e
);
return;
}
warn_non_hot_changes(&old_cfg, &new_cfg);
log_changes(&old_hot, &new_hot, &new_cfg, log_tx, detected_ip_v4, detected_ip_v6);
config_tx.send(Arc::new(new_cfg)).ok();

View File

@@ -75,6 +75,23 @@ fn push_unique_nonempty(target: &mut Vec<String>, value: String) {
}
}
fn is_valid_ad_tag(tag: &str) -> bool {
tag.len() == 32 && tag.chars().all(|ch| ch.is_ascii_hexdigit())
}
fn sanitize_ad_tag(ad_tag: &mut Option<String>) {
let Some(tag) = ad_tag.as_ref() else {
return;
};
if !is_valid_ad_tag(tag) {
warn!(
"Invalid general.ad_tag value, expected exactly 32 hex chars; ad_tag is disabled"
);
*ad_tag = None;
}
}
// ============= Main Config =============
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -184,6 +201,8 @@ impl ProxyConfig {
}
}
sanitize_ad_tag(&mut config.general.ad_tag);
if let Some(update_every) = config.general.update_every {
if update_every == 0 {
return Err(ProxyError::Config(
@@ -218,12 +237,50 @@ impl ProxyConfig {
));
}
if config.general.upstream_connect_retry_attempts == 0 {
return Err(ProxyError::Config(
"general.upstream_connect_retry_attempts must be > 0".to_string(),
));
}
if config.general.upstream_unhealthy_fail_threshold == 0 {
return Err(ProxyError::Config(
"general.upstream_unhealthy_fail_threshold must be > 0".to_string(),
));
}
if config.general.me_reinit_every_secs == 0 {
return Err(ProxyError::Config(
"general.me_reinit_every_secs must be > 0".to_string(),
));
}
if config.general.me_single_endpoint_shadow_writers > 32 {
return Err(ProxyError::Config(
"general.me_single_endpoint_shadow_writers must be within [0, 32]".to_string(),
));
}
if config.general.me_single_endpoint_outage_backoff_min_ms == 0 {
return Err(ProxyError::Config(
"general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(),
));
}
if config.general.me_single_endpoint_outage_backoff_max_ms == 0 {
return Err(ProxyError::Config(
"general.me_single_endpoint_outage_backoff_max_ms must be > 0".to_string(),
));
}
if config.general.me_single_endpoint_outage_backoff_min_ms
> config.general.me_single_endpoint_outage_backoff_max_ms
{
return Err(ProxyError::Config(
"general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms".to_string(),
));
}
if config.general.beobachten_minutes == 0 {
return Err(ProxyError::Config(
"general.beobachten_minutes must be > 0".to_string(),
@@ -274,12 +331,24 @@ impl ProxyConfig {
));
}
if config.general.me_snapshot_min_proxy_for_lines == 0 {
return Err(ProxyError::Config(
"general.me_snapshot_min_proxy_for_lines must be > 0".to_string(),
));
}
if config.general.proxy_secret_stable_snapshots == 0 {
return Err(ProxyError::Config(
"general.proxy_secret_stable_snapshots must be > 0".to_string(),
));
}
if config.general.me_reinit_trigger_channel == 0 {
return Err(ProxyError::Config(
"general.me_reinit_trigger_channel must be > 0".to_string(),
));
}
if !(32..=4096).contains(&config.general.proxy_secret_len_max) {
return Err(ProxyError::Config(
"general.proxy_secret_len_max must be within [32, 4096]".to_string(),
@@ -292,6 +361,26 @@ impl ProxyConfig {
));
}
if config.general.me_route_backpressure_base_timeout_ms == 0 {
return Err(ProxyError::Config(
"general.me_route_backpressure_base_timeout_ms must be > 0".to_string(),
));
}
if config.general.me_route_backpressure_high_timeout_ms
< config.general.me_route_backpressure_base_timeout_ms
{
return Err(ProxyError::Config(
"general.me_route_backpressure_high_timeout_ms must be >= general.me_route_backpressure_base_timeout_ms".to_string(),
));
}
if !(1..=100).contains(&config.general.me_route_backpressure_high_watermark_pct) {
return Err(ProxyError::Config(
"general.me_route_backpressure_high_watermark_pct must be within [1, 100]".to_string(),
));
}
if config.general.effective_me_pool_force_close_secs() > 0
&& config.general.effective_me_pool_force_close_secs()
< config.general.me_pool_drain_ttl_secs
@@ -380,6 +469,7 @@ impl ProxyConfig {
}
validate_network_cfg(&mut config.network)?;
crate::network::dns_overrides::validate_entries(&config.network.dns_overrides)?;
if config.general.use_middle_proxy && config.network.ipv6 == Some(true) {
warn!("IPv6 with Middle Proxy is experimental and may cause KDF address mismatch; consider disabling IPv6 or ME");
@@ -480,16 +570,21 @@ impl ProxyConfig {
)));
}
if let Some(tag) = &self.general.ad_tag {
for (user, tag) in &self.access.user_ad_tags {
let zeros = "00000000000000000000000000000000";
if tag == zeros {
warn!("ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel");
if !is_valid_ad_tag(tag) {
return Err(ProxyError::Config(format!(
"access.user_ad_tags['{}'] must be exactly 32 hex characters",
user
)));
}
if tag.len() != 32 || tag.chars().any(|c| !c.is_ascii_hexdigit()) {
warn!("ad_tag is not a 32-char hex string; ensure you use value issued by @MTProxybot");
if tag == zeros {
warn!(user = %user, "user ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel");
}
}
crate::network::dns_overrides::validate_entries(&self.network.dns_overrides)?;
Ok(())
}
}
@@ -509,6 +604,7 @@ mod tests {
let cfg: ProxyConfig = toml::from_str(toml).unwrap();
assert_eq!(cfg.network.ipv6, default_network_ipv6());
assert_eq!(cfg.network.stun_use, default_true());
assert_eq!(cfg.network.stun_tcp_fallback, default_stun_tcp_fallback());
assert_eq!(
cfg.general.middle_proxy_warm_standby,
@@ -522,6 +618,42 @@ mod tests {
cfg.general.me_reconnect_fast_retry_count,
default_me_reconnect_fast_retry_count()
);
assert_eq!(
cfg.general.me_single_endpoint_shadow_writers,
default_me_single_endpoint_shadow_writers()
);
assert_eq!(
cfg.general.me_single_endpoint_outage_mode_enabled,
default_me_single_endpoint_outage_mode_enabled()
);
assert_eq!(
cfg.general.me_single_endpoint_outage_disable_quarantine,
default_me_single_endpoint_outage_disable_quarantine()
);
assert_eq!(
cfg.general.me_single_endpoint_outage_backoff_min_ms,
default_me_single_endpoint_outage_backoff_min_ms()
);
assert_eq!(
cfg.general.me_single_endpoint_outage_backoff_max_ms,
default_me_single_endpoint_outage_backoff_max_ms()
);
assert_eq!(
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
default_me_single_endpoint_shadow_rotate_every_secs()
);
assert_eq!(
cfg.general.upstream_connect_retry_attempts,
default_upstream_connect_retry_attempts()
);
assert_eq!(
cfg.general.upstream_connect_retry_backoff_ms,
default_upstream_connect_retry_backoff_ms()
);
assert_eq!(
cfg.general.upstream_unhealthy_fail_threshold,
default_upstream_unhealthy_fail_threshold()
);
assert_eq!(cfg.general.update_every, default_update_every());
assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4());
assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt());
@@ -532,6 +664,7 @@ mod tests {
fn impl_defaults_are_sourced_from_default_helpers() {
let network = NetworkConfig::default();
assert_eq!(network.ipv6, default_network_ipv6());
assert_eq!(network.stun_use, default_true());
assert_eq!(network.stun_tcp_fallback, default_stun_tcp_fallback());
let general = GeneralConfig::default();
@@ -547,6 +680,42 @@ mod tests {
general.me_reconnect_fast_retry_count,
default_me_reconnect_fast_retry_count()
);
assert_eq!(
general.me_single_endpoint_shadow_writers,
default_me_single_endpoint_shadow_writers()
);
assert_eq!(
general.me_single_endpoint_outage_mode_enabled,
default_me_single_endpoint_outage_mode_enabled()
);
assert_eq!(
general.me_single_endpoint_outage_disable_quarantine,
default_me_single_endpoint_outage_disable_quarantine()
);
assert_eq!(
general.me_single_endpoint_outage_backoff_min_ms,
default_me_single_endpoint_outage_backoff_min_ms()
);
assert_eq!(
general.me_single_endpoint_outage_backoff_max_ms,
default_me_single_endpoint_outage_backoff_max_ms()
);
assert_eq!(
general.me_single_endpoint_shadow_rotate_every_secs,
default_me_single_endpoint_shadow_rotate_every_secs()
);
assert_eq!(
general.upstream_connect_retry_attempts,
default_upstream_connect_retry_attempts()
);
assert_eq!(
general.upstream_connect_retry_backoff_ms,
default_upstream_connect_retry_backoff_ms()
);
assert_eq!(
general.upstream_unhealthy_fail_threshold,
default_upstream_unhealthy_fail_threshold()
);
assert_eq!(general.update_every, default_update_every());
let server = ServerConfig::default();
@@ -719,6 +888,89 @@ mod tests {
let _ = std::fs::remove_file(path);
}
#[test]
fn me_single_endpoint_outage_backoff_range_is_validated() {
let toml = r#"
[general]
me_single_endpoint_outage_backoff_min_ms = 4000
me_single_endpoint_outage_backoff_max_ms = 3000
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_single_endpoint_outage_backoff_range_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains(
"general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms"
));
let _ = std::fs::remove_file(path);
}
#[test]
fn me_single_endpoint_shadow_writers_too_large_is_rejected() {
let toml = r#"
[general]
me_single_endpoint_shadow_writers = 33
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_single_endpoint_shadow_writers_limit_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.me_single_endpoint_shadow_writers must be within [0, 32]"));
let _ = std::fs::remove_file(path);
}
#[test]
fn upstream_connect_retry_attempts_zero_is_rejected() {
let toml = r#"
[general]
upstream_connect_retry_attempts = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_upstream_connect_retry_attempts_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.upstream_connect_retry_attempts must be > 0"));
let _ = std::fs::remove_file(path);
}
#[test]
fn upstream_unhealthy_fail_threshold_zero_is_rejected() {
let toml = r#"
[general]
upstream_unhealthy_fail_threshold = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_upstream_unhealthy_fail_threshold_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.upstream_unhealthy_fail_threshold must be > 0"));
let _ = std::fs::remove_file(path);
}
#[test]
fn me_hardswap_warmup_defaults_are_set() {
let toml = r#"
@@ -934,4 +1186,108 @@ mod tests {
assert_eq!(cfg.general.me_reinit_drain_timeout_secs, 90);
let _ = std::fs::remove_file(path);
}
#[test]
fn invalid_ad_tag_is_disabled_during_load() {
let toml = r#"
[general]
ad_tag = "not_hex"
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_invalid_ad_tag_test.toml");
std::fs::write(&path, toml).unwrap();
let cfg = ProxyConfig::load(&path).unwrap();
assert!(cfg.general.ad_tag.is_none());
let _ = std::fs::remove_file(path);
}
#[test]
fn valid_ad_tag_is_preserved_during_load() {
let toml = r#"
[general]
ad_tag = "00112233445566778899aabbccddeeff"
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_valid_ad_tag_test.toml");
std::fs::write(&path, toml).unwrap();
let cfg = ProxyConfig::load(&path).unwrap();
assert_eq!(
cfg.general.ad_tag.as_deref(),
Some("00112233445566778899aabbccddeeff")
);
let _ = std::fs::remove_file(path);
}
#[test]
fn invalid_user_ad_tag_reports_access_user_ad_tags_key() {
let toml = r#"
[censorship]
tls_domain = "example.com"
[access.users]
alice = "00000000000000000000000000000000"
[access.user_ad_tags]
alice = "not_hex"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_invalid_user_ad_tag_message_test.toml");
std::fs::write(&path, toml).unwrap();
let cfg = ProxyConfig::load(&path).unwrap();
let err = cfg.validate().unwrap_err().to_string();
assert!(err.contains("access.user_ad_tags['alice'] must be exactly 32 hex characters"));
let _ = std::fs::remove_file(path);
}
#[test]
fn invalid_dns_override_is_rejected() {
let toml = r#"
[network]
dns_overrides = ["example.com:443:2001:db8::10"]
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_invalid_dns_override_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("must be bracketed"));
let _ = std::fs::remove_file(path);
}
#[test]
fn valid_dns_override_is_accepted() {
let toml = r#"
[network]
dns_overrides = ["example.com:443:127.0.0.1", "example.net:443:[2001:db8::10]"]
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_valid_dns_override_test.toml");
std::fs::write(&path, toml).unwrap();
let cfg = ProxyConfig::load(&path).unwrap();
assert_eq!(cfg.network.dns_overrides.len(), 2);
let _ = std::fs::remove_file(path);
}
}

View File

@@ -59,6 +59,126 @@ impl std::fmt::Display for LogLevel {
}
}
/// Middle-End telemetry verbosity level.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum MeTelemetryLevel {
#[default]
Normal,
Silent,
Debug,
}
impl MeTelemetryLevel {
pub fn as_u8(self) -> u8 {
match self {
MeTelemetryLevel::Silent => 0,
MeTelemetryLevel::Normal => 1,
MeTelemetryLevel::Debug => 2,
}
}
pub fn from_u8(raw: u8) -> Self {
match raw {
0 => MeTelemetryLevel::Silent,
2 => MeTelemetryLevel::Debug,
_ => MeTelemetryLevel::Normal,
}
}
pub fn allows_normal(self) -> bool {
!matches!(self, MeTelemetryLevel::Silent)
}
pub fn allows_debug(self) -> bool {
matches!(self, MeTelemetryLevel::Debug)
}
}
impl std::fmt::Display for MeTelemetryLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
MeTelemetryLevel::Silent => write!(f, "silent"),
MeTelemetryLevel::Normal => write!(f, "normal"),
MeTelemetryLevel::Debug => write!(f, "debug"),
}
}
}
/// Middle-End SOCKS KDF fallback policy.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum MeSocksKdfPolicy {
#[default]
Strict,
Compat,
}
impl MeSocksKdfPolicy {
pub fn as_u8(self) -> u8 {
match self {
MeSocksKdfPolicy::Strict => 0,
MeSocksKdfPolicy::Compat => 1,
}
}
pub fn from_u8(raw: u8) -> Self {
match raw {
1 => MeSocksKdfPolicy::Compat,
_ => MeSocksKdfPolicy::Strict,
}
}
}
/// Stale ME writer bind policy during drain window.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
pub enum MeBindStaleMode {
Never,
#[default]
Ttl,
Always,
}
impl MeBindStaleMode {
pub fn as_u8(self) -> u8 {
match self {
MeBindStaleMode::Never => 0,
MeBindStaleMode::Ttl => 1,
MeBindStaleMode::Always => 2,
}
}
pub fn from_u8(raw: u8) -> Self {
match raw {
0 => MeBindStaleMode::Never,
2 => MeBindStaleMode::Always,
_ => MeBindStaleMode::Ttl,
}
}
}
/// Telemetry controls for hot-path counters and ME diagnostics.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TelemetryConfig {
#[serde(default = "default_true")]
pub core_enabled: bool,
#[serde(default = "default_true")]
pub user_enabled: bool,
#[serde(default)]
pub me_level: MeTelemetryLevel,
}
impl Default for TelemetryConfig {
fn default() -> Self {
Self {
core_enabled: default_true(),
user_enabled: default_true(),
me_level: MeTelemetryLevel::Normal,
}
}
}
// ============= Sub-Configs =============
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -97,6 +217,11 @@ pub struct NetworkConfig {
#[serde(default)]
pub multipath: bool,
/// Global switch for STUN probing.
/// When false, STUN is fully disabled and only non-STUN detection remains.
#[serde(default = "default_true")]
pub stun_use: bool,
/// STUN servers list for public IP discovery.
#[serde(default = "default_stun_servers")]
pub stun_servers: Vec<String>,
@@ -112,6 +237,11 @@ pub struct NetworkConfig {
/// Cache file path for detected public IP.
#[serde(default = "default_cache_public_ip_path")]
pub cache_public_ip_path: String,
/// Runtime DNS overrides in `host:port:ip` format.
/// IPv6 IP values must be bracketed: `[2001:db8::1]`.
#[serde(default)]
pub dns_overrides: Vec<String>,
}
impl Default for NetworkConfig {
@@ -121,10 +251,12 @@ impl Default for NetworkConfig {
ipv6: default_network_ipv6(),
prefer: default_prefer_4(),
multipath: false,
stun_use: default_true(),
stun_servers: default_stun_servers(),
stun_tcp_fallback: default_stun_tcp_fallback(),
http_ip_detect_urls: default_http_ip_detect_urls(),
cache_public_ip_path: default_cache_public_ip_path(),
dns_overrides: Vec::new(),
}
}
}
@@ -143,14 +275,15 @@ pub struct GeneralConfig {
#[serde(default = "default_true")]
pub use_middle_proxy: bool,
#[serde(default)]
pub ad_tag: Option<String>,
/// Path to proxy-secret binary file (auto-downloaded if absent).
/// Infrastructure secret from https://core.telegram.org/getProxySecret.
#[serde(default = "default_proxy_secret_path")]
pub proxy_secret_path: Option<String>,
/// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags.
#[serde(default)]
pub ad_tag: Option<String>,
/// Public IP override for middle-proxy NAT environments.
/// When set, this IP is used in ME key derivation and RPC_PROXY_REQ "our_addr".
#[serde(default)]
@@ -261,6 +394,43 @@ pub struct GeneralConfig {
#[serde(default = "default_me_reconnect_fast_retry_count")]
pub me_reconnect_fast_retry_count: u32,
/// Number of additional reserve writers for DC groups with exactly one endpoint.
#[serde(default = "default_me_single_endpoint_shadow_writers")]
pub me_single_endpoint_shadow_writers: u8,
/// Enable aggressive outage recovery mode for single-endpoint DC groups.
#[serde(default = "default_me_single_endpoint_outage_mode_enabled")]
pub me_single_endpoint_outage_mode_enabled: bool,
/// Ignore endpoint quarantine while in single-endpoint outage mode.
#[serde(default = "default_me_single_endpoint_outage_disable_quarantine")]
pub me_single_endpoint_outage_disable_quarantine: bool,
/// Minimum reconnect backoff in ms for single-endpoint outage mode.
#[serde(default = "default_me_single_endpoint_outage_backoff_min_ms")]
pub me_single_endpoint_outage_backoff_min_ms: u64,
/// Maximum reconnect backoff in ms for single-endpoint outage mode.
#[serde(default = "default_me_single_endpoint_outage_backoff_max_ms")]
pub me_single_endpoint_outage_backoff_max_ms: u64,
/// Periodic shadow writer rotation interval in seconds for single-endpoint DC groups.
/// Set to 0 to disable periodic shadow rotation.
#[serde(default = "default_me_single_endpoint_shadow_rotate_every_secs")]
pub me_single_endpoint_shadow_rotate_every_secs: u64,
/// Connect attempts for the selected upstream before returning error/fallback.
#[serde(default = "default_upstream_connect_retry_attempts")]
pub upstream_connect_retry_attempts: u32,
/// Delay in milliseconds between upstream connect attempts.
#[serde(default = "default_upstream_connect_retry_backoff_ms")]
pub upstream_connect_retry_backoff_ms: u64,
/// Consecutive failed requests before upstream is marked unhealthy.
#[serde(default = "default_upstream_unhealthy_fail_threshold")]
pub upstream_unhealthy_fail_threshold: u32,
/// Ignore STUN/interface IP mismatch (keep using Middle Proxy even if NAT detected).
#[serde(default)]
pub stun_iface_mismatch_ignore: bool,
@@ -276,6 +446,26 @@ pub struct GeneralConfig {
#[serde(default)]
pub disable_colors: bool,
/// Runtime telemetry controls for counters/metrics in hot paths.
#[serde(default)]
pub telemetry: TelemetryConfig,
/// SOCKS-bound KDF policy for Middle-End handshake.
#[serde(default)]
pub me_socks_kdf_policy: MeSocksKdfPolicy,
/// Base backpressure timeout in milliseconds for ME route channel send.
#[serde(default = "default_me_route_backpressure_base_timeout_ms")]
pub me_route_backpressure_base_timeout_ms: u64,
/// High backpressure timeout in milliseconds when queue occupancy is above watermark.
#[serde(default = "default_me_route_backpressure_high_timeout_ms")]
pub me_route_backpressure_high_timeout_ms: u64,
/// Queue occupancy percent threshold for high backpressure timeout.
#[serde(default = "default_me_route_backpressure_high_watermark_pct")]
pub me_route_backpressure_high_watermark_pct: u8,
/// [general.links] — proxy link generation overrides.
#[serde(default)]
pub links: LinksConfig,
@@ -317,6 +507,18 @@ pub struct GeneralConfig {
#[serde(default = "default_me_config_apply_cooldown_secs")]
pub me_config_apply_cooldown_secs: u64,
/// Ensure getProxyConfig snapshots are applied only for 2xx HTTP responses.
#[serde(default = "default_me_snapshot_require_http_2xx")]
pub me_snapshot_require_http_2xx: bool,
/// Reject empty getProxyConfig snapshots instead of marking them applied.
#[serde(default = "default_me_snapshot_reject_empty_map")]
pub me_snapshot_reject_empty_map: bool,
/// Minimum parsed `proxy_for` rows required to accept a snapshot.
#[serde(default = "default_me_snapshot_min_proxy_for_lines")]
pub me_snapshot_min_proxy_for_lines: u32,
/// Number of identical getProxySecret snapshots required before runtime secret rotation.
#[serde(default = "default_proxy_secret_stable_snapshots")]
pub proxy_secret_stable_snapshots: u8,
@@ -325,6 +527,10 @@ pub struct GeneralConfig {
#[serde(default = "default_proxy_secret_rotate_runtime")]
pub proxy_secret_rotate_runtime: bool,
/// Keep key-selector and secret bytes from one snapshot during ME handshake.
#[serde(default = "default_me_secret_atomic_snapshot")]
pub me_secret_atomic_snapshot: bool,
/// Maximum allowed proxy-secret length in bytes for startup and runtime refresh.
#[serde(default = "default_proxy_secret_len_max")]
pub proxy_secret_len_max: usize,
@@ -334,6 +540,14 @@ pub struct GeneralConfig {
#[serde(default = "default_me_pool_drain_ttl_secs")]
pub me_pool_drain_ttl_secs: u64,
/// Policy for new binds on stale draining writers.
#[serde(default)]
pub me_bind_stale_mode: MeBindStaleMode,
/// TTL for stale bind allowance when `me_bind_stale_mode = \"ttl\"`.
#[serde(default = "default_me_bind_stale_ttl_secs")]
pub me_bind_stale_ttl_secs: u64,
/// Minimum desired-DC coverage ratio required before draining stale writers.
/// Range: 0.0..=1.0.
#[serde(default = "default_me_pool_min_fresh_ratio")]
@@ -354,6 +568,22 @@ pub struct GeneralConfig {
#[serde(default = "default_proxy_config_reload_secs")]
pub proxy_config_auto_reload_secs: u64,
/// Serialize ME reinit cycles across all trigger sources.
#[serde(default = "default_me_reinit_singleflight")]
pub me_reinit_singleflight: bool,
/// Trigger queue capacity for reinit scheduler.
#[serde(default = "default_me_reinit_trigger_channel")]
pub me_reinit_trigger_channel: usize,
/// Trigger coalescing window before starting a reinit cycle.
#[serde(default = "default_me_reinit_coalesce_window_ms")]
pub me_reinit_coalesce_window_ms: u64,
/// Deterministic candidate sort for ME writer binding path.
#[serde(default = "default_me_deterministic_writer_sort")]
pub me_deterministic_writer_sort: bool,
/// Enable NTP drift check at startup.
#[serde(default = "default_ntp_check")]
pub ntp_check: bool,
@@ -398,10 +628,24 @@ impl Default for GeneralConfig {
me_reconnect_backoff_base_ms: default_reconnect_backoff_base_ms(),
me_reconnect_backoff_cap_ms: default_reconnect_backoff_cap_ms(),
me_reconnect_fast_retry_count: default_me_reconnect_fast_retry_count(),
me_single_endpoint_shadow_writers: default_me_single_endpoint_shadow_writers(),
me_single_endpoint_outage_mode_enabled: default_me_single_endpoint_outage_mode_enabled(),
me_single_endpoint_outage_disable_quarantine: default_me_single_endpoint_outage_disable_quarantine(),
me_single_endpoint_outage_backoff_min_ms: default_me_single_endpoint_outage_backoff_min_ms(),
me_single_endpoint_outage_backoff_max_ms: default_me_single_endpoint_outage_backoff_max_ms(),
me_single_endpoint_shadow_rotate_every_secs: default_me_single_endpoint_shadow_rotate_every_secs(),
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
stun_iface_mismatch_ignore: false,
unknown_dc_log_path: default_unknown_dc_log_path(),
log_level: LogLevel::Normal,
disable_colors: false,
telemetry: TelemetryConfig::default(),
me_socks_kdf_policy: MeSocksKdfPolicy::Strict,
me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(),
me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(),
me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(),
links: LinksConfig::default(),
crypto_pending_buffer: default_crypto_pending_buffer(),
max_client_frame: default_max_client_frame(),
@@ -420,14 +664,24 @@ impl Default for GeneralConfig {
me_hardswap_warmup_pass_backoff_base_ms: default_me_hardswap_warmup_pass_backoff_base_ms(),
me_config_stable_snapshots: default_me_config_stable_snapshots(),
me_config_apply_cooldown_secs: default_me_config_apply_cooldown_secs(),
me_snapshot_require_http_2xx: default_me_snapshot_require_http_2xx(),
me_snapshot_reject_empty_map: default_me_snapshot_reject_empty_map(),
me_snapshot_min_proxy_for_lines: default_me_snapshot_min_proxy_for_lines(),
proxy_secret_stable_snapshots: default_proxy_secret_stable_snapshots(),
proxy_secret_rotate_runtime: default_proxy_secret_rotate_runtime(),
me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(),
proxy_secret_len_max: default_proxy_secret_len_max(),
me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(),
me_bind_stale_mode: MeBindStaleMode::default(),
me_bind_stale_ttl_secs: default_me_bind_stale_ttl_secs(),
me_pool_min_fresh_ratio: default_me_pool_min_fresh_ratio(),
me_reinit_drain_timeout_secs: default_me_reinit_drain_timeout_secs(),
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
me_reinit_singleflight: default_me_reinit_singleflight(),
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
ntp_check: default_ntp_check(),
ntp_servers: default_ntp_servers(),
auto_degradation_enabled: default_true(),
@@ -663,6 +917,10 @@ pub struct AccessConfig {
#[serde(default = "default_access_users")]
pub users: HashMap<String, String>,
/// Per-user ad_tag (32 hex chars from @MTProxybot).
#[serde(default)]
pub user_ad_tags: HashMap<String, String>,
#[serde(default)]
pub user_max_tcp_conns: HashMap<String, usize>,
@@ -689,6 +947,7 @@ impl Default for AccessConfig {
fn default() -> Self {
Self {
users: default_access_users(),
user_ad_tags: HashMap::new(),
user_max_tcp_conns: HashMap::new(),
user_expirations: HashMap::new(),
user_data_quota: HashMap::new(),

View File

@@ -8,7 +8,7 @@ use std::time::Duration;
use rand::Rng;
use tokio::net::TcpListener;
use tokio::signal;
use tokio::sync::Semaphore;
use tokio::sync::{Semaphore, mpsc};
use tracing::{debug, error, info, warn};
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
#[cfg(unix)]
@@ -36,10 +36,12 @@ use crate::ip_tracker::UserIpTracker;
use crate::network::probe::{decide_network_capabilities, log_probe_result, run_probe};
use crate::proxy::ClientHandler;
use crate::stats::beobachten::BeobachtenStore;
use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{ReplayChecker, Stats};
use crate::stream::BufferPool;
use crate::transport::middle_proxy::{
MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, format_sample_line,
MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, MeReinitTrigger, format_sample_line,
format_me_route,
};
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
use crate::tls_front::TlsFrontCache;
@@ -193,6 +195,11 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
std::process::exit(1);
}
if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) {
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
std::process::exit(1);
}
let has_rust_log = std::env::var("RUST_LOG").is_ok();
let effective_log_level = if cli_silent {
LogLevel::Silent
@@ -254,7 +261,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
warn!("Using default tls_domain. Consider setting a custom domain.");
}
let upstream_manager = Arc::new(UpstreamManager::new(config.upstreams.clone()));
let upstream_manager = Arc::new(UpstreamManager::new(
config.upstreams.clone(),
config.general.upstream_connect_retry_attempts,
config.general.upstream_connect_retry_backoff_ms,
config.general.upstream_unhealthy_fail_threshold,
));
let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
tls_domains.push(config.censorship.tls_domain.clone());
@@ -280,17 +292,20 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
.mask_host
.clone()
.unwrap_or_else(|| config.censorship.tls_domain.clone());
let mask_unix_sock = config.censorship.mask_unix_sock.clone();
let fetch_timeout = Duration::from_secs(5);
let cache_initial = cache.clone();
let domains_initial = tls_domains.clone();
let host_initial = mask_host.clone();
let unix_sock_initial = mask_unix_sock.clone();
let upstream_initial = upstream_manager.clone();
tokio::spawn(async move {
let mut join = tokio::task::JoinSet::new();
for domain in domains_initial {
let cache_domain = cache_initial.clone();
let host_domain = host_initial.clone();
let unix_sock_domain = unix_sock_initial.clone();
let upstream_domain = upstream_initial.clone();
join.spawn(async move {
match crate::tls_front::fetcher::fetch_real_tls(
@@ -300,6 +315,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
fetch_timeout,
Some(upstream_domain),
proxy_protocol,
unix_sock_domain.as_deref(),
)
.await
{
@@ -339,6 +355,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let cache_refresh = cache.clone();
let domains_refresh = tls_domains.clone();
let host_refresh = mask_host.clone();
let unix_sock_refresh = mask_unix_sock.clone();
let upstream_refresh = upstream_manager.clone();
tokio::spawn(async move {
loop {
@@ -350,6 +367,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
for domain in domains_refresh.clone() {
let cache_domain = cache_refresh.clone();
let host_domain = host_refresh.clone();
let unix_sock_domain = unix_sock_refresh.clone();
let upstream_domain = upstream_refresh.clone();
join.spawn(async move {
match crate::tls_front::fetcher::fetch_real_tls(
@@ -359,6 +377,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
fetch_timeout,
Some(upstream_domain),
proxy_protocol,
unix_sock_domain.as_deref(),
)
.await
{
@@ -393,6 +412,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let prefer_ipv6 = decision.prefer_ipv6();
let mut use_middle_proxy = config.general.use_middle_proxy && (decision.ipv4_me || decision.ipv6_me);
let stats = Arc::new(Stats::new());
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
let beobachten = Arc::new(BeobachtenStore::new());
let rng = Arc::new(SecureRandom::new());
@@ -403,6 +423,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
if !config.access.user_max_unique_ips.is_empty() {
info!("IP limits configured for {} users", config.access.user_max_unique_ips.len());
}
if !config.network.dns_overrides.is_empty() {
info!(
"Runtime DNS overrides configured: {} entries",
config.network.dns_overrides.len()
);
}
// Connection concurrency limit
let max_connections = Arc::new(Semaphore::new(10_000));
@@ -417,14 +443,17 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
// =====================================================================
let me_pool: Option<Arc<MePool>> = if use_middle_proxy {
info!("=== Middle Proxy Mode ===");
let me_nat_probe = config.general.middle_proxy_nat_probe && config.network.stun_use;
if config.general.middle_proxy_nat_probe && !config.network.stun_use {
info!("Middle-proxy STUN probing disabled by network.stun_use=false");
}
// ad_tag (proxy_tag) for advertising
let proxy_tag = config.general.ad_tag.as_ref().map(|tag| {
hex::decode(tag).unwrap_or_else(|_| {
warn!("Invalid ad_tag hex, middle proxy ad_tag will be empty");
Vec::new()
})
});
// Global ad_tag (pool default). Used when user has no per-user tag in access.user_ad_tags.
let proxy_tag = config
.general
.ad_tag
.as_ref()
.map(|tag| hex::decode(tag).expect("general.ad_tag must be validated before startup"));
// =============================================================
// CRITICAL: Download Telegram proxy-secret (NOT user secret!)
@@ -484,7 +513,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
proxy_tag,
proxy_secret,
config.general.middle_proxy_nat_ip,
config.general.middle_proxy_nat_probe,
me_nat_probe,
None,
config.network.stun_servers.clone(),
config.general.stun_nat_probe_concurrency,
@@ -495,6 +524,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
cfg_v6.map.clone(),
cfg_v4.default_dc.or(cfg_v6.default_dc),
decision.clone(),
Some(upstream_manager.clone()),
rng.clone(),
stats.clone(),
config.general.me_keepalive_enabled,
@@ -508,6 +538,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
config.general.me_reconnect_backoff_base_ms,
config.general.me_reconnect_backoff_cap_ms,
config.general.me_reconnect_fast_retry_count,
config.general.me_single_endpoint_shadow_writers,
config.general.me_single_endpoint_outage_mode_enabled,
config.general.me_single_endpoint_outage_disable_quarantine,
config.general.me_single_endpoint_outage_backoff_min_ms,
config.general.me_single_endpoint_outage_backoff_max_ms,
config.general.me_single_endpoint_shadow_rotate_every_secs,
config.general.hardswap,
config.general.me_pool_drain_ttl_secs,
config.general.effective_me_pool_force_close_secs(),
@@ -516,6 +552,14 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
config.general.me_hardswap_warmup_delay_max_ms,
config.general.me_hardswap_warmup_extra_passes,
config.general.me_hardswap_warmup_pass_backoff_base_ms,
config.general.me_bind_stale_mode,
config.general.me_bind_stale_ttl_secs,
config.general.me_secret_atomic_snapshot,
config.general.me_deterministic_writer_sort,
config.general.me_socks_kdf_policy,
config.general.me_route_backpressure_base_timeout_ms,
config.general.me_route_backpressure_high_timeout_ms,
config.general.me_route_backpressure_high_watermark_pct,
);
let pool_size = config.general.middle_proxy_pool_size.max(1);
@@ -602,7 +646,15 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} else {
info!(" No ME connectivity");
}
info!(" via direct");
let me_route = format_me_route(
&config.upstreams,
&me_results,
prefer_ipv6,
v4_ok,
v6_ok,
)
.await;
info!(" via {}", me_route);
info!("============================================================");
use std::collections::BTreeMap;
@@ -728,12 +780,14 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
// Background tasks
let um_clone = upstream_manager.clone();
let decision_clone = decision.clone();
let dc_overrides_for_health = config.dc_overrides.clone();
tokio::spawn(async move {
um_clone
.run_health_checks(
prefer_ipv6,
decision_clone.ipv4_dc,
decision_clone.ipv6_dc,
dc_overrides_for_health,
)
.await;
});
@@ -763,6 +817,27 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
detected_ip_v6,
);
let stats_policy = stats.clone();
let mut config_rx_policy = config_rx.clone();
let me_pool_policy = me_pool.clone();
tokio::spawn(async move {
loop {
if config_rx_policy.changed().await.is_err() {
break;
}
let cfg = config_rx_policy.borrow_and_update().clone();
stats_policy.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
if let Some(pool) = &me_pool_policy {
pool.update_runtime_transport_policy(
cfg.general.me_socks_kdf_policy,
cfg.general.me_route_backpressure_base_timeout_ms,
cfg.general.me_route_backpressure_high_timeout_ms,
cfg.general.me_route_backpressure_high_watermark_pct,
);
}
}
});
let beobachten_writer = beobachten.clone();
let config_rx_beobachten = config_rx.clone();
tokio::spawn(async move {
@@ -784,26 +859,43 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
});
if let Some(ref pool) = me_pool {
let pool_clone = pool.clone();
let rng_clone = rng.clone();
let config_rx_clone = config_rx.clone();
let reinit_trigger_capacity = config
.general
.me_reinit_trigger_channel
.max(1);
let (reinit_tx, reinit_rx) = mpsc::channel::<MeReinitTrigger>(reinit_trigger_capacity);
let pool_clone_sched = pool.clone();
let rng_clone_sched = rng.clone();
let config_rx_clone_sched = config_rx.clone();
tokio::spawn(async move {
crate::transport::middle_proxy::me_config_updater(
pool_clone,
rng_clone,
config_rx_clone,
crate::transport::middle_proxy::me_reinit_scheduler(
pool_clone_sched,
rng_clone_sched,
config_rx_clone_sched,
reinit_rx,
)
.await;
});
let pool_clone = pool.clone();
let config_rx_clone = config_rx.clone();
let reinit_tx_updater = reinit_tx.clone();
tokio::spawn(async move {
crate::transport::middle_proxy::me_config_updater(
pool_clone,
config_rx_clone,
reinit_tx_updater,
)
.await;
});
let pool_clone_rot = pool.clone();
let rng_clone_rot = rng.clone();
let config_rx_clone_rot = config_rx.clone();
let reinit_tx_rotation = reinit_tx.clone();
tokio::spawn(async move {
crate::transport::middle_proxy::me_rotation_task(
pool_clone_rot,
rng_clone_rot,
config_rx_clone_rot,
reinit_tx_rotation,
)
.await;
});
@@ -1037,9 +1129,18 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let stats = stats.clone();
let beobachten = beobachten.clone();
let config_rx_metrics = config_rx.clone();
let ip_tracker_metrics = ip_tracker.clone();
let whitelist = config.server.metrics_whitelist.clone();
tokio::spawn(async move {
metrics::serve(port, stats, beobachten, config_rx_metrics, whitelist).await;
metrics::serve(
port,
stats,
beobachten,
ip_tracker_metrics,
config_rx_metrics,
whitelist,
)
.await;
});
}

View File

@@ -1,4 +1,5 @@
use std::convert::Infallible;
use std::collections::{BTreeSet, HashMap};
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
@@ -13,6 +14,7 @@ use tokio::net::TcpListener;
use tracing::{info, warn, debug};
use crate::config::ProxyConfig;
use crate::ip_tracker::UserIpTracker;
use crate::stats::beobachten::BeobachtenStore;
use crate::stats::Stats;
@@ -20,6 +22,7 @@ pub async fn serve(
port: u16,
stats: Arc<Stats>,
beobachten: Arc<BeobachtenStore>,
ip_tracker: Arc<UserIpTracker>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Vec<IpNetwork>,
) {
@@ -49,13 +52,15 @@ pub async fn serve(
let stats = stats.clone();
let beobachten = beobachten.clone();
let ip_tracker = ip_tracker.clone();
let config_rx_conn = config_rx.clone();
tokio::spawn(async move {
let svc = service_fn(move |req| {
let stats = stats.clone();
let beobachten = beobachten.clone();
let ip_tracker = ip_tracker.clone();
let config = config_rx_conn.borrow().clone();
async move { handle(req, &stats, &beobachten, &config) }
async move { handle(req, &stats, &beobachten, &ip_tracker, &config).await }
});
if let Err(e) = http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
@@ -67,14 +72,15 @@ pub async fn serve(
}
}
fn handle<B>(
async fn handle<B>(
req: Request<B>,
stats: &Stats,
beobachten: &BeobachtenStore,
ip_tracker: &UserIpTracker,
config: &ProxyConfig,
) -> Result<Response<Full<Bytes>>, Infallible> {
if req.uri().path() == "/metrics" {
let body = render_metrics(stats);
let body = render_metrics(stats, config, ip_tracker).await;
let resp = Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
@@ -109,123 +115,599 @@ fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> Stri
beobachten.snapshot_text(ttl)
}
fn render_metrics(stats: &Stats) -> String {
async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIpTracker) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(4096);
let telemetry = stats.telemetry_policy();
let core_enabled = telemetry.core_enabled;
let user_enabled = telemetry.user_enabled;
let me_allows_normal = telemetry.me_level.allows_normal();
let me_allows_debug = telemetry.me_level.allows_debug();
let _ = writeln!(out, "# HELP telemt_uptime_seconds Proxy uptime");
let _ = writeln!(out, "# TYPE telemt_uptime_seconds gauge");
let _ = writeln!(out, "telemt_uptime_seconds {:.1}", stats.uptime_secs());
let _ = writeln!(out, "# HELP telemt_telemetry_core_enabled Runtime core telemetry switch");
let _ = writeln!(out, "# TYPE telemt_telemetry_core_enabled gauge");
let _ = writeln!(
out,
"telemt_telemetry_core_enabled {}",
if core_enabled { 1 } else { 0 }
);
let _ = writeln!(out, "# HELP telemt_telemetry_user_enabled Runtime per-user telemetry switch");
let _ = writeln!(out, "# TYPE telemt_telemetry_user_enabled gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_enabled {}",
if user_enabled { 1 } else { 0 }
);
let _ = writeln!(out, "# HELP telemt_telemetry_me_level Runtime ME telemetry level flag");
let _ = writeln!(out, "# TYPE telemt_telemetry_me_level gauge");
let _ = writeln!(
out,
"telemt_telemetry_me_level{{level=\"silent\"}} {}",
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Silent) {
1
} else {
0
}
);
let _ = writeln!(
out,
"telemt_telemetry_me_level{{level=\"normal\"}} {}",
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Normal) {
1
} else {
0
}
);
let _ = writeln!(
out,
"telemt_telemetry_me_level{{level=\"debug\"}} {}",
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Debug) {
1
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_connections_total Total accepted connections");
let _ = writeln!(out, "# TYPE telemt_connections_total counter");
let _ = writeln!(out, "telemt_connections_total {}", stats.get_connects_all());
let _ = writeln!(
out,
"telemt_connections_total {}",
if core_enabled { stats.get_connects_all() } else { 0 }
);
let _ = writeln!(out, "# HELP telemt_connections_bad_total Bad/rejected connections");
let _ = writeln!(out, "# TYPE telemt_connections_bad_total counter");
let _ = writeln!(out, "telemt_connections_bad_total {}", stats.get_connects_bad());
let _ = writeln!(
out,
"telemt_connections_bad_total {}",
if core_enabled { stats.get_connects_bad() } else { 0 }
);
let _ = writeln!(out, "# HELP telemt_handshake_timeouts_total Handshake timeouts");
let _ = writeln!(out, "# TYPE telemt_handshake_timeouts_total counter");
let _ = writeln!(out, "telemt_handshake_timeouts_total {}", stats.get_handshake_timeouts());
let _ = writeln!(
out,
"telemt_handshake_timeouts_total {}",
if core_enabled {
stats.get_handshake_timeouts()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_keepalive_sent_total ME keepalive frames sent");
let _ = writeln!(out, "# TYPE telemt_me_keepalive_sent_total counter");
let _ = writeln!(out, "telemt_me_keepalive_sent_total {}", stats.get_me_keepalive_sent());
let _ = writeln!(
out,
"telemt_me_keepalive_sent_total {}",
if me_allows_debug {
stats.get_me_keepalive_sent()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_keepalive_failed_total ME keepalive send failures");
let _ = writeln!(out, "# TYPE telemt_me_keepalive_failed_total counter");
let _ = writeln!(out, "telemt_me_keepalive_failed_total {}", stats.get_me_keepalive_failed());
let _ = writeln!(
out,
"telemt_me_keepalive_failed_total {}",
if me_allows_normal {
stats.get_me_keepalive_failed()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_keepalive_pong_total ME keepalive pong replies");
let _ = writeln!(out, "# TYPE telemt_me_keepalive_pong_total counter");
let _ = writeln!(out, "telemt_me_keepalive_pong_total {}", stats.get_me_keepalive_pong());
let _ = writeln!(
out,
"telemt_me_keepalive_pong_total {}",
if me_allows_debug {
stats.get_me_keepalive_pong()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_keepalive_timeout_total ME keepalive ping timeouts");
let _ = writeln!(out, "# TYPE telemt_me_keepalive_timeout_total counter");
let _ = writeln!(out, "telemt_me_keepalive_timeout_total {}", stats.get_me_keepalive_timeout());
let _ = writeln!(
out,
"telemt_me_keepalive_timeout_total {}",
if me_allows_normal {
stats.get_me_keepalive_timeout()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_reconnect_attempts_total ME reconnect attempts");
let _ = writeln!(out, "# TYPE telemt_me_reconnect_attempts_total counter");
let _ = writeln!(out, "telemt_me_reconnect_attempts_total {}", stats.get_me_reconnect_attempts());
let _ = writeln!(
out,
"telemt_me_reconnect_attempts_total {}",
if me_allows_normal {
stats.get_me_reconnect_attempts()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_reconnect_success_total ME reconnect successes");
let _ = writeln!(out, "# TYPE telemt_me_reconnect_success_total counter");
let _ = writeln!(out, "telemt_me_reconnect_success_total {}", stats.get_me_reconnect_success());
let _ = writeln!(
out,
"telemt_me_reconnect_success_total {}",
if me_allows_normal {
stats.get_me_reconnect_success()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_handshake_reject_total ME handshake rejects from upstream");
let _ = writeln!(out, "# TYPE telemt_me_handshake_reject_total counter");
let _ = writeln!(
out,
"telemt_me_handshake_reject_total {}",
if me_allows_normal {
stats.get_me_handshake_reject_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_handshake_error_code_total ME handshake reject errors by code");
let _ = writeln!(out, "# TYPE telemt_me_handshake_error_code_total counter");
if me_allows_normal {
for (error_code, count) in stats.get_me_handshake_error_code_counts() {
let _ = writeln!(
out,
"telemt_me_handshake_error_code_total{{error_code=\"{}\"}} {}",
error_code,
count
);
}
}
let _ = writeln!(out, "# HELP telemt_me_reader_eof_total ME reader EOF terminations");
let _ = writeln!(out, "# TYPE telemt_me_reader_eof_total counter");
let _ = writeln!(
out,
"telemt_me_reader_eof_total {}",
if me_allows_normal {
stats.get_me_reader_eof_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches");
let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter");
let _ = writeln!(out, "telemt_me_crc_mismatch_total {}", stats.get_me_crc_mismatch());
let _ = writeln!(
out,
"telemt_me_crc_mismatch_total {}",
if me_allows_normal {
stats.get_me_crc_mismatch()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_seq_mismatch_total ME sequence mismatches");
let _ = writeln!(out, "# TYPE telemt_me_seq_mismatch_total counter");
let _ = writeln!(out, "telemt_me_seq_mismatch_total {}", stats.get_me_seq_mismatch());
let _ = writeln!(
out,
"telemt_me_seq_mismatch_total {}",
if me_allows_normal {
stats.get_me_seq_mismatch()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_route_drop_no_conn_total ME route drops: no conn");
let _ = writeln!(out, "# TYPE telemt_me_route_drop_no_conn_total counter");
let _ = writeln!(out, "telemt_me_route_drop_no_conn_total {}", stats.get_me_route_drop_no_conn());
let _ = writeln!(
out,
"telemt_me_route_drop_no_conn_total {}",
if me_allows_normal {
stats.get_me_route_drop_no_conn()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_route_drop_channel_closed_total ME route drops: channel closed");
let _ = writeln!(out, "# TYPE telemt_me_route_drop_channel_closed_total counter");
let _ = writeln!(out, "telemt_me_route_drop_channel_closed_total {}", stats.get_me_route_drop_channel_closed());
let _ = writeln!(
out,
"telemt_me_route_drop_channel_closed_total {}",
if me_allows_normal {
stats.get_me_route_drop_channel_closed()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_route_drop_queue_full_total ME route drops: queue full");
let _ = writeln!(out, "# TYPE telemt_me_route_drop_queue_full_total counter");
let _ = writeln!(out, "telemt_me_route_drop_queue_full_total {}", stats.get_me_route_drop_queue_full());
let _ = writeln!(
out,
"telemt_me_route_drop_queue_full_total {}",
if me_allows_normal {
stats.get_me_route_drop_queue_full()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_route_drop_queue_full_profile_total ME route drops: queue full by adaptive profile"
);
let _ = writeln!(
out,
"# TYPE telemt_me_route_drop_queue_full_profile_total counter"
);
let _ = writeln!(
out,
"telemt_me_route_drop_queue_full_profile_total{{profile=\"base\"}} {}",
if me_allows_normal {
stats.get_me_route_drop_queue_full_base()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_route_drop_queue_full_profile_total{{profile=\"high\"}} {}",
if me_allows_normal {
stats.get_me_route_drop_queue_full_high()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
);
let _ = writeln!(out, "# TYPE telemt_me_socks_kdf_policy_total counter");
let _ = writeln!(
out,
"telemt_me_socks_kdf_policy_total{{policy=\"strict\",outcome=\"reject\"}} {}",
if me_allows_normal {
stats.get_me_socks_kdf_strict_reject()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_socks_kdf_policy_total{{policy=\"compat\",outcome=\"fallback\"}} {}",
if me_allows_debug {
stats.get_me_socks_kdf_compat_fallback()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_endpoint_quarantine_total ME endpoint quarantines due to rapid flaps"
);
let _ = writeln!(out, "# TYPE telemt_me_endpoint_quarantine_total counter");
let _ = writeln!(
out,
"telemt_me_endpoint_quarantine_total {}",
if me_allows_normal {
stats.get_me_endpoint_quarantine_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_kdf_drift_total ME KDF input drift detections");
let _ = writeln!(out, "# TYPE telemt_me_kdf_drift_total counter");
let _ = writeln!(
out,
"telemt_me_kdf_drift_total {}",
if me_allows_normal {
stats.get_me_kdf_drift_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation"
);
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_reuse_total counter");
let _ = writeln!(
out,
"telemt_me_hardswap_pending_reuse_total {}",
if me_allows_debug {
stats.get_me_hardswap_pending_reuse_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_ttl_expired_total Pending hardswap generations reset by TTL expiration"
);
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_ttl_expired_total counter");
let _ = writeln!(
out,
"telemt_me_hardswap_pending_ttl_expired_total {}",
if me_allows_normal {
stats.get_me_hardswap_pending_ttl_expired_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_outage_enter_total Single-endpoint DC outage transitions to active state"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_outage_enter_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_outage_enter_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_outage_enter_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_outage_exit_total Single-endpoint DC outage recovery transitions"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_outage_exit_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_outage_exit_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_outage_exit_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_outage_reconnect_attempt_total Reconnect attempts performed during single-endpoint outages"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_outage_reconnect_attempt_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_outage_reconnect_attempt_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_outage_reconnect_attempt_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_outage_reconnect_success_total Successful reconnect attempts during single-endpoint outages"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_outage_reconnect_success_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_outage_reconnect_success_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_outage_reconnect_success_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_quarantine_bypass_total Outage reconnect attempts that bypassed quarantine"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_quarantine_bypass_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_quarantine_bypass_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_quarantine_bypass_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_single_endpoint_shadow_rotate_total Successful periodic shadow rotations for single-endpoint DC groups"
);
let _ = writeln!(
out,
"# TYPE telemt_me_single_endpoint_shadow_rotate_total counter"
);
let _ = writeln!(
out,
"telemt_me_single_endpoint_shadow_rotate_total {}",
if me_allows_normal {
stats.get_me_single_endpoint_shadow_rotate_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths");
let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter");
let _ = writeln!(out, "telemt_secure_padding_invalid_total {}", stats.get_secure_padding_invalid());
let _ = writeln!(
out,
"telemt_secure_padding_invalid_total {}",
if me_allows_normal {
stats.get_secure_padding_invalid()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_desync_total Total crypto-desync detections");
let _ = writeln!(out, "# TYPE telemt_desync_total counter");
let _ = writeln!(out, "telemt_desync_total {}", stats.get_desync_total());
let _ = writeln!(
out,
"telemt_desync_total {}",
if me_allows_normal {
stats.get_desync_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_desync_full_logged_total Full forensic desync logs emitted");
let _ = writeln!(out, "# TYPE telemt_desync_full_logged_total counter");
let _ = writeln!(out, "telemt_desync_full_logged_total {}", stats.get_desync_full_logged());
let _ = writeln!(
out,
"telemt_desync_full_logged_total {}",
if me_allows_normal {
stats.get_desync_full_logged()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_desync_suppressed_total Suppressed desync forensic events");
let _ = writeln!(out, "# TYPE telemt_desync_suppressed_total counter");
let _ = writeln!(out, "telemt_desync_suppressed_total {}", stats.get_desync_suppressed());
let _ = writeln!(
out,
"telemt_desync_suppressed_total {}",
if me_allows_normal {
stats.get_desync_suppressed()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_desync_frames_bucket_total Desync count by frames_ok bucket");
let _ = writeln!(out, "# TYPE telemt_desync_frames_bucket_total counter");
let _ = writeln!(
out,
"telemt_desync_frames_bucket_total{{bucket=\"0\"}} {}",
stats.get_desync_frames_bucket_0()
if me_allows_normal {
stats.get_desync_frames_bucket_0()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_desync_frames_bucket_total{{bucket=\"1_2\"}} {}",
stats.get_desync_frames_bucket_1_2()
if me_allows_normal {
stats.get_desync_frames_bucket_1_2()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_desync_frames_bucket_total{{bucket=\"3_10\"}} {}",
stats.get_desync_frames_bucket_3_10()
if me_allows_normal {
stats.get_desync_frames_bucket_3_10()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_desync_frames_bucket_total{{bucket=\"gt_10\"}} {}",
stats.get_desync_frames_bucket_gt_10()
if me_allows_normal {
stats.get_desync_frames_bucket_gt_10()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_pool_swap_total Successful ME pool swaps");
let _ = writeln!(out, "# TYPE telemt_pool_swap_total counter");
let _ = writeln!(out, "telemt_pool_swap_total {}", stats.get_pool_swap_total());
let _ = writeln!(
out,
"telemt_pool_swap_total {}",
if me_allows_debug {
stats.get_pool_swap_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_pool_drain_active Active draining ME writers");
let _ = writeln!(out, "# TYPE telemt_pool_drain_active gauge");
let _ = writeln!(out, "telemt_pool_drain_active {}", stats.get_pool_drain_active());
let _ = writeln!(
out,
"telemt_pool_drain_active {}",
if me_allows_debug {
stats.get_pool_drain_active()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_pool_force_close_total Forced close events for draining writers");
let _ = writeln!(out, "# TYPE telemt_pool_force_close_total counter");
let _ = writeln!(
out,
"telemt_pool_force_close_total {}",
stats.get_pool_force_close_total()
if me_allows_normal {
stats.get_pool_force_close_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_pool_stale_pick_total Stale writer fallback picks for new binds");
@@ -233,7 +715,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_pool_stale_pick_total {}",
stats.get_pool_stale_pick_total()
if me_allows_normal {
stats.get_pool_stale_pick_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_writer_removed_total Total ME writer removals");
@@ -241,7 +727,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_writer_removed_total {}",
stats.get_me_writer_removed_total()
if me_allows_debug {
stats.get_me_writer_removed_total()
} else {
0
}
);
let _ = writeln!(
@@ -252,7 +742,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_writer_removed_unexpected_total {}",
stats.get_me_writer_removed_unexpected_total()
if me_allows_normal {
stats.get_me_writer_removed_unexpected_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_refill_triggered_total Immediate ME refill runs started");
@@ -260,7 +754,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_refill_triggered_total {}",
stats.get_me_refill_triggered_total()
if me_allows_debug {
stats.get_me_refill_triggered_total()
} else {
0
}
);
let _ = writeln!(
@@ -271,7 +769,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_refill_skipped_inflight_total {}",
stats.get_me_refill_skipped_inflight_total()
if me_allows_debug {
stats.get_me_refill_skipped_inflight_total()
} else {
0
}
);
let _ = writeln!(out, "# HELP telemt_me_refill_failed_total Immediate ME refill failures");
@@ -279,7 +781,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_refill_failed_total {}",
stats.get_me_refill_failed_total()
if me_allows_normal {
stats.get_me_refill_failed_total()
} else {
0
}
);
let _ = writeln!(
@@ -290,7 +796,11 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_writer_restored_same_endpoint_total {}",
stats.get_me_writer_restored_same_endpoint_total()
if me_allows_normal {
stats.get_me_writer_restored_same_endpoint_total()
} else {
0
}
);
let _ = writeln!(
@@ -301,16 +811,24 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(
out,
"telemt_me_writer_restored_fallback_total {}",
stats.get_me_writer_restored_fallback_total()
if me_allows_normal {
stats.get_me_writer_restored_fallback_total()
} else {
0
}
);
let unresolved_writer_losses = stats
.get_me_writer_removed_unexpected_total()
.saturating_sub(
stats
.get_me_writer_restored_same_endpoint_total()
.saturating_add(stats.get_me_writer_restored_fallback_total()),
);
let unresolved_writer_losses = if me_allows_normal {
stats
.get_me_writer_removed_unexpected_total()
.saturating_sub(
stats
.get_me_writer_restored_same_endpoint_total()
.saturating_add(stats.get_me_writer_restored_fallback_total()),
)
} else {
0
};
let _ = writeln!(
out,
"# HELP telemt_me_writer_removed_unexpected_minus_restored_total Unexpected writer removals not yet compensated by restore"
@@ -337,16 +855,63 @@ fn render_metrics(stats: &Stats) -> String {
let _ = writeln!(out, "# TYPE telemt_user_msgs_from_client counter");
let _ = writeln!(out, "# HELP telemt_user_msgs_to_client Per-user messages sent");
let _ = writeln!(out, "# TYPE telemt_user_msgs_to_client counter");
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_suppressed {}",
if user_enabled { 0 } else { 1 }
);
for entry in stats.iter_user_stats() {
let user = entry.key();
let s = entry.value();
let _ = writeln!(out, "telemt_user_connections_total{{user=\"{}\"}} {}", user, s.connects.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_connections_current{{user=\"{}\"}} {}", user, s.curr_connects.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_octets_from_client{{user=\"{}\"}} {}", user, s.octets_from_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_octets_to_client{{user=\"{}\"}} {}", user, s.octets_to_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_msgs_from_client{{user=\"{}\"}} {}", user, s.msgs_from_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_msgs_to_client{{user=\"{}\"}} {}", user, s.msgs_to_client.load(std::sync::atomic::Ordering::Relaxed));
if user_enabled {
for entry in stats.iter_user_stats() {
let user = entry.key();
let s = entry.value();
let _ = writeln!(out, "telemt_user_connections_total{{user=\"{}\"}} {}", user, s.connects.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_connections_current{{user=\"{}\"}} {}", user, s.curr_connects.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_octets_from_client{{user=\"{}\"}} {}", user, s.octets_from_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_octets_to_client{{user=\"{}\"}} {}", user, s.octets_to_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_msgs_from_client{{user=\"{}\"}} {}", user, s.msgs_from_client.load(std::sync::atomic::Ordering::Relaxed));
let _ = writeln!(out, "telemt_user_msgs_to_client{{user=\"{}\"}} {}", user, s.msgs_to_client.load(std::sync::atomic::Ordering::Relaxed));
}
let ip_stats = ip_tracker.get_stats().await;
let ip_counts: HashMap<String, usize> = ip_stats
.into_iter()
.map(|(user, count, _)| (user, count))
.collect();
let mut unique_users = BTreeSet::new();
unique_users.extend(config.access.user_max_unique_ips.keys().cloned());
unique_users.extend(ip_counts.keys().cloned());
let _ = writeln!(out, "# HELP telemt_user_unique_ips_current Per-user current number of unique active IPs");
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_current gauge");
let _ = writeln!(out, "# HELP telemt_user_unique_ips_limit Per-user configured unique IP limit (0 means unlimited)");
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_limit gauge");
let _ = writeln!(out, "# HELP telemt_user_unique_ips_utilization Per-user unique IP usage ratio (0 for unlimited)");
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge");
for user in unique_users {
let current = ip_counts.get(&user).copied().unwrap_or(0);
let limit = config.access.user_max_unique_ips.get(&user).copied().unwrap_or(0);
let utilization = if limit > 0 {
current as f64 / limit as f64
} else {
0.0
};
let _ = writeln!(out, "telemt_user_unique_ips_current{{user=\"{}\"}} {}", user, current);
let _ = writeln!(out, "telemt_user_unique_ips_limit{{user=\"{}\"}} {}", user, limit);
let _ = writeln!(
out,
"telemt_user_unique_ips_utilization{{user=\"{}\"}} {:.6}",
user,
utilization
);
}
}
out
@@ -358,9 +923,16 @@ mod tests {
use std::net::IpAddr;
use http_body_util::BodyExt;
#[test]
fn test_render_metrics_format() {
#[tokio::test]
async fn test_render_metrics_format() {
let stats = Arc::new(Stats::new());
let tracker = UserIpTracker::new();
let mut config = ProxyConfig::default();
config
.access
.user_max_unique_ips
.insert("alice".to_string(), 4);
stats.increment_connects_all();
stats.increment_connects_all();
stats.increment_connects_bad();
@@ -372,8 +944,12 @@ mod tests {
stats.increment_user_msgs_from("alice");
stats.increment_user_msgs_to("alice");
stats.increment_user_msgs_to("alice");
tracker
.check_and_add("alice", "203.0.113.10".parse().unwrap())
.await
.unwrap();
let output = render_metrics(&stats);
let output = render_metrics(&stats, &config, &tracker).await;
assert!(output.contains("telemt_connections_total 2"));
assert!(output.contains("telemt_connections_bad_total 1"));
@@ -384,22 +960,29 @@ mod tests {
assert!(output.contains("telemt_user_octets_to_client{user=\"alice\"} 2048"));
assert!(output.contains("telemt_user_msgs_from_client{user=\"alice\"} 1"));
assert!(output.contains("telemt_user_msgs_to_client{user=\"alice\"} 2"));
assert!(output.contains("telemt_user_unique_ips_current{user=\"alice\"} 1"));
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 4"));
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.250000"));
}
#[test]
fn test_render_empty_stats() {
#[tokio::test]
async fn test_render_empty_stats() {
let stats = Stats::new();
let output = render_metrics(&stats);
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &config, &tracker).await;
assert!(output.contains("telemt_connections_total 0"));
assert!(output.contains("telemt_connections_bad_total 0"));
assert!(output.contains("telemt_handshake_timeouts_total 0"));
assert!(!output.contains("user="));
}
#[test]
fn test_render_has_type_annotations() {
#[tokio::test]
async fn test_render_has_type_annotations() {
let stats = Stats::new();
let output = render_metrics(&stats);
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &config, &tracker).await;
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
assert!(output.contains("# TYPE telemt_connections_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
@@ -408,12 +991,16 @@ mod tests {
assert!(output.contains(
"# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge"
));
assert!(output.contains("# TYPE telemt_user_unique_ips_current gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge"));
}
#[tokio::test]
async fn test_endpoint_integration() {
let stats = Arc::new(Stats::new());
let beobachten = Arc::new(BeobachtenStore::new());
let tracker = UserIpTracker::new();
let mut config = ProxyConfig::default();
stats.increment_connects_all();
stats.increment_connects_all();
@@ -423,7 +1010,7 @@ mod tests {
.uri("/metrics")
.body(())
.unwrap();
let resp = handle(req, &stats, &beobachten, &config).unwrap();
let resp = handle(req, &stats, &beobachten, &tracker, &config).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = resp.into_body().collect().await.unwrap().to_bytes();
assert!(std::str::from_utf8(body.as_ref()).unwrap().contains("telemt_connections_total 3"));
@@ -439,7 +1026,9 @@ mod tests {
.uri("/beobachten")
.body(())
.unwrap();
let resp_beob = handle(req_beob, &stats, &beobachten, &config).unwrap();
let resp_beob = handle(req_beob, &stats, &beobachten, &tracker, &config)
.await
.unwrap();
assert_eq!(resp_beob.status(), StatusCode::OK);
let body_beob = resp_beob.into_body().collect().await.unwrap().to_bytes();
let beob_text = std::str::from_utf8(body_beob.as_ref()).unwrap();
@@ -450,7 +1039,9 @@ mod tests {
.uri("/other")
.body(())
.unwrap();
let resp404 = handle(req404, &stats, &beobachten, &config).unwrap();
let resp404 = handle(req404, &stats, &beobachten, &tracker, &config)
.await
.unwrap();
assert_eq!(resp404.status(), StatusCode::NOT_FOUND);
}
}

View File

@@ -0,0 +1,197 @@
//! Runtime DNS overrides for `host:port` targets.
use std::collections::HashMap;
use std::net::{IpAddr, Ipv6Addr, SocketAddr};
use std::sync::{OnceLock, RwLock};
use crate::error::{ProxyError, Result};
type OverrideMap = HashMap<(String, u16), IpAddr>;
static DNS_OVERRIDES: OnceLock<RwLock<OverrideMap>> = OnceLock::new();
fn overrides_store() -> &'static RwLock<OverrideMap> {
DNS_OVERRIDES.get_or_init(|| RwLock::new(HashMap::new()))
}
fn parse_ip_spec(ip_spec: &str) -> Result<IpAddr> {
if ip_spec.starts_with('[') && ip_spec.ends_with(']') {
let inner = &ip_spec[1..ip_spec.len() - 1];
let ipv6 = inner.parse::<Ipv6Addr>().map_err(|_| {
ProxyError::Config(format!(
"network.dns_overrides IPv6 override is invalid: '{ip_spec}'"
))
})?;
return Ok(IpAddr::V6(ipv6));
}
let ip = ip_spec.parse::<IpAddr>().map_err(|_| {
ProxyError::Config(format!(
"network.dns_overrides IP is invalid: '{ip_spec}'"
))
})?;
if matches!(ip, IpAddr::V6(_)) {
return Err(ProxyError::Config(format!(
"network.dns_overrides IPv6 must be bracketed: '{ip_spec}'"
)));
}
Ok(ip)
}
fn parse_entry(entry: &str) -> Result<((String, u16), IpAddr)> {
let trimmed = entry.trim();
if trimmed.is_empty() {
return Err(ProxyError::Config(
"network.dns_overrides entry cannot be empty".to_string(),
));
}
let first_sep = trimmed.find(':').ok_or_else(|| {
ProxyError::Config(format!(
"network.dns_overrides entry must use host:port:ip format: '{trimmed}'"
))
})?;
let second_sep = trimmed[first_sep + 1..]
.find(':')
.map(|idx| first_sep + 1 + idx)
.ok_or_else(|| {
ProxyError::Config(format!(
"network.dns_overrides entry must use host:port:ip format: '{trimmed}'"
))
})?;
let host = trimmed[..first_sep].trim();
let port_str = trimmed[first_sep + 1..second_sep].trim();
let ip_str = trimmed[second_sep + 1..].trim();
if host.is_empty() {
return Err(ProxyError::Config(format!(
"network.dns_overrides host cannot be empty: '{trimmed}'"
)));
}
if host.contains(':') {
return Err(ProxyError::Config(format!(
"network.dns_overrides host must be a domain name without ':' in this format: '{trimmed}'"
)));
}
let port = port_str.parse::<u16>().map_err(|_| {
ProxyError::Config(format!(
"network.dns_overrides port is invalid: '{trimmed}'"
))
})?;
let ip = parse_ip_spec(ip_str)?;
Ok(((host.to_ascii_lowercase(), port), ip))
}
fn parse_entries(entries: &[String]) -> Result<OverrideMap> {
let mut parsed = HashMap::new();
for entry in entries {
let (key, ip) = parse_entry(entry)?;
parsed.insert(key, ip);
}
Ok(parsed)
}
/// Validate `network.dns_overrides` entries without updating runtime state.
pub fn validate_entries(entries: &[String]) -> Result<()> {
let _ = parse_entries(entries)?;
Ok(())
}
/// Replace runtime DNS overrides with a new validated snapshot.
pub fn install_entries(entries: &[String]) -> Result<()> {
let parsed = parse_entries(entries)?;
let mut guard = overrides_store()
.write()
.map_err(|_| ProxyError::Config("network.dns_overrides runtime lock is poisoned".to_string()))?;
*guard = parsed;
Ok(())
}
/// Resolve a hostname override for `(host, port)` if present.
pub fn resolve(host: &str, port: u16) -> Option<IpAddr> {
let key = (host.to_ascii_lowercase(), port);
overrides_store()
.read()
.ok()
.and_then(|guard| guard.get(&key).copied())
}
/// Resolve a hostname override and construct a socket address when present.
pub fn resolve_socket_addr(host: &str, port: u16) -> Option<SocketAddr> {
resolve(host, port).map(|ip| SocketAddr::new(ip, port))
}
/// Parse a runtime endpoint in `host:port` format.
///
/// Supports:
/// - `example.com:443`
/// - `[2001:db8::1]:443`
pub fn split_host_port(endpoint: &str) -> Option<(String, u16)> {
if endpoint.starts_with('[') {
let bracket_end = endpoint.find(']')?;
if endpoint.as_bytes().get(bracket_end + 1) != Some(&b':') {
return None;
}
let host = endpoint[1..bracket_end].trim();
let port = endpoint[bracket_end + 2..].trim().parse::<u16>().ok()?;
if host.is_empty() {
return None;
}
return Some((host.to_ascii_lowercase(), port));
}
let split_idx = endpoint.rfind(':')?;
let host = endpoint[..split_idx].trim();
let port = endpoint[split_idx + 1..].trim().parse::<u16>().ok()?;
if host.is_empty() || host.contains(':') {
return None;
}
Some((host.to_ascii_lowercase(), port))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn validate_accepts_ipv4_and_bracketed_ipv6() {
let entries = vec![
"example.com:443:127.0.0.1".to_string(),
"example.net:8443:[2001:db8::10]".to_string(),
];
assert!(validate_entries(&entries).is_ok());
}
#[test]
fn validate_rejects_unbracketed_ipv6() {
let entries = vec!["example.net:443:2001:db8::10".to_string()];
let err = validate_entries(&entries).unwrap_err().to_string();
assert!(err.contains("must be bracketed"));
}
#[test]
fn install_and_resolve_are_case_insensitive_for_host() {
let entries = vec!["MyPetrovich.ru:8443:127.0.0.1".to_string()];
install_entries(&entries).unwrap();
let resolved = resolve("mypetrovich.ru", 8443);
assert_eq!(resolved, Some("127.0.0.1".parse().unwrap()));
}
#[test]
fn split_host_port_parses_supported_shapes() {
assert_eq!(
split_host_port("example.com:443"),
Some(("example.com".to_string(), 443))
);
assert_eq!(
split_host_port("[2001:db8::1]:443"),
Some(("2001:db8::1".to_string(), 443))
);
assert_eq!(split_host_port("2001:db8::1:443"), None);
}
}

View File

@@ -1,3 +1,4 @@
pub mod dns_overrides;
pub mod probe;
pub mod stun;

View File

@@ -68,7 +68,7 @@ pub async fn run_probe(
probe.ipv4_is_bogon = probe.detected_ipv4.map(is_bogon_v4).unwrap_or(false);
probe.ipv6_is_bogon = probe.detected_ipv6.map(is_bogon_v6).unwrap_or(false);
let stun_res = if nat_probe {
let stun_res = if nat_probe && config.stun_use {
let servers = collect_stun_servers(config);
if servers.is_empty() {
warn!("STUN probe is enabled but network.stun_servers is empty");
@@ -80,6 +80,9 @@ pub async fn run_probe(
)
.await
}
} else if nat_probe {
info!("STUN probe is disabled by network.stun_use=false");
DualStunResult::default()
} else {
DualStunResult::default()
};

View File

@@ -7,6 +7,7 @@ use tokio::net::{lookup_host, UdpSocket};
use tokio::time::{timeout, Duration, sleep};
use crate::error::{ProxyError, Result};
use crate::network::dns_overrides::{resolve, split_host_port};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum IpFamily {
@@ -40,16 +41,31 @@ pub async fn stun_probe_dual(stun_addr: &str) -> Result<DualStunResult> {
}
pub async fn stun_probe_family(stun_addr: &str, family: IpFamily) -> Result<Option<StunProbeResult>> {
stun_probe_family_with_bind(stun_addr, family, None).await
}
pub async fn stun_probe_family_with_bind(
stun_addr: &str,
family: IpFamily,
bind_ip: Option<IpAddr>,
) -> Result<Option<StunProbeResult>> {
use rand::RngCore;
let bind_addr = match family {
IpFamily::V4 => "0.0.0.0:0",
IpFamily::V6 => "[::]:0",
let bind_addr = match (family, bind_ip) {
(IpFamily::V4, Some(IpAddr::V4(ip))) => SocketAddr::new(IpAddr::V4(ip), 0),
(IpFamily::V6, Some(IpAddr::V6(ip))) => SocketAddr::new(IpAddr::V6(ip), 0),
(IpFamily::V4, Some(IpAddr::V6(_))) | (IpFamily::V6, Some(IpAddr::V4(_))) => {
return Ok(None);
}
(IpFamily::V4, None) => SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0),
(IpFamily::V6, None) => SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0),
};
let socket = UdpSocket::bind(bind_addr)
.await
.map_err(|e| ProxyError::Proxy(format!("STUN bind failed: {e}")))?;
let socket = match UdpSocket::bind(bind_addr).await {
Ok(socket) => socket,
Err(_) if bind_ip.is_some() => return Ok(None),
Err(e) => return Err(ProxyError::Proxy(format!("STUN bind failed: {e}"))),
};
let target_addr = resolve_stun_addr(stun_addr, family).await?;
if let Some(addr) = target_addr {
@@ -198,6 +214,16 @@ async fn resolve_stun_addr(stun_addr: &str, family: IpFamily) -> Result<Option<S
});
}
if let Some((host, port)) = split_host_port(stun_addr)
&& let Some(ip) = resolve(&host, port)
{
let addr = SocketAddr::new(ip, port);
return Ok(match (addr.is_ipv4(), family) {
(true, IpFamily::V4) | (false, IpFamily::V6) => Some(addr),
_ => None,
});
}
let mut addrs = lookup_host(stun_addr)
.await
.map_err(|e| ProxyError::Proxy(format!("STUN resolve failed: {e}")))?;

View File

@@ -91,6 +91,11 @@ where
stats.increment_connects_all();
let mut real_peer = normalize_ip(peer);
// For non-TCP streams, use a synthetic local address; may be overridden by PROXY protocol dst
let mut local_addr: SocketAddr = format!("0.0.0.0:{}", config.server.port)
.parse()
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
if proxy_protocol_enabled {
match parse_proxy_protocol(&mut stream, peer).await {
Ok(info) => {
@@ -101,6 +106,9 @@ where
"PROXY protocol header parsed"
);
real_peer = normalize_ip(info.src_addr);
if let Some(dst) = info.dst_addr {
local_addr = dst;
}
}
Err(e) => {
stats.increment_connects_bad();
@@ -119,11 +127,6 @@ where
let beobachten_for_timeout = beobachten.clone();
let peer_for_timeout = real_peer.ip();
// For non-TCP streams, use a synthetic local address
let local_addr: SocketAddr = format!("0.0.0.0:{}", config.server.port)
.parse()
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
// Phase 1: handshake (with timeout)
let outcome = match timeout(handshake_timeout, async {
let mut first_bytes = [0u8; 5];
@@ -144,6 +147,7 @@ where
writer,
&first_bytes,
real_peer,
local_addr,
&config,
&beobachten,
)
@@ -169,6 +173,7 @@ where
writer,
&handshake,
real_peer,
local_addr,
&config,
&beobachten,
)
@@ -213,6 +218,7 @@ where
writer,
&first_bytes,
real_peer,
local_addr,
&config,
&beobachten,
)
@@ -238,6 +244,7 @@ where
writer,
&handshake,
real_peer,
local_addr,
&config,
&beobachten,
)
@@ -405,6 +412,8 @@ impl RunningClientHandler {
}
async fn do_handshake(mut self) -> Result<HandshakeOutcome> {
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
if self.proxy_protocol_enabled {
match parse_proxy_protocol(&mut self.stream, self.peer).await {
Ok(info) => {
@@ -415,6 +424,9 @@ impl RunningClientHandler {
"PROXY protocol header parsed"
);
self.peer = normalize_ip(info.src_addr);
if let Some(dst) = info.dst_addr {
local_addr = dst;
}
}
Err(e) => {
self.stats.increment_connects_bad();
@@ -440,13 +452,13 @@ impl RunningClientHandler {
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
if is_tls {
self.handle_tls_client(first_bytes).await
self.handle_tls_client(first_bytes, local_addr).await
} else {
self.handle_direct_client(first_bytes).await
self.handle_direct_client(first_bytes, local_addr).await
}
}
async fn handle_tls_client(mut self, first_bytes: [u8; 5]) -> Result<HandshakeOutcome> {
async fn handle_tls_client(mut self, first_bytes: [u8; 5], local_addr: SocketAddr) -> Result<HandshakeOutcome> {
let peer = self.peer;
let _ip_tracker = self.ip_tracker.clone();
@@ -463,6 +475,7 @@ impl RunningClientHandler {
writer,
&first_bytes,
peer,
local_addr,
&self.config,
&self.beobachten,
)
@@ -479,7 +492,6 @@ impl RunningClientHandler {
let stats = self.stats.clone();
let buffer_pool = self.buffer_pool.clone();
let local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
let (read_half, write_half) = self.stream.into_split();
let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake(
@@ -502,6 +514,7 @@ impl RunningClientHandler {
writer,
&handshake,
peer,
local_addr,
&config,
&self.beobachten,
)
@@ -558,7 +571,7 @@ impl RunningClientHandler {
)))
}
async fn handle_direct_client(mut self, first_bytes: [u8; 5]) -> Result<HandshakeOutcome> {
async fn handle_direct_client(mut self, first_bytes: [u8; 5], local_addr: SocketAddr) -> Result<HandshakeOutcome> {
let peer = self.peer;
let _ip_tracker = self.ip_tracker.clone();
@@ -571,6 +584,7 @@ impl RunningClientHandler {
writer,
&first_bytes,
peer,
local_addr,
&self.config,
&self.beobachten,
)
@@ -587,7 +601,6 @@ impl RunningClientHandler {
let stats = self.stats.clone();
let buffer_pool = self.buffer_pool.clone();
let local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
let (read_half, write_half) = self.stream.into_split();
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
@@ -609,6 +622,7 @@ impl RunningClientHandler {
writer,
&handshake,
peer,
local_addr,
&config,
&self.beobachten,
)

View File

@@ -10,6 +10,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncReadExt, AsyncWriteExt};
use tokio::time::timeout;
use tracing::debug;
use crate::config::ProxyConfig;
use crate::network::dns_overrides::resolve_socket_addr;
use crate::stats::beobachten::BeobachtenStore;
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
@@ -54,6 +55,7 @@ pub async fn handle_bad_client<R, W>(
writer: W,
initial_data: &[u8],
peer: SocketAddr,
local_addr: SocketAddr,
config: &ProxyConfig,
beobachten: &BeobachtenStore,
)
@@ -86,7 +88,29 @@ where
let connect_result = timeout(MASK_TIMEOUT, UnixStream::connect(sock_path)).await;
match connect_result {
Ok(Ok(stream)) => {
let (mask_read, mask_write) = stream.into_split();
let (mask_read, mut mask_write) = stream.into_split();
let proxy_header: Option<Vec<u8>> = match config.censorship.mask_proxy_protocol {
0 => None,
version => {
let header = match version {
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
_ => match (peer, local_addr) {
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
_ =>
ProxyProtocolV1Builder::new().build(),
},
};
Some(header)
}
};
if let Some(header) = proxy_header {
if mask_write.write_all(&header).await.is_err() {
return;
}
}
if timeout(MASK_RELAY_TIMEOUT, relay_to_mask(reader, writer, mask_read, mask_write, initial_data)).await.is_err() {
debug!("Mask relay timed out (unix socket)");
}
@@ -115,31 +139,26 @@ where
"Forwarding bad client to mask host"
);
// Connect to mask host
let mask_addr = format!("{}:{}", mask_host, mask_port);
// Apply runtime DNS override for mask target when configured.
let mask_addr = resolve_socket_addr(mask_host, mask_port)
.map(|addr| addr.to_string())
.unwrap_or_else(|| format!("{}:{}", mask_host, mask_port));
let connect_result = timeout(MASK_TIMEOUT, TcpStream::connect(&mask_addr)).await;
match connect_result {
Ok(Ok(stream)) => {
let proxy_header: Option<Vec<u8>> = match config.censorship.mask_proxy_protocol {
0 => None,
version => {
let header = if let Ok(local_addr) = stream.local_addr() {
match version {
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
_ => match (peer, local_addr) {
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
_ =>
ProxyProtocolV1Builder::new().build(),
},
}
} else {
match version {
2 => ProxyProtocolV2Builder::new().build(),
_ => ProxyProtocolV1Builder::new().build(),
}
let header = match version {
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
_ => match (peer, local_addr) {
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
_ =>
ProxyProtocolV1Builder::new().build(),
},
};
Some(header)
}

View File

@@ -26,6 +26,9 @@ enum C2MeCommand {
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
const C2ME_CHANNEL_CAPACITY: usize = 1024;
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new();
struct RelayForensicsState {
@@ -166,6 +169,27 @@ fn report_desync_frame_too_large(
))
}
fn should_yield_c2me_sender(sent_since_yield: usize, has_backlog: bool) -> bool {
has_backlog && sent_since_yield >= C2ME_SENDER_FAIRNESS_BUDGET
}
async fn enqueue_c2me_command(
tx: &mpsc::Sender<C2MeCommand>,
cmd: C2MeCommand,
) -> std::result::Result<(), mpsc::error::SendError<C2MeCommand>> {
match tx.try_send(cmd) {
Ok(()) => Ok(()),
Err(mpsc::error::TrySendError::Closed(cmd)) => Err(mpsc::error::SendError(cmd)),
Err(mpsc::error::TrySendError::Full(cmd)) => {
// Cooperative yield reduces burst catch-up when the per-conn queue is near saturation.
if tx.capacity() <= C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS {
tokio::task::yield_now().await;
}
tx.send(cmd).await
}
}
}
pub(crate) async fn handle_via_middle_proxy<R, W>(
mut crypto_reader: CryptoReader<R>,
crypto_writer: CryptoWriter<W>,
@@ -214,7 +238,22 @@ where
stats.increment_user_connects(&user);
stats.increment_user_curr_connects(&user);
let proto_flags = proto_flags_for_tag(proto_tag, me_pool.has_proxy_tag());
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
let user_tag: Option<Vec<u8>> = config
.access
.user_ad_tags
.get(&user)
.and_then(|s| hex::decode(s).ok())
.filter(|v| v.len() == 16);
let global_tag: Option<Vec<u8>> = config
.general
.ad_tag
.as_ref()
.and_then(|s| hex::decode(s).ok())
.filter(|v| v.len() == 16);
let effective_tag = user_tag.or(global_tag);
let proto_flags = proto_flags_for_tag(proto_tag, effective_tag.is_some());
debug!(
trace_id = format_args!("0x{:016x}", trace_id),
user = %user,
@@ -230,9 +269,11 @@ where
let frame_limit = config.general.max_client_frame;
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(1024);
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(C2ME_CHANNEL_CAPACITY);
let me_pool_c2me = me_pool.clone();
let effective_tag = effective_tag;
let c2me_sender = tokio::spawn(async move {
let mut sent_since_yield = 0usize;
while let Some(cmd) = c2me_rx.recv().await {
match cmd {
C2MeCommand::Data { payload, flags } => {
@@ -243,7 +284,13 @@ where
translated_local_addr,
&payload,
flags,
effective_tag.as_deref(),
).await?;
sent_since_yield = sent_since_yield.saturating_add(1);
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
sent_since_yield = 0;
tokio::task::yield_now().await;
}
}
C2MeCommand::Close => {
let _ = me_pool_c2me.send_close(conn_id).await;
@@ -360,8 +407,7 @@ where
flags |= RPC_FLAG_NOT_ENCRYPTED;
}
// Keep client read loop lightweight: route heavy ME send path via a dedicated task.
if c2me_tx
.send(C2MeCommand::Data { payload, flags })
if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags })
.await
.is_err()
{
@@ -372,7 +418,7 @@ where
Ok(None) => {
debug!(conn_id, "Client EOF");
client_closed = true;
let _ = c2me_tx.send(C2MeCommand::Close).await;
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await;
break;
}
Err(e) => {
@@ -647,3 +693,84 @@ where
// ACK should remain low-latency.
client_writer.flush().await.map_err(ProxyError::Io)
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::time::{Duration as TokioDuration, timeout};
#[test]
fn should_yield_sender_only_on_budget_with_backlog() {
assert!(!should_yield_c2me_sender(0, true));
assert!(!should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET - 1, true));
assert!(!should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET, false));
assert!(should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET, true));
}
#[tokio::test]
async fn enqueue_c2me_command_uses_try_send_fast_path() {
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(2);
enqueue_c2me_command(
&tx,
C2MeCommand::Data {
payload: vec![1, 2, 3],
flags: 0,
},
)
.await
.unwrap();
let recv = timeout(TokioDuration::from_millis(50), rx.recv())
.await
.unwrap()
.unwrap();
match recv {
C2MeCommand::Data { payload, flags } => {
assert_eq!(payload, vec![1, 2, 3]);
assert_eq!(flags, 0);
}
C2MeCommand::Close => panic!("unexpected close command"),
}
}
#[tokio::test]
async fn enqueue_c2me_command_falls_back_to_send_when_queue_is_full() {
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(1);
tx.send(C2MeCommand::Data {
payload: vec![9],
flags: 9,
})
.await
.unwrap();
let tx2 = tx.clone();
let producer = tokio::spawn(async move {
enqueue_c2me_command(
&tx2,
C2MeCommand::Data {
payload: vec![7, 7],
flags: 7,
},
)
.await
.unwrap();
});
let _ = timeout(TokioDuration::from_millis(100), rx.recv())
.await
.unwrap();
producer.await.unwrap();
let recv = timeout(TokioDuration::from_millis(100), rx.recv())
.await
.unwrap()
.unwrap();
match recv {
C2MeCommand::Data { payload, flags } => {
assert_eq!(payload, vec![7, 7]);
assert_eq!(flags, 7);
}
C2MeCommand::Close => panic!("unexpected close command"),
}
}
}

View File

@@ -3,8 +3,9 @@
#![allow(dead_code)]
pub mod beobachten;
pub mod telemetry;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
use std::time::{Instant, Duration};
use dashmap::DashMap;
use parking_lot::Mutex;
@@ -15,6 +16,9 @@ use std::collections::hash_map::DefaultHasher;
use std::collections::VecDeque;
use tracing::debug;
use crate::config::MeTelemetryLevel;
use self::telemetry::TelemetryPolicy;
// ============= Stats =============
#[derive(Default)]
@@ -28,11 +32,28 @@ pub struct Stats {
me_keepalive_timeout: AtomicU64,
me_reconnect_attempts: AtomicU64,
me_reconnect_success: AtomicU64,
me_handshake_reject_total: AtomicU64,
me_reader_eof_total: AtomicU64,
me_crc_mismatch: AtomicU64,
me_seq_mismatch: AtomicU64,
me_endpoint_quarantine_total: AtomicU64,
me_kdf_drift_total: AtomicU64,
me_hardswap_pending_reuse_total: AtomicU64,
me_hardswap_pending_ttl_expired_total: AtomicU64,
me_single_endpoint_outage_enter_total: AtomicU64,
me_single_endpoint_outage_exit_total: AtomicU64,
me_single_endpoint_outage_reconnect_attempt_total: AtomicU64,
me_single_endpoint_outage_reconnect_success_total: AtomicU64,
me_single_endpoint_quarantine_bypass_total: AtomicU64,
me_single_endpoint_shadow_rotate_total: AtomicU64,
me_handshake_error_codes: DashMap<i32, AtomicU64>,
me_route_drop_no_conn: AtomicU64,
me_route_drop_channel_closed: AtomicU64,
me_route_drop_queue_full: AtomicU64,
me_route_drop_queue_full_base: AtomicU64,
me_route_drop_queue_full_high: AtomicU64,
me_socks_kdf_strict_reject: AtomicU64,
me_socks_kdf_compat_fallback: AtomicU64,
secure_padding_invalid: AtomicU64,
desync_total: AtomicU64,
desync_full_logged: AtomicU64,
@@ -52,6 +73,9 @@ pub struct Stats {
me_refill_failed_total: AtomicU64,
me_writer_restored_same_endpoint_total: AtomicU64,
me_writer_restored_fallback_total: AtomicU64,
telemetry_core_enabled: AtomicBool,
telemetry_user_enabled: AtomicBool,
telemetry_me_level: AtomicU8,
user_stats: DashMap<String, UserStats>,
start_time: parking_lot::RwLock<Option<Instant>>,
}
@@ -69,44 +93,187 @@ pub struct UserStats {
impl Stats {
pub fn new() -> Self {
let stats = Self::default();
stats.apply_telemetry_policy(TelemetryPolicy::default());
*stats.start_time.write() = Some(Instant::now());
stats
}
pub fn increment_connects_all(&self) { self.connects_all.fetch_add(1, Ordering::Relaxed); }
pub fn increment_connects_bad(&self) { self.connects_bad.fetch_add(1, Ordering::Relaxed); }
pub fn increment_handshake_timeouts(&self) { self.handshake_timeouts.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_keepalive_sent(&self) { self.me_keepalive_sent.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_keepalive_failed(&self) { self.me_keepalive_failed.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_keepalive_pong(&self) { self.me_keepalive_pong.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_keepalive_timeout(&self) { self.me_keepalive_timeout.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_keepalive_timeout_by(&self, value: u64) {
self.me_keepalive_timeout.fetch_add(value, Ordering::Relaxed);
fn telemetry_me_level(&self) -> MeTelemetryLevel {
MeTelemetryLevel::from_u8(self.telemetry_me_level.load(Ordering::Relaxed))
}
fn telemetry_core_enabled(&self) -> bool {
self.telemetry_core_enabled.load(Ordering::Relaxed)
}
fn telemetry_user_enabled(&self) -> bool {
self.telemetry_user_enabled.load(Ordering::Relaxed)
}
fn telemetry_me_allows_normal(&self) -> bool {
self.telemetry_me_level().allows_normal()
}
fn telemetry_me_allows_debug(&self) -> bool {
self.telemetry_me_level().allows_debug()
}
pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) {
self.telemetry_core_enabled
.store(policy.core_enabled, Ordering::Relaxed);
self.telemetry_user_enabled
.store(policy.user_enabled, Ordering::Relaxed);
self.telemetry_me_level
.store(policy.me_level.as_u8(), Ordering::Relaxed);
}
pub fn telemetry_policy(&self) -> TelemetryPolicy {
TelemetryPolicy {
core_enabled: self.telemetry_core_enabled(),
user_enabled: self.telemetry_user_enabled(),
me_level: self.telemetry_me_level(),
}
}
pub fn increment_connects_all(&self) {
if self.telemetry_core_enabled() {
self.connects_all.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_connects_bad(&self) {
if self.telemetry_core_enabled() {
self.connects_bad.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_handshake_timeouts(&self) {
if self.telemetry_core_enabled() {
self.handshake_timeouts.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_keepalive_sent(&self) {
if self.telemetry_me_allows_debug() {
self.me_keepalive_sent.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_keepalive_failed(&self) {
if self.telemetry_me_allows_normal() {
self.me_keepalive_failed.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_keepalive_pong(&self) {
if self.telemetry_me_allows_debug() {
self.me_keepalive_pong.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_keepalive_timeout(&self) {
if self.telemetry_me_allows_normal() {
self.me_keepalive_timeout.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_keepalive_timeout_by(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_keepalive_timeout.fetch_add(value, Ordering::Relaxed);
}
}
pub fn increment_me_reconnect_attempt(&self) {
if self.telemetry_me_allows_normal() {
self.me_reconnect_attempts.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_reconnect_success(&self) {
if self.telemetry_me_allows_normal() {
self.me_reconnect_success.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_handshake_reject_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_handshake_reject_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_handshake_error_code(&self, code: i32) {
if !self.telemetry_me_allows_normal() {
return;
}
let entry = self
.me_handshake_error_codes
.entry(code)
.or_insert_with(|| AtomicU64::new(0));
entry.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_me_reader_eof_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_reader_eof_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_crc_mismatch(&self) {
if self.telemetry_me_allows_normal() {
self.me_crc_mismatch.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_seq_mismatch(&self) {
if self.telemetry_me_allows_normal() {
self.me_seq_mismatch.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_route_drop_no_conn(&self) {
if self.telemetry_me_allows_normal() {
self.me_route_drop_no_conn.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_reconnect_attempt(&self) { self.me_reconnect_attempts.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_reconnect_success(&self) { self.me_reconnect_success.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_crc_mismatch(&self) { self.me_crc_mismatch.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_seq_mismatch(&self) { self.me_seq_mismatch.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_route_drop_no_conn(&self) { self.me_route_drop_no_conn.fetch_add(1, Ordering::Relaxed); }
pub fn increment_me_route_drop_channel_closed(&self) {
self.me_route_drop_channel_closed.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_route_drop_channel_closed.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_route_drop_queue_full(&self) {
self.me_route_drop_queue_full.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_route_drop_queue_full.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_route_drop_queue_full_base(&self) {
if self.telemetry_me_allows_normal() {
self.me_route_drop_queue_full_base.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_route_drop_queue_full_high(&self) {
if self.telemetry_me_allows_normal() {
self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_socks_kdf_strict_reject(&self) {
if self.telemetry_me_allows_normal() {
self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_socks_kdf_compat_fallback(&self) {
if self.telemetry_me_allows_debug() {
self.me_socks_kdf_compat_fallback.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_secure_padding_invalid(&self) {
self.secure_padding_invalid.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.secure_padding_invalid.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_desync_total(&self) {
self.desync_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.desync_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_desync_full_logged(&self) {
self.desync_full_logged.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.desync_full_logged.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_desync_suppressed(&self) {
self.desync_suppressed.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.desync_suppressed.fetch_add(1, Ordering::Relaxed);
}
}
pub fn observe_desync_frames_ok(&self, frames_ok: u64) {
if !self.telemetry_me_allows_normal() {
return;
}
match frames_ok {
0 => {
self.desync_frames_bucket_0.fetch_add(1, Ordering::Relaxed);
@@ -123,12 +290,19 @@ impl Stats {
}
}
pub fn increment_pool_swap_total(&self) {
self.pool_swap_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_debug() {
self.pool_swap_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_pool_drain_active(&self) {
self.pool_drain_active.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_debug() {
self.pool_drain_active.fetch_add(1, Ordering::Relaxed);
}
}
pub fn decrement_pool_drain_active(&self) {
if !self.telemetry_me_allows_debug() {
return;
}
let mut current = self.pool_drain_active.load(Ordering::Relaxed);
loop {
if current == 0 {
@@ -146,31 +320,110 @@ impl Stats {
}
}
pub fn increment_pool_force_close_total(&self) {
self.pool_force_close_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.pool_force_close_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_pool_stale_pick_total(&self) {
self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_writer_removed_total(&self) {
self.me_writer_removed_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_debug() {
self.me_writer_removed_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_writer_removed_unexpected_total(&self) {
self.me_writer_removed_unexpected_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_writer_removed_unexpected_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_refill_triggered_total(&self) {
self.me_refill_triggered_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_debug() {
self.me_refill_triggered_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_refill_skipped_inflight_total(&self) {
self.me_refill_skipped_inflight_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_debug() {
self.me_refill_skipped_inflight_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_refill_failed_total(&self) {
self.me_refill_failed_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_refill_failed_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_writer_restored_same_endpoint_total(&self) {
self.me_writer_restored_same_endpoint_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_writer_restored_same_endpoint_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_writer_restored_fallback_total(&self) {
self.me_writer_restored_fallback_total.fetch_add(1, Ordering::Relaxed);
if self.telemetry_me_allows_normal() {
self.me_writer_restored_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_endpoint_quarantine_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_endpoint_quarantine_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_kdf_drift_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_hardswap_pending_reuse_total(&self) {
if self.telemetry_me_allows_debug() {
self.me_hardswap_pending_reuse_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_hardswap_pending_ttl_expired_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_hardswap_pending_ttl_expired_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_outage_enter_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_outage_enter_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_outage_exit_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_outage_exit_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_outage_reconnect_attempt_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_outage_reconnect_attempt_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_outage_reconnect_success_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_outage_reconnect_success_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_quarantine_bypass_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_quarantine_bypass_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_single_endpoint_shadow_rotate_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_single_endpoint_shadow_rotate_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) }
pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) }
@@ -180,8 +433,61 @@ impl Stats {
pub fn get_me_keepalive_timeout(&self) -> u64 { self.me_keepalive_timeout.load(Ordering::Relaxed) }
pub fn get_me_reconnect_attempts(&self) -> u64 { self.me_reconnect_attempts.load(Ordering::Relaxed) }
pub fn get_me_reconnect_success(&self) -> u64 { self.me_reconnect_success.load(Ordering::Relaxed) }
pub fn get_me_handshake_reject_total(&self) -> u64 {
self.me_handshake_reject_total.load(Ordering::Relaxed)
}
pub fn get_me_reader_eof_total(&self) -> u64 {
self.me_reader_eof_total.load(Ordering::Relaxed)
}
pub fn get_me_crc_mismatch(&self) -> u64 { self.me_crc_mismatch.load(Ordering::Relaxed) }
pub fn get_me_seq_mismatch(&self) -> u64 { self.me_seq_mismatch.load(Ordering::Relaxed) }
pub fn get_me_endpoint_quarantine_total(&self) -> u64 {
self.me_endpoint_quarantine_total.load(Ordering::Relaxed)
}
pub fn get_me_kdf_drift_total(&self) -> u64 {
self.me_kdf_drift_total.load(Ordering::Relaxed)
}
pub fn get_me_hardswap_pending_reuse_total(&self) -> u64 {
self.me_hardswap_pending_reuse_total
.load(Ordering::Relaxed)
}
pub fn get_me_hardswap_pending_ttl_expired_total(&self) -> u64 {
self.me_hardswap_pending_ttl_expired_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_outage_enter_total(&self) -> u64 {
self.me_single_endpoint_outage_enter_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_outage_exit_total(&self) -> u64 {
self.me_single_endpoint_outage_exit_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_outage_reconnect_attempt_total(&self) -> u64 {
self.me_single_endpoint_outage_reconnect_attempt_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_outage_reconnect_success_total(&self) -> u64 {
self.me_single_endpoint_outage_reconnect_success_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_quarantine_bypass_total(&self) -> u64 {
self.me_single_endpoint_quarantine_bypass_total
.load(Ordering::Relaxed)
}
pub fn get_me_single_endpoint_shadow_rotate_total(&self) -> u64 {
self.me_single_endpoint_shadow_rotate_total
.load(Ordering::Relaxed)
}
pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> {
let mut out: Vec<(i32, u64)> = self
.me_handshake_error_codes
.iter()
.map(|entry| (*entry.key(), entry.value().load(Ordering::Relaxed)))
.collect();
out.sort_by_key(|(code, _)| *code);
out
}
pub fn get_me_route_drop_no_conn(&self) -> u64 { self.me_route_drop_no_conn.load(Ordering::Relaxed) }
pub fn get_me_route_drop_channel_closed(&self) -> u64 {
self.me_route_drop_channel_closed.load(Ordering::Relaxed)
@@ -189,6 +495,18 @@ impl Stats {
pub fn get_me_route_drop_queue_full(&self) -> u64 {
self.me_route_drop_queue_full.load(Ordering::Relaxed)
}
pub fn get_me_route_drop_queue_full_base(&self) -> u64 {
self.me_route_drop_queue_full_base.load(Ordering::Relaxed)
}
pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
}
pub fn get_me_socks_kdf_strict_reject(&self) -> u64 {
self.me_socks_kdf_strict_reject.load(Ordering::Relaxed)
}
pub fn get_me_socks_kdf_compat_fallback(&self) -> u64 {
self.me_socks_kdf_compat_fallback.load(Ordering::Relaxed)
}
pub fn get_secure_padding_invalid(&self) -> u64 {
self.secure_padding_invalid.load(Ordering::Relaxed)
}
@@ -248,11 +566,17 @@ impl Stats {
}
pub fn increment_user_connects(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.connects.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_user_curr_connects(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.curr_connects.fetch_add(1, Ordering::Relaxed);
}
@@ -285,21 +609,33 @@ impl Stats {
}
pub fn add_user_octets_from(&self, user: &str, bytes: u64) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
}
pub fn add_user_octets_to(&self, user: &str, bytes: u64) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
}
pub fn increment_user_msgs_from(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.msgs_from_client.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_user_msgs_to(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
self.user_stats.entry(user.to_string()).or_default()
.msgs_to_client.fetch_add(1, Ordering::Relaxed);
}
@@ -548,6 +884,7 @@ impl ReplayStats {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::MeTelemetryLevel;
use std::sync::Arc;
#[test]
@@ -558,6 +895,40 @@ mod tests {
stats.increment_connects_all();
assert_eq!(stats.get_connects_all(), 3);
}
#[test]
fn test_telemetry_policy_disables_core_and_user_counters() {
let stats = Stats::new();
stats.apply_telemetry_policy(TelemetryPolicy {
core_enabled: false,
user_enabled: false,
me_level: MeTelemetryLevel::Normal,
});
stats.increment_connects_all();
stats.increment_user_connects("alice");
stats.add_user_octets_from("alice", 1024);
assert_eq!(stats.get_connects_all(), 0);
assert_eq!(stats.get_user_curr_connects("alice"), 0);
assert_eq!(stats.get_user_total_octets("alice"), 0);
}
#[test]
fn test_telemetry_policy_me_silent_blocks_me_counters() {
let stats = Stats::new();
stats.apply_telemetry_policy(TelemetryPolicy {
core_enabled: true,
user_enabled: true,
me_level: MeTelemetryLevel::Silent,
});
stats.increment_me_crc_mismatch();
stats.increment_me_keepalive_sent();
stats.increment_me_route_drop_queue_full();
assert_eq!(stats.get_me_crc_mismatch(), 0);
assert_eq!(stats.get_me_keepalive_sent(), 0);
assert_eq!(stats.get_me_route_drop_queue_full(), 0);
}
#[test]
fn test_replay_checker_basic() {

29
src/stats/telemetry.rs Normal file
View File

@@ -0,0 +1,29 @@
use crate::config::{MeTelemetryLevel, TelemetryConfig};
/// Runtime telemetry policy used by hot-path counters.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TelemetryPolicy {
pub core_enabled: bool,
pub user_enabled: bool,
pub me_level: MeTelemetryLevel,
}
impl Default for TelemetryPolicy {
fn default() -> Self {
Self {
core_enabled: true,
user_enabled: true,
me_level: MeTelemetryLevel::Normal,
}
}
}
impl TelemetryPolicy {
pub fn from_config(cfg: &TelemetryConfig) -> Self {
Self {
core_enabled: cfg.core_enabled,
user_enabled: cfg.user_enabled,
me_level: cfg.me_level,
}
}
}

View File

@@ -336,22 +336,35 @@ impl PendingCiphertext {
}
fn remaining_capacity(&self) -> usize {
self.max_len.saturating_sub(self.buf.len())
self.max_len.saturating_sub(self.pending_len())
}
fn compact_consumed_prefix(&mut self) {
if self.pos == 0 {
return;
}
if self.pos >= self.buf.len() {
self.buf.clear();
self.pos = 0;
return;
}
let _ = self.buf.split_to(self.pos);
self.pos = 0;
}
fn advance(&mut self, n: usize) {
self.pos = (self.pos + n).min(self.buf.len());
if self.pos == self.buf.len() {
self.buf.clear();
self.pos = 0;
self.compact_consumed_prefix();
return;
}
// Compact when a large prefix was consumed.
if self.pos >= 16 * 1024 {
let _ = self.buf.split_to(self.pos);
self.pos = 0;
self.compact_consumed_prefix();
}
}
@@ -379,6 +392,11 @@ impl PendingCiphertext {
));
}
// Reclaim consumed prefix when physical storage is the only limiter.
if self.pos > 0 && self.buf.len() + plaintext.len() > self.max_len {
self.compact_consumed_prefix();
}
let start = self.buf.len();
self.buf.reserve(plaintext.len());
self.buf.extend_from_slice(plaintext);
@@ -777,3 +795,70 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for PassthroughStream<S> {
Pin::new(&mut self.inner).poll_shutdown(cx)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn test_ctr() -> AesCtr {
AesCtr::new(&[0x11; 32], 0x0102_0304_0506_0708_1112_1314_1516_1718)
}
#[test]
fn pending_capacity_reclaims_after_partial_advance_without_compaction_threshold() {
let mut pending = PendingCiphertext::new(1024);
let mut ctr = test_ctr();
let payload = vec![0x41; 900];
pending.push_encrypted(&mut ctr, &payload).unwrap();
// Keep position below compaction threshold to validate logical-capacity accounting.
pending.advance(800);
assert_eq!(pending.pending_len(), 100);
assert_eq!(pending.remaining_capacity(), 924);
}
#[test]
fn push_encrypted_respects_pending_limit() {
let mut pending = PendingCiphertext::new(64);
let mut ctr = test_ctr();
pending.push_encrypted(&mut ctr, &[0x10; 64]).unwrap();
let err = pending.push_encrypted(&mut ctr, &[0x20]).unwrap_err();
assert_eq!(err.kind(), ErrorKind::WouldBlock);
}
#[test]
fn push_encrypted_compacts_prefix_when_physical_buffer_would_overflow() {
let mut pending = PendingCiphertext::new(64);
let mut ctr = test_ctr();
pending.push_encrypted(&mut ctr, &[0x22; 60]).unwrap();
pending.advance(30);
pending.push_encrypted(&mut ctr, &[0x33; 30]).unwrap();
assert_eq!(pending.pending_len(), 60);
assert!(pending.buf.len() <= 64);
}
#[test]
fn pending_ciphertext_preserves_stream_order_across_drain_and_append() {
let mut pending = PendingCiphertext::new(128);
let mut ctr = test_ctr();
let first = vec![0xA1; 80];
let second = vec![0xB2; 40];
pending.push_encrypted(&mut ctr, &first).unwrap();
pending.advance(50);
pending.push_encrypted(&mut ctr, &second).unwrap();
let mut baseline_ctr = test_ctr();
let mut baseline_plain = Vec::with_capacity(first.len() + second.len());
baseline_plain.extend_from_slice(&first);
baseline_plain.extend_from_slice(&second);
baseline_ctr.apply(&mut baseline_plain);
let expected = &baseline_plain[50..];
assert_eq!(pending.pending_slice(), expected);
}
}

View File

@@ -2,8 +2,10 @@ use std::sync::Arc;
use std::time::Duration;
use anyhow::{Result, anyhow};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::net::TcpStream;
#[cfg(unix)]
use tokio::net::UnixStream;
use tokio::time::timeout;
use tokio_rustls::client::TlsStream;
use tokio_rustls::TlsConnector;
@@ -18,6 +20,7 @@ use x509_parser::prelude::FromDer;
use x509_parser::certificate::X509Certificate;
use crate::crypto::SecureRandom;
use crate::network::dns_overrides::resolve_socket_addr;
use crate::protocol::constants::{TLS_RECORD_APPLICATION, TLS_RECORD_HANDSHAKE};
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
use crate::tls_front::types::{
@@ -211,7 +214,10 @@ fn gen_key_share(rng: &SecureRandom) -> [u8; 32] {
key
}
async fn read_tls_record(stream: &mut TcpStream) -> Result<(u8, Vec<u8>)> {
async fn read_tls_record<S>(stream: &mut S) -> Result<(u8, Vec<u8>)>
where
S: AsyncRead + Unpin,
{
let mut header = [0u8; 5];
stream.read_exact(&mut header).await?;
let len = u16::from_be_bytes([header[3], header[4]]) as usize;
@@ -333,6 +339,55 @@ fn u24_bytes(value: usize) -> Option<[u8; 3]> {
])
}
async fn connect_with_dns_override(
host: &str,
port: u16,
connect_timeout: Duration,
) -> Result<TcpStream> {
if let Some(addr) = resolve_socket_addr(host, port) {
return Ok(timeout(connect_timeout, TcpStream::connect(addr)).await??);
}
Ok(timeout(connect_timeout, TcpStream::connect((host, port))).await??)
}
async fn connect_tcp_with_upstream(
host: &str,
port: u16,
connect_timeout: Duration,
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
) -> Result<TcpStream> {
if let Some(manager) = upstream {
if let Some(addr) = resolve_socket_addr(host, port) {
match manager.connect(addr, None, None).await {
Ok(stream) => return Ok(stream),
Err(e) => {
warn!(
host = %host,
port = port,
error = %e,
"Upstream connect failed, using direct connect"
);
}
}
} else if let Ok(mut addrs) = tokio::net::lookup_host((host, port)).await {
if let Some(addr) = addrs.find(|a| a.is_ipv4()) {
match manager.connect(addr, None, None).await {
Ok(stream) => return Ok(stream),
Err(e) => {
warn!(
host = %host,
port = port,
error = %e,
"Upstream connect failed, using direct connect"
);
}
}
}
}
}
connect_with_dns_override(host, port, connect_timeout).await
}
fn encode_tls13_certificate_message(cert_chain_der: &[Vec<u8>]) -> Option<Vec<u8>> {
if cert_chain_der.is_empty() {
return None;
@@ -362,16 +417,15 @@ fn encode_tls13_certificate_message(cert_chain_der: &[Vec<u8>]) -> Option<Vec<u8
Some(message)
}
async fn fetch_via_raw_tls(
host: &str,
port: u16,
async fn fetch_via_raw_tls_stream<S>(
mut stream: S,
sni: &str,
connect_timeout: Duration,
proxy_protocol: u8,
) -> Result<TlsFetchResult> {
let addr = format!("{host}:{port}");
let mut stream = timeout(connect_timeout, TcpStream::connect(addr)).await??;
) -> Result<TlsFetchResult>
where
S: AsyncRead + AsyncWrite + Unpin,
{
let rng = SecureRandom::new();
let client_hello = build_client_hello(sni, &rng);
timeout(connect_timeout, async {
@@ -427,36 +481,61 @@ async fn fetch_via_raw_tls(
})
}
async fn fetch_via_rustls(
async fn fetch_via_raw_tls(
host: &str,
port: u16,
sni: &str,
connect_timeout: Duration,
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
proxy_protocol: u8,
unix_sock: Option<&str>,
) -> Result<TlsFetchResult> {
// rustls handshake path for certificate and basic negotiated metadata.
let mut stream = if let Some(manager) = upstream {
// Resolve host to SocketAddr
if let Ok(mut addrs) = tokio::net::lookup_host((host, port)).await {
if let Some(addr) = addrs.find(|a| a.is_ipv4()) {
match manager.connect(addr, None, None).await {
Ok(s) => s,
Err(e) => {
warn!(sni = %sni, error = %e, "Upstream connect failed, using direct connect");
timeout(connect_timeout, TcpStream::connect((host, port))).await??
}
}
} else {
timeout(connect_timeout, TcpStream::connect((host, port))).await??
#[cfg(unix)]
if let Some(sock_path) = unix_sock {
match timeout(connect_timeout, UnixStream::connect(sock_path)).await {
Ok(Ok(stream)) => {
debug!(
sni = %sni,
sock = %sock_path,
"Raw TLS fetch using mask unix socket"
);
return fetch_via_raw_tls_stream(stream, sni, connect_timeout, proxy_protocol).await;
}
Ok(Err(e)) => {
warn!(
sni = %sni,
sock = %sock_path,
error = %e,
"Raw TLS unix socket connect failed, falling back to TCP"
);
}
Err(_) => {
warn!(
sni = %sni,
sock = %sock_path,
"Raw TLS unix socket connect timed out, falling back to TCP"
);
}
} else {
timeout(connect_timeout, TcpStream::connect((host, port))).await??
}
} else {
timeout(connect_timeout, TcpStream::connect((host, port))).await??
};
}
#[cfg(not(unix))]
let _ = unix_sock;
let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream).await?;
fetch_via_raw_tls_stream(stream, sni, connect_timeout, proxy_protocol).await
}
async fn fetch_via_rustls_stream<S>(
mut stream: S,
host: &str,
sni: &str,
proxy_protocol: u8,
) -> Result<TlsFetchResult>
where
S: AsyncRead + AsyncWrite + Unpin,
{
// rustls handshake path for certificate and basic negotiated metadata.
if proxy_protocol > 0 {
let header = match proxy_protocol {
2 => ProxyProtocolV2Builder::new().build(),
@@ -473,7 +552,7 @@ async fn fetch_via_rustls(
.or_else(|_| ServerName::try_from(host.to_owned()))
.map_err(|_| RustlsError::General("invalid SNI".into()))?;
let tls_stream: TlsStream<TcpStream> = connector.connect(server_name, stream).await?;
let tls_stream: TlsStream<S> = connector.connect(server_name, stream).await?;
// Extract negotiated parameters and certificates
let (_io, session) = tls_stream.get_ref();
@@ -534,6 +613,51 @@ async fn fetch_via_rustls(
})
}
async fn fetch_via_rustls(
host: &str,
port: u16,
sni: &str,
connect_timeout: Duration,
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
proxy_protocol: u8,
unix_sock: Option<&str>,
) -> Result<TlsFetchResult> {
#[cfg(unix)]
if let Some(sock_path) = unix_sock {
match timeout(connect_timeout, UnixStream::connect(sock_path)).await {
Ok(Ok(stream)) => {
debug!(
sni = %sni,
sock = %sock_path,
"Rustls fetch using mask unix socket"
);
return fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await;
}
Ok(Err(e)) => {
warn!(
sni = %sni,
sock = %sock_path,
error = %e,
"Rustls unix socket connect failed, falling back to TCP"
);
}
Err(_) => {
warn!(
sni = %sni,
sock = %sock_path,
"Rustls unix socket connect timed out, falling back to TCP"
);
}
}
}
#[cfg(not(unix))]
let _ = unix_sock;
let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream).await?;
fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await
}
/// Fetch real TLS metadata for the given SNI.
///
/// Strategy:
@@ -547,8 +671,19 @@ pub async fn fetch_real_tls(
connect_timeout: Duration,
upstream: Option<std::sync::Arc<crate::transport::UpstreamManager>>,
proxy_protocol: u8,
unix_sock: Option<&str>,
) -> Result<TlsFetchResult> {
let raw_result = match fetch_via_raw_tls(host, port, sni, connect_timeout, proxy_protocol).await {
let raw_result = match fetch_via_raw_tls(
host,
port,
sni,
connect_timeout,
upstream.clone(),
proxy_protocol,
unix_sock,
)
.await
{
Ok(res) => Some(res),
Err(e) => {
warn!(sni = %sni, error = %e, "Raw TLS fetch failed");
@@ -556,7 +691,17 @@ pub async fn fetch_real_tls(
}
};
match fetch_via_rustls(host, port, sni, connect_timeout, upstream, proxy_protocol).await {
match fetch_via_rustls(
host,
port,
sni,
connect_timeout,
upstream,
proxy_protocol,
unix_sock,
)
.await
{
Ok(rustls_result) => {
if let Some(mut raw) = raw_result {
raw.cert_info = rustls_result.cert_info;

View File

@@ -5,15 +5,15 @@ use std::sync::Arc;
use std::time::Duration;
use httpdate;
use tokio::sync::watch;
use tokio::sync::{mpsc, watch};
use tracing::{debug, info, warn};
use crate::config::ProxyConfig;
use crate::error::Result;
use super::MePool;
use super::rotation::{MeReinitTrigger, enqueue_reinit_trigger};
use super::secret::download_proxy_secret_with_max_len;
use crate::crypto::SecureRandom;
use std::time::SystemTime;
async fn retry_fetch(url: &str) -> Option<ProxyConfigData> {
@@ -38,6 +38,8 @@ async fn retry_fetch(url: &str) -> Option<ProxyConfigData> {
pub struct ProxyConfigData {
pub map: HashMap<i32, Vec<(IpAddr, u16)>>,
pub default_dc: Option<i32>,
pub http_status: u16,
pub proxy_for_lines: u32,
}
#[derive(Debug, Default)]
@@ -172,6 +174,7 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
.await
.map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config GET failed: {e}")))?
;
let http_status = resp.status().as_u16();
if let Some(date) = resp.headers().get(reqwest::header::DATE)
&& let Ok(date_str) = date.to_str()
@@ -194,9 +197,11 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
.map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config read failed: {e}")))?;
let mut map: HashMap<i32, Vec<(IpAddr, u16)>> = HashMap::new();
let mut proxy_for_lines: u32 = 0;
for line in text.lines() {
if let Some((dc, ip, port)) = parse_proxy_line(line) {
map.entry(dc).or_default().push((ip, port));
proxy_for_lines = proxy_for_lines.saturating_add(1);
}
}
@@ -214,14 +219,49 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
None
});
Ok(ProxyConfigData { map, default_dc })
Ok(ProxyConfigData {
map,
default_dc,
http_status,
proxy_for_lines,
})
}
fn snapshot_passes_guards(
cfg: &ProxyConfig,
snapshot: &ProxyConfigData,
snapshot_name: &'static str,
) -> bool {
if cfg.general.me_snapshot_require_http_2xx
&& !(200..=299).contains(&snapshot.http_status)
{
warn!(
snapshot = snapshot_name,
http_status = snapshot.http_status,
"ME snapshot rejected by non-2xx HTTP status"
);
return false;
}
let min_proxy_for = cfg.general.me_snapshot_min_proxy_for_lines;
if snapshot.proxy_for_lines < min_proxy_for {
warn!(
snapshot = snapshot_name,
parsed_proxy_for_lines = snapshot.proxy_for_lines,
min_proxy_for_lines = min_proxy_for,
"ME snapshot rejected by proxy_for line floor"
);
return false;
}
true
}
async fn run_update_cycle(
pool: &Arc<MePool>,
rng: &Arc<SecureRandom>,
cfg: &ProxyConfig,
state: &mut UpdaterState,
reinit_tx: &mpsc::Sender<MeReinitTrigger>,
) {
pool.update_runtime_reinit_policy(
cfg.general.hardswap,
@@ -232,6 +272,16 @@ async fn run_update_cycle(
cfg.general.me_hardswap_warmup_delay_max_ms,
cfg.general.me_hardswap_warmup_extra_passes,
cfg.general.me_hardswap_warmup_pass_backoff_base_ms,
cfg.general.me_bind_stale_mode,
cfg.general.me_bind_stale_ttl_secs,
cfg.general.me_secret_atomic_snapshot,
cfg.general.me_deterministic_writer_sort,
cfg.general.me_single_endpoint_shadow_writers,
cfg.general.me_single_endpoint_outage_mode_enabled,
cfg.general.me_single_endpoint_outage_disable_quarantine,
cfg.general.me_single_endpoint_outage_backoff_min_ms,
cfg.general.me_single_endpoint_outage_backoff_max_ms,
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
);
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
@@ -242,44 +292,48 @@ async fn run_update_cycle(
let mut ready_v4: Option<(ProxyConfigData, u64)> = None;
let cfg_v4 = retry_fetch("https://core.telegram.org/getProxyConfig").await;
if let Some(cfg_v4) = cfg_v4 {
let cfg_v4_hash = hash_proxy_config(&cfg_v4);
let stable_hits = state.config_v4.observe(cfg_v4_hash);
if stable_hits < required_cfg_snapshots {
debug!(
stable_hits,
required_cfg_snapshots,
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
"ME config v4 candidate observed"
);
} else if state.config_v4.is_applied(cfg_v4_hash) {
debug!(
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
"ME config v4 stable snapshot already applied"
);
} else {
ready_v4 = Some((cfg_v4, cfg_v4_hash));
if snapshot_passes_guards(cfg, &cfg_v4, "getProxyConfig") {
let cfg_v4_hash = hash_proxy_config(&cfg_v4);
let stable_hits = state.config_v4.observe(cfg_v4_hash);
if stable_hits < required_cfg_snapshots {
debug!(
stable_hits,
required_cfg_snapshots,
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
"ME config v4 candidate observed"
);
} else if state.config_v4.is_applied(cfg_v4_hash) {
debug!(
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
"ME config v4 stable snapshot already applied"
);
} else {
ready_v4 = Some((cfg_v4, cfg_v4_hash));
}
}
}
let mut ready_v6: Option<(ProxyConfigData, u64)> = None;
let cfg_v6 = retry_fetch("https://core.telegram.org/getProxyConfigV6").await;
if let Some(cfg_v6) = cfg_v6 {
let cfg_v6_hash = hash_proxy_config(&cfg_v6);
let stable_hits = state.config_v6.observe(cfg_v6_hash);
if stable_hits < required_cfg_snapshots {
debug!(
stable_hits,
required_cfg_snapshots,
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
"ME config v6 candidate observed"
);
} else if state.config_v6.is_applied(cfg_v6_hash) {
debug!(
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
"ME config v6 stable snapshot already applied"
);
} else {
ready_v6 = Some((cfg_v6, cfg_v6_hash));
if snapshot_passes_guards(cfg, &cfg_v6, "getProxyConfigV6") {
let cfg_v6_hash = hash_proxy_config(&cfg_v6);
let stable_hits = state.config_v6.observe(cfg_v6_hash);
if stable_hits < required_cfg_snapshots {
debug!(
stable_hits,
required_cfg_snapshots,
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
"ME config v6 candidate observed"
);
} else if state.config_v6.is_applied(cfg_v6_hash) {
debug!(
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
"ME config v6 stable snapshot already applied"
);
} else {
ready_v6 = Some((cfg_v6, cfg_v6_hash));
}
}
}
@@ -292,28 +346,40 @@ async fn run_update_cycle(
let update_v6 = ready_v6
.as_ref()
.map(|(snapshot, _)| snapshot.map.clone());
let changed = pool.update_proxy_maps(update_v4, update_v6).await;
if let Some((snapshot, hash)) = ready_v4 {
if let Some(dc) = snapshot.default_dc {
pool.default_dc
.store(dc, std::sync::atomic::Ordering::Relaxed);
}
state.config_v4.mark_applied(hash);
}
if let Some((_snapshot, hash)) = ready_v6 {
state.config_v6.mark_applied(hash);
}
state.last_map_apply_at = Some(tokio::time::Instant::now());
if changed {
maps_changed = true;
info!("ME config update applied after stable-gate");
let update_is_empty =
update_v4.is_empty() && update_v6.as_ref().is_none_or(|v| v.is_empty());
let apply_outcome = if update_is_empty && !cfg.general.me_snapshot_reject_empty_map {
super::pool_config::SnapshotApplyOutcome::AppliedNoDelta
} else {
debug!("ME config stable-gate applied with no map delta");
pool.update_proxy_maps(update_v4, update_v6).await
};
if matches!(
apply_outcome,
super::pool_config::SnapshotApplyOutcome::RejectedEmpty
) {
warn!("ME config stable snapshot rejected (empty endpoint map)");
} else {
if let Some((snapshot, hash)) = ready_v4 {
if let Some(dc) = snapshot.default_dc {
pool.default_dc
.store(dc, std::sync::atomic::Ordering::Relaxed);
}
state.config_v4.mark_applied(hash);
}
if let Some((_snapshot, hash)) = ready_v6 {
state.config_v6.mark_applied(hash);
}
state.last_map_apply_at = Some(tokio::time::Instant::now());
if apply_outcome.changed() {
maps_changed = true;
info!("ME config update applied after stable-gate");
} else {
debug!("ME config stable-gate applied with no map delta");
}
}
} else if let Some(last) = state.last_map_apply_at {
let wait_secs = map_apply_cooldown_remaining_secs(last, apply_cooldown);
@@ -325,8 +391,7 @@ async fn run_update_cycle(
}
if maps_changed {
pool.zero_downtime_reinit_after_map_change(rng.as_ref())
.await;
enqueue_reinit_trigger(reinit_tx, MeReinitTrigger::MapChanged);
}
pool.reset_stun_state();
@@ -367,8 +432,8 @@ async fn run_update_cycle(
pub async fn me_config_updater(
pool: Arc<MePool>,
rng: Arc<SecureRandom>,
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
reinit_tx: mpsc::Sender<MeReinitTrigger>,
) {
let mut state = UpdaterState::default();
let mut update_every_secs = config_rx
@@ -387,7 +452,7 @@ pub async fn me_config_updater(
tokio::select! {
_ = &mut sleep => {
let cfg = config_rx.borrow().clone();
run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await;
run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await;
let refreshed_secs = cfg.general.effective_update_every_secs().max(1);
if refreshed_secs != update_every_secs {
info!(
@@ -415,6 +480,16 @@ pub async fn me_config_updater(
cfg.general.me_hardswap_warmup_delay_max_ms,
cfg.general.me_hardswap_warmup_extra_passes,
cfg.general.me_hardswap_warmup_pass_backoff_base_ms,
cfg.general.me_bind_stale_mode,
cfg.general.me_bind_stale_ttl_secs,
cfg.general.me_secret_atomic_snapshot,
cfg.general.me_deterministic_writer_sort,
cfg.general.me_single_endpoint_shadow_writers,
cfg.general.me_single_endpoint_outage_mode_enabled,
cfg.general.me_single_endpoint_outage_disable_quarantine,
cfg.general.me_single_endpoint_outage_backoff_min_ms,
cfg.general.me_single_endpoint_outage_backoff_max_ms,
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
);
let new_secs = cfg.general.effective_update_every_secs().max(1);
if new_secs == update_every_secs {
@@ -429,7 +504,7 @@ pub async fn me_config_updater(
);
update_every_secs = new_secs;
update_every = Duration::from_secs(update_every_secs);
run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await;
run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await;
next_tick = tokio::time::Instant::now() + update_every;
} else {
info!(

View File

@@ -1,5 +1,8 @@
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use socket2::{SockRef, TcpKeepalive};
#[cfg(target_os = "linux")]
use libc;
@@ -14,13 +17,16 @@ use tokio::net::{TcpStream, TcpSocket};
use tokio::time::timeout;
use tracing::{debug, info, warn};
use crate::config::MeSocksKdfPolicy;
use crate::crypto::{SecureRandom, build_middleproxy_prekey, derive_middleproxy_keys, sha256};
use crate::error::{ProxyError, Result};
use crate::network::IpFamily;
use crate::network::probe::is_bogon;
use crate::protocol::constants::{
ME_CONNECT_TIMEOUT_SECS, ME_HANDSHAKE_TIMEOUT_SECS, RPC_CRYPTO_AES_U32,
RPC_HANDSHAKE_ERROR_U32, rpc_crypto_flags,
};
use crate::transport::{UpstreamEgressInfo, UpstreamRouteKind};
use super::codec::{
RpcChecksumMode, build_handshake_payload, build_nonce_payload, build_rpc_frame,
@@ -30,6 +36,8 @@ use super::codec::{
use super::wire::{extract_ip_material, IpMaterial};
use super::MePool;
const ME_KDF_DRIFT_STRICT: bool = false;
/// Result of a successful ME handshake with timings.
pub(crate) struct HandshakeOutput {
pub rd: ReadHalf<TcpStream>,
@@ -43,33 +51,141 @@ pub(crate) struct HandshakeOutput {
}
impl MePool {
/// TCP connect with timeout + return RTT in milliseconds.
pub(crate) async fn connect_tcp(&self, addr: SocketAddr) -> Result<(TcpStream, f64)> {
let start = Instant::now();
let connect_fut = async {
if addr.is_ipv6()
&& let Some(v6) = self.detected_ipv6
{
match TcpSocket::new_v6() {
Ok(sock) => {
if let Err(e) = sock.bind(SocketAddr::new(IpAddr::V6(v6), 0)) {
debug!(error = %e, bind_ip = %v6, "ME IPv6 bind failed, falling back to default bind");
} else {
match sock.connect(addr).await {
Ok(stream) => return Ok(stream),
Err(e) => debug!(error = %e, target = %addr, "ME IPv6 bound connect failed, retrying default connect"),
}
}
fn kdf_material_fingerprint(
local_addr_nat: SocketAddr,
peer_addr_nat: SocketAddr,
client_port_for_kdf: u16,
reflected: Option<SocketAddr>,
socks_bound_addr: Option<SocketAddr>,
) -> u64 {
let mut hasher = DefaultHasher::new();
local_addr_nat.hash(&mut hasher);
peer_addr_nat.hash(&mut hasher);
client_port_for_kdf.hash(&mut hasher);
reflected.hash(&mut hasher);
socks_bound_addr.hash(&mut hasher);
hasher.finish()
}
async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option<i16> {
if addr.is_ipv4() {
let map = self.proxy_map_v4.read().await;
for (dc, addrs) in map.iter() {
if addrs
.iter()
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
{
let abs_dc = dc.abs();
if abs_dc > 0
&& let Ok(dc_idx) = i16::try_from(abs_dc)
{
return Some(dc_idx);
}
Err(e) => debug!(error = %e, "ME IPv6 socket creation failed, falling back to default connect"),
}
}
TcpStream::connect(addr).await
} else {
let map = self.proxy_map_v6.read().await;
for (dc, addrs) in map.iter() {
if addrs
.iter()
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
{
let abs_dc = dc.abs();
if abs_dc > 0
&& let Ok(dc_idx) = i16::try_from(abs_dc)
{
return Some(dc_idx);
}
}
}
}
None
}
fn direct_bind_ip_for_stun(
family: IpFamily,
upstream_egress: Option<UpstreamEgressInfo>,
) -> Option<IpAddr> {
let info = upstream_egress?;
if info.route_kind != UpstreamRouteKind::Direct {
return None;
}
match (family, info.direct_bind_ip) {
(IpFamily::V4, Some(IpAddr::V4(ip))) => Some(IpAddr::V4(ip)),
(IpFamily::V6, Some(IpAddr::V6(ip))) => Some(IpAddr::V6(ip)),
_ => None,
}
}
fn select_socks_bound_addr(
family: IpFamily,
upstream_egress: Option<UpstreamEgressInfo>,
) -> Option<SocketAddr> {
let info = upstream_egress?;
if !matches!(
info.route_kind,
UpstreamRouteKind::Socks4 | UpstreamRouteKind::Socks5
) {
return None;
}
let bound = info.socks_bound_addr?;
let family_matches = matches!(
(family, bound.ip()),
(IpFamily::V4, IpAddr::V4(_)) | (IpFamily::V6, IpAddr::V6(_))
);
if !family_matches || is_bogon(bound.ip()) || bound.ip().is_unspecified() {
return None;
}
Some(bound)
}
fn is_socks_route(upstream_egress: Option<UpstreamEgressInfo>) -> bool {
matches!(
upstream_egress.map(|info| info.route_kind),
Some(UpstreamRouteKind::Socks4 | UpstreamRouteKind::Socks5)
)
}
/// TCP connect with timeout + return RTT in milliseconds.
pub(crate) async fn connect_tcp(
&self,
addr: SocketAddr,
) -> Result<(TcpStream, f64, Option<UpstreamEgressInfo>)> {
let start = Instant::now();
let (stream, upstream_egress) = if let Some(upstream) = &self.upstream {
let dc_idx = self.resolve_dc_idx_for_endpoint(addr).await;
let (stream, egress) = upstream.connect_with_details(addr, dc_idx, None).await?;
(stream, Some(egress))
} else {
let connect_fut = async {
if addr.is_ipv6()
&& let Some(v6) = self.detected_ipv6
{
match TcpSocket::new_v6() {
Ok(sock) => {
if let Err(e) = sock.bind(SocketAddr::new(IpAddr::V6(v6), 0)) {
debug!(error = %e, bind_ip = %v6, "ME IPv6 bind failed, falling back to default bind");
} else {
match sock.connect(addr).await {
Ok(stream) => return Ok(stream),
Err(e) => debug!(error = %e, target = %addr, "ME IPv6 bound connect failed, retrying default connect"),
}
}
}
Err(e) => debug!(error = %e, "ME IPv6 socket creation failed, falling back to default connect"),
}
}
TcpStream::connect(addr).await
};
let stream = timeout(Duration::from_secs(ME_CONNECT_TIMEOUT_SECS), connect_fut)
.await
.map_err(|_| ProxyError::ConnectionTimeout {
addr: addr.to_string(),
})??;
(stream, None)
};
let stream = timeout(Duration::from_secs(ME_CONNECT_TIMEOUT_SECS), connect_fut)
.await
.map_err(|_| ProxyError::ConnectionTimeout { addr: addr.to_string() })??;
let connect_ms = start.elapsed().as_secs_f64() * 1000.0;
stream.set_nodelay(true).ok();
if let Err(e) = Self::configure_keepalive(&stream) {
@@ -79,7 +195,7 @@ impl MePool {
if let Err(e) = Self::configure_user_timeout(stream.as_raw_fd()) {
warn!(error = %e, "ME TCP_USER_TIMEOUT setup failed");
}
Ok((stream, connect_ms))
Ok((stream, connect_ms, upstream_egress))
}
fn configure_keepalive(stream: &TcpStream) -> std::io::Result<()> {
@@ -117,12 +233,14 @@ impl MePool {
&self,
stream: TcpStream,
addr: SocketAddr,
upstream_egress: Option<UpstreamEgressInfo>,
rng: &SecureRandom,
) -> Result<HandshakeOutput> {
let hs_start = Instant::now();
let local_addr = stream.local_addr().map_err(ProxyError::Io)?;
let peer_addr = stream.peer_addr().map_err(ProxyError::Io)?;
let transport_peer_addr = stream.peer_addr().map_err(ProxyError::Io)?;
let peer_addr = addr;
let _ = self.maybe_detect_nat_ip(local_addr.ip()).await;
let family = if local_addr.ip().is_ipv4() {
@@ -130,8 +248,32 @@ impl MePool {
} else {
IpFamily::V6
};
let reflected = if self.nat_probe {
self.maybe_reflect_public_addr(family).await
let is_socks_route = Self::is_socks_route(upstream_egress);
let socks_bound_addr = Self::select_socks_bound_addr(family, upstream_egress);
let reflected = if let Some(bound) = socks_bound_addr {
Some(bound)
} else if is_socks_route {
match self.socks_kdf_policy() {
MeSocksKdfPolicy::Strict => {
self.stats.increment_me_socks_kdf_strict_reject();
return Err(ProxyError::InvalidHandshake(
"SOCKS route returned no valid BND.ADDR for ME KDF (strict policy)"
.to_string(),
));
}
MeSocksKdfPolicy::Compat => {
self.stats.increment_me_socks_kdf_compat_fallback();
if self.nat_probe {
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
self.maybe_reflect_public_addr(family, bind_ip).await
} else {
None
}
}
}
} else if self.nat_probe {
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
self.maybe_reflect_public_addr(family, bind_ip).await
} else {
None
};
@@ -146,7 +288,16 @@ impl MePool {
.unwrap_or_default()
.as_secs() as u32;
let ks = self.key_selector().await;
let secret_atomic_snapshot = self.secret_atomic_snapshot.load(Ordering::Relaxed);
let (ks, secret) = if secret_atomic_snapshot {
let snapshot = self.secret_snapshot().await;
(snapshot.key_selector, snapshot.secret)
} else {
// Backward-compatible mode: key selector and secret may come from different updates.
let key_selector = self.key_selector().await;
let secret = self.secret_snapshot().await.secret;
(key_selector, secret)
};
let nonce_payload = build_nonce_payload(ks, crypto_ts, &my_nonce);
let nonce_frame = build_rpc_frame(-2, &nonce_payload, RpcChecksumMode::Crc32);
let dump = hex_dump(&nonce_frame[..nonce_frame.len().min(44)]);
@@ -197,7 +348,9 @@ impl MePool {
%local_addr_nat,
reflected_ip = reflected.map(|r| r.ip()).as_ref().map(ToString::to_string),
%peer_addr,
%transport_peer_addr,
%peer_addr_nat,
socks_bound_addr = socks_bound_addr.map(|v| v.to_string()),
key_selector = format_args!("0x{ks:08x}"),
crypto_schema = format_args!("0x{schema:08x}"),
skew_secs = skew,
@@ -206,7 +359,38 @@ impl MePool {
let ts_bytes = crypto_ts.to_le_bytes();
let server_port_bytes = peer_addr_nat.port().to_le_bytes();
let client_port_bytes = local_addr_nat.port().to_le_bytes();
let client_port_for_kdf = socks_bound_addr
.map(|bound| bound.port())
.filter(|port| *port != 0)
.unwrap_or(local_addr_nat.port());
let kdf_fingerprint = Self::kdf_material_fingerprint(
local_addr_nat,
peer_addr_nat,
client_port_for_kdf,
reflected,
socks_bound_addr,
);
let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await;
if let Some(prev_fingerprint) = kdf_fingerprint_guard.get(&peer_addr_nat).copied()
&& prev_fingerprint != kdf_fingerprint
{
self.stats.increment_me_kdf_drift_total();
warn!(
%peer_addr_nat,
%local_addr_nat,
client_port_for_kdf,
"ME KDF input drift detected for endpoint"
);
if ME_KDF_DRIFT_STRICT {
return Err(ProxyError::InvalidHandshake(
"ME KDF input drift detected (strict mode)".to_string(),
));
}
}
kdf_fingerprint_guard.insert(peer_addr_nat, kdf_fingerprint);
drop(kdf_fingerprint_guard);
let client_port_bytes = client_port_for_kdf.to_le_bytes();
let server_ip = extract_ip_material(peer_addr_nat);
let client_ip = extract_ip_material(local_addr_nat);
@@ -230,8 +414,6 @@ impl MePool {
let diag_level: u8 = std::env::var("ME_DIAG").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
let secret: Vec<u8> = self.proxy_secret.read().await.clone();
let prekey_client = build_middleproxy_prekey(
&srv_nonce,
&my_nonce,
@@ -405,6 +587,8 @@ impl MePool {
} else {
-1
};
self.stats.increment_me_handshake_reject_total();
self.stats.increment_me_handshake_error_code(err_code);
return Err(ProxyError::InvalidHandshake(format!(
"ME rejected handshake (error={err_code})"
)));

View File

@@ -1,10 +1,11 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tracing::{debug, info, warn};
use rand::Rng;
use tracing::{debug, info, warn};
use crate::crypto::SecureRandom;
use crate::network::IpFamily;
@@ -15,11 +16,16 @@ const HEALTH_INTERVAL_SECS: u64 = 1;
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
#[allow(dead_code)]
const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1;
const SHADOW_ROTATE_RETRY_SECS: u64 = 30;
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut inflight: HashMap<(i32, IpFamily), usize> = HashMap::new();
let mut outage_backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new();
let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new();
loop {
tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await;
pool.prune_closed_writers().await;
@@ -30,6 +36,10 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
&mut backoff,
&mut next_attempt,
&mut inflight,
&mut outage_backoff,
&mut outage_next_attempt,
&mut single_endpoint_outage,
&mut shadow_rotate_deadline,
)
.await;
check_family(
@@ -39,6 +49,10 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
&mut backoff,
&mut next_attempt,
&mut inflight,
&mut outage_backoff,
&mut outage_next_attempt,
&mut single_endpoint_outage,
&mut shadow_rotate_deadline,
)
.await;
}
@@ -51,6 +65,10 @@ async fn check_family(
backoff: &mut HashMap<(i32, IpFamily), u64>,
next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
inflight: &mut HashMap<(i32, IpFamily), usize>,
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
single_endpoint_outage: &mut HashSet<(i32, IpFamily)>,
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
) {
let enabled = match family {
IpFamily::V4 => pool.decision.ipv4_me,
@@ -78,31 +96,86 @@ async fn check_family(
}
let mut live_addr_counts = HashMap::<SocketAddr, usize>::new();
for writer in pool
.writers
.read()
.await
.iter()
.filter(|w| !w.draining.load(std::sync::atomic::Ordering::Relaxed))
{
let mut live_writer_ids_by_addr = HashMap::<SocketAddr, Vec<u64>>::new();
for writer in pool.writers.read().await.iter().filter(|w| {
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
}) {
*live_addr_counts.entry(writer.addr).or_insert(0) += 1;
live_writer_ids_by_addr
.entry(writer.addr)
.or_default()
.push(writer.id);
}
for (dc, endpoints) in dc_endpoints {
if endpoints.is_empty() {
continue;
}
let required = MePool::required_writers_for_dc(endpoints.len());
let required = pool.required_writers_for_dc(endpoints.len());
let alive = endpoints
.iter()
.map(|addr| *live_addr_counts.get(addr).unwrap_or(&0))
.sum::<usize>();
let key = (dc, family);
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
if single_endpoint_outage.insert(key) {
pool.stats.increment_me_single_endpoint_outage_enter_total();
warn!(
dc = %dc,
?family,
required,
endpoint_count = endpoints.len(),
"Single-endpoint DC outage detected"
);
}
recover_single_endpoint_outage(
pool,
rng,
key,
endpoints[0],
required,
outage_backoff,
outage_next_attempt,
)
.await;
continue;
}
if single_endpoint_outage.remove(&key) {
pool.stats.increment_me_single_endpoint_outage_exit_total();
outage_backoff.remove(&key);
outage_next_attempt.remove(&key);
shadow_rotate_deadline.remove(&key);
info!(
dc = %dc,
?family,
alive,
required,
endpoint_count = endpoints.len(),
"Single-endpoint DC outage recovered"
);
}
if alive >= required {
maybe_rotate_single_endpoint_shadow(
pool,
rng,
key,
dc,
family,
&endpoints,
alive,
required,
&live_writer_ids_by_addr,
shadow_rotate_deadline,
)
.await;
continue;
}
let missing = required - alive;
let key = (dc, family);
let now = Instant::now();
if let Some(ts) = next_attempt.get(&key)
&& now < *ts
@@ -112,7 +185,18 @@ async fn check_family(
let max_concurrent = pool.me_reconnect_max_concurrent_per_dc.max(1) as usize;
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
return;
continue;
}
if pool.has_refill_inflight_for_endpoints(&endpoints).await {
debug!(
dc = %dc,
?family,
alive,
required,
endpoint_count = endpoints.len(),
"Skipping health reconnect: immediate refill is already in flight for this DC group"
);
continue;
}
*inflight.entry(key).or_insert(0) += 1;
@@ -177,3 +261,207 @@ async fn check_family(
}
}
}
async fn recover_single_endpoint_outage(
pool: &Arc<MePool>,
rng: &Arc<SecureRandom>,
key: (i32, IpFamily),
endpoint: SocketAddr,
required: usize,
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
) {
let now = Instant::now();
if let Some(ts) = outage_next_attempt.get(&key)
&& now < *ts
{
return;
}
let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms();
pool.stats
.increment_me_single_endpoint_outage_reconnect_attempt_total();
let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine();
let attempt_ok = if bypass_quarantine {
pool.stats
.increment_me_single_endpoint_quarantine_bypass_total();
match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
Ok(Ok(())) => true,
Ok(Err(e)) => {
debug!(
dc = %key.0,
family = ?key.1,
%endpoint,
error = %e,
"Single-endpoint outage reconnect failed (quarantine bypass path)"
);
false
}
Err(_) => {
debug!(
dc = %key.0,
family = ?key.1,
%endpoint,
"Single-endpoint outage reconnect timed out (quarantine bypass path)"
);
false
}
}
} else {
let one_endpoint = [endpoint];
match tokio::time::timeout(
pool.me_one_timeout,
pool.connect_endpoints_round_robin(&one_endpoint, rng.as_ref()),
)
.await
{
Ok(ok) => ok,
Err(_) => {
debug!(
dc = %key.0,
family = ?key.1,
%endpoint,
"Single-endpoint outage reconnect timed out"
);
false
}
}
};
if attempt_ok {
pool.stats
.increment_me_single_endpoint_outage_reconnect_success_total();
pool.stats.increment_me_reconnect_success();
outage_backoff.insert(key, min_backoff_ms);
let jitter = min_backoff_ms / JITTER_FRAC_NUM;
let wait = Duration::from_millis(min_backoff_ms)
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
outage_next_attempt.insert(key, now + wait);
info!(
dc = %key.0,
family = ?key.1,
%endpoint,
required,
backoff_ms = min_backoff_ms,
"Single-endpoint outage reconnect succeeded"
);
return;
}
pool.stats.increment_me_reconnect_attempt();
let current_ms = *outage_backoff.get(&key).unwrap_or(&min_backoff_ms);
let next_ms = current_ms.saturating_mul(2).min(max_backoff_ms);
outage_backoff.insert(key, next_ms);
let jitter = next_ms / JITTER_FRAC_NUM;
let wait = Duration::from_millis(next_ms)
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
outage_next_attempt.insert(key, now + wait);
warn!(
dc = %key.0,
family = ?key.1,
%endpoint,
required,
backoff_ms = next_ms,
"Single-endpoint outage reconnect scheduled"
);
}
async fn maybe_rotate_single_endpoint_shadow(
pool: &Arc<MePool>,
rng: &Arc<SecureRandom>,
key: (i32, IpFamily),
dc: i32,
family: IpFamily,
endpoints: &[SocketAddr],
alive: usize,
required: usize,
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
) {
if endpoints.len() != 1 || alive < required {
return;
}
let Some(interval) = pool.single_endpoint_shadow_rotate_interval() else {
return;
};
let now = Instant::now();
if let Some(deadline) = shadow_rotate_deadline.get(&key)
&& now < *deadline
{
return;
}
let endpoint = endpoints[0];
let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else {
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
return;
};
let mut candidate_writer_id = None;
for writer_id in writer_ids {
if pool.registry.is_writer_empty(*writer_id).await {
candidate_writer_id = Some(*writer_id);
break;
}
}
let Some(old_writer_id) = candidate_writer_id else {
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
debug!(
dc = %dc,
?family,
%endpoint,
alive,
required,
"Single-endpoint shadow rotation skipped: no empty writer candidate"
);
return;
};
let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
Ok(Ok(())) => true,
Ok(Err(e)) => {
debug!(
dc = %dc,
?family,
%endpoint,
error = %e,
"Single-endpoint shadow rotation connect failed"
);
false
}
Err(_) => {
debug!(
dc = %dc,
?family,
%endpoint,
"Single-endpoint shadow rotation connect timed out"
);
false
}
};
if !rotate_ok {
shadow_rotate_deadline.insert(
key,
now + interval.min(Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)),
);
return;
}
pool.mark_writer_draining_with_timeout(old_writer_id, pool.force_close_timeout(), false)
.await;
pool.stats.increment_me_single_endpoint_shadow_rotate_total();
shadow_rotate_deadline.insert(key, now + interval);
info!(
dc = %dc,
?family,
%endpoint,
old_writer_id,
rotate_every_secs = interval.as_secs(),
"Single-endpoint shadow writer rotated"
);
}

View File

@@ -23,14 +23,14 @@ use bytes::Bytes;
pub use health::me_health_monitor;
#[allow(unused_imports)]
pub use ping::{run_me_ping, format_sample_line, MePingReport, MePingSample, MePingFamily};
pub use ping::{run_me_ping, format_sample_line, format_me_route, MePingReport, MePingSample, MePingFamily};
pub use pool::MePool;
#[allow(unused_imports)]
pub use pool_nat::{stun_probe, detect_public_ip};
pub use registry::ConnRegistry;
pub use secret::fetch_proxy_secret;
pub use config_updater::{fetch_proxy_config, me_config_updater};
pub use rotation::me_rotation_task;
pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task};
pub use wire::proto_flags_for_tag;
#[derive(Debug)]

View File

@@ -2,8 +2,12 @@ use std::collections::HashMap;
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use tokio::net::UdpSocket;
use crate::config::{UpstreamConfig, UpstreamType};
use crate::crypto::SecureRandom;
use crate::error::ProxyError;
use crate::transport::{UpstreamEgressInfo, UpstreamRouteKind};
use super::MePool;
@@ -17,6 +21,7 @@ pub enum MePingFamily {
pub struct MePingSample {
pub dc: i32,
pub addr: SocketAddr,
pub route: Option<String>,
pub connect_ms: Option<f64>,
pub handshake_ms: Option<f64>,
pub error: Option<String>,
@@ -50,6 +55,208 @@ pub fn format_sample_line(sample: &MePingSample) -> String {
}
}
fn format_direct_with_config(
interface: &Option<String>,
bind_addresses: &Option<Vec<String>>,
) -> Option<String> {
let mut direct_parts: Vec<String> = Vec::new();
if let Some(dev) = interface.as_deref().filter(|v| !v.is_empty()) {
direct_parts.push(format!("dev={dev}"));
}
if let Some(src) = bind_addresses.as_ref().filter(|v| !v.is_empty()) {
direct_parts.push(format!("src={}", src.join(",")));
}
if direct_parts.is_empty() {
None
} else {
Some(format!("direct {}", direct_parts.join(" ")))
}
}
fn pick_target_for_family(reports: &[MePingReport], family: MePingFamily) -> Option<SocketAddr> {
reports.iter().find_map(|report| {
if report.family != family {
return None;
}
report
.samples
.iter()
.find(|s| s.error.is_none() && s.handshake_ms.is_some())
.map(|s| s.addr)
})
}
fn route_from_egress(egress: Option<UpstreamEgressInfo>) -> Option<String> {
let info = egress?;
match info.route_kind {
UpstreamRouteKind::Direct => {
let src_ip = info
.direct_bind_ip
.or_else(|| info.local_addr.map(|addr| addr.ip()));
let ip = src_ip?;
let mut parts = Vec::new();
if let Some(dev) = detect_interface_for_ip(ip) {
parts.push(format!("dev={dev}"));
}
parts.push(format!("src={ip}"));
Some(format!("direct {}", parts.join(" ")))
}
UpstreamRouteKind::Socks4 => {
let route = info
.socks_proxy_addr
.map(|addr| format!("socks4://{addr}"))
.unwrap_or_else(|| "socks4://unknown".to_string());
Some(match info.socks_bound_addr {
Some(bound) => format!("{route} bnd={bound}"),
None => route,
})
}
UpstreamRouteKind::Socks5 => {
let route = info
.socks_proxy_addr
.map(|addr| format!("socks5://{addr}"))
.unwrap_or_else(|| "socks5://unknown".to_string());
Some(match info.socks_bound_addr {
Some(bound) => format!("{route} bnd={bound}"),
None => route,
})
}
}
}
#[cfg(unix)]
fn detect_interface_for_ip(ip: IpAddr) -> Option<String> {
use nix::ifaddrs::getifaddrs;
if let Ok(addrs) = getifaddrs() {
for iface in addrs {
if let Some(address) = iface.address {
if let Some(v4) = address.as_sockaddr_in() {
if IpAddr::V4(v4.ip()) == ip {
return Some(iface.interface_name);
}
} else if let Some(v6) = address.as_sockaddr_in6() {
if IpAddr::V6(v6.ip()) == ip {
return Some(iface.interface_name);
}
}
}
}
}
None
}
#[cfg(not(unix))]
fn detect_interface_for_ip(_ip: IpAddr) -> Option<String> {
None
}
async fn detect_direct_route_details(
reports: &[MePingReport],
prefer_ipv6: bool,
v4_ok: bool,
v6_ok: bool,
) -> Option<String> {
let target_addr = if prefer_ipv6 && v6_ok {
pick_target_for_family(reports, MePingFamily::V6)
.or_else(|| pick_target_for_family(reports, MePingFamily::V4))
} else if v4_ok {
pick_target_for_family(reports, MePingFamily::V4)
.or_else(|| pick_target_for_family(reports, MePingFamily::V6))
} else {
pick_target_for_family(reports, MePingFamily::V6)
.or_else(|| pick_target_for_family(reports, MePingFamily::V4))
}?;
let local_ip = if target_addr.is_ipv4() {
let sock = UdpSocket::bind("0.0.0.0:0").await.ok()?;
sock.connect(target_addr).await.ok()?;
sock.local_addr().ok().map(|a| a.ip())
} else {
let sock = UdpSocket::bind("[::]:0").await.ok()?;
sock.connect(target_addr).await.ok()?;
sock.local_addr().ok().map(|a| a.ip())
};
let mut parts = Vec::new();
if let Some(ip) = local_ip {
if let Some(dev) = detect_interface_for_ip(ip) {
parts.push(format!("dev={dev}"));
}
parts.push(format!("src={ip}"));
}
if parts.is_empty() {
None
} else {
Some(format!("direct {}", parts.join(" ")))
}
}
pub async fn format_me_route(
upstreams: &[UpstreamConfig],
reports: &[MePingReport],
prefer_ipv6: bool,
v4_ok: bool,
v6_ok: bool,
) -> String {
if let Some(route) = reports
.iter()
.flat_map(|report| report.samples.iter())
.find(|sample| sample.error.is_none() && sample.handshake_ms.is_some())
.and_then(|sample| sample.route.clone())
{
return route;
}
let enabled_upstreams: Vec<_> = upstreams.iter().filter(|u| u.enabled).collect();
if enabled_upstreams.is_empty() {
return detect_direct_route_details(reports, prefer_ipv6, v4_ok, v6_ok)
.await
.unwrap_or_else(|| "direct".to_string());
}
if enabled_upstreams.len() == 1 {
return match &enabled_upstreams[0].upstream_type {
UpstreamType::Direct {
interface,
bind_addresses,
} => {
if let Some(route) = format_direct_with_config(interface, bind_addresses) {
route
} else {
detect_direct_route_details(reports, prefer_ipv6, v4_ok, v6_ok)
.await
.unwrap_or_else(|| "direct".to_string())
}
}
UpstreamType::Socks4 { address, .. } => format!("socks4://{address}"),
UpstreamType::Socks5 { address, .. } => format!("socks5://{address}"),
};
}
let has_direct = enabled_upstreams
.iter()
.any(|u| matches!(u.upstream_type, UpstreamType::Direct { .. }));
let has_socks4 = enabled_upstreams
.iter()
.any(|u| matches!(u.upstream_type, UpstreamType::Socks4 { .. }));
let has_socks5 = enabled_upstreams
.iter()
.any(|u| matches!(u.upstream_type, UpstreamType::Socks5 { .. }));
let mut kinds = Vec::new();
if has_direct {
kinds.push("direct");
}
if has_socks4 {
kinds.push("socks4");
}
if has_socks5 {
kinds.push("socks5");
}
format!("mixed upstreams ({})", kinds.join(", "))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -64,6 +271,7 @@ mod tests {
let s = sample(MePingSample {
dc: 4,
addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)), 8888),
route: Some("direct src=1.2.3.4".to_string()),
connect_ms: Some(12.3),
handshake_ms: Some(34.7),
error: None,
@@ -80,6 +288,7 @@ mod tests {
let s = sample(MePingSample {
dc: -5,
addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(5, 6, 7, 8)), 80),
route: Some("socks5".to_string()),
connect_ms: Some(10.0),
handshake_ms: None,
error: Some("handshake timeout".to_string()),
@@ -120,11 +329,13 @@ pub async fn run_me_ping(pool: &Arc<MePool>, rng: &SecureRandom) -> Vec<MePingRe
let mut connect_ms = None;
let mut handshake_ms = None;
let mut error = None;
let mut route = None;
match pool.connect_tcp(addr).await {
Ok((stream, conn_rtt)) => {
Ok((stream, conn_rtt, upstream_egress)) => {
connect_ms = Some(conn_rtt);
match pool.handshake_only(stream, addr, rng).await {
route = route_from_egress(upstream_egress);
match pool.handshake_only(stream, addr, upstream_egress, rng).await {
Ok(hs) => {
handshake_ms = Some(hs.handshake_ms);
// drop halves to close
@@ -144,6 +355,7 @@ pub async fn run_me_ping(pool: &Arc<MePool>, rng: &SecureRandom) -> Vec<MePingRe
samples.push(MePingSample {
dc,
addr,
route,
connect_ms,
handshake_ms,
error,

View File

@@ -1,24 +1,34 @@
use std::collections::{HashMap, HashSet};
use std::net::{IpAddr, Ipv6Addr, SocketAddr};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::{Mutex, Notify, RwLock, mpsc};
use tokio_util::sync::CancellationToken;
use crate::config::{MeBindStaleMode, MeSocksKdfPolicy};
use crate::crypto::SecureRandom;
use crate::network::IpFamily;
use crate::network::probe::NetworkDecision;
use crate::transport::UpstreamManager;
use super::ConnRegistry;
use super::codec::WriterCommand;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(super) struct RefillDcKey {
pub dc: i32,
pub family: IpFamily,
}
#[derive(Clone)]
pub struct MeWriter {
pub id: u64,
pub addr: SocketAddr,
pub generation: u64,
pub contour: Arc<AtomicU8>,
pub created_at: Instant,
pub tx: mpsc::Sender<WriterCommand>,
pub cancel: CancellationToken,
pub degraded: Arc<AtomicBool>,
@@ -27,15 +37,46 @@ pub struct MeWriter {
pub allow_drain_fallback: Arc<AtomicBool>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub(super) enum WriterContour {
Warm = 0,
Active = 1,
Draining = 2,
}
impl WriterContour {
pub(super) fn as_u8(self) -> u8 {
self as u8
}
pub(super) fn from_u8(value: u8) -> Self {
match value {
0 => Self::Warm,
1 => Self::Active,
2 => Self::Draining,
_ => Self::Draining,
}
}
}
#[derive(Debug, Clone)]
pub struct SecretSnapshot {
pub epoch: u64,
pub key_selector: u32,
pub secret: Vec<u8>,
}
#[allow(dead_code)]
pub struct MePool {
pub(super) registry: Arc<ConnRegistry>,
pub(super) writers: Arc<RwLock<Vec<MeWriter>>>,
pub(super) rr: AtomicU64,
pub(super) decision: NetworkDecision,
pub(super) upstream: Option<Arc<UpstreamManager>>,
pub(super) rng: Arc<SecureRandom>,
pub(super) proxy_tag: Option<Vec<u8>>,
pub(super) proxy_secret: Arc<RwLock<Vec<u8>>>,
pub(super) proxy_secret: Arc<RwLock<SecretSnapshot>>,
pub(super) nat_ip_cfg: Option<IpAddr>,
pub(super) nat_ip_detected: Arc<RwLock<Option<IpAddr>>>,
pub(super) nat_probe: bool,
@@ -60,6 +101,12 @@ pub struct MePool {
pub(super) me_reconnect_backoff_base: Duration,
pub(super) me_reconnect_backoff_cap: Duration,
pub(super) me_reconnect_fast_retry_count: u32,
pub(super) me_single_endpoint_shadow_writers: AtomicU8,
pub(super) me_single_endpoint_outage_mode_enabled: AtomicBool,
pub(super) me_single_endpoint_outage_disable_quarantine: AtomicBool,
pub(super) me_single_endpoint_outage_backoff_min_ms: AtomicU64,
pub(super) me_single_endpoint_outage_backoff_max_ms: AtomicU64,
pub(super) me_single_endpoint_shadow_rotate_every_secs: AtomicU64,
pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
pub(super) default_dc: AtomicI32,
@@ -69,10 +116,18 @@ pub struct MePool {
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
pub(super) writer_available: Arc<Notify>,
pub(super) refill_inflight: Arc<Mutex<HashSet<SocketAddr>>>,
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
pub(super) conn_count: AtomicUsize,
pub(super) stats: Arc<crate::stats::Stats>,
pub(super) generation: AtomicU64,
pub(super) active_generation: AtomicU64,
pub(super) warm_generation: AtomicU64,
pub(super) pending_hardswap_generation: AtomicU64,
pub(super) pending_hardswap_started_at_epoch_secs: AtomicU64,
pub(super) pending_hardswap_map_hash: AtomicU64,
pub(super) hardswap: AtomicBool,
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
pub(super) kdf_material_fingerprint: Arc<Mutex<HashMap<SocketAddr, u64>>>,
pub(super) me_pool_drain_ttl_secs: AtomicU64,
pub(super) me_pool_force_close_secs: AtomicU64,
pub(super) me_pool_min_fresh_ratio_permille: AtomicU32,
@@ -80,6 +135,11 @@ pub struct MePool {
pub(super) me_hardswap_warmup_delay_max_ms: AtomicU64,
pub(super) me_hardswap_warmup_extra_passes: AtomicU32,
pub(super) me_hardswap_warmup_pass_backoff_base_ms: AtomicU64,
pub(super) me_bind_stale_mode: AtomicU8,
pub(super) me_bind_stale_ttl_secs: AtomicU64,
pub(super) secret_atomic_snapshot: AtomicBool,
pub(super) me_deterministic_writer_sort: AtomicBool,
pub(super) me_socks_kdf_policy: AtomicU8,
pool_size: usize,
}
@@ -121,6 +181,7 @@ impl MePool {
proxy_map_v6: HashMap<i32, Vec<(IpAddr, u16)>>,
default_dc: Option<i32>,
decision: NetworkDecision,
upstream: Option<Arc<UpstreamManager>>,
rng: Arc<SecureRandom>,
stats: Arc<crate::stats::Stats>,
me_keepalive_enabled: bool,
@@ -134,6 +195,12 @@ impl MePool {
me_reconnect_backoff_base_ms: u64,
me_reconnect_backoff_cap_ms: u64,
me_reconnect_fast_retry_count: u32,
me_single_endpoint_shadow_writers: u8,
me_single_endpoint_outage_mode_enabled: bool,
me_single_endpoint_outage_disable_quarantine: bool,
me_single_endpoint_outage_backoff_min_ms: u64,
me_single_endpoint_outage_backoff_max_ms: u64,
me_single_endpoint_shadow_rotate_every_secs: u64,
hardswap: bool,
me_pool_drain_ttl_secs: u64,
me_pool_force_close_secs: u64,
@@ -142,15 +209,43 @@ impl MePool {
me_hardswap_warmup_delay_max_ms: u64,
me_hardswap_warmup_extra_passes: u8,
me_hardswap_warmup_pass_backoff_base_ms: u64,
me_bind_stale_mode: MeBindStaleMode,
me_bind_stale_ttl_secs: u64,
me_secret_atomic_snapshot: bool,
me_deterministic_writer_sort: bool,
me_socks_kdf_policy: MeSocksKdfPolicy,
me_route_backpressure_base_timeout_ms: u64,
me_route_backpressure_high_timeout_ms: u64,
me_route_backpressure_high_watermark_pct: u8,
) -> Arc<Self> {
let registry = Arc::new(ConnRegistry::new());
registry.update_route_backpressure_policy(
me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms,
me_route_backpressure_high_watermark_pct,
);
Arc::new(Self {
registry: Arc::new(ConnRegistry::new()),
registry,
writers: Arc::new(RwLock::new(Vec::new())),
rr: AtomicU64::new(0),
decision,
upstream,
rng,
proxy_tag,
proxy_secret: Arc::new(RwLock::new(proxy_secret)),
proxy_secret: Arc::new(RwLock::new(SecretSnapshot {
epoch: 1,
key_selector: if proxy_secret.len() >= 4 {
u32::from_le_bytes([
proxy_secret[0],
proxy_secret[1],
proxy_secret[2],
proxy_secret[3],
])
} else {
0
},
secret: proxy_secret,
})),
nat_ip_cfg: nat_ip,
nat_ip_detected: Arc::new(RwLock::new(None)),
nat_probe,
@@ -176,6 +271,22 @@ impl MePool {
me_reconnect_backoff_base: Duration::from_millis(me_reconnect_backoff_base_ms),
me_reconnect_backoff_cap: Duration::from_millis(me_reconnect_backoff_cap_ms),
me_reconnect_fast_retry_count,
me_single_endpoint_shadow_writers: AtomicU8::new(me_single_endpoint_shadow_writers),
me_single_endpoint_outage_mode_enabled: AtomicBool::new(
me_single_endpoint_outage_mode_enabled,
),
me_single_endpoint_outage_disable_quarantine: AtomicBool::new(
me_single_endpoint_outage_disable_quarantine,
),
me_single_endpoint_outage_backoff_min_ms: AtomicU64::new(
me_single_endpoint_outage_backoff_min_ms,
),
me_single_endpoint_outage_backoff_max_ms: AtomicU64::new(
me_single_endpoint_outage_backoff_max_ms,
),
me_single_endpoint_shadow_rotate_every_secs: AtomicU64::new(
me_single_endpoint_shadow_rotate_every_secs,
),
pool_size: 2,
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
@@ -186,9 +297,17 @@ impl MePool {
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
writer_available: Arc::new(Notify::new()),
refill_inflight: Arc::new(Mutex::new(HashSet::new())),
refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())),
conn_count: AtomicUsize::new(0),
generation: AtomicU64::new(1),
active_generation: AtomicU64::new(1),
warm_generation: AtomicU64::new(0),
pending_hardswap_generation: AtomicU64::new(0),
pending_hardswap_started_at_epoch_secs: AtomicU64::new(0),
pending_hardswap_map_hash: AtomicU64::new(0),
hardswap: AtomicBool::new(hardswap),
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
kdf_material_fingerprint: Arc::new(Mutex::new(HashMap::new())),
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs),
me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille(
@@ -200,15 +319,16 @@ impl MePool {
me_hardswap_warmup_pass_backoff_base_ms: AtomicU64::new(
me_hardswap_warmup_pass_backoff_base_ms,
),
me_bind_stale_mode: AtomicU8::new(me_bind_stale_mode.as_u8()),
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot),
me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort),
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
})
}
pub fn has_proxy_tag(&self) -> bool {
self.proxy_tag.is_some()
}
pub fn current_generation(&self) -> u64 {
self.generation.load(Ordering::Relaxed)
self.active_generation.load(Ordering::Relaxed)
}
pub fn update_runtime_reinit_policy(
@@ -221,6 +341,16 @@ impl MePool {
hardswap_warmup_delay_max_ms: u64,
hardswap_warmup_extra_passes: u8,
hardswap_warmup_pass_backoff_base_ms: u64,
bind_stale_mode: MeBindStaleMode,
bind_stale_ttl_secs: u64,
secret_atomic_snapshot: bool,
deterministic_writer_sort: bool,
single_endpoint_shadow_writers: u8,
single_endpoint_outage_mode_enabled: bool,
single_endpoint_outage_disable_quarantine: bool,
single_endpoint_outage_backoff_min_ms: u64,
single_endpoint_outage_backoff_max_ms: u64,
single_endpoint_shadow_rotate_every_secs: u64,
) {
self.hardswap.store(hardswap, Ordering::Relaxed);
self.me_pool_drain_ttl_secs
@@ -237,6 +367,26 @@ impl MePool {
.store(hardswap_warmup_extra_passes as u32, Ordering::Relaxed);
self.me_hardswap_warmup_pass_backoff_base_ms
.store(hardswap_warmup_pass_backoff_base_ms, Ordering::Relaxed);
self.me_bind_stale_mode
.store(bind_stale_mode.as_u8(), Ordering::Relaxed);
self.me_bind_stale_ttl_secs
.store(bind_stale_ttl_secs, Ordering::Relaxed);
self.secret_atomic_snapshot
.store(secret_atomic_snapshot, Ordering::Relaxed);
self.me_deterministic_writer_sort
.store(deterministic_writer_sort, Ordering::Relaxed);
self.me_single_endpoint_shadow_writers
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
self.me_single_endpoint_outage_mode_enabled
.store(single_endpoint_outage_mode_enabled, Ordering::Relaxed);
self.me_single_endpoint_outage_disable_quarantine
.store(single_endpoint_outage_disable_quarantine, Ordering::Relaxed);
self.me_single_endpoint_outage_backoff_min_ms
.store(single_endpoint_outage_backoff_min_ms, Ordering::Relaxed);
self.me_single_endpoint_outage_backoff_max_ms
.store(single_endpoint_outage_backoff_max_ms, Ordering::Relaxed);
self.me_single_endpoint_shadow_rotate_every_secs
.store(single_endpoint_shadow_rotate_every_secs, Ordering::Relaxed);
}
pub fn reset_stun_state(&self) {
@@ -256,6 +406,26 @@ impl MePool {
&self.registry
}
pub fn update_runtime_transport_policy(
&self,
socks_kdf_policy: MeSocksKdfPolicy,
route_backpressure_base_timeout_ms: u64,
route_backpressure_high_timeout_ms: u64,
route_backpressure_high_watermark_pct: u8,
) {
self.me_socks_kdf_policy
.store(socks_kdf_policy.as_u8(), Ordering::Relaxed);
self.registry.update_route_backpressure_policy(
route_backpressure_base_timeout_ms,
route_backpressure_high_timeout_ms,
route_backpressure_high_watermark_pct,
);
}
pub(super) fn socks_kdf_policy(&self) -> MeSocksKdfPolicy {
MeSocksKdfPolicy::from_u8(self.me_socks_kdf_policy.load(Ordering::Relaxed))
}
pub(super) fn writers_arc(&self) -> Arc<RwLock<Vec<MeWriter>>> {
self.writers.clone()
}
@@ -270,11 +440,62 @@ impl MePool {
}
pub(super) async fn key_selector(&self) -> u32 {
let secret = self.proxy_secret.read().await;
if secret.len() >= 4 {
u32::from_le_bytes([secret[0], secret[1], secret[2], secret[3]])
self.proxy_secret.read().await.key_selector
}
pub(super) async fn secret_snapshot(&self) -> SecretSnapshot {
self.proxy_secret.read().await.clone()
}
pub(super) fn bind_stale_mode(&self) -> MeBindStaleMode {
MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed))
}
pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize {
if endpoint_count == 0 {
return 0;
}
if endpoint_count == 1 {
let shadow = self
.me_single_endpoint_shadow_writers
.load(Ordering::Relaxed) as usize;
return (1 + shadow).max(3);
}
endpoint_count.max(3)
}
pub(super) fn single_endpoint_outage_mode_enabled(&self) -> bool {
self.me_single_endpoint_outage_mode_enabled
.load(Ordering::Relaxed)
}
pub(super) fn single_endpoint_outage_disable_quarantine(&self) -> bool {
self.me_single_endpoint_outage_disable_quarantine
.load(Ordering::Relaxed)
}
pub(super) fn single_endpoint_outage_backoff_bounds_ms(&self) -> (u64, u64) {
let min_ms = self
.me_single_endpoint_outage_backoff_min_ms
.load(Ordering::Relaxed);
let max_ms = self
.me_single_endpoint_outage_backoff_max_ms
.load(Ordering::Relaxed);
if min_ms <= max_ms {
(min_ms, max_ms)
} else {
0
(max_ms, min_ms)
}
}
pub(super) fn single_endpoint_shadow_rotate_interval(&self) -> Option<Duration> {
let secs = self
.me_single_endpoint_shadow_rotate_every_secs
.load(Ordering::Relaxed);
if secs == 0 {
None
} else {
Some(Duration::from_secs(secs))
}
}

View File

@@ -7,12 +7,29 @@ use tracing::warn;
use super::pool::MePool;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SnapshotApplyOutcome {
AppliedChanged,
AppliedNoDelta,
RejectedEmpty,
}
impl SnapshotApplyOutcome {
pub fn changed(self) -> bool {
matches!(self, SnapshotApplyOutcome::AppliedChanged)
}
}
impl MePool {
pub async fn update_proxy_maps(
&self,
new_v4: HashMap<i32, Vec<(IpAddr, u16)>>,
new_v6: Option<HashMap<i32, Vec<(IpAddr, u16)>>>,
) -> bool {
) -> SnapshotApplyOutcome {
if new_v4.is_empty() && new_v6.as_ref().is_none_or(|v| v.is_empty()) {
return SnapshotApplyOutcome::RejectedEmpty;
}
let mut changed = false;
{
let mut guard = self.proxy_map_v4.write().await;
@@ -51,7 +68,11 @@ impl MePool {
}
}
}
changed
if changed {
SnapshotApplyOutcome::AppliedChanged
} else {
SnapshotApplyOutcome::AppliedNoDelta
}
}
pub async fn update_secret(self: &Arc<Self>, new_secret: Vec<u8>) -> bool {
@@ -60,8 +81,19 @@ impl MePool {
return false;
}
let mut guard = self.proxy_secret.write().await;
if *guard != new_secret {
*guard = new_secret;
if guard.secret != new_secret {
guard.secret = new_secret;
guard.key_selector = if guard.secret.len() >= 4 {
u32::from_le_bytes([
guard.secret[0],
guard.secret[1],
guard.secret[2],
guard.secret[3],
])
} else {
0
};
guard.epoch = guard.epoch.saturating_add(1);
drop(guard);
self.reconnect_all().await;
return true;

View File

@@ -19,7 +19,7 @@ impl MePool {
me_servers = self.proxy_map_v4.read().await.len(),
pool_size,
key_selector = format_args!("0x{ks:08x}"),
secret_len = self.proxy_secret.read().await.len(),
secret_len = self.proxy_secret.read().await.secret.len(),
"Initializing ME pool"
);

View File

@@ -8,7 +8,7 @@ use tracing::{debug, info, warn};
use crate::error::{ProxyError, Result};
use crate::network::probe::is_bogon;
use crate::network::stun::{stun_probe_dual, IpFamily, StunProbeResult};
use crate::network::stun::{stun_probe_dual, stun_probe_family_with_bind, IpFamily};
use super::MePool;
use std::time::Instant;
@@ -52,6 +52,7 @@ impl MePool {
servers: &[String],
family: IpFamily,
attempt: u8,
bind_ip: Option<IpAddr>,
) -> (Vec<String>, Option<std::net::SocketAddr>) {
let mut join_set = JoinSet::new();
let mut next_idx = 0usize;
@@ -64,7 +65,11 @@ impl MePool {
let stun_addr = servers[next_idx].clone();
next_idx += 1;
join_set.spawn(async move {
let res = timeout(STUN_BATCH_TIMEOUT, stun_probe_dual(&stun_addr)).await;
let res = timeout(
STUN_BATCH_TIMEOUT,
stun_probe_family_with_bind(&stun_addr, family, bind_ip),
)
.await;
(stun_addr, res)
});
}
@@ -74,12 +79,7 @@ impl MePool {
};
match task {
Ok((stun_addr, Ok(Ok(res)))) => {
let picked: Option<StunProbeResult> = match family {
IpFamily::V4 => res.v4,
IpFamily::V6 => res.v6,
};
Ok((stun_addr, Ok(Ok(picked)))) => {
if let Some(result) = picked {
live_servers.push(stun_addr.clone());
let entry = best_by_ip
@@ -207,10 +207,21 @@ impl MePool {
pub(super) async fn maybe_reflect_public_addr(
&self,
family: IpFamily,
bind_ip: Option<IpAddr>,
) -> Option<std::net::SocketAddr> {
const STUN_CACHE_TTL: Duration = Duration::from_secs(600);
let use_shared_cache = bind_ip.is_none();
if !use_shared_cache {
match (family, bind_ip) {
(IpFamily::V4, Some(IpAddr::V4(_)))
| (IpFamily::V6, Some(IpAddr::V6(_)))
| (_, None) => {}
_ => return None,
}
}
// Backoff window
if let Some(until) = *self.stun_backoff_until.read().await
if use_shared_cache
&& let Some(until) = *self.stun_backoff_until.read().await
&& Instant::now() < until
{
if let Ok(cache) = self.nat_reflection_cache.try_lock() {
@@ -223,7 +234,9 @@ impl MePool {
return None;
}
if let Ok(mut cache) = self.nat_reflection_cache.try_lock() {
if use_shared_cache
&& let Ok(mut cache) = self.nat_reflection_cache.try_lock()
{
let slot = match family {
IpFamily::V4 => &mut cache.v4,
IpFamily::V6 => &mut cache.v6,
@@ -235,7 +248,11 @@ impl MePool {
}
}
let attempt = self.nat_probe_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let attempt = if use_shared_cache {
self.nat_probe_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
} else {
0
};
let configured_servers = self.configured_stun_servers();
let live_snapshot = self.nat_stun_live_servers.read().await.clone();
let primary_servers = if live_snapshot.is_empty() {
@@ -245,12 +262,12 @@ impl MePool {
};
let (mut live_servers, mut selected_reflected) = self
.probe_stun_batch_for_family(&primary_servers, family, attempt)
.probe_stun_batch_for_family(&primary_servers, family, attempt, bind_ip)
.await;
if selected_reflected.is_none() && !configured_servers.is_empty() && primary_servers != configured_servers {
let (rediscovered_live, rediscovered_reflected) = self
.probe_stun_batch_for_family(&configured_servers, family, attempt)
.probe_stun_batch_for_family(&configured_servers, family, attempt, bind_ip)
.await;
live_servers = rediscovered_live;
selected_reflected = rediscovered_reflected;
@@ -264,14 +281,18 @@ impl MePool {
}
if let Some(reflected_addr) = selected_reflected {
self.nat_probe_attempts.store(0, std::sync::atomic::Ordering::Relaxed);
if use_shared_cache {
self.nat_probe_attempts.store(0, std::sync::atomic::Ordering::Relaxed);
}
info!(
family = ?family,
live_servers = live_server_count,
"STUN-Quorum reached, IP: {}",
reflected_addr.ip()
);
if let Ok(mut cache) = self.nat_reflection_cache.try_lock() {
if use_shared_cache
&& let Ok(mut cache) = self.nat_reflection_cache.try_lock()
{
let slot = match family {
IpFamily::V4 => &mut cache.v4,
IpFamily::V6 => &mut cache.v6,
@@ -281,8 +302,10 @@ impl MePool {
return Some(reflected_addr);
}
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
*self.stun_backoff_until.write().await = Some(Instant::now() + backoff);
if use_shared_cache {
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
*self.stun_backoff_until.write().await = Some(Instant::now() + backoff);
}
None
}
}

View File

@@ -2,27 +2,173 @@ use std::collections::HashSet;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use tracing::{debug, info, warn};
use crate::crypto::SecureRandom;
use crate::network::IpFamily;
use super::pool::MePool;
use super::pool::{MePool, RefillDcKey, WriterContour};
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
impl MePool {
pub(super) async fn maybe_quarantine_flapping_endpoint(
&self,
addr: SocketAddr,
uptime: Duration,
) {
if uptime > Duration::from_secs(ME_FLAP_UPTIME_THRESHOLD_SECS) {
return;
}
let until = Instant::now() + Duration::from_secs(ME_FLAP_QUARANTINE_SECS);
let mut guard = self.endpoint_quarantine.lock().await;
guard.retain(|_, expiry| *expiry > Instant::now());
guard.insert(addr, until);
self.stats.increment_me_endpoint_quarantine_total();
warn!(
%addr,
uptime_ms = uptime.as_millis(),
quarantine_secs = ME_FLAP_QUARANTINE_SECS,
"ME endpoint temporarily quarantined due to rapid writer flap"
);
}
async fn is_endpoint_quarantined(&self, addr: SocketAddr) -> bool {
let mut guard = self.endpoint_quarantine.lock().await;
let now = Instant::now();
guard.retain(|_, expiry| *expiry > now);
guard.contains_key(&addr)
}
async fn connectable_endpoints(&self, endpoints: &[SocketAddr]) -> Vec<SocketAddr> {
if endpoints.is_empty() {
return Vec::new();
}
let mut guard = self.endpoint_quarantine.lock().await;
let now = Instant::now();
guard.retain(|_, expiry| *expiry > now);
let mut ready = Vec::<SocketAddr>::with_capacity(endpoints.len());
let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None;
for addr in endpoints {
if let Some(expiry) = guard.get(addr).copied() {
match earliest_quarantine {
Some((_, current_expiry)) if current_expiry <= expiry => {}
_ => earliest_quarantine = Some((*addr, expiry)),
}
} else {
ready.push(*addr);
}
}
if !ready.is_empty() {
return ready;
}
if let Some((addr, expiry)) = earliest_quarantine {
debug!(
%addr,
wait_ms = expiry.saturating_duration_since(now).as_millis(),
"All ME endpoints are quarantined for the DC group; retrying earliest one"
);
return vec![addr];
}
Vec::new()
}
pub(super) async fn has_refill_inflight_for_endpoints(&self, endpoints: &[SocketAddr]) -> bool {
if endpoints.is_empty() {
return false;
}
{
let guard = self.refill_inflight.lock().await;
if endpoints.iter().any(|addr| guard.contains(addr)) {
return true;
}
}
let dc_keys = self.resolve_refill_dc_keys_for_endpoints(endpoints).await;
if dc_keys.is_empty() {
return false;
}
let guard = self.refill_inflight_dc.lock().await;
dc_keys.iter().any(|key| guard.contains(key))
}
async fn resolve_refill_dc_key_for_addr(&self, addr: SocketAddr) -> Option<RefillDcKey> {
let family = if addr.is_ipv4() {
IpFamily::V4
} else {
IpFamily::V6
};
let map = self.proxy_map_for_family(family).await;
for (dc, endpoints) in map {
if endpoints
.into_iter()
.any(|(ip, port)| SocketAddr::new(ip, port) == addr)
{
return Some(RefillDcKey {
dc: dc.abs(),
family,
});
}
}
None
}
async fn resolve_refill_dc_keys_for_endpoints(
&self,
endpoints: &[SocketAddr],
) -> HashSet<RefillDcKey> {
let mut out = HashSet::<RefillDcKey>::new();
for addr in endpoints {
if let Some(key) = self.resolve_refill_dc_key_for_addr(*addr).await {
out.insert(key);
}
}
out
}
pub(super) async fn connect_endpoints_round_robin(
self: &Arc<Self>,
endpoints: &[SocketAddr],
rng: &SecureRandom,
) -> bool {
if endpoints.is_empty() {
self.connect_endpoints_round_robin_with_generation_contour(
endpoints,
rng,
self.current_generation(),
WriterContour::Active,
)
.await
}
pub(super) async fn connect_endpoints_round_robin_with_generation_contour(
self: &Arc<Self>,
endpoints: &[SocketAddr],
rng: &SecureRandom,
generation: u64,
contour: WriterContour,
) -> bool {
let candidates = self.connectable_endpoints(endpoints).await;
if candidates.is_empty() {
return false;
}
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % endpoints.len();
for offset in 0..endpoints.len() {
let idx = (start + offset) % endpoints.len();
let addr = endpoints[idx];
match self.connect_one(addr, rng).await {
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len();
for offset in 0..candidates.len() {
let idx = (start + offset) % candidates.len();
let addr = candidates[idx];
match self
.connect_one_with_generation_contour(addr, rng, generation, contour)
.await
{
Ok(()) => return true,
Err(e) => debug!(%addr, error = %e, "ME connect failed during round-robin warmup"),
}
@@ -83,29 +229,37 @@ impl MePool {
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr) -> bool {
let fast_retries = self.me_reconnect_fast_retry_count.max(1);
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
for attempt in 0..fast_retries {
self.stats.increment_me_reconnect_attempt();
match self.connect_one(addr, self.rng.as_ref()).await {
Ok(()) => {
self.stats.increment_me_reconnect_success();
self.stats.increment_me_writer_restored_same_endpoint_total();
info!(
%addr,
attempt = attempt + 1,
"ME writer restored on the same endpoint"
);
return true;
}
Err(e) => {
debug!(
%addr,
attempt = attempt + 1,
error = %e,
"ME immediate same-endpoint reconnect failed"
);
if !same_endpoint_quarantined {
for attempt in 0..fast_retries {
self.stats.increment_me_reconnect_attempt();
match self.connect_one(addr, self.rng.as_ref()).await {
Ok(()) => {
self.stats.increment_me_reconnect_success();
self.stats.increment_me_writer_restored_same_endpoint_total();
info!(
%addr,
attempt = attempt + 1,
"ME writer restored on the same endpoint"
);
return true;
}
Err(e) => {
debug!(
%addr,
attempt = attempt + 1,
error = %e,
"ME immediate same-endpoint reconnect failed"
);
}
}
}
} else {
debug!(
%addr,
"Skipping immediate same-endpoint reconnect because endpoint is quarantined"
);
}
let dc_endpoints = self.endpoints_for_same_dc(addr).await;
@@ -138,6 +292,9 @@ impl MePool {
pub(crate) fn trigger_immediate_refill(self: &Arc<Self>, addr: SocketAddr) {
let pool = Arc::clone(self);
tokio::spawn(async move {
let dc_endpoints = pool.endpoints_for_same_dc(addr).await;
let dc_keys = pool.resolve_refill_dc_keys_for_endpoints(&dc_endpoints).await;
{
let mut guard = pool.refill_inflight.lock().await;
if !guard.insert(addr) {
@@ -145,6 +302,19 @@ impl MePool {
return;
}
}
if !dc_keys.is_empty() {
let mut dc_guard = pool.refill_inflight_dc.lock().await;
if dc_keys.iter().any(|key| dc_guard.contains(key)) {
pool.stats.increment_me_refill_skipped_inflight_total();
drop(dc_guard);
let mut guard = pool.refill_inflight.lock().await;
guard.remove(&addr);
return;
}
dc_guard.extend(dc_keys.iter().copied());
}
pool.stats.increment_me_refill_triggered_total();
let restored = pool.refill_writer_after_loss(addr).await;
@@ -154,6 +324,13 @@ impl MePool {
let mut guard = pool.refill_inflight.lock().await;
guard.remove(&addr);
drop(guard);
if !dc_keys.is_empty() {
let mut dc_guard = pool.refill_inflight_dc.lock().await;
for key in &dc_keys {
dc_guard.remove(key);
}
}
});
}
}

View File

@@ -1,4 +1,5 @@
use std::collections::{HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
@@ -7,12 +8,58 @@ use std::time::Duration;
use rand::Rng;
use rand::seq::SliceRandom;
use tracing::{debug, info, warn};
use std::collections::hash_map::DefaultHasher;
use crate::crypto::SecureRandom;
use super::pool::MePool;
use super::pool::{MePool, WriterContour};
const ME_HARDSWAP_PENDING_TTL_SECS: u64 = 1800;
impl MePool {
fn desired_map_hash(desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>) -> u64 {
let mut hasher = DefaultHasher::new();
let mut dcs: Vec<i32> = desired_by_dc.keys().copied().collect();
dcs.sort_unstable();
for dc in dcs {
dc.hash(&mut hasher);
let mut endpoints: Vec<SocketAddr> = desired_by_dc
.get(&dc)
.map(|set| set.iter().copied().collect())
.unwrap_or_default();
endpoints.sort_unstable();
for endpoint in endpoints {
endpoint.hash(&mut hasher);
}
}
hasher.finish()
}
fn clear_pending_hardswap_state(&self) {
self.pending_hardswap_generation.store(0, Ordering::Relaxed);
self.pending_hardswap_started_at_epoch_secs
.store(0, Ordering::Relaxed);
self.pending_hardswap_map_hash.store(0, Ordering::Relaxed);
self.warm_generation.store(0, Ordering::Relaxed);
}
async fn promote_warm_generation_to_active(&self, generation: u64) {
self.active_generation.store(generation, Ordering::Relaxed);
self.warm_generation.store(0, Ordering::Relaxed);
let ws = self.writers.read().await;
for writer in ws.iter() {
if writer.draining.load(Ordering::Relaxed) {
continue;
}
if writer.generation == generation {
writer
.contour
.store(WriterContour::Active.as_u8(), Ordering::Relaxed);
}
}
}
fn coverage_ratio(
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
active_writer_addrs: &HashSet<SocketAddr>,
@@ -101,10 +148,6 @@ impl MePool {
out
}
pub(super) fn required_writers_for_dc(endpoint_count: usize) -> usize {
endpoint_count.max(3)
}
fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
let min_ms = self.me_hardswap_warmup_delay_min_ms.load(Ordering::Relaxed);
let max_ms = self.me_hardswap_warmup_delay_max_ms.load(Ordering::Relaxed);
@@ -174,7 +217,7 @@ impl MePool {
let mut endpoint_list: Vec<SocketAddr> = endpoints.iter().copied().collect();
endpoint_list.sort_unstable();
let required = Self::required_writers_for_dc(endpoint_list.len());
let required = self.required_writers_for_dc(endpoint_list.len());
let mut completed = false;
let mut last_fresh_count = self
.fresh_writer_count_for_endpoints(generation, endpoints)
@@ -202,7 +245,14 @@ impl MePool {
let delay_ms = self.hardswap_warmup_connect_delay_ms();
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
let connected = self.connect_endpoints_round_robin(&endpoint_list, rng).await;
let connected = self
.connect_endpoints_round_robin_with_generation_contour(
&endpoint_list,
rng,
generation,
WriterContour::Warm,
)
.await;
debug!(
dc = *dc,
pass = pass_idx + 1,
@@ -265,11 +315,61 @@ impl MePool {
return;
}
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
let now_epoch_secs = Self::now_epoch_secs();
let previous_generation = self.current_generation();
let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
let hardswap = self.hardswap.load(Ordering::Relaxed);
let generation = if hardswap {
let pending_generation = self.pending_hardswap_generation.load(Ordering::Relaxed);
let pending_started_at = self
.pending_hardswap_started_at_epoch_secs
.load(Ordering::Relaxed);
let pending_map_hash = self.pending_hardswap_map_hash.load(Ordering::Relaxed);
let pending_age_secs = now_epoch_secs.saturating_sub(pending_started_at);
let pending_ttl_expired = pending_started_at > 0 && pending_age_secs > ME_HARDSWAP_PENDING_TTL_SECS;
let pending_matches_map = pending_map_hash != 0 && pending_map_hash == desired_map_hash;
if pending_generation != 0
&& pending_generation >= previous_generation
&& pending_matches_map
&& !pending_ttl_expired
{
self.stats.increment_me_hardswap_pending_reuse_total();
debug!(
previous_generation,
generation = pending_generation,
pending_age_secs,
"ME hardswap continues with pending generation"
);
pending_generation
} else {
if pending_generation != 0 && pending_ttl_expired {
self.stats.increment_me_hardswap_pending_ttl_expired_total();
warn!(
previous_generation,
generation = pending_generation,
pending_age_secs,
pending_ttl_secs = ME_HARDSWAP_PENDING_TTL_SECS,
"ME hardswap pending generation expired by TTL; starting fresh generation"
);
}
let next_generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
self.pending_hardswap_generation
.store(next_generation, Ordering::Relaxed);
self.pending_hardswap_started_at_epoch_secs
.store(now_epoch_secs, Ordering::Relaxed);
self.pending_hardswap_map_hash
.store(desired_map_hash, Ordering::Relaxed);
self.warm_generation.store(next_generation, Ordering::Relaxed);
next_generation
}
} else {
self.clear_pending_hardswap_state();
self.generation.fetch_add(1, Ordering::Relaxed) + 1
};
if hardswap {
self.warm_generation.store(generation, Ordering::Relaxed);
self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc)
.await;
} else {
@@ -305,7 +405,7 @@ impl MePool {
if endpoints.is_empty() {
continue;
}
let required = Self::required_writers_for_dc(endpoints.len());
let required = self.required_writers_for_dc(endpoints.len());
let fresh_count = writers
.iter()
.filter(|w| !w.draining.load(Ordering::Relaxed))
@@ -334,6 +434,10 @@ impl MePool {
return;
}
if hardswap {
self.promote_warm_generation_to_active(generation).await;
}
let desired_addrs: HashSet<SocketAddr> = desired_by_dc
.values()
.flat_map(|set| set.iter().copied())
@@ -354,6 +458,9 @@ impl MePool {
drop(writers);
if stale_writer_ids.is_empty() {
if hardswap {
self.clear_pending_hardswap_state();
}
debug!("ME reinit cycle completed with no stale writers");
return;
}
@@ -375,6 +482,9 @@ impl MePool {
self.mark_writer_draining_with_timeout(writer_id, drain_timeout, !hardswap)
.await;
}
if hardswap {
self.clear_pending_hardswap_state();
}
}
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) {

View File

@@ -1,6 +1,6 @@
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use bytes::BytesMut;
@@ -9,12 +9,13 @@ use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use crate::config::MeBindStaleMode;
use crate::crypto::SecureRandom;
use crate::error::{ProxyError, Result};
use crate::protocol::constants::RPC_PING_U32;
use super::codec::{RpcWriter, WriterCommand};
use super::pool::{MePool, MeWriter};
use super::pool::{MePool, MeWriter, WriterContour};
use super::reader::reader_loop;
use super::registry::BoundConn;
@@ -42,16 +43,32 @@ impl MePool {
}
pub(crate) async fn connect_one(self: &Arc<Self>, addr: SocketAddr, rng: &SecureRandom) -> Result<()> {
let secret_len = self.proxy_secret.read().await.len();
self.connect_one_with_generation_contour(
addr,
rng,
self.current_generation(),
WriterContour::Active,
)
.await
}
pub(super) async fn connect_one_with_generation_contour(
self: &Arc<Self>,
addr: SocketAddr,
rng: &SecureRandom,
generation: u64,
contour: WriterContour,
) -> Result<()> {
let secret_len = self.proxy_secret.read().await.secret.len();
if secret_len < 32 {
return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into()));
}
let (stream, _connect_ms) = self.connect_tcp(addr).await?;
let hs = self.handshake_only(stream, addr, rng).await?;
let (stream, _connect_ms, upstream_egress) = self.connect_tcp(addr).await?;
let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?;
let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed);
let generation = self.current_generation();
let contour = Arc::new(AtomicU8::new(contour.as_u8()));
let cancel = CancellationToken::new();
let degraded = Arc::new(AtomicBool::new(false));
let draining = Arc::new(AtomicBool::new(false));
@@ -88,6 +105,8 @@ impl MePool {
id: writer_id,
addr,
generation,
contour: contour.clone(),
created_at: Instant::now(),
tx: tx.clone(),
cancel: cancel.clone(),
degraded: degraded.clone(),
@@ -248,6 +267,7 @@ impl MePool {
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
let mut close_tx: Option<mpsc::Sender<WriterCommand>> = None;
let mut removed_addr: Option<SocketAddr> = None;
let mut removed_uptime: Option<Duration> = None;
let mut trigger_refill = false;
{
let mut ws = self.writers.write().await;
@@ -260,6 +280,7 @@ impl MePool {
self.stats.increment_me_writer_removed_total();
w.cancel.cancel();
removed_addr = Some(w.addr);
removed_uptime = Some(w.created_at.elapsed());
trigger_refill = !was_draining;
if trigger_refill {
self.stats.increment_me_writer_removed_unexpected_total();
@@ -274,6 +295,9 @@ impl MePool {
if trigger_refill
&& let Some(addr) = removed_addr
{
if let Some(uptime) = removed_uptime {
self.maybe_quarantine_flapping_endpoint(addr, uptime).await;
}
self.trigger_immediate_refill(addr);
}
self.rtt_stats.lock().await.remove(&writer_id);
@@ -298,6 +322,8 @@ impl MePool {
if !already_draining {
self.stats.increment_pool_drain_active();
}
w.contour
.store(WriterContour::Draining.as_u8(), Ordering::Relaxed);
w.draining.store(true, Ordering::Relaxed);
true
} else {
@@ -351,16 +377,22 @@ impl MePool {
return false;
}
let ttl_secs = self.me_pool_drain_ttl_secs.load(Ordering::Relaxed);
if ttl_secs == 0 {
return true;
}
match self.bind_stale_mode() {
MeBindStaleMode::Never => false,
MeBindStaleMode::Always => true,
MeBindStaleMode::Ttl => {
let ttl_secs = self.me_bind_stale_ttl_secs.load(Ordering::Relaxed);
if ttl_secs == 0 {
return true;
}
let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed);
if started == 0 {
return false;
}
let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed);
if started == 0 {
return false;
}
Self::now_epoch_secs().saturating_sub(started) <= ttl_secs
Self::now_epoch_secs().saturating_sub(started) <= ttl_secs
}
}
}
}

View File

@@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::io::ErrorKind;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
@@ -45,7 +46,11 @@ pub(crate) async fn reader_loop(
_ = cancel.cancelled() => return Ok(()),
};
if n == 0 {
return Ok(());
stats.increment_me_reader_eof_total();
return Err(ProxyError::Io(std::io::Error::new(
ErrorKind::UnexpectedEof,
"ME socket closed by peer",
)));
}
raw.extend_from_slice(&tmp[..n]);
@@ -124,7 +129,14 @@ pub(crate) async fn reader_loop(
match routed {
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
RouteResult::ChannelClosed => stats.increment_me_route_drop_channel_closed(),
RouteResult::QueueFull => stats.increment_me_route_drop_queue_full(),
RouteResult::QueueFullBase => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_base();
}
RouteResult::QueueFullHigh => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
}
RouteResult::Routed => {}
}
reg.unregister(cid).await;
@@ -140,7 +152,14 @@ pub(crate) async fn reader_loop(
match routed {
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
RouteResult::ChannelClosed => stats.increment_me_route_drop_channel_closed(),
RouteResult::QueueFull => stats.increment_me_route_drop_queue_full(),
RouteResult::QueueFullBase => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_base();
}
RouteResult::QueueFullHigh => {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
}
RouteResult::Routed => {}
}
reg.unregister(cid).await;

View File

@@ -1,6 +1,6 @@
use std::collections::{HashMap, HashSet};
use std::net::SocketAddr;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
use std::time::Duration;
use tokio::sync::{mpsc, RwLock};
@@ -10,14 +10,17 @@ use super::codec::WriterCommand;
use super::MeResponse;
const ROUTE_CHANNEL_CAPACITY: usize = 4096;
const ROUTE_BACKPRESSURE_TIMEOUT: Duration = Duration::from_millis(25);
const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25;
const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120;
const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteResult {
Routed,
NoConn,
ChannelClosed,
QueueFull,
QueueFullBase,
QueueFullHigh,
}
#[derive(Clone)]
@@ -65,6 +68,9 @@ impl RegistryInner {
pub struct ConnRegistry {
inner: RwLock<RegistryInner>,
next_id: AtomicU64,
route_backpressure_base_timeout_ms: AtomicU64,
route_backpressure_high_timeout_ms: AtomicU64,
route_backpressure_high_watermark_pct: AtomicU8,
}
impl ConnRegistry {
@@ -73,9 +79,35 @@ impl ConnRegistry {
Self {
inner: RwLock::new(RegistryInner::new()),
next_id: AtomicU64::new(start),
route_backpressure_base_timeout_ms: AtomicU64::new(
ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS,
),
route_backpressure_high_timeout_ms: AtomicU64::new(
ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS,
),
route_backpressure_high_watermark_pct: AtomicU8::new(
ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT,
),
}
}
pub fn update_route_backpressure_policy(
&self,
base_timeout_ms: u64,
high_timeout_ms: u64,
high_watermark_pct: u8,
) {
let base = base_timeout_ms.max(1);
let high = high_timeout_ms.max(base);
let watermark = high_watermark_pct.clamp(1, 100);
self.route_backpressure_base_timeout_ms
.store(base, Ordering::Relaxed);
self.route_backpressure_high_timeout_ms
.store(high, Ordering::Relaxed);
self.route_backpressure_high_watermark_pct
.store(watermark, Ordering::Relaxed);
}
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = mpsc::channel(ROUTE_CHANNEL_CAPACITY);
@@ -112,10 +144,40 @@ impl ConnRegistry {
Err(TrySendError::Closed(_)) => RouteResult::ChannelClosed,
Err(TrySendError::Full(resp)) => {
// Absorb short bursts without dropping/closing the session immediately.
match tokio::time::timeout(ROUTE_BACKPRESSURE_TIMEOUT, tx.send(resp)).await {
let base_timeout_ms =
self.route_backpressure_base_timeout_ms.load(Ordering::Relaxed).max(1);
let high_timeout_ms = self
.route_backpressure_high_timeout_ms
.load(Ordering::Relaxed)
.max(base_timeout_ms);
let high_watermark_pct = self
.route_backpressure_high_watermark_pct
.load(Ordering::Relaxed)
.clamp(1, 100);
let used = ROUTE_CHANNEL_CAPACITY.saturating_sub(tx.capacity());
let used_pct = if ROUTE_CHANNEL_CAPACITY == 0 {
100
} else {
(used.saturating_mul(100) / ROUTE_CHANNEL_CAPACITY) as u8
};
let high_profile = used_pct >= high_watermark_pct;
let timeout_ms = if high_profile {
high_timeout_ms
} else {
base_timeout_ms
};
let timeout_dur = Duration::from_millis(timeout_ms);
match tokio::time::timeout(timeout_dur, tx.send(resp)).await {
Ok(Ok(())) => RouteResult::Routed,
Ok(Err(_)) => RouteResult::ChannelClosed,
Err(_) => RouteResult::QueueFull,
Err(_) => {
if high_profile {
RouteResult::QueueFullHigh
} else {
RouteResult::QueueFullBase
}
}
}
}
}

View File

@@ -1,19 +1,111 @@
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::watch;
use tracing::{info, warn};
use tokio::sync::{mpsc, watch};
use tracing::{debug, info, warn};
use crate::config::ProxyConfig;
use crate::crypto::SecureRandom;
use super::MePool;
/// Periodically reinitialize ME generations and swap them after full warmup.
pub async fn me_rotation_task(
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MeReinitTrigger {
Periodic,
MapChanged,
}
impl MeReinitTrigger {
fn as_str(self) -> &'static str {
match self {
MeReinitTrigger::Periodic => "periodic",
MeReinitTrigger::MapChanged => "map-change",
}
}
}
pub fn enqueue_reinit_trigger(
tx: &mpsc::Sender<MeReinitTrigger>,
trigger: MeReinitTrigger,
) {
match tx.try_send(trigger) {
Ok(()) => {}
Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {
debug!(trigger = trigger.as_str(), "ME reinit trigger dropped (queue full)");
}
Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {
warn!(trigger = trigger.as_str(), "ME reinit trigger dropped (scheduler closed)");
}
}
}
pub async fn me_reinit_scheduler(
pool: Arc<MePool>,
rng: Arc<SecureRandom>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
) {
info!("ME reinit scheduler started");
loop {
let Some(first_trigger) = trigger_rx.recv().await else {
warn!("ME reinit scheduler stopped: trigger channel closed");
break;
};
let mut map_change_seen = matches!(first_trigger, MeReinitTrigger::MapChanged);
let mut periodic_seen = matches!(first_trigger, MeReinitTrigger::Periodic);
let cfg = config_rx.borrow().clone();
let coalesce_window = Duration::from_millis(cfg.general.me_reinit_coalesce_window_ms);
if !coalesce_window.is_zero() {
let deadline = tokio::time::Instant::now() + coalesce_window;
loop {
let now = tokio::time::Instant::now();
if now >= deadline {
break;
}
match tokio::time::timeout(deadline - now, trigger_rx.recv()).await {
Ok(Some(next)) => {
if next == MeReinitTrigger::MapChanged {
map_change_seen = true;
} else {
periodic_seen = true;
}
}
Ok(None) => break,
Err(_) => break,
}
}
}
let reason = if map_change_seen && periodic_seen {
"map-change+periodic"
} else if map_change_seen {
"map-change"
} else {
"periodic"
};
if cfg.general.me_reinit_singleflight {
debug!(reason, "ME reinit scheduled (single-flight)");
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
} else {
debug!(reason, "ME reinit scheduled (concurrent mode)");
let pool_clone = pool.clone();
let rng_clone = rng.clone();
tokio::spawn(async move {
pool_clone
.zero_downtime_reinit_periodic(rng_clone.as_ref())
.await;
});
}
}
}
/// Periodically enqueue reinitialization triggers for ME generations.
pub async fn me_rotation_task(
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
reinit_tx: mpsc::Sender<MeReinitTrigger>,
) {
let mut interval_secs = config_rx
.borrow()
@@ -31,7 +123,7 @@ pub async fn me_rotation_task(
tokio::select! {
_ = &mut sleep => {
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic);
let refreshed_secs = config_rx
.borrow()
.general
@@ -70,7 +162,7 @@ pub async fn me_rotation_task(
);
interval_secs = new_secs;
interval = Duration::from_secs(interval_secs);
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic);
next_tick = tokio::time::Instant::now() + interval;
} else {
info!(

View File

@@ -1,8 +1,10 @@
use std::cmp::Reverse;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use tokio::sync::mpsc::error::TrySendError;
use tracing::{debug, warn};
use crate::error::{ProxyError, Result};
@@ -11,11 +13,13 @@ use crate::protocol::constants::RPC_CLOSE_EXT_U32;
use super::MePool;
use super::codec::WriterCommand;
use super::pool::WriterContour;
use super::wire::build_proxy_req_payload;
use rand::seq::SliceRandom;
use super::registry::ConnMeta;
impl MePool {
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
pub async fn send_proxy_req(
self: &Arc<Self>,
conn_id: u64,
@@ -24,13 +28,15 @@ impl MePool {
our_addr: SocketAddr,
data: &[u8],
proto_flags: u32,
tag_override: Option<&[u8]>,
) -> Result<()> {
let tag = tag_override.or(self.proxy_tag.as_deref());
let payload = build_proxy_req_payload(
conn_id,
client_addr,
our_addr,
data,
self.proxy_tag.as_deref(),
tag,
proto_flags,
);
let meta = ConnMeta {
@@ -43,15 +49,17 @@ impl MePool {
loop {
if let Some(current) = self.registry.get_writer(conn_id).await {
let send_res = {
current
.tx
.send(WriterCommand::Data(payload.clone()))
.await
};
match send_res {
match current.tx.try_send(WriterCommand::Data(payload.clone())) {
Ok(()) => return Ok(()),
Err(_) => {
Err(TrySendError::Full(cmd)) => {
if current.tx.send(cmd).await.is_ok() {
return Ok(());
}
warn!(writer_id = current.writer_id, "ME writer channel closed");
self.remove_writer_and_close_clients(current.writer_id).await;
continue;
}
Err(TrySendError::Closed(_)) => {
warn!(writer_id = current.writer_id, "ME writer channel closed");
self.remove_writer_and_close_clients(current.writer_id).await;
continue;
@@ -94,7 +102,14 @@ impl MePool {
ws.clone()
};
let mut candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await;
let mut candidate_indices = self
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
.await;
if candidate_indices.is_empty() {
candidate_indices = self
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
.await;
}
if candidate_indices.is_empty() {
// Emergency connect-on-demand
if emergency_attempts >= 3 {
@@ -120,7 +135,14 @@ impl MePool {
let ws2 = self.writers.read().await;
writers_snapshot = ws2.clone();
drop(ws2);
candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await;
candidate_indices = self
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
.await;
if candidate_indices.is_empty() {
candidate_indices = self
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
.await;
}
if !candidate_indices.is_empty() {
break;
}
@@ -131,14 +153,44 @@ impl MePool {
}
}
candidate_indices.sort_by_key(|idx| {
let w = &writers_snapshot[*idx];
let degraded = w.degraded.load(Ordering::Relaxed);
let stale = (w.generation < self.current_generation()) as usize;
(stale, degraded as usize)
});
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
candidate_indices.sort_by(|lhs, rhs| {
let left = &writers_snapshot[*lhs];
let right = &writers_snapshot[*rhs];
let left_key = (
self.writer_contour_rank_for_selection(left),
(left.generation < self.current_generation()) as usize,
left.degraded.load(Ordering::Relaxed) as usize,
Reverse(left.tx.capacity()),
left.addr,
left.id,
);
let right_key = (
self.writer_contour_rank_for_selection(right),
(right.generation < self.current_generation()) as usize,
right.degraded.load(Ordering::Relaxed) as usize,
Reverse(right.tx.capacity()),
right.addr,
right.id,
);
left_key.cmp(&right_key)
});
} else {
candidate_indices.sort_by_key(|idx| {
let w = &writers_snapshot[*idx];
let degraded = w.degraded.load(Ordering::Relaxed);
let stale = (w.generation < self.current_generation()) as usize;
(
self.writer_contour_rank_for_selection(w),
stale,
degraded as usize,
Reverse(w.tx.capacity()),
)
});
}
let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len();
let mut fallback_blocking_idx: Option<usize> = None;
for offset in 0..candidate_indices.len() {
let idx = candidate_indices[(start + offset) % candidate_indices.len()];
@@ -146,29 +198,41 @@ impl MePool {
if !self.writer_accepts_new_binding(w) {
continue;
}
if w.tx.send(WriterCommand::Data(payload.clone())).await.is_ok() {
self.registry
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
.await;
if w.generation < self.current_generation() {
self.stats.increment_pool_stale_pick_total();
debug!(
conn_id,
writer_id = w.id,
writer_generation = w.generation,
current_generation = self.current_generation(),
"Selected stale ME writer for fallback bind"
);
match w.tx.try_send(WriterCommand::Data(payload.clone())) {
Ok(()) => {
self.registry
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
.await;
if w.generation < self.current_generation() {
self.stats.increment_pool_stale_pick_total();
debug!(
conn_id,
writer_id = w.id,
writer_generation = w.generation,
current_generation = self.current_generation(),
"Selected stale ME writer for fallback bind"
);
}
return Ok(());
}
Err(TrySendError::Full(_)) => {
if fallback_blocking_idx.is_none() {
fallback_blocking_idx = Some(idx);
}
}
Err(TrySendError::Closed(_)) => {
warn!(writer_id = w.id, "ME writer channel closed");
self.remove_writer_and_close_clients(w.id).await;
continue;
}
return Ok(());
} else {
warn!(writer_id = w.id, "ME writer channel closed");
self.remove_writer_and_close_clients(w.id).await;
continue;
}
}
let w = writers_snapshot[candidate_indices[start]].clone();
let Some(blocking_idx) = fallback_blocking_idx else {
continue;
};
let w = writers_snapshot[blocking_idx].clone();
if !self.writer_accepts_new_binding(&w) {
continue;
}
@@ -215,6 +279,7 @@ impl MePool {
&self,
writers: &[super::pool::MeWriter],
target_dc: i16,
include_warm: bool,
) -> Vec<usize> {
let key = target_dc as i32;
let mut preferred = Vec::<SocketAddr>::new();
@@ -258,13 +323,13 @@ impl MePool {
if preferred.is_empty() {
return (0..writers.len())
.filter(|i| self.writer_accepts_new_binding(&writers[*i]))
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
.collect();
}
let mut out = Vec::new();
for (idx, w) in writers.iter().enumerate() {
if !self.writer_accepts_new_binding(w) {
if !self.writer_eligible_for_selection(w, include_warm) {
continue;
}
if preferred.contains(&w.addr) {
@@ -273,10 +338,33 @@ impl MePool {
}
if out.is_empty() {
return (0..writers.len())
.filter(|i| self.writer_accepts_new_binding(&writers[*i]))
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
.collect();
}
out
}
fn writer_eligible_for_selection(
&self,
writer: &super::pool::MeWriter,
include_warm: bool,
) -> bool {
if !self.writer_accepts_new_binding(writer) {
return false;
}
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
WriterContour::Active => true,
WriterContour::Warm => include_warm,
WriterContour::Draining => true,
}
}
fn writer_contour_rank_for_selection(&self, writer: &super::pool::MeWriter) -> usize {
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
WriterContour::Active => 0,
WriterContour::Warm => 1,
WriterContour::Draining => 2,
}
}
}

View File

@@ -14,5 +14,5 @@ pub use socket::*;
#[allow(unused_imports)]
pub use socks::*;
#[allow(unused_imports)]
pub use upstream::{DcPingResult, StartupPingResult, UpstreamManager};
pub use upstream::{DcPingResult, StartupPingResult, UpstreamEgressInfo, UpstreamManager, UpstreamRouteKind};
pub mod middle_proxy;

View File

@@ -5,11 +5,16 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use crate::error::{ProxyError, Result};
#[derive(Debug, Clone, Copy)]
pub struct SocksBoundAddr {
pub addr: SocketAddr,
}
pub async fn connect_socks4(
stream: &mut TcpStream,
target: SocketAddr,
user_id: Option<&str>,
) -> Result<()> {
) -> Result<SocksBoundAddr> {
let ip = match target.ip() {
IpAddr::V4(ip) => ip,
IpAddr::V6(_) => return Err(ProxyError::Proxy("SOCKS4 does not support IPv6".to_string())),
@@ -36,8 +41,13 @@ pub async fn connect_socks4(
if resp[1] != 90 {
return Err(ProxyError::Proxy(format!("SOCKS4 request rejected: code {}", resp[1])));
}
Ok(())
let bound_port = u16::from_be_bytes([resp[2], resp[3]]);
let bound_ip = IpAddr::from([resp[4], resp[5], resp[6], resp[7]]);
Ok(SocksBoundAddr {
addr: SocketAddr::new(bound_ip, bound_port),
})
}
pub async fn connect_socks5(
@@ -45,7 +55,7 @@ pub async fn connect_socks5(
target: SocketAddr,
username: Option<&str>,
password: Option<&str>,
) -> Result<()> {
) -> Result<SocksBoundAddr> {
// 1. Auth negotiation
// VER (1) | NMETHODS (1) | METHODS (variable)
let mut methods = vec![0u8]; // No auth
@@ -122,24 +132,36 @@ pub async fn connect_socks5(
return Err(ProxyError::Proxy(format!("SOCKS5 request failed: code {}", head[1])));
}
// Skip address part of response
match head[3] {
// Parse bound address from response.
let bound_addr = match head[3] {
1 => { // IPv4
let mut addr = [0u8; 4 + 2];
stream.read_exact(&mut addr).await.map_err(ProxyError::Io)?;
let ip = IpAddr::from([addr[0], addr[1], addr[2], addr[3]]);
let port = u16::from_be_bytes([addr[4], addr[5]]);
SocketAddr::new(ip, port)
},
3 => { // Domain
let mut len = [0u8; 1];
stream.read_exact(&mut len).await.map_err(ProxyError::Io)?;
let mut addr = vec![0u8; len[0] as usize + 2];
stream.read_exact(&mut addr).await.map_err(ProxyError::Io)?;
// Domain-bound response is not useful for KDF IP material.
let port_pos = addr.len().saturating_sub(2);
let port = u16::from_be_bytes([addr[port_pos], addr[port_pos + 1]]);
SocketAddr::new(IpAddr::from([0, 0, 0, 0]), port)
},
4 => { // IPv6
let mut addr = [0u8; 16 + 2];
stream.read_exact(&mut addr).await.map_err(ProxyError::Io)?;
let ip = IpAddr::from(<[u8; 16]>::try_from(&addr[..16]).map_err(|_| {
ProxyError::Proxy("Invalid SOCKS5 IPv6 bound address".to_string())
})?);
let port = u16::from_be_bytes([addr[16], addr[17]]);
SocketAddr::new(ip, port)
},
_ => return Err(ProxyError::Proxy("Invalid address type in SOCKS5 response".to_string())),
}
Ok(())
}
};
Ok(SocksBoundAddr { addr: bound_addr })
}

View File

@@ -4,7 +4,7 @@
#![allow(deprecated)]
use std::collections::HashMap;
use std::collections::{BTreeSet, HashMap};
use std::net::{SocketAddr, IpAddr};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -17,6 +17,7 @@ use tracing::{debug, warn, info, trace};
use crate::config::{UpstreamConfig, UpstreamType};
use crate::error::{Result, ProxyError};
use crate::network::dns_overrides::{resolve_socket_addr, split_host_port};
use crate::protocol::constants::{TG_DATACENTERS_V4, TG_DATACENTERS_V6, TG_DATACENTER_PORT};
use crate::transport::socket::{create_outgoing_socket_bound, resolve_interface_ip};
use crate::transport::socks::{connect_socks4, connect_socks5};
@@ -28,6 +29,12 @@ const NUM_DCS: usize = 5;
const DC_PING_TIMEOUT_SECS: u64 = 5;
/// Timeout for direct TG DC TCP connect readiness.
const DIRECT_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Interval between upstream health-check cycles.
const HEALTH_CHECK_INTERVAL_SECS: u64 = 30;
/// Timeout for a single health-check connect attempt.
const HEALTH_CHECK_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Upstream is considered healthy when at least this many DC groups are reachable.
const MIN_HEALTHY_DC_GROUPS: usize = 3;
// ============= RTT Tracking =============
@@ -150,15 +157,46 @@ pub struct StartupPingResult {
pub both_available: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UpstreamRouteKind {
Direct,
Socks4,
Socks5,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UpstreamEgressInfo {
pub route_kind: UpstreamRouteKind,
pub local_addr: Option<SocketAddr>,
pub direct_bind_ip: Option<IpAddr>,
pub socks_bound_addr: Option<SocketAddr>,
pub socks_proxy_addr: Option<SocketAddr>,
}
#[derive(Debug, Clone)]
struct HealthCheckGroup {
dc_idx: i16,
primary: Vec<SocketAddr>,
fallback: Vec<SocketAddr>,
}
// ============= Upstream Manager =============
#[derive(Clone)]
pub struct UpstreamManager {
upstreams: Arc<RwLock<Vec<UpstreamState>>>,
connect_retry_attempts: u32,
connect_retry_backoff: Duration,
unhealthy_fail_threshold: u32,
}
impl UpstreamManager {
pub fn new(configs: Vec<UpstreamConfig>) -> Self {
pub fn new(
configs: Vec<UpstreamConfig>,
connect_retry_attempts: u32,
connect_retry_backoff_ms: u64,
unhealthy_fail_threshold: u32,
) -> Self {
let states = configs.into_iter()
.filter(|c| c.enabled)
.map(UpstreamState::new)
@@ -166,24 +204,88 @@ impl UpstreamManager {
Self {
upstreams: Arc::new(RwLock::new(states)),
connect_retry_attempts: connect_retry_attempts.max(1),
connect_retry_backoff: Duration::from_millis(connect_retry_backoff_ms),
unhealthy_fail_threshold: unhealthy_fail_threshold.max(1),
}
}
#[cfg(unix)]
fn resolve_interface_addrs(name: &str, want_ipv6: bool) -> Vec<IpAddr> {
use nix::ifaddrs::getifaddrs;
let mut out = Vec::new();
if let Ok(addrs) = getifaddrs() {
for iface in addrs {
if iface.interface_name != name {
continue;
}
if let Some(address) = iface.address {
if let Some(v4) = address.as_sockaddr_in() {
if !want_ipv6 {
out.push(IpAddr::V4(v4.ip()));
}
} else if let Some(v6) = address.as_sockaddr_in6()
&& want_ipv6
{
out.push(IpAddr::V6(v6.ip()));
}
}
}
}
out.sort_unstable();
out.dedup();
out
}
fn resolve_bind_address(
interface: &Option<String>,
bind_addresses: &Option<Vec<String>>,
target: SocketAddr,
rr: Option<&AtomicUsize>,
validate_ip_on_interface: bool,
) -> Option<IpAddr> {
let want_ipv6 = target.is_ipv6();
if let Some(addrs) = bind_addresses {
let candidates: Vec<IpAddr> = addrs
let mut candidates: Vec<IpAddr> = addrs
.iter()
.filter_map(|s| s.parse::<IpAddr>().ok())
.filter(|ip| ip.is_ipv6() == want_ipv6)
.collect();
// Explicit bind IP has strict priority over interface auto-selection.
if validate_ip_on_interface
&& let Some(iface) = interface
&& iface.parse::<IpAddr>().is_err()
{
#[cfg(unix)]
{
let iface_addrs = Self::resolve_interface_addrs(iface, want_ipv6);
if !iface_addrs.is_empty() {
candidates.retain(|ip| {
let ok = iface_addrs.contains(ip);
if !ok {
warn!(
interface = %iface,
bind_ip = %ip,
target = %target,
"Configured bind address is not assigned to interface"
);
}
ok
});
} else if !candidates.is_empty() {
warn!(
interface = %iface,
target = %target,
"Configured interface has no addresses for target family; falling back to direct connect without bind"
);
candidates.clear();
}
}
}
if !candidates.is_empty() {
if let Some(counter) = rr {
let idx = counter.fetch_add(1, Ordering::Relaxed) % candidates.len();
@@ -191,6 +293,19 @@ impl UpstreamManager {
}
return candidates.first().copied();
}
if validate_ip_on_interface
&& interface
.as_ref()
.is_some_and(|iface| iface.parse::<IpAddr>().is_err())
{
warn!(
interface = interface.as_deref().unwrap_or(""),
target = %target,
"No valid bind_addresses left for interface; falling back to direct connect without bind"
);
return None;
}
}
if let Some(iface) = interface {
@@ -209,6 +324,31 @@ impl UpstreamManager {
None
}
async fn connect_hostname_with_dns_override(
address: &str,
connect_timeout: Duration,
) -> Result<TcpStream> {
if let Some((host, port)) = split_host_port(address)
&& let Some(addr) = resolve_socket_addr(&host, port)
{
return match tokio::time::timeout(connect_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(e)) => Err(ProxyError::Io(e)),
Err(_) => Err(ProxyError::ConnectionTimeout {
addr: addr.to_string(),
}),
};
}
match tokio::time::timeout(connect_timeout, TcpStream::connect(address)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(e)) => Err(ProxyError::Io(e)),
Err(_) => Err(ProxyError::ConnectionTimeout {
addr: address.to_string(),
}),
}
}
/// Select upstream using latency-weighted random selection.
async fn select_upstream(&self, dc_idx: Option<i16>, scope: Option<&str>) -> Option<usize> {
let upstreams = self.upstreams.read().await;
@@ -290,6 +430,17 @@ impl UpstreamManager {
/// Connect to target through a selected upstream.
pub async fn connect(&self, target: SocketAddr, dc_idx: Option<i16>, scope: Option<&str>) -> Result<TcpStream> {
let (stream, _) = self.connect_with_details(target, dc_idx, scope).await?;
Ok(stream)
}
/// Connect to target through a selected upstream and return egress details.
pub async fn connect_with_details(
&self,
target: SocketAddr,
dc_idx: Option<i16>,
scope: Option<&str>,
) -> Result<(TcpStream, UpstreamEgressInfo)> {
let idx = self.select_upstream(dc_idx, scope).await
.ok_or_else(|| ProxyError::Config("No upstreams available".to_string()))?;
@@ -303,43 +454,83 @@ impl UpstreamManager {
upstream.selected_scope = s.to_string();
}
let start = Instant::now();
let bind_rr = {
let guard = self.upstreams.read().await;
guard.get(idx).map(|u| u.bind_rr.clone())
};
match self.connect_via_upstream(&upstream, target, bind_rr).await {
Ok(stream) => {
let rtt_ms = start.elapsed().as_secs_f64() * 1000.0;
let mut guard = self.upstreams.write().await;
if let Some(u) = guard.get_mut(idx) {
if !u.healthy {
debug!(rtt_ms = format!("{:.1}", rtt_ms), "Upstream recovered");
}
u.healthy = true;
u.fails = 0;
let mut last_error: Option<ProxyError> = None;
for attempt in 1..=self.connect_retry_attempts {
let start = Instant::now();
match self
.connect_via_upstream(&upstream, target, bind_rr.clone())
.await
{
Ok((stream, egress)) => {
let rtt_ms = start.elapsed().as_secs_f64() * 1000.0;
let mut guard = self.upstreams.write().await;
if let Some(u) = guard.get_mut(idx) {
if !u.healthy {
debug!(rtt_ms = format!("{:.1}", rtt_ms), "Upstream recovered");
}
if attempt > 1 {
debug!(
attempt,
attempts = self.connect_retry_attempts,
rtt_ms = format!("{:.1}", rtt_ms),
"Upstream connect recovered after retry"
);
}
u.healthy = true;
u.fails = 0;
if let Some(di) = dc_idx.and_then(UpstreamState::dc_array_idx) {
u.dc_latency[di].update(rtt_ms);
if let Some(di) = dc_idx.and_then(UpstreamState::dc_array_idx) {
u.dc_latency[di].update(rtt_ms);
}
}
return Ok((stream, egress));
}
Ok(stream)
},
Err(e) => {
let mut guard = self.upstreams.write().await;
if let Some(u) = guard.get_mut(idx) {
u.fails += 1;
warn!(fails = u.fails, "Upstream failed: {}", e);
if u.fails > 3 {
u.healthy = false;
warn!("Upstream marked unhealthy");
Err(e) => {
if attempt < self.connect_retry_attempts {
debug!(
attempt,
attempts = self.connect_retry_attempts,
target = %target,
error = %e,
"Upstream connect attempt failed, retrying"
);
if !self.connect_retry_backoff.is_zero() {
tokio::time::sleep(self.connect_retry_backoff).await;
}
}
last_error = Some(e);
}
Err(e)
}
}
let error = last_error.unwrap_or_else(|| {
ProxyError::Config("Upstream connect attempts exhausted".to_string())
});
let mut guard = self.upstreams.write().await;
if let Some(u) = guard.get_mut(idx) {
u.fails += 1;
warn!(
fails = u.fails,
attempts = self.connect_retry_attempts,
"Upstream failed after retries: {}",
error
);
if u.fails >= self.unhealthy_fail_threshold {
u.healthy = false;
warn!(
fails = u.fails,
threshold = self.unhealthy_fail_threshold,
"Upstream marked unhealthy"
);
}
}
Err(error)
}
async fn connect_via_upstream(
@@ -347,7 +538,7 @@ impl UpstreamManager {
config: &UpstreamConfig,
target: SocketAddr,
bind_rr: Option<Arc<AtomicUsize>>,
) -> Result<TcpStream> {
) -> Result<(TcpStream, UpstreamEgressInfo)> {
match &config.upstream_type {
UpstreamType::Direct { interface, bind_addresses } => {
let bind_ip = Self::resolve_bind_address(
@@ -355,6 +546,7 @@ impl UpstreamManager {
bind_addresses,
target,
bind_rr.as_deref(),
true,
);
let socket = create_outgoing_socket_bound(target, bind_ip)?;
@@ -388,7 +580,17 @@ impl UpstreamManager {
return Err(ProxyError::Io(e));
}
Ok(stream)
let local_addr = stream.local_addr().ok();
Ok((
stream,
UpstreamEgressInfo {
route_kind: UpstreamRouteKind::Direct,
local_addr,
direct_bind_ip: bind_ip,
socks_bound_addr: None,
socks_proxy_addr: None,
},
))
},
UpstreamType::Socks4 { address, interface, user_id } => {
let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS);
@@ -400,6 +602,7 @@ impl UpstreamManager {
&None,
proxy_addr,
bind_rr.as_deref(),
false,
);
let socket = create_outgoing_socket_bound(proxy_addr, bind_ip)?;
@@ -433,15 +636,7 @@ impl UpstreamManager {
if interface.is_some() {
warn!("SOCKS4 interface binding is not supported for hostname addresses, ignoring");
}
match tokio::time::timeout(connect_timeout, TcpStream::connect(address)).await {
Ok(Ok(stream)) => stream,
Ok(Err(e)) => return Err(ProxyError::Io(e)),
Err(_) => {
return Err(ProxyError::ConnectionTimeout {
addr: address.clone(),
});
}
}
Self::connect_hostname_with_dns_override(address, connect_timeout).await?
};
// replace socks user_id with config.selected_scope, if set
@@ -449,16 +644,32 @@ impl UpstreamManager {
.filter(|s| !s.is_empty());
let _user_id: Option<&str> = scope.or(user_id.as_deref());
match tokio::time::timeout(connect_timeout, connect_socks4(&mut stream, target, _user_id)).await {
Ok(Ok(())) => {}
let bound = match tokio::time::timeout(
connect_timeout,
connect_socks4(&mut stream, target, _user_id),
)
.await
{
Ok(Ok(bound)) => bound,
Ok(Err(e)) => return Err(e),
Err(_) => {
return Err(ProxyError::ConnectionTimeout {
addr: target.to_string(),
});
}
}
Ok(stream)
};
let local_addr = stream.local_addr().ok();
let socks_proxy_addr = stream.peer_addr().ok();
Ok((
stream,
UpstreamEgressInfo {
route_kind: UpstreamRouteKind::Socks4,
local_addr,
direct_bind_ip: None,
socks_bound_addr: Some(bound.addr),
socks_proxy_addr,
},
))
},
UpstreamType::Socks5 { address, interface, username, password } => {
let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS);
@@ -470,6 +681,7 @@ impl UpstreamManager {
&None,
proxy_addr,
bind_rr.as_deref(),
false,
);
let socket = create_outgoing_socket_bound(proxy_addr, bind_ip)?;
@@ -503,15 +715,7 @@ impl UpstreamManager {
if interface.is_some() {
warn!("SOCKS5 interface binding is not supported for hostname addresses, ignoring");
}
match tokio::time::timeout(connect_timeout, TcpStream::connect(address)).await {
Ok(Ok(stream)) => stream,
Ok(Err(e)) => return Err(ProxyError::Io(e)),
Err(_) => {
return Err(ProxyError::ConnectionTimeout {
addr: address.clone(),
});
}
}
Self::connect_hostname_with_dns_override(address, connect_timeout).await?
};
debug!(config = ?config, "Socks5 connection");
@@ -521,21 +725,32 @@ impl UpstreamManager {
let _username: Option<&str> = scope.or(username.as_deref());
let _password: Option<&str> = scope.or(password.as_deref());
match tokio::time::timeout(
let bound = match tokio::time::timeout(
connect_timeout,
connect_socks5(&mut stream, target, _username, _password),
)
.await
{
Ok(Ok(())) => {}
Ok(Ok(bound)) => bound,
Ok(Err(e)) => return Err(e),
Err(_) => {
return Err(ProxyError::ConnectionTimeout {
addr: target.to_string(),
});
}
}
Ok(stream)
};
let local_addr = stream.local_addr().ok();
let socks_proxy_addr = stream.peer_addr().ok();
Ok((
stream,
UpstreamEgressInfo {
route_kind: UpstreamRouteKind::Socks5,
local_addr,
direct_bind_ip: None,
socks_bound_addr: Some(bound.addr),
socks_proxy_addr,
},
))
},
}
}
@@ -562,8 +777,22 @@ impl UpstreamManager {
for (upstream_idx, upstream_config, bind_rr) in &upstreams {
let upstream_name = match &upstream_config.upstream_type {
UpstreamType::Direct { interface, .. } => {
format!("direct{}", interface.as_ref().map(|i| format!(" ({})", i)).unwrap_or_default())
UpstreamType::Direct {
interface,
bind_addresses,
} => {
let mut direct_parts = Vec::new();
if let Some(dev) = interface.as_deref().filter(|v| !v.is_empty()) {
direct_parts.push(format!("dev={dev}"));
}
if let Some(src) = bind_addresses.as_ref().filter(|v| !v.is_empty()) {
direct_parts.push(format!("src={}", src.join(",")));
}
if direct_parts.is_empty() {
"direct".to_string()
} else {
format!("direct {}", direct_parts.join(" "))
}
}
UpstreamType::Socks4 { address, .. } => format!("socks4://{}", address),
UpstreamType::Socks5 { address, .. } => format!("socks5://{}", address),
@@ -767,45 +996,148 @@ impl UpstreamManager {
target: SocketAddr,
) -> Result<f64> {
let start = Instant::now();
let _stream = self.connect_via_upstream(config, target, bind_rr).await?;
let _ = self.connect_via_upstream(config, target, bind_rr).await?;
Ok(start.elapsed().as_secs_f64() * 1000.0)
}
fn required_healthy_group_count(total_groups: usize) -> usize {
if total_groups == 0 {
0
} else {
total_groups.min(MIN_HEALTHY_DC_GROUPS)
}
}
fn build_health_check_groups(
prefer_ipv6: bool,
ipv4_enabled: bool,
ipv6_enabled: bool,
dc_overrides: &HashMap<String, Vec<String>>,
) -> Vec<HealthCheckGroup> {
let mut v4_by_dc: HashMap<i16, Vec<SocketAddr>> = HashMap::new();
let mut v6_by_dc: HashMap<i16, Vec<SocketAddr>> = HashMap::new();
if ipv4_enabled {
for (idx, dc_ip) in TG_DATACENTERS_V4.iter().enumerate() {
let dc_idx = (idx + 1) as i16;
v4_by_dc
.entry(dc_idx)
.or_default()
.push(SocketAddr::new(*dc_ip, TG_DATACENTER_PORT));
}
}
if ipv6_enabled {
for (idx, dc_ip) in TG_DATACENTERS_V6.iter().enumerate() {
let dc_idx = (idx + 1) as i16;
v6_by_dc
.entry(dc_idx)
.or_default()
.push(SocketAddr::new(*dc_ip, TG_DATACENTER_PORT));
}
}
for (dc_key, addrs) in dc_overrides {
let dc_idx = match dc_key.parse::<i16>() {
Ok(v) if v > 0 => v,
_ => {
warn!(dc = %dc_key, "Invalid dc_overrides key for health-check, skipping");
continue;
}
};
for addr_str in addrs {
match addr_str.parse::<SocketAddr>() {
Ok(addr) if addr.is_ipv6() => {
if ipv6_enabled {
v6_by_dc.entry(dc_idx).or_default().push(addr);
}
}
Ok(addr) => {
if ipv4_enabled {
v4_by_dc.entry(dc_idx).or_default().push(addr);
}
}
Err(_) => {
warn!(
dc = %dc_idx,
addr = %addr_str,
"Invalid dc_overrides address for health-check, skipping"
);
}
}
}
}
for addrs in v4_by_dc.values_mut() {
addrs.sort_unstable();
addrs.dedup();
}
for addrs in v6_by_dc.values_mut() {
addrs.sort_unstable();
addrs.dedup();
}
let mut all_dcs = BTreeSet::new();
all_dcs.extend(v4_by_dc.keys().copied());
all_dcs.extend(v6_by_dc.keys().copied());
let mut groups = Vec::with_capacity(all_dcs.len());
for dc_idx in all_dcs {
let v4_endpoints = v4_by_dc.remove(&dc_idx).unwrap_or_default();
let v6_endpoints = v6_by_dc.remove(&dc_idx).unwrap_or_default();
let (primary, fallback) = if prefer_ipv6 {
(v6_endpoints, v4_endpoints)
} else {
(v4_endpoints, v6_endpoints)
};
if primary.is_empty() && fallback.is_empty() {
continue;
}
groups.push(HealthCheckGroup {
dc_idx,
primary,
fallback,
});
}
groups
}
// ============= Health Checks =============
/// Background health check: rotates through DCs, 30s interval.
/// Uses preferred IP version based on config.
pub async fn run_health_checks(&self, prefer_ipv6: bool, ipv4_enabled: bool, ipv6_enabled: bool) {
let mut dc_rotation = 0usize;
/// Background health check based on reachable DC groups through each upstream.
/// Upstream stays healthy while at least `MIN_HEALTHY_DC_GROUPS` groups are reachable.
pub async fn run_health_checks(
&self,
prefer_ipv6: bool,
ipv4_enabled: bool,
ipv6_enabled: bool,
dc_overrides: HashMap<String, Vec<String>>,
) {
let groups = Self::build_health_check_groups(
prefer_ipv6,
ipv4_enabled,
ipv6_enabled,
&dc_overrides,
);
let required_healthy_groups = Self::required_healthy_group_count(groups.len());
let mut endpoint_rotation: HashMap<(usize, i16, bool), usize> = HashMap::new();
if groups.is_empty() {
warn!("No DC groups available for upstream health-checks");
}
loop {
tokio::time::sleep(Duration::from_secs(30)).await;
tokio::time::sleep(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECS)).await;
let dc_zero_idx = dc_rotation % NUM_DCS;
dc_rotation += 1;
let primary_v6 = SocketAddr::new(TG_DATACENTERS_V6[dc_zero_idx], TG_DATACENTER_PORT);
let primary_v4 = SocketAddr::new(TG_DATACENTERS_V4[dc_zero_idx], TG_DATACENTER_PORT);
let dc_addr = if prefer_ipv6 && ipv6_enabled {
primary_v6
} else if ipv4_enabled {
primary_v4
} else if ipv6_enabled {
primary_v6
} else {
if groups.is_empty() || required_healthy_groups == 0 {
continue;
};
let fallback_addr = if dc_addr.is_ipv6() && ipv4_enabled {
Some(primary_v4)
} else if dc_addr.is_ipv4() && ipv6_enabled {
Some(primary_v6)
} else {
None
};
}
let count = self.upstreams.read().await.len();
for i in 0..count {
let (config, bind_rr) = {
let guard = self.upstreams.read().await;
@@ -813,92 +1145,123 @@ impl UpstreamManager {
(u.config.clone(), u.bind_rr.clone())
};
let start = Instant::now();
let result = tokio::time::timeout(
Duration::from_secs(10),
self.connect_via_upstream(&config, dc_addr, Some(bind_rr.clone()))
).await;
let mut healthy_groups = 0usize;
let mut latency_updates: Vec<(usize, f64)> = Vec::new();
match result {
Ok(Ok(_stream)) => {
let rtt_ms = start.elapsed().as_secs_f64() * 1000.0;
let mut guard = self.upstreams.write().await;
let u = &mut guard[i];
u.dc_latency[dc_zero_idx].update(rtt_ms);
for group in &groups {
let mut group_ok = false;
let mut group_rtt_ms = None;
if !u.healthy {
info!(
rtt = format!("{:.0} ms", rtt_ms),
dc = dc_zero_idx + 1,
"Upstream recovered"
);
}
u.healthy = true;
u.fails = 0;
u.last_check = std::time::Instant::now();
}
Ok(Err(_)) | Err(_) => {
// Try fallback
debug!(dc = dc_zero_idx + 1, "Health check failed, trying fallback");
if let Some(fallback_addr) = fallback_addr {
let start2 = Instant::now();
let result2 = tokio::time::timeout(
Duration::from_secs(10),
self.connect_via_upstream(&config, fallback_addr, Some(bind_rr.clone()))
).await;
let mut guard = self.upstreams.write().await;
let u = &mut guard[i];
match result2 {
Ok(Ok(_stream)) => {
let rtt_ms = start2.elapsed().as_secs_f64() * 1000.0;
u.dc_latency[dc_zero_idx].update(rtt_ms);
if !u.healthy {
info!(
rtt = format!("{:.0} ms", rtt_ms),
dc = dc_zero_idx + 1,
"Upstream recovered (fallback)"
);
}
u.healthy = true;
u.fails = 0;
}
Ok(Err(e)) => {
u.fails += 1;
debug!(dc = dc_zero_idx + 1, fails = u.fails,
"Health check failed (both): {}", e);
if u.fails > 3 {
u.healthy = false;
warn!("Upstream unhealthy (fails)");
}
}
Err(_) => {
u.fails += 1;
debug!(dc = dc_zero_idx + 1, fails = u.fails,
"Health check timeout (both)");
if u.fails > 3 {
u.healthy = false;
warn!("Upstream unhealthy (timeout)");
}
}
}
u.last_check = std::time::Instant::now();
for (is_primary, endpoints) in [(true, &group.primary), (false, &group.fallback)] {
if endpoints.is_empty() {
continue;
}
let mut guard = self.upstreams.write().await;
let u = &mut guard[i];
u.fails += 1;
if u.fails > 3 {
u.healthy = false;
warn!("Upstream unhealthy (no fallback family)");
let rotation_key = (i, group.dc_idx, is_primary);
let start_idx = *endpoint_rotation.entry(rotation_key).or_insert(0) % endpoints.len();
let mut next_idx = (start_idx + 1) % endpoints.len();
for step in 0..endpoints.len() {
let endpoint_idx = (start_idx + step) % endpoints.len();
let endpoint = endpoints[endpoint_idx];
let start = Instant::now();
let result = tokio::time::timeout(
Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS),
self.connect_via_upstream(&config, endpoint, Some(bind_rr.clone())),
)
.await;
match result {
Ok(Ok(_stream)) => {
group_ok = true;
group_rtt_ms = Some(start.elapsed().as_secs_f64() * 1000.0);
next_idx = (endpoint_idx + 1) % endpoints.len();
break;
}
Ok(Err(e)) => {
debug!(
upstream = i,
dc = group.dc_idx,
endpoint = %endpoint,
primary = is_primary,
error = %e,
"Health-check endpoint failed"
);
}
Err(_) => {
debug!(
upstream = i,
dc = group.dc_idx,
endpoint = %endpoint,
primary = is_primary,
"Health-check endpoint timed out"
);
}
}
}
endpoint_rotation.insert(rotation_key, next_idx);
if group_ok {
break;
}
}
if group_ok {
healthy_groups += 1;
if let (Some(dc_array_idx), Some(rtt_ms)) =
(UpstreamState::dc_array_idx(group.dc_idx), group_rtt_ms)
{
latency_updates.push((dc_array_idx, rtt_ms));
}
u.last_check = std::time::Instant::now();
}
}
let mut guard = self.upstreams.write().await;
let u = &mut guard[i];
for (dc_array_idx, rtt_ms) in latency_updates {
u.dc_latency[dc_array_idx].update(rtt_ms);
}
if healthy_groups >= required_healthy_groups {
if !u.healthy {
info!(
upstream = i,
healthy_groups,
total_groups = groups.len(),
required_groups = required_healthy_groups,
"Upstream recovered by DC-group health threshold"
);
}
u.healthy = true;
u.fails = 0;
} else {
u.fails += 1;
debug!(
upstream = i,
healthy_groups,
total_groups = groups.len(),
required_groups = required_healthy_groups,
fails = u.fails,
"Upstream health-check below DC-group threshold"
);
if u.fails >= self.unhealthy_fail_threshold {
u.healthy = false;
warn!(
upstream = i,
healthy_groups,
total_groups = groups.len(),
required_groups = required_healthy_groups,
fails = u.fails,
threshold = self.unhealthy_fail_threshold,
"Upstream unhealthy (insufficient reachable DC groups)"
);
}
}
u.last_check = std::time::Instant::now();
}
}
}
@@ -929,3 +1292,76 @@ impl UpstreamManager {
Some(SocketAddr::new(ip, TG_DATACENTER_PORT))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn required_healthy_group_count_applies_three_group_threshold() {
assert_eq!(UpstreamManager::required_healthy_group_count(0), 0);
assert_eq!(UpstreamManager::required_healthy_group_count(1), 1);
assert_eq!(UpstreamManager::required_healthy_group_count(2), 2);
assert_eq!(UpstreamManager::required_healthy_group_count(3), 3);
assert_eq!(UpstreamManager::required_healthy_group_count(5), 3);
}
#[test]
fn build_health_check_groups_merges_family_endpoints_with_preference() {
let mut overrides = HashMap::new();
overrides.insert(
"2".to_string(),
vec![
"203.0.113.10:443".to_string(),
"203.0.113.11:443".to_string(),
"[2001:db8::10]:443".to_string(),
],
);
let groups = UpstreamManager::build_health_check_groups(true, true, true, &overrides);
let dc2 = groups
.iter()
.find(|g| g.dc_idx == 2)
.expect("dc2 must be present");
assert!(dc2.primary.iter().all(|addr| addr.is_ipv6()));
assert!(dc2.fallback.iter().all(|addr| addr.is_ipv4()));
assert!(dc2
.primary
.contains(&"[2001:db8::10]:443".parse::<SocketAddr>().unwrap()));
assert!(dc2
.fallback
.contains(&"203.0.113.10:443".parse::<SocketAddr>().unwrap()));
assert!(dc2
.fallback
.contains(&"203.0.113.11:443".parse::<SocketAddr>().unwrap()));
}
#[test]
fn build_health_check_groups_keeps_multiple_endpoints_per_group() {
let mut overrides = HashMap::new();
overrides.insert(
"9".to_string(),
vec![
"198.51.100.1:443".to_string(),
"198.51.100.2:443".to_string(),
"198.51.100.1:443".to_string(),
],
);
let groups = UpstreamManager::build_health_check_groups(false, true, false, &overrides);
let dc9 = groups
.iter()
.find(|g| g.dc_idx == 9)
.expect("override-only dc group must be present");
assert_eq!(dc9.primary.len(), 2);
assert!(dc9
.primary
.contains(&"198.51.100.1:443".parse::<SocketAddr>().unwrap()));
assert!(dc9
.primary
.contains(&"198.51.100.2:443".parse::<SocketAddr>().unwrap()));
assert!(dc9.fallback.is_empty());
}
}