mirror of
https://github.com/telemt/telemt.git
synced 2026-04-15 09:34:10 +03:00
Compare commits
65 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb0832b803 | ||
|
|
c01ca40b6d | ||
|
|
cfec6dbb3c | ||
|
|
1fe1acadd4 | ||
|
|
225fc3e4ea | ||
|
|
4a0d88ad43 | ||
|
|
58ff0c7971 | ||
|
|
7d39bf1698 | ||
|
|
3b8eea762b | ||
|
|
07ec84d071 | ||
|
|
235642459a | ||
|
|
3799fc13c4 | ||
|
|
71261522bd | ||
|
|
762deac511 | ||
|
|
4300720d35 | ||
|
|
b7a8e759eb | ||
|
|
1a68dc1c2d | ||
|
|
a6d22e8a57 | ||
|
|
9477103f89 | ||
|
|
e589891706 | ||
|
|
fad4b652c4 | ||
|
|
96bfc223fe | ||
|
|
265b9a5f11 | ||
|
|
74ad9037de | ||
|
|
49f4a7bb22 | ||
|
|
ac453638b8 | ||
|
|
e7773b2bda | ||
|
|
6f1980dfd7 | ||
|
|
427fbef50f | ||
|
|
08609f4b6d | ||
|
|
501d802b8d | ||
|
|
e8ff39d2ae | ||
|
|
6c1b837d5b | ||
|
|
b112908c86 | ||
|
|
1e400d4cc2 | ||
|
|
a11c8b659b | ||
|
|
bc432f06e2 | ||
|
|
338636ede6 | ||
|
|
c05779208e | ||
|
|
7ba21ec5a8 | ||
|
|
d997c0b216 | ||
|
|
62cf4f0a1c | ||
|
|
e710fefed2 | ||
|
|
edef06edb5 | ||
|
|
7a0b015e65 | ||
|
|
8b2ec35c46 | ||
|
|
d324d84ec7 | ||
|
|
47b12f9489 | ||
|
|
a5967d0ca3 | ||
|
|
44cdfd4b23 | ||
|
|
25ffcf6081 | ||
|
|
60322807b6 | ||
|
|
ed93b0a030 | ||
|
|
2370c8d5e4 | ||
|
|
a3197b0fe1 | ||
|
|
e27ef04c3d | ||
|
|
cf7e2ebf4b | ||
|
|
685bfafe74 | ||
|
|
0f6fcf49a7 | ||
|
|
036f0e1569 | ||
|
|
291c22583f | ||
|
|
ee5b01bb31 | ||
|
|
ccacf78890 | ||
|
|
42db1191a8 | ||
|
|
9ce26d16cb |
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -2087,7 +2087,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "telemt"
|
||||
version = "3.0.13"
|
||||
version = "3.1.3"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"anyhow",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.1.3"
|
||||
version = "3.1.5"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -215,10 +215,12 @@ hello = "00000000000000000000000000000000"
|
||||
|
||||
```
|
||||
### Advanced
|
||||
#### Adtag
|
||||
To use channel advertising and usage statistics from Telegram, get Adtag from [@mtproxybot](https://t.me/mtproxybot), add this parameter to section `[General]`
|
||||
#### Adtag (per-user)
|
||||
To use channel advertising and usage statistics from Telegram, get an Adtag from [@mtproxybot](https://t.me/mtproxybot). Set it per user in `[access.user_ad_tags]` (32 hex chars):
|
||||
```toml
|
||||
ad_tag = "00000000000000000000000000000000" # Replace zeros to your adtag from @mtproxybot
|
||||
[access.user_ad_tags]
|
||||
username1 = "11111111111111111111111111111111" # Replace with your tag from @mtproxybot
|
||||
username2 = "22222222222222222222222222222222"
|
||||
```
|
||||
#### Listening and Announce IPs
|
||||
To specify listening address and/or address in links, add to section `[[server.listeners]]` of config.toml:
|
||||
|
||||
676
config.full.toml
676
config.full.toml
@@ -1,176 +1,669 @@
|
||||
# Telemt full config with default values.
|
||||
# Examples are kept in comments after '#'.
|
||||
# ==============================================================================
|
||||
#
|
||||
# TELEMT — Advanced Rust-based Telegram MTProto Proxy
|
||||
# Full Configuration Reference
|
||||
#
|
||||
# This file is both a working config and a complete documentation.
|
||||
# Every parameter is explained. Read it top to bottom before deploying.
|
||||
#
|
||||
# Quick Start:
|
||||
# 1. Set [server].port to your desired port (443 recommended)
|
||||
# 2. Generate a secret: openssl rand -hex 16
|
||||
# 3. Put it in [access.users] under a name you choose
|
||||
# 4. Set [censorship].tls_domain to a popular unblocked HTTPS site
|
||||
# 5. Set your public IP in [general].middle_proxy_nat_ip
|
||||
# and [general.links].public_host
|
||||
# 6. Set announce IP in [[server.listeners]]
|
||||
# 7. Run Telemt. It prints a tg:// link. Send it to your users.
|
||||
#
|
||||
# Modes of Operation:
|
||||
# Direct Mode (use_middle_proxy = false)
|
||||
# Connects straight to Telegram DCs via TCP. Simple, fast, low overhead.
|
||||
# No ad_tag support. No CDN DC support (203, etc).
|
||||
#
|
||||
# Middle-Proxy Mode (use_middle_proxy = true)
|
||||
# Connects to Telegram Middle-End servers via RPC protocol.
|
||||
# Required for ad_tag monetization and CDN support.
|
||||
# Requires proxy_secret_path and a valid public IP.
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
# Top-level legacy field.
|
||||
show_link = [] # example: "*" or ["alice", "bob"]
|
||||
# default_dc = 2 # example: default DC for unmapped non-standard DCs
|
||||
|
||||
# ==============================================================================
|
||||
# LEGACY TOP-LEVEL FIELDS
|
||||
# ==============================================================================
|
||||
|
||||
# Deprecated. Use [general.links].show instead.
|
||||
# Accepts "*" for all users, or an array like ["alice", "bob"].
|
||||
show_link = ["0"]
|
||||
|
||||
# Fallback Datacenter index (1-5) when a client requests an unknown DC ID.
|
||||
# DC 2 is Amsterdam (Europe), closest for most CIS users.
|
||||
# default_dc = 2
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# GENERAL SETTINGS
|
||||
# ==============================================================================
|
||||
|
||||
[general]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Core Protocol
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Coalesce the MTProto handshake and first data payload into a single TCP packet.
|
||||
# Significantly reduces connection latency. No reason to disable.
|
||||
fast_mode = true
|
||||
use_middle_proxy = false
|
||||
# ad_tag = "00000000000000000000000000000000" # example
|
||||
# proxy_secret_path = "proxy-secret" # example custom path
|
||||
# middle_proxy_nat_ip = "203.0.113.10" # example public NAT IP override
|
||||
|
||||
# How the proxy connects to Telegram servers.
|
||||
# false = Direct TCP to Telegram DCs (simple, low overhead)
|
||||
# true = Middle-End RPC protocol (required for ad_tag and CDN DCs)
|
||||
use_middle_proxy = true
|
||||
|
||||
# 32-char hex Ad-Tag from @MTProxybot for sponsored channel injection.
|
||||
# Only works when use_middle_proxy = true.
|
||||
# Obtain yours: message @MTProxybot on Telegram, register your proxy.
|
||||
# ad_tag = "00000000000000000000000000000000"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Middle-End Authentication
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Path to the Telegram infrastructure AES key file.
|
||||
# Auto-downloaded from https://core.telegram.org/getProxySecret on first run.
|
||||
# This key authenticates your proxy with Middle-End servers.
|
||||
proxy_secret_path = "proxy-secret"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public IP Configuration (Critical for Middle-Proxy Mode)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Your server's PUBLIC IPv4 address.
|
||||
# Middle-End servers need this for the cryptographic Key Derivation Function.
|
||||
# If your server has a direct public IP, set it here.
|
||||
# If behind NAT (AWS, Docker, etc.), this MUST be your external IP.
|
||||
# If omitted, Telemt uses STUN to auto-detect (see middle_proxy_nat_probe).
|
||||
# middle_proxy_nat_ip = "203.0.113.10"
|
||||
|
||||
# Auto-detect public IP via STUN servers defined in [network].
|
||||
# Set to false if you hardcoded middle_proxy_nat_ip above.
|
||||
# Set to true if you want automatic detection.
|
||||
middle_proxy_nat_probe = true
|
||||
# middle_proxy_nat_stun = "stun.l.google.com:19302" # example
|
||||
# middle_proxy_nat_stun_servers = [] # example: ["stun1.l.google.com:19302", "stun2.l.google.com:19302"]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Middle-End Connection Pool
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Number of persistent multiplexed RPC connections to ME servers.
|
||||
# All client traffic is routed through these "fat pipes".
|
||||
# 8 handles thousands of concurrent users comfortably.
|
||||
middle_proxy_pool_size = 8
|
||||
|
||||
# Legacy field. Connections kept initialized but idle as warm standby.
|
||||
middle_proxy_warm_standby = 16
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Middle-End Keepalive
|
||||
# Telegram ME servers aggressively kill idle TCP connections.
|
||||
# These settings send periodic RPC_PING frames to keep pipes alive.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
me_keepalive_enabled = true
|
||||
|
||||
# Base interval between pings in seconds.
|
||||
me_keepalive_interval_secs = 25
|
||||
|
||||
# Random jitter added to interval to prevent all connections pinging simultaneously.
|
||||
me_keepalive_jitter_secs = 5
|
||||
|
||||
# Randomize ping payload bytes to prevent DPI from fingerprinting ping patterns.
|
||||
me_keepalive_payload_random = true
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Client-Side Limits
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Max buffered ciphertext per client (bytes) when upstream is slow.
|
||||
# Acts as backpressure to prevent memory exhaustion. 256KB is safe.
|
||||
crypto_pending_buffer = 262144
|
||||
|
||||
# Maximum single MTProto frame size from client. 16MB is protocol standard.
|
||||
max_client_frame = 16777216
|
||||
desync_all_full = false
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Crypto Desynchronization Logging
|
||||
# Desync errors usually mean DPI/GFW is tampering with connections.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# true = full forensics (trace ID, IP hash, hex dumps) for EVERY desync event
|
||||
# false = deduplicated logging, one entry per time window (prevents log spam)
|
||||
# Set true if you are actively debugging DPI interference.
|
||||
desync_all_full = true
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Beobachten — Built-in Honeypot / Active Probe Tracker
|
||||
# Tracks IPs that fail handshakes or behave like TLS scanners.
|
||||
# Output file can be fed into fail2ban or iptables for auto-blocking.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
beobachten = true
|
||||
beobachten_minutes = 10
|
||||
|
||||
# How long (minutes) to remember a suspicious IP before expiring it.
|
||||
beobachten_minutes = 30
|
||||
|
||||
# How often (seconds) to flush tracker state to disk.
|
||||
beobachten_flush_secs = 15
|
||||
|
||||
# File path for the tracker output.
|
||||
beobachten_file = "cache/beobachten.txt"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Hardswap — Zero-Downtime ME Pool Rotation
|
||||
# When Telegram updates ME server IPs, Hardswap creates a completely new pool,
|
||||
# waits until it is fully ready, migrates traffic, then kills the old pool.
|
||||
# Users experience zero interruption.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
hardswap = true
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# ME Pool Warmup Staggering
|
||||
# When creating a new pool, connections are opened one by one with delays
|
||||
# to avoid a burst of SYN packets that could trigger ISP flood protection.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
me_warmup_stagger_enabled = true
|
||||
|
||||
# Delay between each connection creation (milliseconds).
|
||||
me_warmup_step_delay_ms = 500
|
||||
|
||||
# Random jitter added to the delay (milliseconds).
|
||||
me_warmup_step_jitter_ms = 300
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# ME Reconnect Backoff
|
||||
# If an ME server drops the connection, Telemt retries with this strategy.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Max simultaneous reconnect attempts per DC.
|
||||
me_reconnect_max_concurrent_per_dc = 8
|
||||
|
||||
# Exponential backoff base (milliseconds).
|
||||
me_reconnect_backoff_base_ms = 500
|
||||
|
||||
# Backoff ceiling (milliseconds). Will never wait longer than this.
|
||||
me_reconnect_backoff_cap_ms = 30000
|
||||
|
||||
# Number of instant retries before switching to exponential backoff.
|
||||
me_reconnect_fast_retry_count = 12
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# NAT Mismatch Behavior
|
||||
# If STUN-detected IP differs from local interface IP (you are behind NAT).
|
||||
# false = abort ME mode (safe default)
|
||||
# true = force ME mode anyway (use if you know your NAT setup is correct)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
stun_iface_mismatch_ignore = false
|
||||
unknown_dc_log_path = "unknown-dc.txt" # to disable: set to null
|
||||
log_level = "normal" # debug | verbose | normal | silent
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# File to log unknown DC requests (DC IDs outside standard 1-5).
|
||||
unknown_dc_log_path = "unknown-dc.txt"
|
||||
|
||||
# Verbosity: "debug" | "verbose" | "normal" | "silent"
|
||||
log_level = "normal"
|
||||
|
||||
# Disable ANSI color codes in log output (useful for file logging).
|
||||
disable_colors = false
|
||||
fast_mode_min_tls_record = 0
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# FakeTLS Record Sizing
|
||||
# Buffer small MTProto packets into larger TLS records to mimic real HTTPS.
|
||||
# Real HTTPS servers send records close to MTU size (~1400 bytes).
|
||||
# A stream of tiny TLS records is a strong DPI signal.
|
||||
# Set to 0 to disable. Set to 1400 for realistic HTTPS emulation.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
fast_mode_min_tls_record = 1400
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Periodic Updates
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# How often (seconds) to re-fetch ME server lists and proxy secrets
|
||||
# from core.telegram.org. Keeps your proxy in sync with Telegram infrastructure.
|
||||
update_every = 300
|
||||
me_reinit_every_secs = 900
|
||||
|
||||
# How often (seconds) to force a Hardswap even if the ME map is unchanged.
|
||||
# Shorter intervals mean shorter-lived TCP flows, harder for DPI to profile.
|
||||
me_reinit_every_secs = 600
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Hardswap Warmup Tuning
|
||||
# Fine-grained control over how the new pool is warmed up before traffic switch.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
me_hardswap_warmup_delay_min_ms = 1000
|
||||
me_hardswap_warmup_delay_max_ms = 2000
|
||||
me_hardswap_warmup_extra_passes = 3
|
||||
me_hardswap_warmup_pass_backoff_base_ms = 500
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Config Update Debouncing
|
||||
# Telegram sometimes pushes transient/broken configs. Debouncing requires
|
||||
# N consecutive identical fetches before applying a change.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# ME server list must be identical for this many fetches before applying.
|
||||
me_config_stable_snapshots = 2
|
||||
|
||||
# Minimum seconds between config applications.
|
||||
me_config_apply_cooldown_secs = 300
|
||||
|
||||
# Proxy secret must be identical for this many fetches before applying.
|
||||
proxy_secret_stable_snapshots = 2
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Proxy Secret Rotation
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Apply newly downloaded secrets at runtime without restart.
|
||||
proxy_secret_rotate_runtime = true
|
||||
|
||||
# Maximum acceptable secret length (bytes). Rejects abnormally large secrets.
|
||||
proxy_secret_len_max = 256
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Hardswap Drain Settings
|
||||
# Controls graceful shutdown of old ME connections during pool rotation.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Seconds to keep old connections alive for in-flight data before force-closing.
|
||||
me_pool_drain_ttl_secs = 90
|
||||
|
||||
# Minimum ratio of healthy connections in new pool before draining old pool.
|
||||
# 0.8 = at least 80% of new pool must be ready.
|
||||
me_pool_min_fresh_ratio = 0.8
|
||||
|
||||
# Maximum seconds to wait for drain to complete before force-killing.
|
||||
me_reinit_drain_timeout_secs = 120
|
||||
# Legacy compatibility fields used when update_every is omitted.
|
||||
proxy_secret_auto_reload_secs = 3600
|
||||
proxy_config_auto_reload_secs = 3600
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# NTP Clock Check
|
||||
# MTProto uses timestamps. Clock drift > 30 seconds breaks handshakes.
|
||||
# Telemt checks on startup and warns if out of sync.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
ntp_check = true
|
||||
ntp_servers = ["pool.ntp.org"] # example: ["pool.ntp.org", "time.cloudflare.com"]
|
||||
ntp_servers = ["pool.ntp.org"]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Auto-Degradation
|
||||
# If ME servers become completely unreachable (ISP blocking),
|
||||
# automatically fall back to Direct Mode so users stay connected.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
auto_degradation_enabled = true
|
||||
|
||||
# Number of DC groups that must be unreachable before triggering fallback.
|
||||
degradation_min_unavailable_dc_groups = 2
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# ALLOWED CLIENT PROTOCOLS
|
||||
# Only enable what you need. In censored regions, TLS-only is safest.
|
||||
# ==============================================================================
|
||||
|
||||
[general.modes]
|
||||
|
||||
# Classic MTProto. Unobfuscated length prefixes. Trivially detected by DPI.
|
||||
# No reason to enable unless you have ancient clients.
|
||||
classic = false
|
||||
|
||||
# Obfuscated MTProto with randomized padding. Better than classic, but
|
||||
# still detectable by statistical analysis of packet sizes.
|
||||
secure = false
|
||||
|
||||
# FakeTLS (ee-secrets). Wraps MTProto in TLS 1.3 framing.
|
||||
# To DPI, it looks like a normal HTTPS connection.
|
||||
# This should be the ONLY enabled mode in censored environments.
|
||||
tls = true
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# STARTUP LINK GENERATION
|
||||
# Controls what tg:// invite links are printed to console on startup.
|
||||
# ==============================================================================
|
||||
|
||||
[general.links]
|
||||
show ="*" # example: "*" or ["alice", "bob"]
|
||||
# public_host = "proxy.example.com" # example explicit host/IP for tg:// links
|
||||
# public_port = 443 # example explicit port for tg:// links
|
||||
|
||||
# Which users to generate links for.
|
||||
# "*" = all users, or an array like ["alice", "bob"].
|
||||
show = "*"
|
||||
|
||||
# IP or domain to embed in the tg:// link.
|
||||
# If omitted, Telemt uses STUN to auto-detect.
|
||||
# Set this to your server's public IP or domain for reliable links.
|
||||
# public_host = "proxy.example.com"
|
||||
|
||||
# Port to embed in the tg:// link.
|
||||
# If omitted, uses [server].port.
|
||||
# public_port = 443
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# NETWORK & IP RESOLUTION
|
||||
# ==============================================================================
|
||||
|
||||
[network]
|
||||
|
||||
# Enable IPv4 for outbound connections to Telegram.
|
||||
ipv4 = true
|
||||
ipv6 = false # set true to enable IPv6
|
||||
prefer = 4 # 4 or 6
|
||||
|
||||
# Enable IPv6 for outbound connections to Telegram.
|
||||
ipv6 = false
|
||||
|
||||
# Prefer IPv4 (4) or IPv6 (6) when both are available.
|
||||
prefer = 4
|
||||
|
||||
# Experimental: use both IPv4 and IPv6 ME servers simultaneously.
|
||||
# May improve reliability but doubles connection count.
|
||||
multipath = false
|
||||
|
||||
# STUN servers for external IP discovery.
|
||||
# Used for Middle-Proxy KDF (if nat_probe=true) and link generation.
|
||||
stun_servers = [
|
||||
"stun.l.google.com:5349",
|
||||
"stun1.l.google.com:3478",
|
||||
"stun.gmx.net:3478",
|
||||
"stun.l.google.com:19302",
|
||||
"stun.1und1.de:3478",
|
||||
"stun1.l.google.com:19302",
|
||||
"stun2.l.google.com:19302",
|
||||
"stun3.l.google.com:19302",
|
||||
"stun4.l.google.com:19302",
|
||||
"stun.services.mozilla.com:3478",
|
||||
"stun.stunprotocol.org:3478",
|
||||
"stun.nextcloud.com:3478",
|
||||
"stun.voip.eutelia.it:3478",
|
||||
"stun.l.google.com:5349",
|
||||
"stun1.l.google.com:3478",
|
||||
"stun.gmx.net:3478",
|
||||
"stun.l.google.com:19302"
|
||||
]
|
||||
|
||||
# If UDP STUN is blocked, attempt TCP-based STUN as fallback.
|
||||
stun_tcp_fallback = true
|
||||
http_ip_detect_urls = ["https://ifconfig.me/ip", "https://api.ipify.org"]
|
||||
|
||||
# If all STUN fails, use HTTP APIs to discover public IP.
|
||||
http_ip_detect_urls = [
|
||||
"https://ifconfig.me/ip",
|
||||
"https://api.ipify.org"
|
||||
]
|
||||
|
||||
# Cache discovered public IP to this file to survive restarts.
|
||||
cache_public_ip_path = "cache/public_ip.txt"
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# SERVER BINDING & METRICS
|
||||
# ==============================================================================
|
||||
|
||||
[server]
|
||||
|
||||
# TCP port to listen on.
|
||||
# 443 is recommended (looks like normal HTTPS traffic).
|
||||
port = 443
|
||||
|
||||
# IPv4 bind address. "0.0.0.0" = all interfaces.
|
||||
listen_addr_ipv4 = "0.0.0.0"
|
||||
|
||||
# IPv6 bind address. "::" = all interfaces.
|
||||
listen_addr_ipv6 = "::"
|
||||
# listen_unix_sock = "/var/run/telemt.sock" # example
|
||||
# listen_unix_sock_perm = "0660" # example unix socket mode
|
||||
# listen_tcp = true # example explicit override (auto-detected when omitted)
|
||||
|
||||
# Unix socket listener (for reverse proxy setups with Nginx/HAProxy).
|
||||
# listen_unix_sock = "/var/run/telemt.sock"
|
||||
# listen_unix_sock_perm = "0660"
|
||||
|
||||
# Enable PROXY protocol header parsing.
|
||||
# Set true ONLY if Telemt is behind HAProxy/Nginx that injects PROXY headers.
|
||||
# If enabled without a proxy in front, clients will fail to connect.
|
||||
proxy_protocol = false
|
||||
# metrics_port = 9090 # example
|
||||
metrics_whitelist = ["127.0.0.1/32", "::1/128"]
|
||||
# Example explicit listeners (default: omitted, auto-generated from listen_addr_*):
|
||||
|
||||
# Prometheus metrics HTTP endpoint port.
|
||||
# Uncomment to enable. Access at http://your-server:9090/metrics
|
||||
# metrics_port = 9090
|
||||
|
||||
# IP ranges allowed to access the metrics endpoint.
|
||||
metrics_whitelist = [
|
||||
"127.0.0.1/32",
|
||||
"::1/128"
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Listener Overrides
|
||||
# Define explicit listeners with specific bind IPs and announce IPs.
|
||||
# The announce IP is what gets embedded in tg:// links and sent to ME servers.
|
||||
# You MUST set announce to your server's public IP for ME mode to work.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# [[server.listeners]]
|
||||
# ip = "0.0.0.0"
|
||||
# announce = "proxy-v4.example.com"
|
||||
# # announce_ip = "203.0.113.10" # deprecated alias
|
||||
# proxy_protocol = false
|
||||
# reuse_allow = false
|
||||
#
|
||||
# [[server.listeners]]
|
||||
# ip = "::"
|
||||
# announce = "proxy-v6.example.com"
|
||||
# proxy_protocol = false
|
||||
# announce = "203.0.113.10"
|
||||
# reuse_allow = false
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# TIMEOUTS (seconds unless noted)
|
||||
# ==============================================================================
|
||||
|
||||
[timeouts]
|
||||
|
||||
# Maximum time for client to complete FakeTLS + MTProto handshake.
|
||||
client_handshake = 15
|
||||
|
||||
# Maximum time to establish TCP connection to upstream Telegram DC.
|
||||
tg_connect = 10
|
||||
|
||||
# TCP keepalive interval for client connections.
|
||||
client_keepalive = 60
|
||||
|
||||
# Maximum client inactivity before dropping the connection.
|
||||
client_ack = 300
|
||||
|
||||
# Instant retry count for a single ME endpoint before giving up on it.
|
||||
me_one_retry = 3
|
||||
|
||||
# Timeout (milliseconds) for a single ME endpoint connection attempt.
|
||||
me_one_timeout_ms = 1500
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# ANTI-CENSORSHIP / FAKETLS / MASKING
|
||||
# This is where Telemt becomes invisible to Deep Packet Inspection.
|
||||
# ==============================================================================
|
||||
|
||||
[censorship]
|
||||
tls_domain = "petrovich.ru"
|
||||
# tls_domains = ["example.com", "cdn.example.net"] # Additional domains for EE links
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# TLS Domain Fronting
|
||||
# The SNI (Server Name Indication) your proxy presents to connecting clients.
|
||||
# Must be a popular, unblocked HTTPS website in your target country.
|
||||
# DPI sees traffic to this domain. Choose carefully.
|
||||
# Good choices: major CDNs, banks, government sites, search engines.
|
||||
# Bad choices: obscure sites, already-blocked domains.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
tls_domain = "www.google.com"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Active Probe Masking
|
||||
# When someone connects but fails the MTProto handshake (wrong secret),
|
||||
# they might be an ISP active prober testing if this is a proxy.
|
||||
#
|
||||
# mask = false: drop the connection (prober knows something is here)
|
||||
# mask = true: transparently proxy them to mask_host (prober sees a real website)
|
||||
#
|
||||
# With mask enabled, your server is indistinguishable from a real web server
|
||||
# to anyone who doesn't have the correct secret.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
mask = true
|
||||
# mask_host = "www.google.com" # example, defaults to tls_domain when both mask_host/mask_unix_sock are unset
|
||||
# mask_unix_sock = "/var/run/nginx.sock" # example, mutually exclusive with mask_host
|
||||
|
||||
# The real web server to forward failed handshakes to.
|
||||
# If omitted, defaults to tls_domain.
|
||||
# mask_host = "www.google.com"
|
||||
|
||||
# Port on the mask host to connect to.
|
||||
mask_port = 443
|
||||
# mask_proxy_protocol = 0 # Send PROXY protocol header to mask_host: 0 = off, 1 = v1 (text), 2 = v2 (binary)
|
||||
fake_cert_len = 2048 # if tls_emulation=false and default value is used, loader may randomize this value at runtime
|
||||
|
||||
# Inject PROXY protocol header when forwarding to mask host.
|
||||
# 0 = disabled, 1 = v1, 2 = v2. Leave disabled unless mask_host expects it.
|
||||
# mask_proxy_protocol = 0
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# TLS Certificate Emulation
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Size (bytes) of the locally generated fake TLS certificate.
|
||||
# Only used when tls_emulation is disabled.
|
||||
fake_cert_len = 2048
|
||||
|
||||
# KILLER FEATURE: Real-Time TLS Emulation.
|
||||
# Telemt connects to tls_domain, fetches its actual TLS 1.3 certificate chain,
|
||||
# and exactly replicates the byte sizes of ServerHello and Certificate records.
|
||||
# Defeats DPI that uses TLS record length heuristics to detect proxies.
|
||||
# Strongly recommended in censored environments.
|
||||
tls_emulation = true
|
||||
|
||||
# Directory to cache fetched TLS certificates.
|
||||
tls_front_dir = "tlsfront"
|
||||
server_hello_delay_min_ms = 0
|
||||
server_hello_delay_max_ms = 0
|
||||
tls_new_session_tickets = 0
|
||||
tls_full_cert_ttl_secs = 90
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# ServerHello Timing
|
||||
# Real web servers take 30-150ms to respond to ClientHello due to network
|
||||
# latency and crypto processing. A proxy responding in <1ms is suspicious.
|
||||
# These settings add realistic delay to mimic genuine server behavior.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Minimum delay before sending ServerHello (milliseconds).
|
||||
server_hello_delay_min_ms = 50
|
||||
|
||||
# Maximum delay before sending ServerHello (milliseconds).
|
||||
server_hello_delay_max_ms = 150
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# TLS Session Tickets
|
||||
# Real TLS 1.3 servers send 1-2 NewSessionTicket messages after handshake.
|
||||
# A server that sends zero tickets is anomalous and may trigger DPI flags.
|
||||
# Set this to match your tls_domain's behavior (usually 2).
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# tls_new_session_tickets = 0
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Full Certificate Frequency
|
||||
# When tls_emulation is enabled, this controls how often (per client IP)
|
||||
# to send the complete emulated certificate chain.
|
||||
#
|
||||
# > 0: Subsequent connections within TTL seconds get a smaller cached version.
|
||||
# Saves bandwidth but creates a detectable size difference between
|
||||
# first and repeat connections.
|
||||
#
|
||||
# = 0: Every connection gets the full certificate. More bandwidth but
|
||||
# perfectly consistent behavior, no anomalies for DPI to detect.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
tls_full_cert_ttl_secs = 0
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# ALPN Enforcement
|
||||
# Ensure ServerHello responds with the exact ALPN protocol the client requested.
|
||||
# Mismatched ALPN (e.g., client asks h2, server says http/1.1) is a DPI red flag.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
alpn_enforce = true
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# ACCESS CONTROL & USERS
|
||||
# ==============================================================================
|
||||
|
||||
[access]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Replay Attack Protection
|
||||
# DPI can record a legitimate user's handshake and replay it later to probe
|
||||
# whether the server is a proxy. Telemt remembers recent handshake nonces
|
||||
# and rejects duplicates.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Number of nonce slots in the replay detection buffer.
|
||||
replay_check_len = 65536
|
||||
|
||||
# How long (seconds) to remember nonces before expiring them.
|
||||
replay_window_secs = 1800
|
||||
|
||||
# Allow clients with incorrect system clocks to connect.
|
||||
# false = reject clients with significant time skew (more secure)
|
||||
# true = accept anyone regardless of clock (more permissive)
|
||||
ignore_time_skew = false
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# User Secrets
|
||||
# Each user needs a unique 32-character hex string as their secret.
|
||||
# Generate with: openssl rand -hex 16
|
||||
#
|
||||
# This secret is embedded in the tg:// link. Anyone with it can connect.
|
||||
# Format: username = "hex_secret"
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[access.users]
|
||||
# format: "username" = "32_hex_chars_secret"
|
||||
hello = "00000000000000000000000000000000"
|
||||
# alice = "11111111111111111111111111111111" # example
|
||||
# alice = "0123456789abcdef0123456789abcdef"
|
||||
# bob = "fedcba9876543210fedcba9876543210"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Per-User Connection Limits
|
||||
# Limits concurrent TCP connections per user to prevent secret sharing.
|
||||
# Uncomment and set for each user as needed.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[access.user_max_tcp_conns]
|
||||
# alice = 100 # example
|
||||
# alice = 100
|
||||
# bob = 50
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Per-User Expiration Dates
|
||||
# Automatically revoke access after the specified date (ISO 8601 format).
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[access.user_expirations]
|
||||
# alice = "2078-01-01T00:00:00Z" # example
|
||||
# alice = "2025-12-31T23:59:59Z"
|
||||
# bob = "2026-06-15T00:00:00Z"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Per-User Data Quotas
|
||||
# Maximum total bytes transferred per user. Connection refused after limit.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[access.user_data_quota]
|
||||
# hello = 10737418240 # example bytes
|
||||
# alice = 10737418240 # example bytes
|
||||
# alice = 107374182400
|
||||
# bob = 53687091200
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Per-User Unique IP Limits
|
||||
# Maximum number of different IP addresses that can use this secret
|
||||
# at the same time. Highly effective against secret leaking/sharing.
|
||||
# Set to 1 for single-device, 2-3 for phone+desktop, etc.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
[access.user_max_unique_ips]
|
||||
# hello = 10 # example
|
||||
# alice = 100 # example
|
||||
# alice = 3
|
||||
# bob = 2
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# UPSTREAM ROUTING
|
||||
# Controls how Telemt connects to Telegram servers (or ME servers).
|
||||
# If omitted entirely, uses the OS default route.
|
||||
# ==============================================================================
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Direct upstream: use the server's own network interface.
|
||||
# You can optionally bind to a specific interface or local IP.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Default behavior if [[upstreams]] is omitted: loader injects one direct upstream.
|
||||
# Example explicit upstreams:
|
||||
# [[upstreams]]
|
||||
# type = "direct"
|
||||
# interface = "eth0"
|
||||
@@ -178,28 +671,27 @@ hello = "00000000000000000000000000000000"
|
||||
# weight = 1
|
||||
# enabled = true
|
||||
# scopes = "*"
|
||||
#
|
||||
# [[upstreams]]
|
||||
# type = "socks4"
|
||||
# address = "198.51.100.20:1080"
|
||||
# interface = "eth0"
|
||||
# user_id = "telemt"
|
||||
# weight = 1
|
||||
# enabled = true
|
||||
# scopes = "*"
|
||||
#
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# SOCKS5 upstream: route Telegram traffic through a SOCKS5 proxy.
|
||||
# Useful if your server's IP is blocked from reaching Telegram DCs.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# [[upstreams]]
|
||||
# type = "socks5"
|
||||
# address = "198.51.100.30:1080"
|
||||
# interface = "eth0"
|
||||
# username = "proxy-user"
|
||||
# password = "proxy-pass"
|
||||
# weight = 1
|
||||
# enabled = true
|
||||
# scopes = "*"
|
||||
|
||||
# === DC Address Overrides ===
|
||||
|
||||
# ==============================================================================
|
||||
# DATACENTER OVERRIDES
|
||||
# Force specific DC IDs to route to specific IP:Port combinations.
|
||||
# DC 203 (CDN) is auto-injected by Telemt if not specified here.
|
||||
# ==============================================================================
|
||||
|
||||
# [dc_overrides]
|
||||
# "201" = "149.154.175.50:443" # example
|
||||
# "202" = ["149.154.167.51:443", "149.154.175.100:443"] # example
|
||||
# "203" = "91.105.192.100:443" # loader auto-adds this one when omitted
|
||||
# "201" = "149.154.175.50:443"
|
||||
# "202" = ["149.154.167.51:443", "149.154.175.100:443"]
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
# === General Settings ===
|
||||
[general]
|
||||
use_middle_proxy = false
|
||||
# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags]
|
||||
# ad_tag = "00000000000000000000000000000000"
|
||||
# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot)
|
||||
|
||||
# === Log Level ===
|
||||
# Log level: debug | verbose | normal | silent
|
||||
|
||||
64
docs/FAQ.ru.md
Normal file
64
docs/FAQ.ru.md
Normal file
@@ -0,0 +1,64 @@
|
||||
## Как настроить канал "спонсор прокси"
|
||||
|
||||
1. Зайти в бота @MTProxybot.
|
||||
2. Ввести команду `/newproxy`
|
||||
3. Отправить IP и порт сервера. Например: 1.2.3.4:443
|
||||
4. Открыть конфиг `nano /etc/telemt.toml`.
|
||||
5. Скопировать и отправить боту секрет пользователя из раздела [access.users].
|
||||
6. Скопировать полученный tag у бота. Например 1234567890abcdef1234567890abcdef.
|
||||
7. Раскомментировать параметр ad_tag и вписать tag, полученный у бота.
|
||||
8. Раскомментировать/добавить параметр use_middle_proxy = true.
|
||||
|
||||
Пример конфига:
|
||||
```toml
|
||||
[general]
|
||||
ad_tag = "1234567890abcdef1234567890abcdef"
|
||||
use_middle_proxy = true
|
||||
```
|
||||
9. Сохранить конфиг. Ctrl+S -> Ctrl+X.
|
||||
10. Перезапустить telemt `systemctl restart telemt`.
|
||||
11. В боте отправить команду /myproxies и выбрать добавленный сервер.
|
||||
12. Нажать кнопку "Set promotion".
|
||||
13. Отправить **публичную ссылку** на канал. Приватный канал добавить нельзя!
|
||||
14. Подождать примерно 1 час, пока информация обновится на серверах Telegram.
|
||||
> [!WARNING]
|
||||
> У вас не будет отображаться "спонсор прокси" если вы уже подписаны на канал.
|
||||
|
||||
## Сколько человек может пользоваться 1 ссылкой
|
||||
|
||||
По умолчанию 1 ссылкой может пользоваться сколько угодно человек.
|
||||
Вы можете ограничить число IP, использующих прокси.
|
||||
```toml
|
||||
[access.user_max_unique_ips]
|
||||
hello = 1
|
||||
```
|
||||
Этот параметр ограничивает, сколько уникальных IP может использовать 1 ссылку одновременно. Если один пользователь отключится, второй сможет подключиться. Также с одного IP может сидеть несколько пользователей.
|
||||
|
||||
## Как сделать несколько разных ссылок
|
||||
|
||||
1. Сгенерируйте нужное число секретов `openssl rand -hex 16`
|
||||
2. Открыть конфиг `nano /etc/telemt.toml`
|
||||
3. Добавить новых пользователей.
|
||||
```toml
|
||||
[access.users]
|
||||
user1 = "00000000000000000000000000000001"
|
||||
user2 = "00000000000000000000000000000002"
|
||||
user3 = "00000000000000000000000000000003"
|
||||
```
|
||||
4. Сохранить конфиг. Ctrl+S -> Ctrl+X. Перезапускать telemt не нужно.
|
||||
5. Получить ссылки через `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
|
||||
|
||||
## Как посмотреть метрики
|
||||
|
||||
1. Открыть конфиг `nano /etc/telemt.toml`
|
||||
2. Добавить следующие параметры
|
||||
```toml
|
||||
[server]
|
||||
metrics_port = 9090
|
||||
metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"]
|
||||
```
|
||||
3. Сохранить конфиг. Ctrl+S -> Ctrl+X.
|
||||
4. Метрики доступны по адресу SERVER_IP:9090/metrics.
|
||||
> [!WARNING]
|
||||
> "0.0.0.0/0" в metrics_whitelist открывает доступ с любого IP. Замените на свой ip. Например "1.2.3.4"
|
||||
|
||||
40
docs/MIDDLE-END-KDF.de.md
Normal file
40
docs/MIDDLE-END-KDF.de.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# Middle-End Proxy
|
||||
|
||||
## KDF-Adressierung — Implementierungs-FAQ
|
||||
|
||||
### Benötigt die C-Referenzimplementierung sowohl externe IP-Adresse als auch Port für die KDF?
|
||||
|
||||
Ja.
|
||||
|
||||
In der C-Referenzimplementierung werden **sowohl IP-Adresse als auch Port in die KDF einbezogen** — auf beiden Seiten der Verbindung.
|
||||
|
||||
In `aes_create_keys()` enthält der KDF-Input:
|
||||
|
||||
- `server_ip + client_port`
|
||||
- `client_ip + server_port`
|
||||
- sowie Secret / Nonces
|
||||
|
||||
Für IPv6:
|
||||
|
||||
- IPv4-Felder werden auf 0 gesetzt
|
||||
- IPv6-Adressen werden ergänzt
|
||||
|
||||
Die **Ports bleiben weiterhin Bestandteil der KDF**.
|
||||
|
||||
> Wenn sich externe IP oder Port (z. B. durch NAT, SOCKS oder Proxy) von den erwarteten Werten unterscheiden, entstehen unterschiedliche Schlüssel — der Handshake schlägt fehl.
|
||||
|
||||
---
|
||||
|
||||
### Kann der Port aus der KDF ausgeschlossen werden (z. B. durch Port = 0)?
|
||||
|
||||
**Nein!**
|
||||
|
||||
Die C-Referenzimplementierung enthält **keine Möglichkeit, den Port zu ignorieren**:
|
||||
- `client_port` und `server_port` sind fester Bestandteil der KDF
|
||||
- Es werden immer reale Socket-Ports übergeben:
|
||||
- `c->our_port`
|
||||
- `c->remote_port`
|
||||
|
||||
Falls ein Port den Wert `0` hat, wird er dennoch als `0` in die KDF übernommen.
|
||||
|
||||
Eine „Port-Ignore“-Logik existiert nicht.
|
||||
41
docs/MIDDLE-END-KDF.en.md
Normal file
41
docs/MIDDLE-END-KDF.en.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Middle-End Proxy
|
||||
|
||||
## KDF Addressing — Implementation FAQ
|
||||
|
||||
### Does the C-implementation require both external IP address and port for the KDF?
|
||||
|
||||
**Yes!**
|
||||
|
||||
In the C reference implementation, **both IP address and port are included in the KDF input** from both sides of the connection.
|
||||
|
||||
Inside `aes_create_keys()`, the KDF input explicitly contains:
|
||||
|
||||
- `server_ip + client_port`
|
||||
- `client_ip + server_port`
|
||||
- followed by shared secret / nonces
|
||||
|
||||
For IPv6:
|
||||
|
||||
- IPv4 fields are zeroed
|
||||
- IPv6 addresses are inserted
|
||||
|
||||
However, **client_port and server_port remain part of the KDF regardless of IP version**.
|
||||
|
||||
> If externally observed IP or port (e.g. due to NAT, SOCKS, or proxy traversal) differs from what the peer expects, the derived keys will not match and the handshake will fail.
|
||||
|
||||
---
|
||||
|
||||
### Can port be excluded from KDF (e.g. by using port = 0)?
|
||||
|
||||
**No!**
|
||||
|
||||
The C-implementation provides **no mechanism to ignore the port**:
|
||||
|
||||
- `client_port` and `server_port` are explicitly included in the KDF input
|
||||
- Real socket ports are always passed:
|
||||
- `c->our_port`
|
||||
- `c->remote_port`
|
||||
|
||||
If a port is `0`, it is still incorporated into the KDF as `0`.
|
||||
|
||||
There is **no conditional logic to exclude ports**
|
||||
41
docs/MIDDLE-END-KDF.ru.md
Normal file
41
docs/MIDDLE-END-KDF.ru.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Middle-End Proxy
|
||||
|
||||
## KDF Addressing — FAQ по реализации
|
||||
|
||||
### Требует ли C-референсная реализация KDF внешний IP и порт?
|
||||
|
||||
**Да**
|
||||
|
||||
В C-референсе **в KDF участвуют и IP-адрес, и порт** — с обеих сторон соединения.
|
||||
|
||||
В `aes_create_keys()` в строку KDF входят:
|
||||
|
||||
- `server_ip + client_port`
|
||||
- `client_ip + server_port`
|
||||
- далее secret / nonces
|
||||
|
||||
Для IPv6:
|
||||
|
||||
- IPv4-поля заполняются нулями
|
||||
- добавляются IPv6-адреса
|
||||
|
||||
Однако **порты client_port и server_port всё равно участвуют в KDF**.
|
||||
|
||||
> Если внешний IP или порт (например, из-за NAT, SOCKS или прокси) не совпадает с ожидаемым другой стороной — ключи расходятся и handshake ломается.
|
||||
|
||||
---
|
||||
|
||||
### Можно ли исключить порт из KDF (например, установив порт = 0)?
|
||||
|
||||
**Нет.**
|
||||
|
||||
В C-референсе **нет механики отключения порта**.
|
||||
|
||||
- `client_port` и `server_port` явно включены в KDF
|
||||
- Передаются реальные порты сокета:
|
||||
- `c->our_port`
|
||||
- `c->remote_port`
|
||||
|
||||
Если порт равен `0`, он всё равно попадёт в KDF как `0`.
|
||||
|
||||
Отдельной логики «игнорировать порт» не предусмотрено.
|
||||
152
docs/QUICK_START_GUIDE.en.md
Normal file
152
docs/QUICK_START_GUIDE.en.md
Normal file
@@ -0,0 +1,152 @@
|
||||
# Telemt via Systemd
|
||||
|
||||
## Installation
|
||||
|
||||
This software is designed for Debian-based OS: in addition to Debian, these are Ubuntu, Mint, Kali, MX and many other Linux
|
||||
|
||||
**1. Download**
|
||||
```bash
|
||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
||||
```
|
||||
**2. Move to the Bin folder**
|
||||
```bash
|
||||
mv telemt /bin
|
||||
```
|
||||
**3. Make the file executable**
|
||||
```bash
|
||||
chmod +x /bin/telemt
|
||||
```
|
||||
|
||||
## How to use?
|
||||
|
||||
**This guide "assumes" that you:**
|
||||
- logged in as root or executed `su -` / `sudo su`
|
||||
- Already have the "telemt" executable file in the /bin folder. Read the **[Installation](#Installation)** section.
|
||||
|
||||
---
|
||||
|
||||
**0. Check port and generate secrets**
|
||||
|
||||
The port you have selected for use should be MISSING from the list, when:
|
||||
```bash
|
||||
netstat -lnp
|
||||
```
|
||||
|
||||
Generate 16 bytes/32 characters HEX with OpenSSL or another way:
|
||||
```bash
|
||||
openssl rand -hex 16
|
||||
```
|
||||
OR
|
||||
```bash
|
||||
xxd -l 16 -p /dev/urandom
|
||||
```
|
||||
OR
|
||||
```bash
|
||||
python3 -c 'import os; print(os.urandom(16).hex())'
|
||||
```
|
||||
Save the obtained result somewhere. You will need it later!
|
||||
|
||||
---
|
||||
|
||||
**1. Place your config to /etc/telemt.toml**
|
||||
|
||||
Open nano
|
||||
```bash
|
||||
nano /etc/telemt.toml
|
||||
```
|
||||
paste your config
|
||||
|
||||
```toml
|
||||
# === General Settings ===
|
||||
[general]
|
||||
# ad_tag = "00000000000000000000000000000000"
|
||||
|
||||
[general.modes]
|
||||
classic = false
|
||||
secure = false
|
||||
tls = true
|
||||
|
||||
# === Anti-Censorship & Masking ===
|
||||
[censorship]
|
||||
tls_domain = "petrovich.ru"
|
||||
|
||||
[access.users]
|
||||
# format: "username" = "32_hex_chars_secret"
|
||||
hello = "00000000000000000000000000000000"
|
||||
```
|
||||
then Ctrl+S -> Ctrl+X to save
|
||||
|
||||
> [!WARNING]
|
||||
> Replace the value of the hello parameter with the value you obtained in step 0.
|
||||
> Replace the value of the tls_domain parameter with another website.
|
||||
|
||||
---
|
||||
|
||||
**2. Create service on /etc/systemd/system/telemt.service**
|
||||
|
||||
Open nano
|
||||
```bash
|
||||
nano /etc/systemd/system/telemt.service
|
||||
```
|
||||
|
||||
paste this Systemd Module
|
||||
```bash
|
||||
[Unit]
|
||||
Description=Telemt
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/bin
|
||||
ExecStart=/bin/telemt /etc/telemt.toml
|
||||
Restart=on-failure
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
then Ctrl+S -> Ctrl+X to save
|
||||
|
||||
|
||||
**3.** To start it, enter the command `systemctl start telemt`
|
||||
|
||||
**4.** To get status information, enter `systemctl status telemt`
|
||||
|
||||
**5.** For automatic startup at system boot, enter `systemctl enable telemt`
|
||||
|
||||
**6.** To get the links, enter `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
|
||||
|
||||
---
|
||||
|
||||
# Telemt via Docker Compose
|
||||
|
||||
**1. Edit `config.toml` in repo root (at least: port, users secrets, tls_domain)**
|
||||
**2. Start container:**
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
**3. Check logs:**
|
||||
```bash
|
||||
docker compose logs -f telemt
|
||||
```
|
||||
**4. Stop:**
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
> [!NOTE]
|
||||
> - `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only)
|
||||
> - By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added)
|
||||
> - If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host`
|
||||
|
||||
**Run without Compose**
|
||||
```bash
|
||||
docker build -t telemt:local .
|
||||
docker run --name telemt --restart unless-stopped \
|
||||
-p 443:443 \
|
||||
-e RUST_LOG=info \
|
||||
-v "$PWD/config.toml:/app/config.toml:ro" \
|
||||
--read-only \
|
||||
--cap-drop ALL --cap-add NET_BIND_SERVICE \
|
||||
--ulimit nofile=65536:65536 \
|
||||
telemt:local
|
||||
```
|
||||
152
docs/QUICK_START_GUIDE.ru.md
Normal file
152
docs/QUICK_START_GUIDE.ru.md
Normal file
@@ -0,0 +1,152 @@
|
||||
# Telemt через Systemd
|
||||
|
||||
## Установка
|
||||
|
||||
Это программное обеспечение разработано для ОС на базе Debian: помимо Debian, это Ubuntu, Mint, Kali, MX и многие другие Linux
|
||||
|
||||
**1. Скачать**
|
||||
```bash
|
||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
||||
```
|
||||
**2. Переместить в папку Bin**
|
||||
```bash
|
||||
mv telemt /bin
|
||||
```
|
||||
**3. Сделать файл исполняемым**
|
||||
```bash
|
||||
chmod +x /bin/telemt
|
||||
```
|
||||
|
||||
## Как правильно использовать?
|
||||
|
||||
**Эта инструкция "предполагает", что вы:**
|
||||
- Авторизовались как пользователь root или выполнил `su -` / `sudo su`
|
||||
- У вас уже есть исполняемый файл "telemt" в папке /bin. Читайте раздел **[Установка](#установка)**
|
||||
|
||||
---
|
||||
|
||||
**0. Проверьте порт и сгенерируйте секреты**
|
||||
|
||||
Порт, который вы выбрали для использования, должен отсутствовать в списке:
|
||||
```bash
|
||||
netstat -lnp
|
||||
```
|
||||
|
||||
Сгенерируйте 16 bytes/32 символа в шестнадцатеричном формате с помощью OpenSSL или другим способом:
|
||||
```bash
|
||||
openssl rand -hex 16
|
||||
```
|
||||
ИЛИ
|
||||
```bash
|
||||
xxd -l 16 -p /dev/urandom
|
||||
```
|
||||
ИЛИ
|
||||
```bash
|
||||
python3 -c 'import os; print(os.urandom(16).hex())'
|
||||
```
|
||||
Полученный результат сохраняем где-нибудь. Он понадобиться вам дальше!
|
||||
|
||||
---
|
||||
|
||||
**1. Поместите свою конфигурацию в файл /etc/telemt.toml**
|
||||
|
||||
Открываем nano
|
||||
```bash
|
||||
nano /etc/telemt.toml
|
||||
```
|
||||
Вставьте свою конфигурацию
|
||||
|
||||
```toml
|
||||
# === General Settings ===
|
||||
[general]
|
||||
# ad_tag = "00000000000000000000000000000000"
|
||||
|
||||
[general.modes]
|
||||
classic = false
|
||||
secure = false
|
||||
tls = true
|
||||
|
||||
# === Anti-Censorship & Masking ===
|
||||
[censorship]
|
||||
tls_domain = "petrovich.ru"
|
||||
|
||||
[access.users]
|
||||
# format: "username" = "32_hex_chars_secret"
|
||||
hello = "00000000000000000000000000000000"
|
||||
```
|
||||
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
|
||||
|
||||
> [!WARNING]
|
||||
> Замените значение параметра hello на значение, которое вы получили в пункте 0.
|
||||
> Так же замените значение параметра tls_domain на другой сайт.
|
||||
|
||||
---
|
||||
|
||||
**2. Создайте службу в /etc/systemd/system/telemt.service**
|
||||
|
||||
Открываем nano
|
||||
```bash
|
||||
nano /etc/systemd/system/telemt.service
|
||||
```
|
||||
|
||||
Вставьте этот модуль Systemd
|
||||
```bash
|
||||
[Unit]
|
||||
Description=Telemt
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/bin
|
||||
ExecStart=/bin/telemt /etc/telemt.toml
|
||||
Restart=on-failure
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
|
||||
|
||||
|
||||
**3.** Для запуска введите команду `systemctl start telemt`
|
||||
|
||||
**4.** Для получения информации о статусе введите `systemctl status telemt`
|
||||
|
||||
**5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt`
|
||||
|
||||
**6.** Для получения ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac`
|
||||
|
||||
---
|
||||
|
||||
# Telemt через Docker Compose
|
||||
|
||||
**1. Отредактируйте `config.toml` в корневом каталоге репозитория (как минимум: порт, пользовательские секреты, tls_domain)**
|
||||
**2. Запустите контейнер:**
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
**3. Проверьте логи:**
|
||||
```bash
|
||||
docker compose logs -f telemt
|
||||
```
|
||||
**4. Остановите контейнер:**
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
> [!NOTE]
|
||||
> - В `docker-compose.yml` файл `./config.toml` монтируется в `/app/config.toml` (доступно только для чтения)
|
||||
> - По умолчанию публикуются порты 443:443, а контейнер запускается со сброшенными привилегиями (добавлена только `NET_BIND_SERVICE`)
|
||||
> - Если вам действительно нужна сеть хоста (обычно это требуется только для некоторых конфигураций IPv6), раскомментируйте `network_mode: host`
|
||||
|
||||
**Запуск в Docker Compose**
|
||||
```bash
|
||||
docker build -t telemt:local .
|
||||
docker run --name telemt --restart unless-stopped \
|
||||
-p 443:443 \
|
||||
-e RUST_LOG=info \
|
||||
-v "$PWD/config.toml:/app/config.toml:ro" \
|
||||
--read-only \
|
||||
--cap-drop ALL --cap-add NET_BIND_SERVICE \
|
||||
--ulimit nofile=65536:65536 \
|
||||
telemt:local
|
||||
```
|
||||
321
docs/XRAY-SINGBOX-ROUTING.ru.md
Normal file
321
docs/XRAY-SINGBOX-ROUTING.ru.md
Normal file
@@ -0,0 +1,321 @@
|
||||
# SNI-маршрутизация в xray-core / sing-box + TLS-fronting
|
||||
|
||||
## Термины (в контексте этого кейса)
|
||||
|
||||
- **TLS-fronting домен** — домен, который фигурирует в TLS ClientHello как **SNI** (например, `petrovich.ru`): он используется как "маска" на L7 и как ключ маршрутизации в прокси-роутере.
|
||||
- **xray-core / sing-box** — локальный или удалённый L7/TLS-роутер (прокси), который:
|
||||
1) принимает входящее TCP/TLS-соединение,
|
||||
2) читает TLS ClientHello,
|
||||
3) извлекает SNI,
|
||||
4) по SNI выбирает outbound/апстрим,
|
||||
5) устанавливает новое TCP-соединение к целевому хосту уже **от себя**.
|
||||
- **SNI (Server Name Indication)** — поле в TLS ClientHello, где клиент Telegram сообщает доменное имя для "маскировки"
|
||||
- **DNS-resolve на стороне L7-роутера** — если выходной адрес задан доменом (или роутер решил "всё равно идти по SNI"), то DNS резолвится **на стороне xray/sing-box**, а не на стороне Telegram-клиента
|
||||
|
||||
---
|
||||
|
||||
## Ключевая идея: куда на самом деле идёт соединение решает не то, что вы указали клиенту, а то как L7-роутер трактует SNI
|
||||
|
||||
Механика:
|
||||
|
||||
1) Telegram-клиенту вы можете указать **IP/домен telemt**,как "сервер".
|
||||
2) Между клиентом и telemt стоит xray-core/sing-box, который принимает TCP, читает TLS ClientHello и видит **SNI=petrovich.ru**
|
||||
3) Дальше роутер говорит: "Вижу SNI - направить на апстрим/маршрут N"
|
||||
4) И устанавливает исходящее соединение не "по тому IP, который пользователь подразумевал", а **по домену из SNI** (или по сопоставлению SNI→outbound), используя для определния его IP собственный DNS-кеш или резолвер
|
||||
5) `petrovich.ru` по A-записи указывает **не на IP telemt**, а значит при L7-маршрутизации трафик уйдёт на "оригинальный" сайт за этим доменом, а не в telemt: Telegram-клиент, естественно, не сможет получить ожидаемое поведение, потому что ответить с handshake на той стороне некому
|
||||
|
||||
---
|
||||
|
||||
## Схема №1 "Как это НЕ работает"
|
||||
|
||||
```text
|
||||
Telegram Client
|
||||
|
|
||||
| (указан IP/домен telemt)
|
||||
v
|
||||
telemt instance
|
||||
````
|
||||
|
||||
Ожидание: "я указал telemt -> значит трафик попадёт в telemt" - **нет!**
|
||||
|
||||
---
|
||||
|
||||
## Схема №2. "Как это реально работает с TLS/L7-роутером и SNI"
|
||||
|
||||
```text
|
||||
Telegram Client
|
||||
|
|
||||
| 1) TCP/TLS connection:
|
||||
| - ClientHello:
|
||||
| - SNI=petrovich.ru
|
||||
v
|
||||
xray-core / sing-box / любой L7 router
|
||||
|
|
||||
| 2) читает ClientHello -> вытаскивает SNI
|
||||
| 3) выбирает маршрут по SNI
|
||||
| 4) делает DNS для petrovich.ru
|
||||
| 5) подключается к полученному IP по TLS с этим SNI
|
||||
v
|
||||
"Оригинальный" сайт, A-запись которого не на telemt
|
||||
|
|
||||
X не telemt -> Telegram-клиент не коннектится как ожидалось
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Почему указанный в клиенте IP/домен telemt "не спасает"
|
||||
|
||||
Потому что в таком режиме xray/sing-box выступает как **точка терминации TCP/TLS**, можно сказать - TLS-инспектор на уровне ClientHello, это означает:
|
||||
|
||||
* TCP-сессия от Telegram-клиента заканчивается на xray/sing-box
|
||||
* Дальше создаётся **новая** TCP-сессия "от имени" xray/sing-box к апстриму
|
||||
* Выбор апстрима делается правилами роутинга, а в TLS-сценариях самый удобный и распространённый ключ — **SNI**
|
||||
|
||||
То есть, "куда идти дальше" определяется логикой L7-роутера:
|
||||
|
||||
* либо правилами вида `if SNI == petrovich.ru -> outbound X`,
|
||||
* либо более "автоматическим" поведением: `подключаться к тому хосту, который указан в SNI`,
|
||||
* плюс кэш DNS и собственные резолверы роутера
|
||||
|
||||
---
|
||||
|
||||
## Что именно извлекается из TLS ClientHello и почему этого достаточно
|
||||
|
||||
TLS ClientHello отправляется **в начале** TLS-сессии и, в классическом TLS без ECH, содержит SNI в открытом виде.
|
||||
|
||||
Упрощённо:
|
||||
|
||||
```text
|
||||
ClientHello:
|
||||
- supported_versions
|
||||
- cipher_suites
|
||||
- extensions:
|
||||
- server_name: petrovich.ru <-- SNI
|
||||
- alpn: h2/http1.1/...
|
||||
- ...
|
||||
```
|
||||
|
||||
Роутеру не нужно расшифровывать трафик и завершать TLS "как сервер" — часто достаточно просто прочитать первые пакеты и распарсить ClientHello, чтобы получить SNI и принять решение
|
||||
|
||||
---
|
||||
|
||||
## Типовой алгоритм SNI-роутинга
|
||||
|
||||
1. Принять входящий TCP.
|
||||
2. Подождать первые байты.
|
||||
3. Определить протокол:
|
||||
|
||||
* если видим TLS ClientHello → парсим SNI/ALPN
|
||||
4. Применить route rules:
|
||||
|
||||
* match по `server_name` / `domain` / `tls.sni`
|
||||
5. Выбрать outbound:
|
||||
|
||||
* direct / proxy / specific upstream / detour
|
||||
6. Установить исходящее соединение:
|
||||
|
||||
* либо на фиксированный IP:порт,
|
||||
* либо на домен через DNS-resolve на стороне роутера
|
||||
7. Начать проксирование данных между входом и выходом
|
||||
|
||||
---
|
||||
|
||||
## Почему "A-запись фронтинг-домена не на telemt" ломает кейс
|
||||
|
||||
### Ситуация
|
||||
|
||||
* В ClientHello: `SNI = petrovich.ru`
|
||||
* DNS: `petrovich.ru -> 203.0.113.77` - "оригинальный" сайт
|
||||
* telemt живёт на: `198.51.100.10`
|
||||
|
||||
### Что делает роутер
|
||||
|
||||
* Видит SNI `petrovich.ru`
|
||||
* Либо:
|
||||
|
||||
* (а) напрямую коннектится к `petrovich.ru:443`, резолвя A-запись в `203.0.113.77`,
|
||||
* либо:
|
||||
* (б) выбирает outbound, который указывает на `petrovich.ru` как destination,
|
||||
* либо:
|
||||
* (в) делает sniffing/override destination по SNI
|
||||
|
||||
В итоге исходящий коннект идёт на `203.0.113.77:443`, а не на telemt!
|
||||
Другой сервер, другой протокол, другая логика, где telemt не участвует
|
||||
|
||||
---
|
||||
|
||||
## "Где именно происходит подмена destination на SNI"
|
||||
|
||||
Это зависит от конфигурации, но типовые варианты:
|
||||
|
||||
### Вариант A: outbound задан доменом (и он совпадает с SNI)
|
||||
|
||||
Правило по SNI выбирает outbound, у которого destination задан доменом фронтинга,
|
||||
тогда DNS резолвится на стороне роутера и вы уходите на "оригинальный" хост
|
||||
|
||||
### Вариант B: destination override / sniffing
|
||||
|
||||
Роутер "снифает" SNI и **перезаписывает** destination на домен из SNI (даже если вход изначально был на IP telemt),
|
||||
это особенно коварно: пользователь видит "я подключаюсь к IP telemt", но роутер после sniffing решает иначе
|
||||
|
||||
### Вариант C: split DNS / кеш / независимый резолвер
|
||||
|
||||
Даже если клиент "где-то" резолвит иначе, это не важно: конечный DNS для исходящего коннекта — на стороне xray/sing-box,
|
||||
который может иметь:
|
||||
|
||||
* свой DoH/DoT,
|
||||
* свой кеш,
|
||||
* свои правила fake-ip / system resolver,
|
||||
* и, как следствие, своя "карта" **домен/SNI -> IP**
|
||||
|
||||
---
|
||||
|
||||
## Признаки того, что трафик "утёк на оригинал", а не попал в telemt
|
||||
|
||||
* На стороне telemt отсутствуют входящие соединения/логи
|
||||
* На стороне роутера видно, что destination — домен фронтинга, а IP соответствует публичному сайту
|
||||
* TLS-метрики/сертификат на выходе соответствует "оригинальному" сайту в записах трафика
|
||||
* Telegram-клиент получает неожиданный тип ответов/ошибку handshaking/timeout в debug-режиме
|
||||
|
||||
---
|
||||
|
||||
## Best-practice решение для этого кейса: свой домен фронтинга + заглушка на telemt + Let's Encrypt
|
||||
|
||||
### Цель
|
||||
|
||||
Сделать так, чтобы:
|
||||
|
||||
* SNI (фронтинг-домен) **резолвился в IP telemt**,
|
||||
* на IP telemt реально был TLS-сервис с валидным сертификатом под этот домен,
|
||||
* даже если кто-то "попробует открыть домен как сайт", он увидит нормальную заглушку, а не "пустоту"
|
||||
|
||||
### Что это даёт
|
||||
|
||||
* xray/sing-box, маршрутизируя по SNI, будет неизбежно приходить на telemt, потому что DNS(SNI-домен) → IP telemt
|
||||
* Внешний вид будет правдоподобным: обычный домен с обычным сертификатом
|
||||
* Устойчивость: меньше сюрпризов от DNS-кеша/перерезолва/"умных" правил роутера
|
||||
|
||||
---
|
||||
|
||||
## Рекомендуемая схема (целевое состояние)
|
||||
|
||||
```text
|
||||
Telegram Client
|
||||
|
|
||||
| TLS ClientHello: SNI = hello.example.com
|
||||
v
|
||||
xray-core / sing-box
|
||||
|
|
||||
| Route by SNI -> outbound -> connect to hello.example.com:443
|
||||
| DNS(hello.example.com) = IP telemt
|
||||
v
|
||||
telemt instance (IP telemt)
|
||||
|
|
||||
| TLS cert for hello.example.com (Let's Encrypt)
|
||||
| + сайт-заглушка / health endpoint
|
||||
v
|
||||
OK
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Практический чеклист (минимальный)
|
||||
|
||||
1. Купить/иметь домен: `hello.example.com`
|
||||
2. В DNS:
|
||||
|
||||
* `A hello.example.com -> <IP telemt>`
|
||||
* (опционально) AAAA, если используете IPv6 и он стабилен
|
||||
3. На telemt-хосте:
|
||||
|
||||
* поднять TLS endpoint на 443 с валидным сертификатом LE под `hello.example.com`
|
||||
* отдать "заглушку" (например, статический сайт), чтобы домен выглядел как обычный веб-сервис
|
||||
4. В xray/sing-box правилах:
|
||||
|
||||
* маршрутизировать нужный трафик по SNI = `hello.example.com` в "правильный" outbound (к telemt)
|
||||
* избегать конфигураций, где destination override уводит на чужой домен
|
||||
5. Важно:
|
||||
|
||||
* если вы используете кеш DNS на роутере — сбросить/обновить его после смены A-записи
|
||||
|
||||
---
|
||||
|
||||
## Пояснение про сайт-заглушку
|
||||
|
||||
Для эмуляции TLS, telemt имеет подсистему TLS-F в `src/tls_front`:
|
||||
- её модуль - fetcher, собирает TLS-профили, чтоб максимально поведенчески корректно повторять TLS конкретно указанного сайта
|
||||
|
||||
Когда вы указываете сайт, который не отвечает по TLS:
|
||||
- fetcher не может собрать TLS-профиль и происходит fallback на `fake_cert_len` - примитивный алгоритм,
|
||||
- он забивает служебную информацию TLS рандомными байтами,
|
||||
- простые системы DPI не распознают это
|
||||
- однако, продвинутые системы, такие как nEdge или Fraud Control в сетях мобильной связи легко заблокируют или замедлят такой трафик
|
||||
|
||||
Создав сайт-заглушку с Let's Encrypt сертификатом, вы даёте TLS-F возможность получить данные сертификата и корректно его "повторять" в дальнейшем
|
||||
|
||||
---
|
||||
|
||||
## Вариант конфиг-подхода: "SNI строго привязываем к telemt - фиксированный IP"
|
||||
|
||||
Чтобы полностью исключить зависимость от DNS если вам это нужно, можно сделать outbound, который ходит на **фиксированный IP telemt**, но при этом выставляет SNI/Host как `hello.example.com`.
|
||||
|
||||
Идея:
|
||||
|
||||
* destination: `IP:443`
|
||||
* SNI: `hello.example.com`
|
||||
* сертификат на telemt именно под `hello.example.com`
|
||||
|
||||
Так вы получаете:
|
||||
|
||||
* TLS выглядит корректно, ведь SNI совпадает с сертификатом,
|
||||
* а routing никогда не уйдёт на "оригинал", потому что A-запись указывает на telemt и контроллируется вами!
|
||||
|
||||
Но в вашем описании проблема как раз в том, что роутер "сам решает по SNI и резолвит домен", поэтому самый универсальный вариант — сделать так, чтобы DNS всегда приводил в telemt
|
||||
|
||||
---
|
||||
|
||||
## Пример логики правил на псевдоконфиге L7-роутера
|
||||
|
||||
```text
|
||||
if inbound is TLS and sni == "hello.example.com":
|
||||
route -> outbound "telemt"
|
||||
else:
|
||||
route -> outbound "default"
|
||||
```
|
||||
|
||||
Outbound `telemt`:
|
||||
|
||||
* destination: `hello.example.com:443`
|
||||
* TLS enabled
|
||||
* SNI: `hello.example.com`
|
||||
|
||||
---
|
||||
|
||||
## Отдельно: что может неожиданно сломать даже "правильный" DNS
|
||||
|
||||
* **Кеширование DNS** на xray/sing-box или на системном резолвере, особенно при смене A-записи
|
||||
* **Split-horizon DNS**: разные ответы внутри/снаружи, попытки подмены/терминирования в других точках
|
||||
* **IPv6**: если есть AAAA и он указывает не туда, роутер может предпочесть IPv6: помните, что поддержка v6 нестабильна и не рекомендуется в prod
|
||||
* **DoH/DoT** на роутере: он может резолвить не тем резолвером, которым вы проверяли
|
||||
|
||||
Минимальная гигиена:
|
||||
|
||||
* контролировать A/AAAA,
|
||||
* держать TTL разумным,
|
||||
* проверять, каким резолвером пользуется именно роутер,
|
||||
* при необходимости отключить/ограничить destination override
|
||||
|
||||
---
|
||||
|
||||
## Итог
|
||||
|
||||
В режиме TLS-fronting с xray-core/sing-box как L7/TLS-роутером **SNI становится приоритетным "source-of-truth" для маршрутизации**
|
||||
|
||||
Если фронтинг-домен по DNS указывает не на IP telemt, роутер честно уводит трафик на "оригинальный" сайт, потому что он строит исходящее соединение "по SNI"
|
||||
|
||||
Надёжное решение для этого кейса:
|
||||
|
||||
* использовать **свой домен** для фронтинга,
|
||||
* направить его **A/AAAA** на IP telemt,
|
||||
* поднять на telemt **TLS-сервис с Let’s Encrypt сертификатом** под этот домен,
|
||||
* (желательно) держать **сайт-заглушку**, чтобы 443 выглядел как обычный HTTPS
|
||||
73
install.sh
Normal file
73
install.sh
Normal file
@@ -0,0 +1,73 @@
|
||||
sudo bash -c '
|
||||
set -e
|
||||
|
||||
# --- Проверка на существующую установку ---
|
||||
if systemctl list-unit-files | grep -q telemt.service; then
|
||||
# --- РЕЖИМ ОБНОВЛЕНИЯ ---
|
||||
echo "--- Обнаружена существующая установка Telemt. Запускаю обновление... ---"
|
||||
|
||||
echo "[*] Остановка службы telemt..."
|
||||
systemctl stop telemt || true # Игнорируем ошибку, если служба уже остановлена
|
||||
|
||||
echo "[1/2] Скачивание последней версии Telemt..."
|
||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
||||
|
||||
echo "[1/2] Замена исполняемого файла в /usr/local/bin..."
|
||||
mv telemt /usr/local/bin/telemt
|
||||
chmod +x /usr/local/bin/telemt
|
||||
|
||||
echo "[2/2] Запуск службы..."
|
||||
systemctl start telemt
|
||||
|
||||
echo "--- Обновление Telemt успешно завершено! ---"
|
||||
echo
|
||||
echo "Для проверки статуса службы выполните:"
|
||||
echo " systemctl status telemt"
|
||||
|
||||
else
|
||||
# --- РЕЖИМ НОВОЙ УСТАНОВКИ ---
|
||||
echo "--- Начало автоматической установки Telemt ---"
|
||||
|
||||
# Шаг 1: Скачивание и установка бинарного файла
|
||||
echo "[1/5] Скачивание последней версии Telemt..."
|
||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
||||
|
||||
echo "[1/5] Перемещение исполняемого файла в /usr/local/bin и установка прав..."
|
||||
mv telemt /usr/local/bin/telemt
|
||||
chmod +x /usr/local/bin/telemt
|
||||
|
||||
# Шаг 2: Генерация секрета
|
||||
echo "[2/5] Генерация секретного ключа..."
|
||||
SECRET=$(openssl rand -hex 16)
|
||||
|
||||
# Шаг 3: Создание файла конфигурации
|
||||
echo "[3/5] Создание файла конфигурации /etc/telemt.toml..."
|
||||
printf "# === General Settings ===\n[general]\n[general.modes]\nclassic = false\nsecure = false\ntls = true\n\n# === Anti-Censorship & Masking ===\n[censorship]\n# !!! ВАЖНО: Замените на ваш домен или домен, который вы хотите использовать для маскировки !!!\ntls_domain = \"petrovich.ru\"\n\n[access.users]\nhello = \"%s\"\n" "$SECRET" > /etc/telemt.toml
|
||||
|
||||
# Шаг 4: Создание службы Systemd
|
||||
echo "[4/5] Создание службы systemd..."
|
||||
printf "[Unit]\nDescription=Telemt Proxy\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/telemt /etc/telemt.toml\nRestart=on-failure\nRestartSec=5\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/telemt.service
|
||||
|
||||
# Шаг 5: Запуск службы
|
||||
echo "[5/5] Перезагрузка systemd, запуск и включение службы telemt..."
|
||||
systemctl daemon-reload
|
||||
systemctl start telemt
|
||||
systemctl enable telemt
|
||||
|
||||
echo "--- Установка и запуск Telemt успешно завершены! ---"
|
||||
echo
|
||||
echo "ВАЖНАЯ ИНФОРМАЦИЯ:"
|
||||
echo "==================="
|
||||
echo "1. Вам НЕОБХОДИМО отредактировать файл /etc/telemt.toml и заменить '\''petrovich.ru'\'' на другой домен"
|
||||
echo " с помощью команды:"
|
||||
echo " nano /etc/telemt.toml"
|
||||
echo " После редактирования файла перезапустите службу командой:"
|
||||
echo " sudo systemctl restart telemt"
|
||||
echo
|
||||
echo "2. Для проверки статуса службы выполните команду:"
|
||||
echo " systemctl status telemt"
|
||||
echo
|
||||
echo "3. Для получения ссылок на подключение выполните команду:"
|
||||
echo " journalctl -u telemt -n -g '\''links'\'' --no-pager -o cat | tac"
|
||||
fi
|
||||
'
|
||||
@@ -8,6 +8,10 @@ const DEFAULT_STUN_TCP_FALLBACK: bool = true;
|
||||
const DEFAULT_MIDDLE_PROXY_WARM_STANDBY: usize = 16;
|
||||
const DEFAULT_ME_RECONNECT_MAX_CONCURRENT_PER_DC: u32 = 8;
|
||||
const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16;
|
||||
const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2;
|
||||
const DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS: u64 = 90;
|
||||
const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT: u8 = 1;
|
||||
const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180;
|
||||
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3;
|
||||
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4;
|
||||
const DEFAULT_LISTEN_ADDR_IPV6: &str = "::";
|
||||
@@ -129,11 +133,11 @@ pub(crate) fn default_middle_proxy_warm_standby() -> usize {
|
||||
}
|
||||
|
||||
pub(crate) fn default_keepalive_interval() -> u64 {
|
||||
25
|
||||
8
|
||||
}
|
||||
|
||||
pub(crate) fn default_keepalive_jitter() -> u64 {
|
||||
5
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_warmup_step_delay_ms() -> u64 {
|
||||
@@ -160,6 +164,42 @@ pub(crate) fn default_me_reconnect_fast_retry_count() -> u32 {
|
||||
DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_shadow_writers() -> u8 {
|
||||
DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_outage_mode_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_outage_disable_quarantine() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_outage_backoff_min_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_outage_backoff_max_ms() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_single_endpoint_shadow_rotate_every_secs() -> u64 {
|
||||
900
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_adaptive_floor_idle_secs() -> u64 {
|
||||
DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_adaptive_floor_min_writers_single_endpoint() -> u8 {
|
||||
DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_adaptive_floor_recover_grace_secs() -> u64 {
|
||||
DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||
}
|
||||
@@ -277,6 +317,18 @@ pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||
15 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||
64
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||
1000
|
||||
}
|
||||
@@ -301,6 +353,18 @@ pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
@@ -309,6 +373,10 @@ pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||
256
|
||||
}
|
||||
@@ -321,10 +389,18 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||
default_me_pool_drain_ttl_secs()
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||
0.8
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_hardswap() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
@@ -4,21 +4,22 @@
|
||||
//!
|
||||
//! # What can be reloaded without restart
|
||||
//!
|
||||
//! | Section | Field | Effect |
|
||||
//! |-----------|-------------------------------|-----------------------------------|
|
||||
//! | `general` | `log_level` | Filter updated via `log_level_tx` |
|
||||
//! | `general` | `ad_tag` | Passed on next connection |
|
||||
//! | `general` | `middle_proxy_pool_size` | Passed on next connection |
|
||||
//! | `general` | `me_keepalive_*` | Passed on next connection |
|
||||
//! | `general` | `desync_all_full` | Applied immediately |
|
||||
//! | `general` | `update_every` | Applied to ME updater immediately |
|
||||
//! | `general` | `hardswap` | Applied on next ME map update |
|
||||
//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update |
|
||||
//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update |
|
||||
//! | `general` | `me_reinit_drain_timeout_secs`| Applied on next ME map update |
|
||||
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
||||
//! | `network` | `dns_overrides` | Applied immediately |
|
||||
//! | `access` | All user/quota fields | Effective immediately |
|
||||
//! | Section | Field | Effect |
|
||||
//! |-----------|--------------------------------|------------------------------------------------|
|
||||
//! | `general` | `log_level` | Filter updated via `log_level_tx` |
|
||||
//! | `access` | `user_ad_tags` | Passed on next connection |
|
||||
//! | `general` | `ad_tag` | Passed on next connection (fallback per-user) |
|
||||
//! | `general` | `middle_proxy_pool_size` | Passed on next connection |
|
||||
//! | `general` | `me_keepalive_*` | Passed on next connection |
|
||||
//! | `general` | `desync_all_full` | Applied immediately |
|
||||
//! | `general` | `update_every` | Applied to ME updater immediately |
|
||||
//! | `general` | `hardswap` | Applied on next ME map update |
|
||||
//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update |
|
||||
//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update |
|
||||
//! | `general` | `me_reinit_drain_timeout_secs` | Applied on next ME map update |
|
||||
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
||||
//! | `network` | `dns_overrides` | Applied immediately |
|
||||
//! | `access` | All user/quota fields | Effective immediately |
|
||||
//!
|
||||
//! Fields that require re-binding sockets (`server.port`, `censorship.*`,
|
||||
//! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted.
|
||||
@@ -31,7 +32,7 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher};
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::config::{LogLevel, MeSocksKdfPolicy, MeTelemetryLevel};
|
||||
use crate::config::{LogLevel, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel};
|
||||
use super::load::ProxyConfig;
|
||||
|
||||
// ── Hot fields ────────────────────────────────────────────────────────────────
|
||||
@@ -57,6 +58,10 @@ pub struct HotFields {
|
||||
pub telemetry_user_enabled: bool,
|
||||
pub telemetry_me_level: MeTelemetryLevel,
|
||||
pub me_socks_kdf_policy: MeSocksKdfPolicy,
|
||||
pub me_floor_mode: MeFloorMode,
|
||||
pub me_adaptive_floor_idle_secs: u64,
|
||||
pub me_adaptive_floor_min_writers_single_endpoint: u8,
|
||||
pub me_adaptive_floor_recover_grace_secs: u64,
|
||||
pub me_route_backpressure_base_timeout_ms: u64,
|
||||
pub me_route_backpressure_high_timeout_ms: u64,
|
||||
pub me_route_backpressure_high_watermark_pct: u8,
|
||||
@@ -84,6 +89,14 @@ impl HotFields {
|
||||
telemetry_user_enabled: cfg.general.telemetry.user_enabled,
|
||||
telemetry_me_level: cfg.general.telemetry.me_level,
|
||||
me_socks_kdf_policy: cfg.general.me_socks_kdf_policy,
|
||||
me_floor_mode: cfg.general.me_floor_mode,
|
||||
me_adaptive_floor_idle_secs: cfg.general.me_adaptive_floor_idle_secs,
|
||||
me_adaptive_floor_min_writers_single_endpoint: cfg
|
||||
.general
|
||||
.me_adaptive_floor_min_writers_single_endpoint,
|
||||
me_adaptive_floor_recover_grace_secs: cfg
|
||||
.general
|
||||
.me_adaptive_floor_recover_grace_secs,
|
||||
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
|
||||
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
|
||||
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
|
||||
@@ -207,14 +220,17 @@ fn log_changes(
|
||||
log_tx.send(new_hot.log_level.clone()).ok();
|
||||
}
|
||||
|
||||
if old_hot.ad_tag != new_hot.ad_tag {
|
||||
if old_hot.access.user_ad_tags != new_hot.access.user_ad_tags {
|
||||
info!(
|
||||
"config reload: ad_tag: {} → {}",
|
||||
old_hot.ad_tag.as_deref().unwrap_or("none"),
|
||||
new_hot.ad_tag.as_deref().unwrap_or("none"),
|
||||
"config reload: user_ad_tags updated ({} entries)",
|
||||
new_hot.access.user_ad_tags.len(),
|
||||
);
|
||||
}
|
||||
|
||||
if old_hot.ad_tag != new_hot.ad_tag {
|
||||
info!("config reload: general.ad_tag updated (applied on next connection)");
|
||||
}
|
||||
|
||||
if old_hot.dns_overrides != new_hot.dns_overrides {
|
||||
info!(
|
||||
"config reload: network.dns_overrides updated ({} entries)",
|
||||
@@ -305,6 +321,22 @@ fn log_changes(
|
||||
);
|
||||
}
|
||||
|
||||
if old_hot.me_floor_mode != new_hot.me_floor_mode
|
||||
|| old_hot.me_adaptive_floor_idle_secs != new_hot.me_adaptive_floor_idle_secs
|
||||
|| old_hot.me_adaptive_floor_min_writers_single_endpoint
|
||||
!= new_hot.me_adaptive_floor_min_writers_single_endpoint
|
||||
|| old_hot.me_adaptive_floor_recover_grace_secs
|
||||
!= new_hot.me_adaptive_floor_recover_grace_secs
|
||||
{
|
||||
info!(
|
||||
"config reload: me_floor: mode={:?} idle={}s min_single={} recover_grace={}s",
|
||||
new_hot.me_floor_mode,
|
||||
new_hot.me_adaptive_floor_idle_secs,
|
||||
new_hot.me_adaptive_floor_min_writers_single_endpoint,
|
||||
new_hot.me_adaptive_floor_recover_grace_secs,
|
||||
);
|
||||
}
|
||||
|
||||
if old_hot.me_route_backpressure_base_timeout_ms
|
||||
!= new_hot.me_route_backpressure_base_timeout_ms
|
||||
|| old_hot.me_route_backpressure_high_timeout_ms
|
||||
|
||||
@@ -255,6 +255,41 @@ impl ProxyConfig {
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_single_endpoint_shadow_writers > 32 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_single_endpoint_shadow_writers must be within [0, 32]".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_adaptive_floor_min_writers_single_endpoint == 0
|
||||
|| config.general.me_adaptive_floor_min_writers_single_endpoint > 32
|
||||
{
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_single_endpoint_outage_backoff_min_ms == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_single_endpoint_outage_backoff_max_ms == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_single_endpoint_outage_backoff_max_ms must be > 0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_single_endpoint_outage_backoff_min_ms
|
||||
> config.general.me_single_endpoint_outage_backoff_max_ms
|
||||
{
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.beobachten_minutes == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.beobachten_minutes must be > 0".to_string(),
|
||||
@@ -305,12 +340,24 @@ impl ProxyConfig {
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_snapshot_min_proxy_for_lines == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_snapshot_min_proxy_for_lines must be > 0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.proxy_secret_stable_snapshots == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.proxy_secret_stable_snapshots must be > 0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_reinit_trigger_channel == 0 {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_reinit_trigger_channel must be > 0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if !(32..=4096).contains(&config.general.proxy_secret_len_max) {
|
||||
return Err(ProxyError::Config(
|
||||
"general.proxy_secret_len_max must be within [32, 4096]".to_string(),
|
||||
@@ -532,15 +579,16 @@ impl ProxyConfig {
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(tag) = &self.general.ad_tag {
|
||||
for (user, tag) in &self.access.user_ad_tags {
|
||||
let zeros = "00000000000000000000000000000000";
|
||||
if !is_valid_ad_tag(tag) {
|
||||
return Err(ProxyError::Config(
|
||||
"general.ad_tag must be exactly 32 hex characters".to_string(),
|
||||
));
|
||||
return Err(ProxyError::Config(format!(
|
||||
"access.user_ad_tags['{}'] must be exactly 32 hex characters",
|
||||
user
|
||||
)));
|
||||
}
|
||||
if tag == zeros {
|
||||
warn!("ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel");
|
||||
warn!(user = %user, "user ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -579,6 +627,43 @@ mod tests {
|
||||
cfg.general.me_reconnect_fast_retry_count,
|
||||
default_me_reconnect_fast_retry_count()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_shadow_writers,
|
||||
default_me_single_endpoint_shadow_writers()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||
default_me_single_endpoint_outage_mode_enabled()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||
default_me_single_endpoint_outage_disable_quarantine()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_outage_backoff_min_ms,
|
||||
default_me_single_endpoint_outage_backoff_min_ms()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_outage_backoff_max_ms,
|
||||
default_me_single_endpoint_outage_backoff_max_ms()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
|
||||
default_me_single_endpoint_shadow_rotate_every_secs()
|
||||
);
|
||||
assert_eq!(cfg.general.me_floor_mode, MeFloorMode::default());
|
||||
assert_eq!(
|
||||
cfg.general.me_adaptive_floor_idle_secs,
|
||||
default_me_adaptive_floor_idle_secs()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_adaptive_floor_min_writers_single_endpoint,
|
||||
default_me_adaptive_floor_min_writers_single_endpoint()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.me_adaptive_floor_recover_grace_secs,
|
||||
default_me_adaptive_floor_recover_grace_secs()
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.general.upstream_connect_retry_attempts,
|
||||
default_upstream_connect_retry_attempts()
|
||||
@@ -617,6 +702,43 @@ mod tests {
|
||||
general.me_reconnect_fast_retry_count,
|
||||
default_me_reconnect_fast_retry_count()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_shadow_writers,
|
||||
default_me_single_endpoint_shadow_writers()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_outage_mode_enabled,
|
||||
default_me_single_endpoint_outage_mode_enabled()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_outage_disable_quarantine,
|
||||
default_me_single_endpoint_outage_disable_quarantine()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_outage_backoff_min_ms,
|
||||
default_me_single_endpoint_outage_backoff_min_ms()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_outage_backoff_max_ms,
|
||||
default_me_single_endpoint_outage_backoff_max_ms()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_single_endpoint_shadow_rotate_every_secs,
|
||||
default_me_single_endpoint_shadow_rotate_every_secs()
|
||||
);
|
||||
assert_eq!(general.me_floor_mode, MeFloorMode::default());
|
||||
assert_eq!(
|
||||
general.me_adaptive_floor_idle_secs,
|
||||
default_me_adaptive_floor_idle_secs()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_adaptive_floor_min_writers_single_endpoint,
|
||||
default_me_adaptive_floor_min_writers_single_endpoint()
|
||||
);
|
||||
assert_eq!(
|
||||
general.me_adaptive_floor_recover_grace_secs,
|
||||
default_me_adaptive_floor_recover_grace_secs()
|
||||
);
|
||||
assert_eq!(
|
||||
general.upstream_connect_retry_attempts,
|
||||
default_upstream_connect_retry_attempts()
|
||||
@@ -801,6 +923,93 @@ mod tests {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn me_single_endpoint_outage_backoff_range_is_validated() {
|
||||
let toml = r#"
|
||||
[general]
|
||||
me_single_endpoint_outage_backoff_min_ms = 4000
|
||||
me_single_endpoint_outage_backoff_max_ms = 3000
|
||||
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#;
|
||||
let dir = std::env::temp_dir();
|
||||
let path = dir.join("telemt_me_single_endpoint_outage_backoff_range_test.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||
assert!(err.contains(
|
||||
"general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms"
|
||||
));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn me_single_endpoint_shadow_writers_too_large_is_rejected() {
|
||||
let toml = r#"
|
||||
[general]
|
||||
me_single_endpoint_shadow_writers = 33
|
||||
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#;
|
||||
let dir = std::env::temp_dir();
|
||||
let path = dir.join("telemt_me_single_endpoint_shadow_writers_limit_test.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||
assert!(err.contains("general.me_single_endpoint_shadow_writers must be within [0, 32]"));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn me_adaptive_floor_min_writers_out_of_range_is_rejected() {
|
||||
let toml = r#"
|
||||
[general]
|
||||
me_adaptive_floor_min_writers_single_endpoint = 0
|
||||
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#;
|
||||
let dir = std::env::temp_dir();
|
||||
let path = dir.join("telemt_me_adaptive_floor_min_writers_out_of_range_test.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains(
|
||||
"general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]"
|
||||
)
|
||||
);
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn me_floor_mode_adaptive_is_parsed() {
|
||||
let toml = r#"
|
||||
[general]
|
||||
me_floor_mode = "adaptive"
|
||||
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
user = "00000000000000000000000000000000"
|
||||
"#;
|
||||
let dir = std::env::temp_dir();
|
||||
let path = dir.join("telemt_me_floor_mode_adaptive_test.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
let cfg = ProxyConfig::load(&path).unwrap();
|
||||
assert_eq!(cfg.general.me_floor_mode, MeFloorMode::Adaptive);
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_connect_retry_attempts_zero_is_rejected() {
|
||||
let toml = r#"
|
||||
@@ -1100,6 +1309,27 @@ mod tests {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_user_ad_tag_reports_access_user_ad_tags_key() {
|
||||
let toml = r#"
|
||||
[censorship]
|
||||
tls_domain = "example.com"
|
||||
|
||||
[access.users]
|
||||
alice = "00000000000000000000000000000000"
|
||||
|
||||
[access.user_ad_tags]
|
||||
alice = "not_hex"
|
||||
"#;
|
||||
let dir = std::env::temp_dir();
|
||||
let path = dir.join("telemt_invalid_user_ad_tag_message_test.toml");
|
||||
std::fs::write(&path, toml).unwrap();
|
||||
let cfg = ProxyConfig::load(&path).unwrap();
|
||||
let err = cfg.validate().unwrap_err().to_string();
|
||||
assert!(err.contains("access.user_ad_tags['alice'] must be exactly 32 hex characters"));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_dns_override_is_rejected() {
|
||||
let toml = r#"
|
||||
|
||||
@@ -130,6 +130,59 @@ impl MeSocksKdfPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stale ME writer bind policy during drain window.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum MeBindStaleMode {
|
||||
Never,
|
||||
#[default]
|
||||
Ttl,
|
||||
Always,
|
||||
}
|
||||
|
||||
impl MeBindStaleMode {
|
||||
pub fn as_u8(self) -> u8 {
|
||||
match self {
|
||||
MeBindStaleMode::Never => 0,
|
||||
MeBindStaleMode::Ttl => 1,
|
||||
MeBindStaleMode::Always => 2,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_u8(raw: u8) -> Self {
|
||||
match raw {
|
||||
0 => MeBindStaleMode::Never,
|
||||
2 => MeBindStaleMode::Always,
|
||||
_ => MeBindStaleMode::Ttl,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Middle-End writer floor policy mode.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum MeFloorMode {
|
||||
#[default]
|
||||
Static,
|
||||
Adaptive,
|
||||
}
|
||||
|
||||
impl MeFloorMode {
|
||||
pub fn as_u8(self) -> u8 {
|
||||
match self {
|
||||
MeFloorMode::Static => 0,
|
||||
MeFloorMode::Adaptive => 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_u8(raw: u8) -> Self {
|
||||
match raw {
|
||||
1 => MeFloorMode::Adaptive,
|
||||
_ => MeFloorMode::Static,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Telemetry controls for hot-path counters and ME diagnostics.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TelemetryConfig {
|
||||
@@ -247,14 +300,15 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub use_middle_proxy: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub ad_tag: Option<String>,
|
||||
|
||||
/// Path to proxy-secret binary file (auto-downloaded if absent).
|
||||
/// Infrastructure secret from https://core.telegram.org/getProxySecret.
|
||||
#[serde(default = "default_proxy_secret_path")]
|
||||
pub proxy_secret_path: Option<String>,
|
||||
|
||||
/// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags.
|
||||
#[serde(default)]
|
||||
pub ad_tag: Option<String>,
|
||||
|
||||
/// Public IP override for middle-proxy NAT environments.
|
||||
/// When set, this IP is used in ME key derivation and RPC_PROXY_REQ "our_addr".
|
||||
#[serde(default)]
|
||||
@@ -365,6 +419,47 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_me_reconnect_fast_retry_count")]
|
||||
pub me_reconnect_fast_retry_count: u32,
|
||||
|
||||
/// Number of additional reserve writers for DC groups with exactly one endpoint.
|
||||
#[serde(default = "default_me_single_endpoint_shadow_writers")]
|
||||
pub me_single_endpoint_shadow_writers: u8,
|
||||
|
||||
/// Enable aggressive outage recovery mode for single-endpoint DC groups.
|
||||
#[serde(default = "default_me_single_endpoint_outage_mode_enabled")]
|
||||
pub me_single_endpoint_outage_mode_enabled: bool,
|
||||
|
||||
/// Ignore endpoint quarantine while in single-endpoint outage mode.
|
||||
#[serde(default = "default_me_single_endpoint_outage_disable_quarantine")]
|
||||
pub me_single_endpoint_outage_disable_quarantine: bool,
|
||||
|
||||
/// Minimum reconnect backoff in ms for single-endpoint outage mode.
|
||||
#[serde(default = "default_me_single_endpoint_outage_backoff_min_ms")]
|
||||
pub me_single_endpoint_outage_backoff_min_ms: u64,
|
||||
|
||||
/// Maximum reconnect backoff in ms for single-endpoint outage mode.
|
||||
#[serde(default = "default_me_single_endpoint_outage_backoff_max_ms")]
|
||||
pub me_single_endpoint_outage_backoff_max_ms: u64,
|
||||
|
||||
/// Periodic shadow writer rotation interval in seconds for single-endpoint DC groups.
|
||||
/// Set to 0 to disable periodic shadow rotation.
|
||||
#[serde(default = "default_me_single_endpoint_shadow_rotate_every_secs")]
|
||||
pub me_single_endpoint_shadow_rotate_every_secs: u64,
|
||||
|
||||
/// Floor policy mode for ME writer targets.
|
||||
#[serde(default)]
|
||||
pub me_floor_mode: MeFloorMode,
|
||||
|
||||
/// Idle time in seconds before adaptive floor can reduce single-endpoint writer target.
|
||||
#[serde(default = "default_me_adaptive_floor_idle_secs")]
|
||||
pub me_adaptive_floor_idle_secs: u64,
|
||||
|
||||
/// Minimum writer target for single-endpoint DC groups in adaptive floor mode.
|
||||
#[serde(default = "default_me_adaptive_floor_min_writers_single_endpoint")]
|
||||
pub me_adaptive_floor_min_writers_single_endpoint: u8,
|
||||
|
||||
/// Grace period in seconds to hold static floor after activity in adaptive mode.
|
||||
#[serde(default = "default_me_adaptive_floor_recover_grace_secs")]
|
||||
pub me_adaptive_floor_recover_grace_secs: u64,
|
||||
|
||||
/// Connect attempts for the selected upstream before returning error/fallback.
|
||||
#[serde(default = "default_upstream_connect_retry_attempts")]
|
||||
pub upstream_connect_retry_attempts: u32,
|
||||
@@ -453,6 +548,18 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_me_config_apply_cooldown_secs")]
|
||||
pub me_config_apply_cooldown_secs: u64,
|
||||
|
||||
/// Ensure getProxyConfig snapshots are applied only for 2xx HTTP responses.
|
||||
#[serde(default = "default_me_snapshot_require_http_2xx")]
|
||||
pub me_snapshot_require_http_2xx: bool,
|
||||
|
||||
/// Reject empty getProxyConfig snapshots instead of marking them applied.
|
||||
#[serde(default = "default_me_snapshot_reject_empty_map")]
|
||||
pub me_snapshot_reject_empty_map: bool,
|
||||
|
||||
/// Minimum parsed `proxy_for` rows required to accept a snapshot.
|
||||
#[serde(default = "default_me_snapshot_min_proxy_for_lines")]
|
||||
pub me_snapshot_min_proxy_for_lines: u32,
|
||||
|
||||
/// Number of identical getProxySecret snapshots required before runtime secret rotation.
|
||||
#[serde(default = "default_proxy_secret_stable_snapshots")]
|
||||
pub proxy_secret_stable_snapshots: u8,
|
||||
@@ -461,6 +568,10 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_proxy_secret_rotate_runtime")]
|
||||
pub proxy_secret_rotate_runtime: bool,
|
||||
|
||||
/// Keep key-selector and secret bytes from one snapshot during ME handshake.
|
||||
#[serde(default = "default_me_secret_atomic_snapshot")]
|
||||
pub me_secret_atomic_snapshot: bool,
|
||||
|
||||
/// Maximum allowed proxy-secret length in bytes for startup and runtime refresh.
|
||||
#[serde(default = "default_proxy_secret_len_max")]
|
||||
pub proxy_secret_len_max: usize,
|
||||
@@ -470,6 +581,14 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_me_pool_drain_ttl_secs")]
|
||||
pub me_pool_drain_ttl_secs: u64,
|
||||
|
||||
/// Policy for new binds on stale draining writers.
|
||||
#[serde(default)]
|
||||
pub me_bind_stale_mode: MeBindStaleMode,
|
||||
|
||||
/// TTL for stale bind allowance when `me_bind_stale_mode = \"ttl\"`.
|
||||
#[serde(default = "default_me_bind_stale_ttl_secs")]
|
||||
pub me_bind_stale_ttl_secs: u64,
|
||||
|
||||
/// Minimum desired-DC coverage ratio required before draining stale writers.
|
||||
/// Range: 0.0..=1.0.
|
||||
#[serde(default = "default_me_pool_min_fresh_ratio")]
|
||||
@@ -490,6 +609,22 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_proxy_config_reload_secs")]
|
||||
pub proxy_config_auto_reload_secs: u64,
|
||||
|
||||
/// Serialize ME reinit cycles across all trigger sources.
|
||||
#[serde(default = "default_me_reinit_singleflight")]
|
||||
pub me_reinit_singleflight: bool,
|
||||
|
||||
/// Trigger queue capacity for reinit scheduler.
|
||||
#[serde(default = "default_me_reinit_trigger_channel")]
|
||||
pub me_reinit_trigger_channel: usize,
|
||||
|
||||
/// Trigger coalescing window before starting a reinit cycle.
|
||||
#[serde(default = "default_me_reinit_coalesce_window_ms")]
|
||||
pub me_reinit_coalesce_window_ms: u64,
|
||||
|
||||
/// Deterministic candidate sort for ME writer binding path.
|
||||
#[serde(default = "default_me_deterministic_writer_sort")]
|
||||
pub me_deterministic_writer_sort: bool,
|
||||
|
||||
/// Enable NTP drift check at startup.
|
||||
#[serde(default = "default_ntp_check")]
|
||||
pub ntp_check: bool,
|
||||
@@ -534,6 +669,16 @@ impl Default for GeneralConfig {
|
||||
me_reconnect_backoff_base_ms: default_reconnect_backoff_base_ms(),
|
||||
me_reconnect_backoff_cap_ms: default_reconnect_backoff_cap_ms(),
|
||||
me_reconnect_fast_retry_count: default_me_reconnect_fast_retry_count(),
|
||||
me_single_endpoint_shadow_writers: default_me_single_endpoint_shadow_writers(),
|
||||
me_single_endpoint_outage_mode_enabled: default_me_single_endpoint_outage_mode_enabled(),
|
||||
me_single_endpoint_outage_disable_quarantine: default_me_single_endpoint_outage_disable_quarantine(),
|
||||
me_single_endpoint_outage_backoff_min_ms: default_me_single_endpoint_outage_backoff_min_ms(),
|
||||
me_single_endpoint_outage_backoff_max_ms: default_me_single_endpoint_outage_backoff_max_ms(),
|
||||
me_single_endpoint_shadow_rotate_every_secs: default_me_single_endpoint_shadow_rotate_every_secs(),
|
||||
me_floor_mode: MeFloorMode::default(),
|
||||
me_adaptive_floor_idle_secs: default_me_adaptive_floor_idle_secs(),
|
||||
me_adaptive_floor_min_writers_single_endpoint: default_me_adaptive_floor_min_writers_single_endpoint(),
|
||||
me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(),
|
||||
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
||||
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
||||
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
|
||||
@@ -564,14 +709,24 @@ impl Default for GeneralConfig {
|
||||
me_hardswap_warmup_pass_backoff_base_ms: default_me_hardswap_warmup_pass_backoff_base_ms(),
|
||||
me_config_stable_snapshots: default_me_config_stable_snapshots(),
|
||||
me_config_apply_cooldown_secs: default_me_config_apply_cooldown_secs(),
|
||||
me_snapshot_require_http_2xx: default_me_snapshot_require_http_2xx(),
|
||||
me_snapshot_reject_empty_map: default_me_snapshot_reject_empty_map(),
|
||||
me_snapshot_min_proxy_for_lines: default_me_snapshot_min_proxy_for_lines(),
|
||||
proxy_secret_stable_snapshots: default_proxy_secret_stable_snapshots(),
|
||||
proxy_secret_rotate_runtime: default_proxy_secret_rotate_runtime(),
|
||||
me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(),
|
||||
proxy_secret_len_max: default_proxy_secret_len_max(),
|
||||
me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(),
|
||||
me_bind_stale_mode: MeBindStaleMode::default(),
|
||||
me_bind_stale_ttl_secs: default_me_bind_stale_ttl_secs(),
|
||||
me_pool_min_fresh_ratio: default_me_pool_min_fresh_ratio(),
|
||||
me_reinit_drain_timeout_secs: default_me_reinit_drain_timeout_secs(),
|
||||
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
|
||||
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
|
||||
me_reinit_singleflight: default_me_reinit_singleflight(),
|
||||
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
||||
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
||||
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
||||
ntp_check: default_ntp_check(),
|
||||
ntp_servers: default_ntp_servers(),
|
||||
auto_degradation_enabled: default_true(),
|
||||
@@ -807,6 +962,10 @@ pub struct AccessConfig {
|
||||
#[serde(default = "default_access_users")]
|
||||
pub users: HashMap<String, String>,
|
||||
|
||||
/// Per-user ad_tag (32 hex chars from @MTProxybot).
|
||||
#[serde(default)]
|
||||
pub user_ad_tags: HashMap<String, String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub user_max_tcp_conns: HashMap<String, usize>,
|
||||
|
||||
@@ -833,6 +992,7 @@ impl Default for AccessConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
users: default_access_users(),
|
||||
user_ad_tags: HashMap::new(),
|
||||
user_max_tcp_conns: HashMap::new(),
|
||||
user_expirations: HashMap::new(),
|
||||
user_data_quota: HashMap::new(),
|
||||
|
||||
61
src/main.rs
61
src/main.rs
@@ -8,7 +8,7 @@ use std::time::Duration;
|
||||
use rand::Rng;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::signal;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::sync::{Semaphore, mpsc};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
||||
#[cfg(unix)]
|
||||
@@ -40,7 +40,7 @@ use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::transport::middle_proxy::{
|
||||
MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, format_sample_line,
|
||||
MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, MeReinitTrigger, format_sample_line,
|
||||
format_me_route,
|
||||
};
|
||||
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
|
||||
@@ -448,7 +448,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
info!("Middle-proxy STUN probing disabled by network.stun_use=false");
|
||||
}
|
||||
|
||||
// ad_tag (proxy_tag) for advertising
|
||||
// Global ad_tag (pool default). Used when user has no per-user tag in access.user_ad_tags.
|
||||
let proxy_tag = config
|
||||
.general
|
||||
.ad_tag
|
||||
@@ -538,6 +538,16 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
config.general.me_reconnect_backoff_base_ms,
|
||||
config.general.me_reconnect_backoff_cap_ms,
|
||||
config.general.me_reconnect_fast_retry_count,
|
||||
config.general.me_single_endpoint_shadow_writers,
|
||||
config.general.me_single_endpoint_outage_mode_enabled,
|
||||
config.general.me_single_endpoint_outage_disable_quarantine,
|
||||
config.general.me_single_endpoint_outage_backoff_min_ms,
|
||||
config.general.me_single_endpoint_outage_backoff_max_ms,
|
||||
config.general.me_single_endpoint_shadow_rotate_every_secs,
|
||||
config.general.me_floor_mode,
|
||||
config.general.me_adaptive_floor_idle_secs,
|
||||
config.general.me_adaptive_floor_min_writers_single_endpoint,
|
||||
config.general.me_adaptive_floor_recover_grace_secs,
|
||||
config.general.hardswap,
|
||||
config.general.me_pool_drain_ttl_secs,
|
||||
config.general.effective_me_pool_force_close_secs(),
|
||||
@@ -546,6 +556,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
config.general.me_hardswap_warmup_delay_max_ms,
|
||||
config.general.me_hardswap_warmup_extra_passes,
|
||||
config.general.me_hardswap_warmup_pass_backoff_base_ms,
|
||||
config.general.me_bind_stale_mode,
|
||||
config.general.me_bind_stale_ttl_secs,
|
||||
config.general.me_secret_atomic_snapshot,
|
||||
config.general.me_deterministic_writer_sort,
|
||||
config.general.me_socks_kdf_policy,
|
||||
config.general.me_route_backpressure_base_timeout_ms,
|
||||
config.general.me_route_backpressure_high_timeout_ms,
|
||||
@@ -770,12 +784,14 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
// Background tasks
|
||||
let um_clone = upstream_manager.clone();
|
||||
let decision_clone = decision.clone();
|
||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||
tokio::spawn(async move {
|
||||
um_clone
|
||||
.run_health_checks(
|
||||
prefer_ipv6,
|
||||
decision_clone.ipv4_dc,
|
||||
decision_clone.ipv6_dc,
|
||||
dc_overrides_for_health,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -847,26 +863,43 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
});
|
||||
|
||||
if let Some(ref pool) = me_pool {
|
||||
let pool_clone = pool.clone();
|
||||
let rng_clone = rng.clone();
|
||||
let config_rx_clone = config_rx.clone();
|
||||
let reinit_trigger_capacity = config
|
||||
.general
|
||||
.me_reinit_trigger_channel
|
||||
.max(1);
|
||||
let (reinit_tx, reinit_rx) = mpsc::channel::<MeReinitTrigger>(reinit_trigger_capacity);
|
||||
|
||||
let pool_clone_sched = pool.clone();
|
||||
let rng_clone_sched = rng.clone();
|
||||
let config_rx_clone_sched = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_config_updater(
|
||||
pool_clone,
|
||||
rng_clone,
|
||||
config_rx_clone,
|
||||
crate::transport::middle_proxy::me_reinit_scheduler(
|
||||
pool_clone_sched,
|
||||
rng_clone_sched,
|
||||
config_rx_clone_sched,
|
||||
reinit_rx,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let pool_clone = pool.clone();
|
||||
let config_rx_clone = config_rx.clone();
|
||||
let reinit_tx_updater = reinit_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_config_updater(
|
||||
pool_clone,
|
||||
config_rx_clone,
|
||||
reinit_tx_updater,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
let pool_clone_rot = pool.clone();
|
||||
let rng_clone_rot = rng.clone();
|
||||
let config_rx_clone_rot = config_rx.clone();
|
||||
let reinit_tx_rotation = reinit_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_rotation_task(
|
||||
pool_clone_rot,
|
||||
rng_clone_rot,
|
||||
config_rx_clone_rot,
|
||||
reinit_tx_rotation,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
295
src/metrics.rs
295
src/metrics.rs
@@ -274,6 +274,43 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_handshake_reject_total ME handshake rejects from upstream");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_reject_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_reject_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_handshake_reject_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_handshake_error_code_total ME handshake reject errors by code");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_error_code_total counter");
|
||||
if me_allows_normal {
|
||||
for (error_code, count) in stats.get_me_handshake_error_code_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_error_code_total{{error_code=\"{}\"}} {}",
|
||||
error_code,
|
||||
count
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_reader_eof_total ME reader EOF terminations");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reader_eof_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reader_eof_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reader_eof_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter");
|
||||
let _ = writeln!(
|
||||
@@ -385,6 +422,262 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_total ME endpoint quarantines due to rapid flaps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_endpoint_quarantine_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_kdf_drift_total ME KDF input drift detections");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_drift_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_kdf_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_kdf_port_only_drift_total ME KDF client-port changes with stable non-port material"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_port_only_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_port_only_drift_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_kdf_port_only_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_reuse_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_reuse_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_hardswap_pending_reuse_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_ttl_expired_total Pending hardswap generations reset by TTL expiration"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_ttl_expired_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_ttl_expired_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_hardswap_pending_ttl_expired_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_enter_total Single-endpoint DC outage transitions to active state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_enter_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_enter_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_enter_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_exit_total Single-endpoint DC outage recovery transitions"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_exit_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_exit_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_exit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_attempt_total Reconnect attempts performed during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_attempt_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_attempt_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_success_total Successful reconnect attempts during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_success_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_success_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_quarantine_bypass_total Outage reconnect attempts that bypassed quarantine"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_quarantine_bypass_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_quarantine_bypass_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_quarantine_bypass_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_total Successful periodic shadow rotations for single-endpoint DC groups"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total Shadow rotations skipped because endpoint is quarantined"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_skipped_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode Runtime ME writer floor policy mode"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode gauge");
|
||||
let floor_mode = config.general.me_floor_mode;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"static\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Static) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"adaptive\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Adaptive) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode_switch_all_total Runtime ME floor mode switches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode_switch_all_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_all_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"static\",to=\"adaptive\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_static_to_adaptive_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"adaptive\",to=\"static\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_adaptive_to_static_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths");
|
||||
let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter");
|
||||
let _ = writeln!(
|
||||
@@ -477,7 +770,7 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_swap_total {}",
|
||||
if me_allows_debug {
|
||||
if me_allows_normal {
|
||||
stats.get_pool_swap_total()
|
||||
} else {
|
||||
0
|
||||
|
||||
@@ -91,6 +91,11 @@ where
|
||||
stats.increment_connects_all();
|
||||
let mut real_peer = normalize_ip(peer);
|
||||
|
||||
// For non-TCP streams, use a synthetic local address; may be overridden by PROXY protocol dst
|
||||
let mut local_addr: SocketAddr = format!("0.0.0.0:{}", config.server.port)
|
||||
.parse()
|
||||
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
|
||||
|
||||
if proxy_protocol_enabled {
|
||||
match parse_proxy_protocol(&mut stream, peer).await {
|
||||
Ok(info) => {
|
||||
@@ -101,6 +106,9 @@ where
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
real_peer = normalize_ip(info.src_addr);
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
stats.increment_connects_bad();
|
||||
@@ -119,11 +127,6 @@ where
|
||||
let beobachten_for_timeout = beobachten.clone();
|
||||
let peer_for_timeout = real_peer.ip();
|
||||
|
||||
// For non-TCP streams, use a synthetic local address
|
||||
let local_addr: SocketAddr = format!("0.0.0.0:{}", config.server.port)
|
||||
.parse()
|
||||
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
|
||||
|
||||
// Phase 1: handshake (with timeout)
|
||||
let outcome = match timeout(handshake_timeout, async {
|
||||
let mut first_bytes = [0u8; 5];
|
||||
@@ -144,6 +147,7 @@ where
|
||||
writer,
|
||||
&first_bytes,
|
||||
real_peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
@@ -169,6 +173,7 @@ where
|
||||
writer,
|
||||
&handshake,
|
||||
real_peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
@@ -213,6 +218,7 @@ where
|
||||
writer,
|
||||
&first_bytes,
|
||||
real_peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
@@ -238,6 +244,7 @@ where
|
||||
writer,
|
||||
&handshake,
|
||||
real_peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
)
|
||||
@@ -405,6 +412,8 @@ impl RunningClientHandler {
|
||||
}
|
||||
|
||||
async fn do_handshake(mut self) -> Result<HandshakeOutcome> {
|
||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||
|
||||
if self.proxy_protocol_enabled {
|
||||
match parse_proxy_protocol(&mut self.stream, self.peer).await {
|
||||
Ok(info) => {
|
||||
@@ -415,6 +424,9 @@ impl RunningClientHandler {
|
||||
"PROXY protocol header parsed"
|
||||
);
|
||||
self.peer = normalize_ip(info.src_addr);
|
||||
if let Some(dst) = info.dst_addr {
|
||||
local_addr = dst;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
self.stats.increment_connects_bad();
|
||||
@@ -440,13 +452,13 @@ impl RunningClientHandler {
|
||||
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
|
||||
|
||||
if is_tls {
|
||||
self.handle_tls_client(first_bytes).await
|
||||
self.handle_tls_client(first_bytes, local_addr).await
|
||||
} else {
|
||||
self.handle_direct_client(first_bytes).await
|
||||
self.handle_direct_client(first_bytes, local_addr).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_tls_client(mut self, first_bytes: [u8; 5]) -> Result<HandshakeOutcome> {
|
||||
async fn handle_tls_client(mut self, first_bytes: [u8; 5], local_addr: SocketAddr) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
let _ip_tracker = self.ip_tracker.clone();
|
||||
|
||||
@@ -463,6 +475,7 @@ impl RunningClientHandler {
|
||||
writer,
|
||||
&first_bytes,
|
||||
peer,
|
||||
local_addr,
|
||||
&self.config,
|
||||
&self.beobachten,
|
||||
)
|
||||
@@ -479,7 +492,6 @@ impl RunningClientHandler {
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake(
|
||||
@@ -502,6 +514,7 @@ impl RunningClientHandler {
|
||||
writer,
|
||||
&handshake,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&self.beobachten,
|
||||
)
|
||||
@@ -558,7 +571,7 @@ impl RunningClientHandler {
|
||||
)))
|
||||
}
|
||||
|
||||
async fn handle_direct_client(mut self, first_bytes: [u8; 5]) -> Result<HandshakeOutcome> {
|
||||
async fn handle_direct_client(mut self, first_bytes: [u8; 5], local_addr: SocketAddr) -> Result<HandshakeOutcome> {
|
||||
let peer = self.peer;
|
||||
let _ip_tracker = self.ip_tracker.clone();
|
||||
|
||||
@@ -571,6 +584,7 @@ impl RunningClientHandler {
|
||||
writer,
|
||||
&first_bytes,
|
||||
peer,
|
||||
local_addr,
|
||||
&self.config,
|
||||
&self.beobachten,
|
||||
)
|
||||
@@ -587,7 +601,6 @@ impl RunningClientHandler {
|
||||
let stats = self.stats.clone();
|
||||
let buffer_pool = self.buffer_pool.clone();
|
||||
|
||||
let local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
|
||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
|
||||
@@ -609,6 +622,7 @@ impl RunningClientHandler {
|
||||
writer,
|
||||
&handshake,
|
||||
peer,
|
||||
local_addr,
|
||||
&config,
|
||||
&self.beobachten,
|
||||
)
|
||||
|
||||
@@ -55,6 +55,7 @@ pub async fn handle_bad_client<R, W>(
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
)
|
||||
@@ -87,7 +88,29 @@ where
|
||||
let connect_result = timeout(MASK_TIMEOUT, UnixStream::connect(sock_path)).await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
let (mask_read, mask_write) = stream.into_split();
|
||||
let (mask_read, mut mask_write) = stream.into_split();
|
||||
let proxy_header: Option<Vec<u8>> = match config.censorship.mask_proxy_protocol {
|
||||
0 => None,
|
||||
version => {
|
||||
let header = match version {
|
||||
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
|
||||
_ => match (peer, local_addr) {
|
||||
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
|
||||
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
|
||||
_ =>
|
||||
ProxyProtocolV1Builder::new().build(),
|
||||
},
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
};
|
||||
if let Some(header) = proxy_header {
|
||||
if mask_write.write_all(&header).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if timeout(MASK_RELAY_TIMEOUT, relay_to_mask(reader, writer, mask_read, mask_write, initial_data)).await.is_err() {
|
||||
debug!("Mask relay timed out (unix socket)");
|
||||
}
|
||||
@@ -126,23 +149,16 @@ where
|
||||
let proxy_header: Option<Vec<u8>> = match config.censorship.mask_proxy_protocol {
|
||||
0 => None,
|
||||
version => {
|
||||
let header = if let Ok(local_addr) = stream.local_addr() {
|
||||
match version {
|
||||
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
|
||||
_ => match (peer, local_addr) {
|
||||
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
|
||||
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
|
||||
_ =>
|
||||
ProxyProtocolV1Builder::new().build(),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
match version {
|
||||
2 => ProxyProtocolV2Builder::new().build(),
|
||||
_ => ProxyProtocolV1Builder::new().build(),
|
||||
}
|
||||
let header = match version {
|
||||
2 => ProxyProtocolV2Builder::new().with_addrs(peer, local_addr).build(),
|
||||
_ => match (peer, local_addr) {
|
||||
(SocketAddr::V4(src), SocketAddr::V4(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp4(src.into(), dst.into()).build(),
|
||||
(SocketAddr::V6(src), SocketAddr::V6(dst)) =>
|
||||
ProxyProtocolV1Builder::new().tcp6(src.into(), dst.into()).build(),
|
||||
_ =>
|
||||
ProxyProtocolV1Builder::new().build(),
|
||||
},
|
||||
};
|
||||
Some(header)
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ enum C2MeCommand {
|
||||
|
||||
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
|
||||
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
|
||||
const C2ME_CHANNEL_CAPACITY: usize = 1024;
|
||||
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
|
||||
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
|
||||
static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new();
|
||||
|
||||
struct RelayForensicsState {
|
||||
@@ -166,6 +169,27 @@ fn report_desync_frame_too_large(
|
||||
))
|
||||
}
|
||||
|
||||
fn should_yield_c2me_sender(sent_since_yield: usize, has_backlog: bool) -> bool {
|
||||
has_backlog && sent_since_yield >= C2ME_SENDER_FAIRNESS_BUDGET
|
||||
}
|
||||
|
||||
async fn enqueue_c2me_command(
|
||||
tx: &mpsc::Sender<C2MeCommand>,
|
||||
cmd: C2MeCommand,
|
||||
) -> std::result::Result<(), mpsc::error::SendError<C2MeCommand>> {
|
||||
match tx.try_send(cmd) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(mpsc::error::TrySendError::Closed(cmd)) => Err(mpsc::error::SendError(cmd)),
|
||||
Err(mpsc::error::TrySendError::Full(cmd)) => {
|
||||
// Cooperative yield reduces burst catch-up when the per-conn queue is near saturation.
|
||||
if tx.capacity() <= C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tx.send(cmd).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn handle_via_middle_proxy<R, W>(
|
||||
mut crypto_reader: CryptoReader<R>,
|
||||
crypto_writer: CryptoWriter<W>,
|
||||
@@ -214,7 +238,22 @@ where
|
||||
stats.increment_user_connects(&user);
|
||||
stats.increment_user_curr_connects(&user);
|
||||
|
||||
let proto_flags = proto_flags_for_tag(proto_tag, me_pool.has_proxy_tag());
|
||||
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
|
||||
let user_tag: Option<Vec<u8>> = config
|
||||
.access
|
||||
.user_ad_tags
|
||||
.get(&user)
|
||||
.and_then(|s| hex::decode(s).ok())
|
||||
.filter(|v| v.len() == 16);
|
||||
let global_tag: Option<Vec<u8>> = config
|
||||
.general
|
||||
.ad_tag
|
||||
.as_ref()
|
||||
.and_then(|s| hex::decode(s).ok())
|
||||
.filter(|v| v.len() == 16);
|
||||
let effective_tag = user_tag.or(global_tag);
|
||||
|
||||
let proto_flags = proto_flags_for_tag(proto_tag, effective_tag.is_some());
|
||||
debug!(
|
||||
trace_id = format_args!("0x{:016x}", trace_id),
|
||||
user = %user,
|
||||
@@ -230,9 +269,11 @@ where
|
||||
|
||||
let frame_limit = config.general.max_client_frame;
|
||||
|
||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(1024);
|
||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(C2ME_CHANNEL_CAPACITY);
|
||||
let me_pool_c2me = me_pool.clone();
|
||||
let effective_tag = effective_tag;
|
||||
let c2me_sender = tokio::spawn(async move {
|
||||
let mut sent_since_yield = 0usize;
|
||||
while let Some(cmd) = c2me_rx.recv().await {
|
||||
match cmd {
|
||||
C2MeCommand::Data { payload, flags } => {
|
||||
@@ -243,7 +284,13 @@ where
|
||||
translated_local_addr,
|
||||
&payload,
|
||||
flags,
|
||||
effective_tag.as_deref(),
|
||||
).await?;
|
||||
sent_since_yield = sent_since_yield.saturating_add(1);
|
||||
if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) {
|
||||
sent_since_yield = 0;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
C2MeCommand::Close => {
|
||||
let _ = me_pool_c2me.send_close(conn_id).await;
|
||||
@@ -360,8 +407,7 @@ where
|
||||
flags |= RPC_FLAG_NOT_ENCRYPTED;
|
||||
}
|
||||
// Keep client read loop lightweight: route heavy ME send path via a dedicated task.
|
||||
if c2me_tx
|
||||
.send(C2MeCommand::Data { payload, flags })
|
||||
if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags })
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
@@ -372,7 +418,7 @@ where
|
||||
Ok(None) => {
|
||||
debug!(conn_id, "Client EOF");
|
||||
client_closed = true;
|
||||
let _ = c2me_tx.send(C2MeCommand::Close).await;
|
||||
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await;
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -647,3 +693,84 @@ where
|
||||
// ACK should remain low-latency.
|
||||
client_writer.flush().await.map_err(ProxyError::Io)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::time::{Duration as TokioDuration, timeout};
|
||||
|
||||
#[test]
|
||||
fn should_yield_sender_only_on_budget_with_backlog() {
|
||||
assert!(!should_yield_c2me_sender(0, true));
|
||||
assert!(!should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET - 1, true));
|
||||
assert!(!should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET, false));
|
||||
assert!(should_yield_c2me_sender(C2ME_SENDER_FAIRNESS_BUDGET, true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_c2me_command_uses_try_send_fast_path() {
|
||||
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(2);
|
||||
enqueue_c2me_command(
|
||||
&tx,
|
||||
C2MeCommand::Data {
|
||||
payload: vec![1, 2, 3],
|
||||
flags: 0,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let recv = timeout(TokioDuration::from_millis(50), rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
match recv {
|
||||
C2MeCommand::Data { payload, flags } => {
|
||||
assert_eq!(payload, vec![1, 2, 3]);
|
||||
assert_eq!(flags, 0);
|
||||
}
|
||||
C2MeCommand::Close => panic!("unexpected close command"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_c2me_command_falls_back_to_send_when_queue_is_full() {
|
||||
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(1);
|
||||
tx.send(C2MeCommand::Data {
|
||||
payload: vec![9],
|
||||
flags: 9,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let producer = tokio::spawn(async move {
|
||||
enqueue_c2me_command(
|
||||
&tx2,
|
||||
C2MeCommand::Data {
|
||||
payload: vec![7, 7],
|
||||
flags: 7,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let _ = timeout(TokioDuration::from_millis(100), rx.recv())
|
||||
.await
|
||||
.unwrap();
|
||||
producer.await.unwrap();
|
||||
|
||||
let recv = timeout(TokioDuration::from_millis(100), rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
match recv {
|
||||
C2MeCommand::Data { payload, flags } => {
|
||||
assert_eq!(payload, vec![7, 7]);
|
||||
assert_eq!(flags, 7);
|
||||
}
|
||||
C2MeCommand::Close => panic!("unexpected close command"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
200
src/stats/mod.rs
200
src/stats/mod.rs
@@ -32,8 +32,26 @@ pub struct Stats {
|
||||
me_keepalive_timeout: AtomicU64,
|
||||
me_reconnect_attempts: AtomicU64,
|
||||
me_reconnect_success: AtomicU64,
|
||||
me_handshake_reject_total: AtomicU64,
|
||||
me_reader_eof_total: AtomicU64,
|
||||
me_crc_mismatch: AtomicU64,
|
||||
me_seq_mismatch: AtomicU64,
|
||||
me_endpoint_quarantine_total: AtomicU64,
|
||||
me_kdf_drift_total: AtomicU64,
|
||||
me_kdf_port_only_drift_total: AtomicU64,
|
||||
me_hardswap_pending_reuse_total: AtomicU64,
|
||||
me_hardswap_pending_ttl_expired_total: AtomicU64,
|
||||
me_single_endpoint_outage_enter_total: AtomicU64,
|
||||
me_single_endpoint_outage_exit_total: AtomicU64,
|
||||
me_single_endpoint_outage_reconnect_attempt_total: AtomicU64,
|
||||
me_single_endpoint_outage_reconnect_success_total: AtomicU64,
|
||||
me_single_endpoint_quarantine_bypass_total: AtomicU64,
|
||||
me_single_endpoint_shadow_rotate_total: AtomicU64,
|
||||
me_single_endpoint_shadow_rotate_skipped_quarantine_total: AtomicU64,
|
||||
me_floor_mode_switch_total: AtomicU64,
|
||||
me_floor_mode_switch_static_to_adaptive_total: AtomicU64,
|
||||
me_floor_mode_switch_adaptive_to_static_total: AtomicU64,
|
||||
me_handshake_error_codes: DashMap<i32, AtomicU64>,
|
||||
me_route_drop_no_conn: AtomicU64,
|
||||
me_route_drop_channel_closed: AtomicU64,
|
||||
me_route_drop_queue_full: AtomicU64,
|
||||
@@ -172,6 +190,26 @@ impl Stats {
|
||||
self.me_reconnect_success.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_handshake_reject_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_handshake_reject_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_handshake_error_code(&self, code: i32) {
|
||||
if !self.telemetry_me_allows_normal() {
|
||||
return;
|
||||
}
|
||||
let entry = self
|
||||
.me_handshake_error_codes
|
||||
.entry(code)
|
||||
.or_insert_with(|| AtomicU64::new(0));
|
||||
entry.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
pub fn increment_me_reader_eof_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_reader_eof_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_crc_mismatch(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_crc_mismatch.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -257,7 +295,7 @@ impl Stats {
|
||||
}
|
||||
}
|
||||
pub fn increment_pool_swap_total(&self) {
|
||||
if self.telemetry_me_allows_debug() {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.pool_swap_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -333,6 +371,95 @@ impl Stats {
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_endpoint_quarantine_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_endpoint_quarantine_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_kdf_drift_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_kdf_port_only_drift_total(&self) {
|
||||
if self.telemetry_me_allows_debug() {
|
||||
self.me_kdf_port_only_drift_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_hardswap_pending_reuse_total(&self) {
|
||||
if self.telemetry_me_allows_debug() {
|
||||
self.me_hardswap_pending_reuse_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_hardswap_pending_ttl_expired_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_hardswap_pending_ttl_expired_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_outage_enter_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_outage_enter_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_outage_exit_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_outage_exit_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_outage_reconnect_attempt_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_outage_reconnect_attempt_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_outage_reconnect_success_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_outage_reconnect_success_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_quarantine_bypass_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_quarantine_bypass_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_shadow_rotate_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_shadow_rotate_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_single_endpoint_shadow_rotate_skipped_quarantine_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_single_endpoint_shadow_rotate_skipped_quarantine_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_floor_mode_switch_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_floor_mode_switch_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_floor_mode_switch_static_to_adaptive_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_floor_mode_switch_static_to_adaptive_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn increment_me_floor_mode_switch_adaptive_to_static_total(&self) {
|
||||
if self.telemetry_me_allows_normal() {
|
||||
self.me_floor_mode_switch_adaptive_to_static_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) }
|
||||
pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) }
|
||||
pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) }
|
||||
@@ -341,8 +468,79 @@ impl Stats {
|
||||
pub fn get_me_keepalive_timeout(&self) -> u64 { self.me_keepalive_timeout.load(Ordering::Relaxed) }
|
||||
pub fn get_me_reconnect_attempts(&self) -> u64 { self.me_reconnect_attempts.load(Ordering::Relaxed) }
|
||||
pub fn get_me_reconnect_success(&self) -> u64 { self.me_reconnect_success.load(Ordering::Relaxed) }
|
||||
pub fn get_me_handshake_reject_total(&self) -> u64 {
|
||||
self.me_handshake_reject_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_reader_eof_total(&self) -> u64 {
|
||||
self.me_reader_eof_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_crc_mismatch(&self) -> u64 { self.me_crc_mismatch.load(Ordering::Relaxed) }
|
||||
pub fn get_me_seq_mismatch(&self) -> u64 { self.me_seq_mismatch.load(Ordering::Relaxed) }
|
||||
pub fn get_me_endpoint_quarantine_total(&self) -> u64 {
|
||||
self.me_endpoint_quarantine_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_kdf_drift_total(&self) -> u64 {
|
||||
self.me_kdf_drift_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_kdf_port_only_drift_total(&self) -> u64 {
|
||||
self.me_kdf_port_only_drift_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_hardswap_pending_reuse_total(&self) -> u64 {
|
||||
self.me_hardswap_pending_reuse_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_hardswap_pending_ttl_expired_total(&self) -> u64 {
|
||||
self.me_hardswap_pending_ttl_expired_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_outage_enter_total(&self) -> u64 {
|
||||
self.me_single_endpoint_outage_enter_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_outage_exit_total(&self) -> u64 {
|
||||
self.me_single_endpoint_outage_exit_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_outage_reconnect_attempt_total(&self) -> u64 {
|
||||
self.me_single_endpoint_outage_reconnect_attempt_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_outage_reconnect_success_total(&self) -> u64 {
|
||||
self.me_single_endpoint_outage_reconnect_success_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_quarantine_bypass_total(&self) -> u64 {
|
||||
self.me_single_endpoint_quarantine_bypass_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_shadow_rotate_total(&self) -> u64 {
|
||||
self.me_single_endpoint_shadow_rotate_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_single_endpoint_shadow_rotate_skipped_quarantine_total(&self) -> u64 {
|
||||
self.me_single_endpoint_shadow_rotate_skipped_quarantine_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_floor_mode_switch_total(&self) -> u64 {
|
||||
self.me_floor_mode_switch_total.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_floor_mode_switch_static_to_adaptive_total(&self) -> u64 {
|
||||
self.me_floor_mode_switch_static_to_adaptive_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_floor_mode_switch_adaptive_to_static_total(&self) -> u64 {
|
||||
self.me_floor_mode_switch_adaptive_to_static_total
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> {
|
||||
let mut out: Vec<(i32, u64)> = self
|
||||
.me_handshake_error_codes
|
||||
.iter()
|
||||
.map(|entry| (*entry.key(), entry.value().load(Ordering::Relaxed)))
|
||||
.collect();
|
||||
out.sort_by_key(|(code, _)| *code);
|
||||
out
|
||||
}
|
||||
pub fn get_me_route_drop_no_conn(&self) -> u64 { self.me_route_drop_no_conn.load(Ordering::Relaxed) }
|
||||
pub fn get_me_route_drop_channel_closed(&self) -> u64 {
|
||||
self.me_route_drop_channel_closed.load(Ordering::Relaxed)
|
||||
|
||||
@@ -336,22 +336,35 @@ impl PendingCiphertext {
|
||||
}
|
||||
|
||||
fn remaining_capacity(&self) -> usize {
|
||||
self.max_len.saturating_sub(self.buf.len())
|
||||
self.max_len.saturating_sub(self.pending_len())
|
||||
}
|
||||
|
||||
fn compact_consumed_prefix(&mut self) {
|
||||
if self.pos == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
if self.pos >= self.buf.len() {
|
||||
self.buf.clear();
|
||||
self.pos = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = self.buf.split_to(self.pos);
|
||||
self.pos = 0;
|
||||
}
|
||||
|
||||
fn advance(&mut self, n: usize) {
|
||||
self.pos = (self.pos + n).min(self.buf.len());
|
||||
|
||||
if self.pos == self.buf.len() {
|
||||
self.buf.clear();
|
||||
self.pos = 0;
|
||||
self.compact_consumed_prefix();
|
||||
return;
|
||||
}
|
||||
|
||||
// Compact when a large prefix was consumed.
|
||||
if self.pos >= 16 * 1024 {
|
||||
let _ = self.buf.split_to(self.pos);
|
||||
self.pos = 0;
|
||||
self.compact_consumed_prefix();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,6 +392,11 @@ impl PendingCiphertext {
|
||||
));
|
||||
}
|
||||
|
||||
// Reclaim consumed prefix when physical storage is the only limiter.
|
||||
if self.pos > 0 && self.buf.len() + plaintext.len() > self.max_len {
|
||||
self.compact_consumed_prefix();
|
||||
}
|
||||
|
||||
let start = self.buf.len();
|
||||
self.buf.reserve(plaintext.len());
|
||||
self.buf.extend_from_slice(plaintext);
|
||||
@@ -777,3 +795,70 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for PassthroughStream<S> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_ctr() -> AesCtr {
|
||||
AesCtr::new(&[0x11; 32], 0x0102_0304_0506_0708_1112_1314_1516_1718)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_capacity_reclaims_after_partial_advance_without_compaction_threshold() {
|
||||
let mut pending = PendingCiphertext::new(1024);
|
||||
let mut ctr = test_ctr();
|
||||
let payload = vec![0x41; 900];
|
||||
pending.push_encrypted(&mut ctr, &payload).unwrap();
|
||||
|
||||
// Keep position below compaction threshold to validate logical-capacity accounting.
|
||||
pending.advance(800);
|
||||
assert_eq!(pending.pending_len(), 100);
|
||||
assert_eq!(pending.remaining_capacity(), 924);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_encrypted_respects_pending_limit() {
|
||||
let mut pending = PendingCiphertext::new(64);
|
||||
let mut ctr = test_ctr();
|
||||
|
||||
pending.push_encrypted(&mut ctr, &[0x10; 64]).unwrap();
|
||||
let err = pending.push_encrypted(&mut ctr, &[0x20]).unwrap_err();
|
||||
assert_eq!(err.kind(), ErrorKind::WouldBlock);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_encrypted_compacts_prefix_when_physical_buffer_would_overflow() {
|
||||
let mut pending = PendingCiphertext::new(64);
|
||||
let mut ctr = test_ctr();
|
||||
|
||||
pending.push_encrypted(&mut ctr, &[0x22; 60]).unwrap();
|
||||
pending.advance(30);
|
||||
pending.push_encrypted(&mut ctr, &[0x33; 30]).unwrap();
|
||||
|
||||
assert_eq!(pending.pending_len(), 60);
|
||||
assert!(pending.buf.len() <= 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_ciphertext_preserves_stream_order_across_drain_and_append() {
|
||||
let mut pending = PendingCiphertext::new(128);
|
||||
let mut ctr = test_ctr();
|
||||
|
||||
let first = vec![0xA1; 80];
|
||||
let second = vec![0xB2; 40];
|
||||
|
||||
pending.push_encrypted(&mut ctr, &first).unwrap();
|
||||
pending.advance(50);
|
||||
pending.push_encrypted(&mut ctr, &second).unwrap();
|
||||
|
||||
let mut baseline_ctr = test_ctr();
|
||||
let mut baseline_plain = Vec::with_capacity(first.len() + second.len());
|
||||
baseline_plain.extend_from_slice(&first);
|
||||
baseline_plain.extend_from_slice(&second);
|
||||
baseline_ctr.apply(&mut baseline_plain);
|
||||
|
||||
let expected = &baseline_plain[50..];
|
||||
assert_eq!(pending.pending_slice(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -499,7 +499,7 @@ async fn fetch_via_raw_tls(
|
||||
sock = %sock_path,
|
||||
"Raw TLS fetch using mask unix socket"
|
||||
);
|
||||
return fetch_via_raw_tls_stream(stream, sni, connect_timeout, 0).await;
|
||||
return fetch_via_raw_tls_stream(stream, sni, connect_timeout, proxy_protocol).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(
|
||||
@@ -631,7 +631,7 @@ async fn fetch_via_rustls(
|
||||
sock = %sock_path,
|
||||
"Rustls fetch using mask unix socket"
|
||||
);
|
||||
return fetch_via_rustls_stream(stream, host, sni, 0).await;
|
||||
return fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(
|
||||
|
||||
@@ -5,15 +5,15 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use httpdate;
|
||||
use tokio::sync::watch;
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::error::Result;
|
||||
|
||||
use super::MePool;
|
||||
use super::rotation::{MeReinitTrigger, enqueue_reinit_trigger};
|
||||
use super::secret::download_proxy_secret_with_max_len;
|
||||
use crate::crypto::SecureRandom;
|
||||
use std::time::SystemTime;
|
||||
|
||||
async fn retry_fetch(url: &str) -> Option<ProxyConfigData> {
|
||||
@@ -38,6 +38,8 @@ async fn retry_fetch(url: &str) -> Option<ProxyConfigData> {
|
||||
pub struct ProxyConfigData {
|
||||
pub map: HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||
pub default_dc: Option<i32>,
|
||||
pub http_status: u16,
|
||||
pub proxy_for_lines: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -172,6 +174,7 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
|
||||
.await
|
||||
.map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config GET failed: {e}")))?
|
||||
;
|
||||
let http_status = resp.status().as_u16();
|
||||
|
||||
if let Some(date) = resp.headers().get(reqwest::header::DATE)
|
||||
&& let Ok(date_str) = date.to_str()
|
||||
@@ -194,9 +197,11 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
|
||||
.map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config read failed: {e}")))?;
|
||||
|
||||
let mut map: HashMap<i32, Vec<(IpAddr, u16)>> = HashMap::new();
|
||||
let mut proxy_for_lines: u32 = 0;
|
||||
for line in text.lines() {
|
||||
if let Some((dc, ip, port)) = parse_proxy_line(line) {
|
||||
map.entry(dc).or_default().push((ip, port));
|
||||
proxy_for_lines = proxy_for_lines.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,14 +219,49 @@ pub async fn fetch_proxy_config(url: &str) -> Result<ProxyConfigData> {
|
||||
None
|
||||
});
|
||||
|
||||
Ok(ProxyConfigData { map, default_dc })
|
||||
Ok(ProxyConfigData {
|
||||
map,
|
||||
default_dc,
|
||||
http_status,
|
||||
proxy_for_lines,
|
||||
})
|
||||
}
|
||||
|
||||
fn snapshot_passes_guards(
|
||||
cfg: &ProxyConfig,
|
||||
snapshot: &ProxyConfigData,
|
||||
snapshot_name: &'static str,
|
||||
) -> bool {
|
||||
if cfg.general.me_snapshot_require_http_2xx
|
||||
&& !(200..=299).contains(&snapshot.http_status)
|
||||
{
|
||||
warn!(
|
||||
snapshot = snapshot_name,
|
||||
http_status = snapshot.http_status,
|
||||
"ME snapshot rejected by non-2xx HTTP status"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let min_proxy_for = cfg.general.me_snapshot_min_proxy_for_lines;
|
||||
if snapshot.proxy_for_lines < min_proxy_for {
|
||||
warn!(
|
||||
snapshot = snapshot_name,
|
||||
parsed_proxy_for_lines = snapshot.proxy_for_lines,
|
||||
min_proxy_for_lines = min_proxy_for,
|
||||
"ME snapshot rejected by proxy_for line floor"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
async fn run_update_cycle(
|
||||
pool: &Arc<MePool>,
|
||||
rng: &Arc<SecureRandom>,
|
||||
cfg: &ProxyConfig,
|
||||
state: &mut UpdaterState,
|
||||
reinit_tx: &mpsc::Sender<MeReinitTrigger>,
|
||||
) {
|
||||
pool.update_runtime_reinit_policy(
|
||||
cfg.general.hardswap,
|
||||
@@ -232,6 +272,20 @@ async fn run_update_cycle(
|
||||
cfg.general.me_hardswap_warmup_delay_max_ms,
|
||||
cfg.general.me_hardswap_warmup_extra_passes,
|
||||
cfg.general.me_hardswap_warmup_pass_backoff_base_ms,
|
||||
cfg.general.me_bind_stale_mode,
|
||||
cfg.general.me_bind_stale_ttl_secs,
|
||||
cfg.general.me_secret_atomic_snapshot,
|
||||
cfg.general.me_deterministic_writer_sort,
|
||||
cfg.general.me_single_endpoint_shadow_writers,
|
||||
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||
cfg.general.me_single_endpoint_outage_backoff_min_ms,
|
||||
cfg.general.me_single_endpoint_outage_backoff_max_ms,
|
||||
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
|
||||
cfg.general.me_floor_mode,
|
||||
cfg.general.me_adaptive_floor_idle_secs,
|
||||
cfg.general.me_adaptive_floor_min_writers_single_endpoint,
|
||||
cfg.general.me_adaptive_floor_recover_grace_secs,
|
||||
);
|
||||
|
||||
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
|
||||
@@ -242,44 +296,48 @@ async fn run_update_cycle(
|
||||
let mut ready_v4: Option<(ProxyConfigData, u64)> = None;
|
||||
let cfg_v4 = retry_fetch("https://core.telegram.org/getProxyConfig").await;
|
||||
if let Some(cfg_v4) = cfg_v4 {
|
||||
let cfg_v4_hash = hash_proxy_config(&cfg_v4);
|
||||
let stable_hits = state.config_v4.observe(cfg_v4_hash);
|
||||
if stable_hits < required_cfg_snapshots {
|
||||
debug!(
|
||||
stable_hits,
|
||||
required_cfg_snapshots,
|
||||
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
|
||||
"ME config v4 candidate observed"
|
||||
);
|
||||
} else if state.config_v4.is_applied(cfg_v4_hash) {
|
||||
debug!(
|
||||
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
|
||||
"ME config v4 stable snapshot already applied"
|
||||
);
|
||||
} else {
|
||||
ready_v4 = Some((cfg_v4, cfg_v4_hash));
|
||||
if snapshot_passes_guards(cfg, &cfg_v4, "getProxyConfig") {
|
||||
let cfg_v4_hash = hash_proxy_config(&cfg_v4);
|
||||
let stable_hits = state.config_v4.observe(cfg_v4_hash);
|
||||
if stable_hits < required_cfg_snapshots {
|
||||
debug!(
|
||||
stable_hits,
|
||||
required_cfg_snapshots,
|
||||
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
|
||||
"ME config v4 candidate observed"
|
||||
);
|
||||
} else if state.config_v4.is_applied(cfg_v4_hash) {
|
||||
debug!(
|
||||
snapshot = format_args!("0x{cfg_v4_hash:016x}"),
|
||||
"ME config v4 stable snapshot already applied"
|
||||
);
|
||||
} else {
|
||||
ready_v4 = Some((cfg_v4, cfg_v4_hash));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut ready_v6: Option<(ProxyConfigData, u64)> = None;
|
||||
let cfg_v6 = retry_fetch("https://core.telegram.org/getProxyConfigV6").await;
|
||||
if let Some(cfg_v6) = cfg_v6 {
|
||||
let cfg_v6_hash = hash_proxy_config(&cfg_v6);
|
||||
let stable_hits = state.config_v6.observe(cfg_v6_hash);
|
||||
if stable_hits < required_cfg_snapshots {
|
||||
debug!(
|
||||
stable_hits,
|
||||
required_cfg_snapshots,
|
||||
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
|
||||
"ME config v6 candidate observed"
|
||||
);
|
||||
} else if state.config_v6.is_applied(cfg_v6_hash) {
|
||||
debug!(
|
||||
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
|
||||
"ME config v6 stable snapshot already applied"
|
||||
);
|
||||
} else {
|
||||
ready_v6 = Some((cfg_v6, cfg_v6_hash));
|
||||
if snapshot_passes_guards(cfg, &cfg_v6, "getProxyConfigV6") {
|
||||
let cfg_v6_hash = hash_proxy_config(&cfg_v6);
|
||||
let stable_hits = state.config_v6.observe(cfg_v6_hash);
|
||||
if stable_hits < required_cfg_snapshots {
|
||||
debug!(
|
||||
stable_hits,
|
||||
required_cfg_snapshots,
|
||||
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
|
||||
"ME config v6 candidate observed"
|
||||
);
|
||||
} else if state.config_v6.is_applied(cfg_v6_hash) {
|
||||
debug!(
|
||||
snapshot = format_args!("0x{cfg_v6_hash:016x}"),
|
||||
"ME config v6 stable snapshot already applied"
|
||||
);
|
||||
} else {
|
||||
ready_v6 = Some((cfg_v6, cfg_v6_hash));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,28 +350,40 @@ async fn run_update_cycle(
|
||||
let update_v6 = ready_v6
|
||||
.as_ref()
|
||||
.map(|(snapshot, _)| snapshot.map.clone());
|
||||
|
||||
let changed = pool.update_proxy_maps(update_v4, update_v6).await;
|
||||
|
||||
if let Some((snapshot, hash)) = ready_v4 {
|
||||
if let Some(dc) = snapshot.default_dc {
|
||||
pool.default_dc
|
||||
.store(dc, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
state.config_v4.mark_applied(hash);
|
||||
}
|
||||
|
||||
if let Some((_snapshot, hash)) = ready_v6 {
|
||||
state.config_v6.mark_applied(hash);
|
||||
}
|
||||
|
||||
state.last_map_apply_at = Some(tokio::time::Instant::now());
|
||||
|
||||
if changed {
|
||||
maps_changed = true;
|
||||
info!("ME config update applied after stable-gate");
|
||||
let update_is_empty =
|
||||
update_v4.is_empty() && update_v6.as_ref().is_none_or(|v| v.is_empty());
|
||||
let apply_outcome = if update_is_empty && !cfg.general.me_snapshot_reject_empty_map {
|
||||
super::pool_config::SnapshotApplyOutcome::AppliedNoDelta
|
||||
} else {
|
||||
debug!("ME config stable-gate applied with no map delta");
|
||||
pool.update_proxy_maps(update_v4, update_v6).await
|
||||
};
|
||||
|
||||
if matches!(
|
||||
apply_outcome,
|
||||
super::pool_config::SnapshotApplyOutcome::RejectedEmpty
|
||||
) {
|
||||
warn!("ME config stable snapshot rejected (empty endpoint map)");
|
||||
} else {
|
||||
if let Some((snapshot, hash)) = ready_v4 {
|
||||
if let Some(dc) = snapshot.default_dc {
|
||||
pool.default_dc
|
||||
.store(dc, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
state.config_v4.mark_applied(hash);
|
||||
}
|
||||
|
||||
if let Some((_snapshot, hash)) = ready_v6 {
|
||||
state.config_v6.mark_applied(hash);
|
||||
}
|
||||
|
||||
state.last_map_apply_at = Some(tokio::time::Instant::now());
|
||||
|
||||
if apply_outcome.changed() {
|
||||
maps_changed = true;
|
||||
info!("ME config update applied after stable-gate");
|
||||
} else {
|
||||
debug!("ME config stable-gate applied with no map delta");
|
||||
}
|
||||
}
|
||||
} else if let Some(last) = state.last_map_apply_at {
|
||||
let wait_secs = map_apply_cooldown_remaining_secs(last, apply_cooldown);
|
||||
@@ -325,8 +395,7 @@ async fn run_update_cycle(
|
||||
}
|
||||
|
||||
if maps_changed {
|
||||
pool.zero_downtime_reinit_after_map_change(rng.as_ref())
|
||||
.await;
|
||||
enqueue_reinit_trigger(reinit_tx, MeReinitTrigger::MapChanged);
|
||||
}
|
||||
|
||||
pool.reset_stun_state();
|
||||
@@ -367,8 +436,8 @@ async fn run_update_cycle(
|
||||
|
||||
pub async fn me_config_updater(
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
reinit_tx: mpsc::Sender<MeReinitTrigger>,
|
||||
) {
|
||||
let mut state = UpdaterState::default();
|
||||
let mut update_every_secs = config_rx
|
||||
@@ -387,7 +456,7 @@ pub async fn me_config_updater(
|
||||
tokio::select! {
|
||||
_ = &mut sleep => {
|
||||
let cfg = config_rx.borrow().clone();
|
||||
run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await;
|
||||
run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await;
|
||||
let refreshed_secs = cfg.general.effective_update_every_secs().max(1);
|
||||
if refreshed_secs != update_every_secs {
|
||||
info!(
|
||||
@@ -415,6 +484,20 @@ pub async fn me_config_updater(
|
||||
cfg.general.me_hardswap_warmup_delay_max_ms,
|
||||
cfg.general.me_hardswap_warmup_extra_passes,
|
||||
cfg.general.me_hardswap_warmup_pass_backoff_base_ms,
|
||||
cfg.general.me_bind_stale_mode,
|
||||
cfg.general.me_bind_stale_ttl_secs,
|
||||
cfg.general.me_secret_atomic_snapshot,
|
||||
cfg.general.me_deterministic_writer_sort,
|
||||
cfg.general.me_single_endpoint_shadow_writers,
|
||||
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||
cfg.general.me_single_endpoint_outage_backoff_min_ms,
|
||||
cfg.general.me_single_endpoint_outage_backoff_max_ms,
|
||||
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
|
||||
cfg.general.me_floor_mode,
|
||||
cfg.general.me_adaptive_floor_idle_secs,
|
||||
cfg.general.me_adaptive_floor_min_writers_single_endpoint,
|
||||
cfg.general.me_adaptive_floor_recover_grace_secs,
|
||||
);
|
||||
let new_secs = cfg.general.effective_update_every_secs().max(1);
|
||||
if new_secs == update_every_secs {
|
||||
@@ -429,7 +512,7 @@ pub async fn me_config_updater(
|
||||
);
|
||||
update_every_secs = new_secs;
|
||||
update_every = Duration::from_secs(update_every_secs);
|
||||
run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await;
|
||||
run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await;
|
||||
next_tick = tokio::time::Instant::now() + update_every;
|
||||
} else {
|
||||
info!(
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use socket2::{SockRef, TcpKeepalive};
|
||||
#[cfg(target_os = "linux")]
|
||||
use libc;
|
||||
@@ -33,6 +36,24 @@ use super::codec::{
|
||||
use super::wire::{extract_ip_material, IpMaterial};
|
||||
use super::MePool;
|
||||
|
||||
const ME_KDF_DRIFT_STRICT: bool = false;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum KdfClientPortSource {
|
||||
LocalSocket = 0,
|
||||
SocksBound = 1,
|
||||
}
|
||||
|
||||
impl KdfClientPortSource {
|
||||
fn from_socks_bound_port(socks_bound_port: Option<u16>) -> Self {
|
||||
if socks_bound_port.is_some() {
|
||||
Self::SocksBound
|
||||
} else {
|
||||
Self::LocalSocket
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a successful ME handshake with timings.
|
||||
pub(crate) struct HandshakeOutput {
|
||||
pub rd: ReadHalf<TcpStream>,
|
||||
@@ -46,6 +67,22 @@ pub(crate) struct HandshakeOutput {
|
||||
}
|
||||
|
||||
impl MePool {
|
||||
fn kdf_material_fingerprint(
|
||||
local_ip_nat: IpAddr,
|
||||
peer_addr_nat: SocketAddr,
|
||||
reflected_ip: Option<IpAddr>,
|
||||
socks_bound_ip: Option<IpAddr>,
|
||||
client_port_source: KdfClientPortSource,
|
||||
) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
local_ip_nat.hash(&mut hasher);
|
||||
peer_addr_nat.hash(&mut hasher);
|
||||
reflected_ip.hash(&mut hasher);
|
||||
socks_bound_ip.hash(&mut hasher);
|
||||
client_port_source.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option<i16> {
|
||||
if addr.is_ipv4() {
|
||||
let map = self.proxy_map_v4.read().await;
|
||||
@@ -267,7 +304,16 @@ impl MePool {
|
||||
.unwrap_or_default()
|
||||
.as_secs() as u32;
|
||||
|
||||
let ks = self.key_selector().await;
|
||||
let secret_atomic_snapshot = self.secret_atomic_snapshot.load(Ordering::Relaxed);
|
||||
let (ks, secret) = if secret_atomic_snapshot {
|
||||
let snapshot = self.secret_snapshot().await;
|
||||
(snapshot.key_selector, snapshot.secret)
|
||||
} else {
|
||||
// Backward-compatible mode: key selector and secret may come from different updates.
|
||||
let key_selector = self.key_selector().await;
|
||||
let secret = self.secret_snapshot().await.secret;
|
||||
(key_selector, secret)
|
||||
};
|
||||
let nonce_payload = build_nonce_payload(ks, crypto_ts, &my_nonce);
|
||||
let nonce_frame = build_rpc_frame(-2, &nonce_payload, RpcChecksumMode::Crc32);
|
||||
let dump = hex_dump(&nonce_frame[..nonce_frame.len().min(44)]);
|
||||
@@ -329,10 +375,50 @@ impl MePool {
|
||||
|
||||
let ts_bytes = crypto_ts.to_le_bytes();
|
||||
let server_port_bytes = peer_addr_nat.port().to_le_bytes();
|
||||
let client_port_for_kdf = socks_bound_addr
|
||||
let socks_bound_port = socks_bound_addr
|
||||
.map(|bound| bound.port())
|
||||
.filter(|port| *port != 0)
|
||||
.unwrap_or(local_addr_nat.port());
|
||||
.filter(|port| *port != 0);
|
||||
let client_port_for_kdf = socks_bound_port.unwrap_or(local_addr_nat.port());
|
||||
let client_port_source = KdfClientPortSource::from_socks_bound_port(socks_bound_port);
|
||||
let kdf_fingerprint = Self::kdf_material_fingerprint(
|
||||
local_addr_nat.ip(),
|
||||
peer_addr_nat,
|
||||
reflected.map(|value| value.ip()),
|
||||
socks_bound_addr.map(|value| value.ip()),
|
||||
client_port_source,
|
||||
);
|
||||
let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await;
|
||||
if let Some((prev_fingerprint, prev_client_port)) =
|
||||
kdf_fingerprint_guard.get(&peer_addr_nat).copied()
|
||||
{
|
||||
if prev_fingerprint != kdf_fingerprint {
|
||||
self.stats.increment_me_kdf_drift_total();
|
||||
warn!(
|
||||
%peer_addr_nat,
|
||||
%local_addr_nat,
|
||||
client_port_for_kdf,
|
||||
client_port_source = ?client_port_source,
|
||||
"ME KDF material drift detected for endpoint"
|
||||
);
|
||||
if ME_KDF_DRIFT_STRICT {
|
||||
return Err(ProxyError::InvalidHandshake(
|
||||
"ME KDF material drift detected (strict mode)".to_string(),
|
||||
));
|
||||
}
|
||||
} else if prev_client_port != client_port_for_kdf {
|
||||
self.stats.increment_me_kdf_port_only_drift_total();
|
||||
debug!(
|
||||
%peer_addr_nat,
|
||||
previous_client_port_for_kdf = prev_client_port,
|
||||
client_port_for_kdf,
|
||||
client_port_source = ?client_port_source,
|
||||
"ME KDF client port changed with stable material"
|
||||
);
|
||||
}
|
||||
}
|
||||
kdf_fingerprint_guard.insert(peer_addr_nat, (kdf_fingerprint, client_port_for_kdf));
|
||||
drop(kdf_fingerprint_guard);
|
||||
|
||||
let client_port_bytes = client_port_for_kdf.to_le_bytes();
|
||||
|
||||
let server_ip = extract_ip_material(peer_addr_nat);
|
||||
@@ -357,8 +443,6 @@ impl MePool {
|
||||
|
||||
let diag_level: u8 = std::env::var("ME_DIAG").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
|
||||
|
||||
let secret: Vec<u8> = self.proxy_secret.read().await.clone();
|
||||
|
||||
let prekey_client = build_middleproxy_prekey(
|
||||
&srv_nonce,
|
||||
&my_nonce,
|
||||
@@ -532,6 +616,8 @@ impl MePool {
|
||||
} else {
|
||||
-1
|
||||
};
|
||||
self.stats.increment_me_handshake_reject_total();
|
||||
self.stats.increment_me_handshake_error_code(err_code);
|
||||
return Err(ProxyError::InvalidHandshake(format!(
|
||||
"ME rejected handshake (error={err_code})"
|
||||
)));
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tracing::{debug, info, warn};
|
||||
use rand::Rng;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::MeFloorMode;
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::network::IpFamily;
|
||||
|
||||
@@ -15,11 +17,18 @@ const HEALTH_INTERVAL_SECS: u64 = 1;
|
||||
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
||||
#[allow(dead_code)]
|
||||
const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1;
|
||||
const SHADOW_ROTATE_RETRY_SECS: u64 = 30;
|
||||
|
||||
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
||||
let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
||||
let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||
let mut inflight: HashMap<(i32, IpFamily), usize> = HashMap::new();
|
||||
let mut outage_backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
||||
let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||
let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new();
|
||||
let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||
let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||
let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await;
|
||||
pool.prune_closed_writers().await;
|
||||
@@ -30,6 +39,12 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
||||
&mut backoff,
|
||||
&mut next_attempt,
|
||||
&mut inflight,
|
||||
&mut outage_backoff,
|
||||
&mut outage_next_attempt,
|
||||
&mut single_endpoint_outage,
|
||||
&mut shadow_rotate_deadline,
|
||||
&mut adaptive_idle_since,
|
||||
&mut adaptive_recover_until,
|
||||
)
|
||||
.await;
|
||||
check_family(
|
||||
@@ -39,6 +54,12 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
||||
&mut backoff,
|
||||
&mut next_attempt,
|
||||
&mut inflight,
|
||||
&mut outage_backoff,
|
||||
&mut outage_next_attempt,
|
||||
&mut single_endpoint_outage,
|
||||
&mut shadow_rotate_deadline,
|
||||
&mut adaptive_idle_since,
|
||||
&mut adaptive_recover_until,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -51,6 +72,12 @@ async fn check_family(
|
||||
backoff: &mut HashMap<(i32, IpFamily), u64>,
|
||||
next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
inflight: &mut HashMap<(i32, IpFamily), usize>,
|
||||
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
|
||||
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
single_endpoint_outage: &mut HashSet<(i32, IpFamily)>,
|
||||
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
) {
|
||||
let enabled = match family {
|
||||
IpFamily::V4 => pool.decision.ipv4_me,
|
||||
@@ -77,32 +104,103 @@ async fn check_family(
|
||||
endpoints.dedup();
|
||||
}
|
||||
|
||||
if pool.floor_mode() == MeFloorMode::Static {
|
||||
adaptive_idle_since.clear();
|
||||
adaptive_recover_until.clear();
|
||||
}
|
||||
|
||||
let mut live_addr_counts = HashMap::<SocketAddr, usize>::new();
|
||||
for writer in pool
|
||||
.writers
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.filter(|w| !w.draining.load(std::sync::atomic::Ordering::Relaxed))
|
||||
{
|
||||
let mut live_writer_ids_by_addr = HashMap::<SocketAddr, Vec<u64>>::new();
|
||||
for writer in pool.writers.read().await.iter().filter(|w| {
|
||||
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}) {
|
||||
*live_addr_counts.entry(writer.addr).or_insert(0) += 1;
|
||||
live_writer_ids_by_addr
|
||||
.entry(writer.addr)
|
||||
.or_default()
|
||||
.push(writer.id);
|
||||
}
|
||||
|
||||
for (dc, endpoints) in dc_endpoints {
|
||||
if endpoints.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let required = MePool::required_writers_for_dc(endpoints.len());
|
||||
let key = (dc, family);
|
||||
let reduce_for_idle = should_reduce_floor_for_idle(
|
||||
pool,
|
||||
key,
|
||||
&endpoints,
|
||||
&live_writer_ids_by_addr,
|
||||
adaptive_idle_since,
|
||||
adaptive_recover_until,
|
||||
)
|
||||
.await;
|
||||
let required = pool.required_writers_for_dc_with_floor_mode(endpoints.len(), reduce_for_idle);
|
||||
let alive = endpoints
|
||||
.iter()
|
||||
.map(|addr| *live_addr_counts.get(addr).unwrap_or(&0))
|
||||
.sum::<usize>();
|
||||
|
||||
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
|
||||
if single_endpoint_outage.insert(key) {
|
||||
pool.stats.increment_me_single_endpoint_outage_enter_total();
|
||||
warn!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
required,
|
||||
endpoint_count = endpoints.len(),
|
||||
"Single-endpoint DC outage detected"
|
||||
);
|
||||
}
|
||||
|
||||
recover_single_endpoint_outage(
|
||||
pool,
|
||||
rng,
|
||||
key,
|
||||
endpoints[0],
|
||||
required,
|
||||
outage_backoff,
|
||||
outage_next_attempt,
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
|
||||
if single_endpoint_outage.remove(&key) {
|
||||
pool.stats.increment_me_single_endpoint_outage_exit_total();
|
||||
outage_backoff.remove(&key);
|
||||
outage_next_attempt.remove(&key);
|
||||
shadow_rotate_deadline.remove(&key);
|
||||
adaptive_idle_since.remove(&key);
|
||||
adaptive_recover_until.remove(&key);
|
||||
info!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
alive,
|
||||
required,
|
||||
endpoint_count = endpoints.len(),
|
||||
"Single-endpoint DC outage recovered"
|
||||
);
|
||||
}
|
||||
|
||||
if alive >= required {
|
||||
maybe_rotate_single_endpoint_shadow(
|
||||
pool,
|
||||
rng,
|
||||
key,
|
||||
dc,
|
||||
family,
|
||||
&endpoints,
|
||||
alive,
|
||||
required,
|
||||
&live_writer_ids_by_addr,
|
||||
shadow_rotate_deadline,
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
let missing = required - alive;
|
||||
|
||||
let key = (dc, family);
|
||||
let now = Instant::now();
|
||||
if let Some(ts) = next_attempt.get(&key)
|
||||
&& now < *ts
|
||||
@@ -112,7 +210,18 @@ async fn check_family(
|
||||
|
||||
let max_concurrent = pool.me_reconnect_max_concurrent_per_dc.max(1) as usize;
|
||||
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
|
||||
return;
|
||||
continue;
|
||||
}
|
||||
if pool.has_refill_inflight_for_endpoints(&endpoints).await {
|
||||
debug!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
alive,
|
||||
required,
|
||||
endpoint_count = endpoints.len(),
|
||||
"Skipping health reconnect: immediate refill is already in flight for this DC group"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
*inflight.entry(key).or_insert(0) += 1;
|
||||
|
||||
@@ -177,3 +286,268 @@ async fn check_family(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn should_reduce_floor_for_idle(
|
||||
pool: &Arc<MePool>,
|
||||
key: (i32, IpFamily),
|
||||
endpoints: &[SocketAddr],
|
||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
) -> bool {
|
||||
if endpoints.len() != 1 || pool.floor_mode() != MeFloorMode::Adaptive {
|
||||
adaptive_idle_since.remove(&key);
|
||||
adaptive_recover_until.remove(&key);
|
||||
return false;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let endpoint = endpoints[0];
|
||||
let writer_ids = live_writer_ids_by_addr
|
||||
.get(&endpoint)
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or(&[]);
|
||||
let has_bound_clients = has_bound_clients_on_endpoint(pool, writer_ids).await;
|
||||
if has_bound_clients {
|
||||
adaptive_idle_since.remove(&key);
|
||||
adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration());
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(recover_until) = adaptive_recover_until.get(&key)
|
||||
&& now < *recover_until
|
||||
{
|
||||
adaptive_idle_since.remove(&key);
|
||||
return false;
|
||||
}
|
||||
adaptive_recover_until.remove(&key);
|
||||
|
||||
let idle_since = adaptive_idle_since.entry(key).or_insert(now);
|
||||
now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration()
|
||||
}
|
||||
|
||||
async fn has_bound_clients_on_endpoint(pool: &Arc<MePool>, writer_ids: &[u64]) -> bool {
|
||||
for writer_id in writer_ids {
|
||||
if !pool.registry.is_writer_empty(*writer_id).await {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn recover_single_endpoint_outage(
|
||||
pool: &Arc<MePool>,
|
||||
rng: &Arc<SecureRandom>,
|
||||
key: (i32, IpFamily),
|
||||
endpoint: SocketAddr,
|
||||
required: usize,
|
||||
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
|
||||
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
) {
|
||||
let now = Instant::now();
|
||||
if let Some(ts) = outage_next_attempt.get(&key)
|
||||
&& now < *ts
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms();
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||
|
||||
let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine();
|
||||
let attempt_ok = if bypass_quarantine {
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||
match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
dc = %key.0,
|
||||
family = ?key.1,
|
||||
%endpoint,
|
||||
error = %e,
|
||||
"Single-endpoint outage reconnect failed (quarantine bypass path)"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
dc = %key.0,
|
||||
family = ?key.1,
|
||||
%endpoint,
|
||||
"Single-endpoint outage reconnect timed out (quarantine bypass path)"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let one_endpoint = [endpoint];
|
||||
match tokio::time::timeout(
|
||||
pool.me_one_timeout,
|
||||
pool.connect_endpoints_round_robin(&one_endpoint, rng.as_ref()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(ok) => ok,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
dc = %key.0,
|
||||
family = ?key.1,
|
||||
%endpoint,
|
||||
"Single-endpoint outage reconnect timed out"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if attempt_ok {
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_outage_reconnect_success_total();
|
||||
pool.stats.increment_me_reconnect_success();
|
||||
outage_backoff.insert(key, min_backoff_ms);
|
||||
let jitter = min_backoff_ms / JITTER_FRAC_NUM;
|
||||
let wait = Duration::from_millis(min_backoff_ms)
|
||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
||||
outage_next_attempt.insert(key, now + wait);
|
||||
info!(
|
||||
dc = %key.0,
|
||||
family = ?key.1,
|
||||
%endpoint,
|
||||
required,
|
||||
backoff_ms = min_backoff_ms,
|
||||
"Single-endpoint outage reconnect succeeded"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
pool.stats.increment_me_reconnect_attempt();
|
||||
let current_ms = *outage_backoff.get(&key).unwrap_or(&min_backoff_ms);
|
||||
let next_ms = current_ms.saturating_mul(2).min(max_backoff_ms);
|
||||
outage_backoff.insert(key, next_ms);
|
||||
let jitter = next_ms / JITTER_FRAC_NUM;
|
||||
let wait = Duration::from_millis(next_ms)
|
||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
||||
outage_next_attempt.insert(key, now + wait);
|
||||
warn!(
|
||||
dc = %key.0,
|
||||
family = ?key.1,
|
||||
%endpoint,
|
||||
required,
|
||||
backoff_ms = next_ms,
|
||||
"Single-endpoint outage reconnect scheduled"
|
||||
);
|
||||
}
|
||||
|
||||
async fn maybe_rotate_single_endpoint_shadow(
|
||||
pool: &Arc<MePool>,
|
||||
rng: &Arc<SecureRandom>,
|
||||
key: (i32, IpFamily),
|
||||
dc: i32,
|
||||
family: IpFamily,
|
||||
endpoints: &[SocketAddr],
|
||||
alive: usize,
|
||||
required: usize,
|
||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
||||
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
||||
) {
|
||||
if endpoints.len() != 1 || alive < required {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(interval) = pool.single_endpoint_shadow_rotate_interval() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let now = Instant::now();
|
||||
if let Some(deadline) = shadow_rotate_deadline.get(&key)
|
||||
&& now < *deadline
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let endpoint = endpoints[0];
|
||||
if pool.is_endpoint_quarantined(endpoint).await {
|
||||
pool.stats
|
||||
.increment_me_single_endpoint_shadow_rotate_skipped_quarantine_total();
|
||||
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
|
||||
debug!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
%endpoint,
|
||||
"Single-endpoint shadow rotation skipped: endpoint is quarantined"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else {
|
||||
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
|
||||
return;
|
||||
};
|
||||
|
||||
let mut candidate_writer_id = None;
|
||||
for writer_id in writer_ids {
|
||||
if pool.registry.is_writer_empty(*writer_id).await {
|
||||
candidate_writer_id = Some(*writer_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(old_writer_id) = candidate_writer_id else {
|
||||
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
|
||||
debug!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
%endpoint,
|
||||
alive,
|
||||
required,
|
||||
"Single-endpoint shadow rotation skipped: no empty writer candidate"
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
%endpoint,
|
||||
error = %e,
|
||||
"Single-endpoint shadow rotation connect failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
%endpoint,
|
||||
"Single-endpoint shadow rotation connect timed out"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if !rotate_ok {
|
||||
shadow_rotate_deadline.insert(
|
||||
key,
|
||||
now + interval.min(Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
pool.mark_writer_draining_with_timeout(old_writer_id, pool.force_close_timeout(), false)
|
||||
.await;
|
||||
pool.stats.increment_me_single_endpoint_shadow_rotate_total();
|
||||
shadow_rotate_deadline.insert(key, now + interval);
|
||||
info!(
|
||||
dc = %dc,
|
||||
?family,
|
||||
%endpoint,
|
||||
old_writer_id,
|
||||
rotate_every_secs = interval.as_secs(),
|
||||
"Single-endpoint shadow writer rotated"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ pub use pool_nat::{stun_probe, detect_public_ip};
|
||||
pub use registry::ConnRegistry;
|
||||
pub use secret::fetch_proxy_secret;
|
||||
pub use config_updater::{fetch_proxy_config, me_config_updater};
|
||||
pub use rotation::me_rotation_task;
|
||||
pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task};
|
||||
pub use wire::proto_flags_for_tag;
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{Mutex, Notify, RwLock, mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::MeSocksKdfPolicy;
|
||||
use crate::config::{MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy};
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::network::IpFamily;
|
||||
use crate::network::probe::NetworkDecision;
|
||||
@@ -16,11 +16,19 @@ use crate::transport::UpstreamManager;
|
||||
use super::ConnRegistry;
|
||||
use super::codec::WriterCommand;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub(super) struct RefillDcKey {
|
||||
pub dc: i32,
|
||||
pub family: IpFamily,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MeWriter {
|
||||
pub id: u64,
|
||||
pub addr: SocketAddr,
|
||||
pub generation: u64,
|
||||
pub contour: Arc<AtomicU8>,
|
||||
pub created_at: Instant,
|
||||
pub tx: mpsc::Sender<WriterCommand>,
|
||||
pub cancel: CancellationToken,
|
||||
pub degraded: Arc<AtomicBool>,
|
||||
@@ -29,6 +37,36 @@ pub struct MeWriter {
|
||||
pub allow_drain_fallback: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub(super) enum WriterContour {
|
||||
Warm = 0,
|
||||
Active = 1,
|
||||
Draining = 2,
|
||||
}
|
||||
|
||||
impl WriterContour {
|
||||
pub(super) fn as_u8(self) -> u8 {
|
||||
self as u8
|
||||
}
|
||||
|
||||
pub(super) fn from_u8(value: u8) -> Self {
|
||||
match value {
|
||||
0 => Self::Warm,
|
||||
1 => Self::Active,
|
||||
2 => Self::Draining,
|
||||
_ => Self::Draining,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SecretSnapshot {
|
||||
pub epoch: u64,
|
||||
pub key_selector: u32,
|
||||
pub secret: Vec<u8>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct MePool {
|
||||
pub(super) registry: Arc<ConnRegistry>,
|
||||
@@ -38,7 +76,7 @@ pub struct MePool {
|
||||
pub(super) upstream: Option<Arc<UpstreamManager>>,
|
||||
pub(super) rng: Arc<SecureRandom>,
|
||||
pub(super) proxy_tag: Option<Vec<u8>>,
|
||||
pub(super) proxy_secret: Arc<RwLock<Vec<u8>>>,
|
||||
pub(super) proxy_secret: Arc<RwLock<SecretSnapshot>>,
|
||||
pub(super) nat_ip_cfg: Option<IpAddr>,
|
||||
pub(super) nat_ip_detected: Arc<RwLock<Option<IpAddr>>>,
|
||||
pub(super) nat_probe: bool,
|
||||
@@ -63,6 +101,16 @@ pub struct MePool {
|
||||
pub(super) me_reconnect_backoff_base: Duration,
|
||||
pub(super) me_reconnect_backoff_cap: Duration,
|
||||
pub(super) me_reconnect_fast_retry_count: u32,
|
||||
pub(super) me_single_endpoint_shadow_writers: AtomicU8,
|
||||
pub(super) me_single_endpoint_outage_mode_enabled: AtomicBool,
|
||||
pub(super) me_single_endpoint_outage_disable_quarantine: AtomicBool,
|
||||
pub(super) me_single_endpoint_outage_backoff_min_ms: AtomicU64,
|
||||
pub(super) me_single_endpoint_outage_backoff_max_ms: AtomicU64,
|
||||
pub(super) me_single_endpoint_shadow_rotate_every_secs: AtomicU64,
|
||||
pub(super) me_floor_mode: AtomicU8,
|
||||
pub(super) me_adaptive_floor_idle_secs: AtomicU64,
|
||||
pub(super) me_adaptive_floor_min_writers_single_endpoint: AtomicU8,
|
||||
pub(super) me_adaptive_floor_recover_grace_secs: AtomicU64,
|
||||
pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
||||
pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
||||
pub(super) default_dc: AtomicI32,
|
||||
@@ -72,10 +120,18 @@ pub struct MePool {
|
||||
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
|
||||
pub(super) writer_available: Arc<Notify>,
|
||||
pub(super) refill_inflight: Arc<Mutex<HashSet<SocketAddr>>>,
|
||||
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
|
||||
pub(super) conn_count: AtomicUsize,
|
||||
pub(super) stats: Arc<crate::stats::Stats>,
|
||||
pub(super) generation: AtomicU64,
|
||||
pub(super) active_generation: AtomicU64,
|
||||
pub(super) warm_generation: AtomicU64,
|
||||
pub(super) pending_hardswap_generation: AtomicU64,
|
||||
pub(super) pending_hardswap_started_at_epoch_secs: AtomicU64,
|
||||
pub(super) pending_hardswap_map_hash: AtomicU64,
|
||||
pub(super) hardswap: AtomicBool,
|
||||
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
|
||||
pub(super) kdf_material_fingerprint: Arc<Mutex<HashMap<SocketAddr, (u64, u16)>>>,
|
||||
pub(super) me_pool_drain_ttl_secs: AtomicU64,
|
||||
pub(super) me_pool_force_close_secs: AtomicU64,
|
||||
pub(super) me_pool_min_fresh_ratio_permille: AtomicU32,
|
||||
@@ -83,6 +139,10 @@ pub struct MePool {
|
||||
pub(super) me_hardswap_warmup_delay_max_ms: AtomicU64,
|
||||
pub(super) me_hardswap_warmup_extra_passes: AtomicU32,
|
||||
pub(super) me_hardswap_warmup_pass_backoff_base_ms: AtomicU64,
|
||||
pub(super) me_bind_stale_mode: AtomicU8,
|
||||
pub(super) me_bind_stale_ttl_secs: AtomicU64,
|
||||
pub(super) secret_atomic_snapshot: AtomicBool,
|
||||
pub(super) me_deterministic_writer_sort: AtomicBool,
|
||||
pub(super) me_socks_kdf_policy: AtomicU8,
|
||||
pool_size: usize,
|
||||
}
|
||||
@@ -139,6 +199,16 @@ impl MePool {
|
||||
me_reconnect_backoff_base_ms: u64,
|
||||
me_reconnect_backoff_cap_ms: u64,
|
||||
me_reconnect_fast_retry_count: u32,
|
||||
me_single_endpoint_shadow_writers: u8,
|
||||
me_single_endpoint_outage_mode_enabled: bool,
|
||||
me_single_endpoint_outage_disable_quarantine: bool,
|
||||
me_single_endpoint_outage_backoff_min_ms: u64,
|
||||
me_single_endpoint_outage_backoff_max_ms: u64,
|
||||
me_single_endpoint_shadow_rotate_every_secs: u64,
|
||||
me_floor_mode: MeFloorMode,
|
||||
me_adaptive_floor_idle_secs: u64,
|
||||
me_adaptive_floor_min_writers_single_endpoint: u8,
|
||||
me_adaptive_floor_recover_grace_secs: u64,
|
||||
hardswap: bool,
|
||||
me_pool_drain_ttl_secs: u64,
|
||||
me_pool_force_close_secs: u64,
|
||||
@@ -147,6 +217,10 @@ impl MePool {
|
||||
me_hardswap_warmup_delay_max_ms: u64,
|
||||
me_hardswap_warmup_extra_passes: u8,
|
||||
me_hardswap_warmup_pass_backoff_base_ms: u64,
|
||||
me_bind_stale_mode: MeBindStaleMode,
|
||||
me_bind_stale_ttl_secs: u64,
|
||||
me_secret_atomic_snapshot: bool,
|
||||
me_deterministic_writer_sort: bool,
|
||||
me_socks_kdf_policy: MeSocksKdfPolicy,
|
||||
me_route_backpressure_base_timeout_ms: u64,
|
||||
me_route_backpressure_high_timeout_ms: u64,
|
||||
@@ -166,7 +240,20 @@ impl MePool {
|
||||
upstream,
|
||||
rng,
|
||||
proxy_tag,
|
||||
proxy_secret: Arc::new(RwLock::new(proxy_secret)),
|
||||
proxy_secret: Arc::new(RwLock::new(SecretSnapshot {
|
||||
epoch: 1,
|
||||
key_selector: if proxy_secret.len() >= 4 {
|
||||
u32::from_le_bytes([
|
||||
proxy_secret[0],
|
||||
proxy_secret[1],
|
||||
proxy_secret[2],
|
||||
proxy_secret[3],
|
||||
])
|
||||
} else {
|
||||
0
|
||||
},
|
||||
secret: proxy_secret,
|
||||
})),
|
||||
nat_ip_cfg: nat_ip,
|
||||
nat_ip_detected: Arc::new(RwLock::new(None)),
|
||||
nat_probe,
|
||||
@@ -192,6 +279,30 @@ impl MePool {
|
||||
me_reconnect_backoff_base: Duration::from_millis(me_reconnect_backoff_base_ms),
|
||||
me_reconnect_backoff_cap: Duration::from_millis(me_reconnect_backoff_cap_ms),
|
||||
me_reconnect_fast_retry_count,
|
||||
me_single_endpoint_shadow_writers: AtomicU8::new(me_single_endpoint_shadow_writers),
|
||||
me_single_endpoint_outage_mode_enabled: AtomicBool::new(
|
||||
me_single_endpoint_outage_mode_enabled,
|
||||
),
|
||||
me_single_endpoint_outage_disable_quarantine: AtomicBool::new(
|
||||
me_single_endpoint_outage_disable_quarantine,
|
||||
),
|
||||
me_single_endpoint_outage_backoff_min_ms: AtomicU64::new(
|
||||
me_single_endpoint_outage_backoff_min_ms,
|
||||
),
|
||||
me_single_endpoint_outage_backoff_max_ms: AtomicU64::new(
|
||||
me_single_endpoint_outage_backoff_max_ms,
|
||||
),
|
||||
me_single_endpoint_shadow_rotate_every_secs: AtomicU64::new(
|
||||
me_single_endpoint_shadow_rotate_every_secs,
|
||||
),
|
||||
me_floor_mode: AtomicU8::new(me_floor_mode.as_u8()),
|
||||
me_adaptive_floor_idle_secs: AtomicU64::new(me_adaptive_floor_idle_secs),
|
||||
me_adaptive_floor_min_writers_single_endpoint: AtomicU8::new(
|
||||
me_adaptive_floor_min_writers_single_endpoint,
|
||||
),
|
||||
me_adaptive_floor_recover_grace_secs: AtomicU64::new(
|
||||
me_adaptive_floor_recover_grace_secs,
|
||||
),
|
||||
pool_size: 2,
|
||||
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
|
||||
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
|
||||
@@ -202,9 +313,17 @@ impl MePool {
|
||||
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
|
||||
writer_available: Arc::new(Notify::new()),
|
||||
refill_inflight: Arc::new(Mutex::new(HashSet::new())),
|
||||
refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())),
|
||||
conn_count: AtomicUsize::new(0),
|
||||
generation: AtomicU64::new(1),
|
||||
active_generation: AtomicU64::new(1),
|
||||
warm_generation: AtomicU64::new(0),
|
||||
pending_hardswap_generation: AtomicU64::new(0),
|
||||
pending_hardswap_started_at_epoch_secs: AtomicU64::new(0),
|
||||
pending_hardswap_map_hash: AtomicU64::new(0),
|
||||
hardswap: AtomicBool::new(hardswap),
|
||||
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
|
||||
kdf_material_fingerprint: Arc::new(Mutex::new(HashMap::new())),
|
||||
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
|
||||
me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs),
|
||||
me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille(
|
||||
@@ -216,16 +335,16 @@ impl MePool {
|
||||
me_hardswap_warmup_pass_backoff_base_ms: AtomicU64::new(
|
||||
me_hardswap_warmup_pass_backoff_base_ms,
|
||||
),
|
||||
me_bind_stale_mode: AtomicU8::new(me_bind_stale_mode.as_u8()),
|
||||
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
|
||||
secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot),
|
||||
me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort),
|
||||
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn has_proxy_tag(&self) -> bool {
|
||||
self.proxy_tag.is_some()
|
||||
}
|
||||
|
||||
pub fn current_generation(&self) -> u64 {
|
||||
self.generation.load(Ordering::Relaxed)
|
||||
self.active_generation.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn update_runtime_reinit_policy(
|
||||
@@ -238,6 +357,20 @@ impl MePool {
|
||||
hardswap_warmup_delay_max_ms: u64,
|
||||
hardswap_warmup_extra_passes: u8,
|
||||
hardswap_warmup_pass_backoff_base_ms: u64,
|
||||
bind_stale_mode: MeBindStaleMode,
|
||||
bind_stale_ttl_secs: u64,
|
||||
secret_atomic_snapshot: bool,
|
||||
deterministic_writer_sort: bool,
|
||||
single_endpoint_shadow_writers: u8,
|
||||
single_endpoint_outage_mode_enabled: bool,
|
||||
single_endpoint_outage_disable_quarantine: bool,
|
||||
single_endpoint_outage_backoff_min_ms: u64,
|
||||
single_endpoint_outage_backoff_max_ms: u64,
|
||||
single_endpoint_shadow_rotate_every_secs: u64,
|
||||
floor_mode: MeFloorMode,
|
||||
adaptive_floor_idle_secs: u64,
|
||||
adaptive_floor_min_writers_single_endpoint: u8,
|
||||
adaptive_floor_recover_grace_secs: u64,
|
||||
) {
|
||||
self.hardswap.store(hardswap, Ordering::Relaxed);
|
||||
self.me_pool_drain_ttl_secs
|
||||
@@ -254,6 +387,49 @@ impl MePool {
|
||||
.store(hardswap_warmup_extra_passes as u32, Ordering::Relaxed);
|
||||
self.me_hardswap_warmup_pass_backoff_base_ms
|
||||
.store(hardswap_warmup_pass_backoff_base_ms, Ordering::Relaxed);
|
||||
self.me_bind_stale_mode
|
||||
.store(bind_stale_mode.as_u8(), Ordering::Relaxed);
|
||||
self.me_bind_stale_ttl_secs
|
||||
.store(bind_stale_ttl_secs, Ordering::Relaxed);
|
||||
self.secret_atomic_snapshot
|
||||
.store(secret_atomic_snapshot, Ordering::Relaxed);
|
||||
self.me_deterministic_writer_sort
|
||||
.store(deterministic_writer_sort, Ordering::Relaxed);
|
||||
self.me_single_endpoint_shadow_writers
|
||||
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
|
||||
self.me_single_endpoint_outage_mode_enabled
|
||||
.store(single_endpoint_outage_mode_enabled, Ordering::Relaxed);
|
||||
self.me_single_endpoint_outage_disable_quarantine
|
||||
.store(single_endpoint_outage_disable_quarantine, Ordering::Relaxed);
|
||||
self.me_single_endpoint_outage_backoff_min_ms
|
||||
.store(single_endpoint_outage_backoff_min_ms, Ordering::Relaxed);
|
||||
self.me_single_endpoint_outage_backoff_max_ms
|
||||
.store(single_endpoint_outage_backoff_max_ms, Ordering::Relaxed);
|
||||
self.me_single_endpoint_shadow_rotate_every_secs
|
||||
.store(single_endpoint_shadow_rotate_every_secs, Ordering::Relaxed);
|
||||
let previous_floor_mode = self.floor_mode();
|
||||
self.me_floor_mode
|
||||
.store(floor_mode.as_u8(), Ordering::Relaxed);
|
||||
self.me_adaptive_floor_idle_secs
|
||||
.store(adaptive_floor_idle_secs, Ordering::Relaxed);
|
||||
self.me_adaptive_floor_min_writers_single_endpoint
|
||||
.store(adaptive_floor_min_writers_single_endpoint, Ordering::Relaxed);
|
||||
self.me_adaptive_floor_recover_grace_secs
|
||||
.store(adaptive_floor_recover_grace_secs, Ordering::Relaxed);
|
||||
if previous_floor_mode != floor_mode {
|
||||
self.stats.increment_me_floor_mode_switch_total();
|
||||
match (previous_floor_mode, floor_mode) {
|
||||
(MeFloorMode::Static, MeFloorMode::Adaptive) => {
|
||||
self.stats
|
||||
.increment_me_floor_mode_switch_static_to_adaptive_total();
|
||||
}
|
||||
(MeFloorMode::Adaptive, MeFloorMode::Static) => {
|
||||
self.stats
|
||||
.increment_me_floor_mode_switch_adaptive_to_static_total();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset_stun_state(&self) {
|
||||
@@ -307,11 +483,96 @@ impl MePool {
|
||||
}
|
||||
|
||||
pub(super) async fn key_selector(&self) -> u32 {
|
||||
let secret = self.proxy_secret.read().await;
|
||||
if secret.len() >= 4 {
|
||||
u32::from_le_bytes([secret[0], secret[1], secret[2], secret[3]])
|
||||
self.proxy_secret.read().await.key_selector
|
||||
}
|
||||
|
||||
pub(super) async fn secret_snapshot(&self) -> SecretSnapshot {
|
||||
self.proxy_secret.read().await.clone()
|
||||
}
|
||||
|
||||
pub(super) fn bind_stale_mode(&self) -> MeBindStaleMode {
|
||||
MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize {
|
||||
if endpoint_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
if endpoint_count == 1 {
|
||||
let shadow = self
|
||||
.me_single_endpoint_shadow_writers
|
||||
.load(Ordering::Relaxed) as usize;
|
||||
return (1 + shadow).max(3);
|
||||
}
|
||||
endpoint_count.max(3)
|
||||
}
|
||||
|
||||
pub(super) fn floor_mode(&self) -> MeFloorMode {
|
||||
MeFloorMode::from_u8(self.me_floor_mode.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
pub(super) fn adaptive_floor_idle_duration(&self) -> Duration {
|
||||
Duration::from_secs(self.me_adaptive_floor_idle_secs.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
pub(super) fn adaptive_floor_recover_grace_duration(&self) -> Duration {
|
||||
Duration::from_secs(
|
||||
self.me_adaptive_floor_recover_grace_secs
|
||||
.load(Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn required_writers_for_dc_with_floor_mode(
|
||||
&self,
|
||||
endpoint_count: usize,
|
||||
reduce_for_idle: bool,
|
||||
) -> usize {
|
||||
let base_required = self.required_writers_for_dc(endpoint_count);
|
||||
if !reduce_for_idle {
|
||||
return base_required;
|
||||
}
|
||||
if endpoint_count != 1 || self.floor_mode() != MeFloorMode::Adaptive {
|
||||
return base_required;
|
||||
}
|
||||
let min_writers = (self
|
||||
.me_adaptive_floor_min_writers_single_endpoint
|
||||
.load(Ordering::Relaxed) as usize)
|
||||
.max(1);
|
||||
base_required.min(min_writers)
|
||||
}
|
||||
|
||||
pub(super) fn single_endpoint_outage_mode_enabled(&self) -> bool {
|
||||
self.me_single_endpoint_outage_mode_enabled
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(super) fn single_endpoint_outage_disable_quarantine(&self) -> bool {
|
||||
self.me_single_endpoint_outage_disable_quarantine
|
||||
.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(super) fn single_endpoint_outage_backoff_bounds_ms(&self) -> (u64, u64) {
|
||||
let min_ms = self
|
||||
.me_single_endpoint_outage_backoff_min_ms
|
||||
.load(Ordering::Relaxed);
|
||||
let max_ms = self
|
||||
.me_single_endpoint_outage_backoff_max_ms
|
||||
.load(Ordering::Relaxed);
|
||||
if min_ms <= max_ms {
|
||||
(min_ms, max_ms)
|
||||
} else {
|
||||
0
|
||||
(max_ms, min_ms)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn single_endpoint_shadow_rotate_interval(&self) -> Option<Duration> {
|
||||
let secs = self
|
||||
.me_single_endpoint_shadow_rotate_every_secs
|
||||
.load(Ordering::Relaxed);
|
||||
if secs == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(Duration::from_secs(secs))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,12 +7,29 @@ use tracing::warn;
|
||||
|
||||
use super::pool::MePool;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SnapshotApplyOutcome {
|
||||
AppliedChanged,
|
||||
AppliedNoDelta,
|
||||
RejectedEmpty,
|
||||
}
|
||||
|
||||
impl SnapshotApplyOutcome {
|
||||
pub fn changed(self) -> bool {
|
||||
matches!(self, SnapshotApplyOutcome::AppliedChanged)
|
||||
}
|
||||
}
|
||||
|
||||
impl MePool {
|
||||
pub async fn update_proxy_maps(
|
||||
&self,
|
||||
new_v4: HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||
new_v6: Option<HashMap<i32, Vec<(IpAddr, u16)>>>,
|
||||
) -> bool {
|
||||
) -> SnapshotApplyOutcome {
|
||||
if new_v4.is_empty() && new_v6.as_ref().is_none_or(|v| v.is_empty()) {
|
||||
return SnapshotApplyOutcome::RejectedEmpty;
|
||||
}
|
||||
|
||||
let mut changed = false;
|
||||
{
|
||||
let mut guard = self.proxy_map_v4.write().await;
|
||||
@@ -51,7 +68,11 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
}
|
||||
changed
|
||||
if changed {
|
||||
SnapshotApplyOutcome::AppliedChanged
|
||||
} else {
|
||||
SnapshotApplyOutcome::AppliedNoDelta
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn update_secret(self: &Arc<Self>, new_secret: Vec<u8>) -> bool {
|
||||
@@ -60,8 +81,19 @@ impl MePool {
|
||||
return false;
|
||||
}
|
||||
let mut guard = self.proxy_secret.write().await;
|
||||
if *guard != new_secret {
|
||||
*guard = new_secret;
|
||||
if guard.secret != new_secret {
|
||||
guard.secret = new_secret;
|
||||
guard.key_selector = if guard.secret.len() >= 4 {
|
||||
u32::from_le_bytes([
|
||||
guard.secret[0],
|
||||
guard.secret[1],
|
||||
guard.secret[2],
|
||||
guard.secret[3],
|
||||
])
|
||||
} else {
|
||||
0
|
||||
};
|
||||
guard.epoch = guard.epoch.saturating_add(1);
|
||||
drop(guard);
|
||||
self.reconnect_all().await;
|
||||
return true;
|
||||
|
||||
@@ -19,7 +19,7 @@ impl MePool {
|
||||
me_servers = self.proxy_map_v4.read().await.len(),
|
||||
pool_size,
|
||||
key_selector = format_args!("0x{ks:08x}"),
|
||||
secret_len = self.proxy_secret.read().await.len(),
|
||||
secret_len = self.proxy_secret.read().await.secret.len(),
|
||||
"Initializing ME pool"
|
||||
);
|
||||
|
||||
|
||||
@@ -2,27 +2,173 @@ use std::collections::HashSet;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::network::IpFamily;
|
||||
|
||||
use super::pool::MePool;
|
||||
use super::pool::{MePool, RefillDcKey, WriterContour};
|
||||
|
||||
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
|
||||
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
|
||||
|
||||
impl MePool {
|
||||
pub(super) async fn maybe_quarantine_flapping_endpoint(
|
||||
&self,
|
||||
addr: SocketAddr,
|
||||
uptime: Duration,
|
||||
) {
|
||||
if uptime > Duration::from_secs(ME_FLAP_UPTIME_THRESHOLD_SECS) {
|
||||
return;
|
||||
}
|
||||
|
||||
let until = Instant::now() + Duration::from_secs(ME_FLAP_QUARANTINE_SECS);
|
||||
let mut guard = self.endpoint_quarantine.lock().await;
|
||||
guard.retain(|_, expiry| *expiry > Instant::now());
|
||||
guard.insert(addr, until);
|
||||
self.stats.increment_me_endpoint_quarantine_total();
|
||||
warn!(
|
||||
%addr,
|
||||
uptime_ms = uptime.as_millis(),
|
||||
quarantine_secs = ME_FLAP_QUARANTINE_SECS,
|
||||
"ME endpoint temporarily quarantined due to rapid writer flap"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) async fn is_endpoint_quarantined(&self, addr: SocketAddr) -> bool {
|
||||
let mut guard = self.endpoint_quarantine.lock().await;
|
||||
let now = Instant::now();
|
||||
guard.retain(|_, expiry| *expiry > now);
|
||||
guard.contains_key(&addr)
|
||||
}
|
||||
|
||||
async fn connectable_endpoints(&self, endpoints: &[SocketAddr]) -> Vec<SocketAddr> {
|
||||
if endpoints.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut guard = self.endpoint_quarantine.lock().await;
|
||||
let now = Instant::now();
|
||||
guard.retain(|_, expiry| *expiry > now);
|
||||
|
||||
let mut ready = Vec::<SocketAddr>::with_capacity(endpoints.len());
|
||||
let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None;
|
||||
for addr in endpoints {
|
||||
if let Some(expiry) = guard.get(addr).copied() {
|
||||
match earliest_quarantine {
|
||||
Some((_, current_expiry)) if current_expiry <= expiry => {}
|
||||
_ => earliest_quarantine = Some((*addr, expiry)),
|
||||
}
|
||||
} else {
|
||||
ready.push(*addr);
|
||||
}
|
||||
}
|
||||
|
||||
if !ready.is_empty() {
|
||||
return ready;
|
||||
}
|
||||
|
||||
if let Some((addr, expiry)) = earliest_quarantine {
|
||||
debug!(
|
||||
%addr,
|
||||
wait_ms = expiry.saturating_duration_since(now).as_millis(),
|
||||
"All ME endpoints are quarantined for the DC group; retrying earliest one"
|
||||
);
|
||||
return vec![addr];
|
||||
}
|
||||
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
pub(super) async fn has_refill_inflight_for_endpoints(&self, endpoints: &[SocketAddr]) -> bool {
|
||||
if endpoints.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
{
|
||||
let guard = self.refill_inflight.lock().await;
|
||||
if endpoints.iter().any(|addr| guard.contains(addr)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
let dc_keys = self.resolve_refill_dc_keys_for_endpoints(endpoints).await;
|
||||
if dc_keys.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let guard = self.refill_inflight_dc.lock().await;
|
||||
dc_keys.iter().any(|key| guard.contains(key))
|
||||
}
|
||||
|
||||
async fn resolve_refill_dc_key_for_addr(&self, addr: SocketAddr) -> Option<RefillDcKey> {
|
||||
let family = if addr.is_ipv4() {
|
||||
IpFamily::V4
|
||||
} else {
|
||||
IpFamily::V6
|
||||
};
|
||||
let map = self.proxy_map_for_family(family).await;
|
||||
for (dc, endpoints) in map {
|
||||
if endpoints
|
||||
.into_iter()
|
||||
.any(|(ip, port)| SocketAddr::new(ip, port) == addr)
|
||||
{
|
||||
return Some(RefillDcKey {
|
||||
dc: dc.abs(),
|
||||
family,
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn resolve_refill_dc_keys_for_endpoints(
|
||||
&self,
|
||||
endpoints: &[SocketAddr],
|
||||
) -> HashSet<RefillDcKey> {
|
||||
let mut out = HashSet::<RefillDcKey>::new();
|
||||
for addr in endpoints {
|
||||
if let Some(key) = self.resolve_refill_dc_key_for_addr(*addr).await {
|
||||
out.insert(key);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) async fn connect_endpoints_round_robin(
|
||||
self: &Arc<Self>,
|
||||
endpoints: &[SocketAddr],
|
||||
rng: &SecureRandom,
|
||||
) -> bool {
|
||||
if endpoints.is_empty() {
|
||||
self.connect_endpoints_round_robin_with_generation_contour(
|
||||
endpoints,
|
||||
rng,
|
||||
self.current_generation(),
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn connect_endpoints_round_robin_with_generation_contour(
|
||||
self: &Arc<Self>,
|
||||
endpoints: &[SocketAddr],
|
||||
rng: &SecureRandom,
|
||||
generation: u64,
|
||||
contour: WriterContour,
|
||||
) -> bool {
|
||||
let candidates = self.connectable_endpoints(endpoints).await;
|
||||
if candidates.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % endpoints.len();
|
||||
for offset in 0..endpoints.len() {
|
||||
let idx = (start + offset) % endpoints.len();
|
||||
let addr = endpoints[idx];
|
||||
match self.connect_one(addr, rng).await {
|
||||
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len();
|
||||
for offset in 0..candidates.len() {
|
||||
let idx = (start + offset) % candidates.len();
|
||||
let addr = candidates[idx];
|
||||
match self
|
||||
.connect_one_with_generation_contour(addr, rng, generation, contour)
|
||||
.await
|
||||
{
|
||||
Ok(()) => return true,
|
||||
Err(e) => debug!(%addr, error = %e, "ME connect failed during round-robin warmup"),
|
||||
}
|
||||
@@ -83,29 +229,37 @@ impl MePool {
|
||||
|
||||
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr) -> bool {
|
||||
let fast_retries = self.me_reconnect_fast_retry_count.max(1);
|
||||
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
||||
|
||||
for attempt in 0..fast_retries {
|
||||
self.stats.increment_me_reconnect_attempt();
|
||||
match self.connect_one(addr, self.rng.as_ref()).await {
|
||||
Ok(()) => {
|
||||
self.stats.increment_me_reconnect_success();
|
||||
self.stats.increment_me_writer_restored_same_endpoint_total();
|
||||
info!(
|
||||
%addr,
|
||||
attempt = attempt + 1,
|
||||
"ME writer restored on the same endpoint"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
%addr,
|
||||
attempt = attempt + 1,
|
||||
error = %e,
|
||||
"ME immediate same-endpoint reconnect failed"
|
||||
);
|
||||
if !same_endpoint_quarantined {
|
||||
for attempt in 0..fast_retries {
|
||||
self.stats.increment_me_reconnect_attempt();
|
||||
match self.connect_one(addr, self.rng.as_ref()).await {
|
||||
Ok(()) => {
|
||||
self.stats.increment_me_reconnect_success();
|
||||
self.stats.increment_me_writer_restored_same_endpoint_total();
|
||||
info!(
|
||||
%addr,
|
||||
attempt = attempt + 1,
|
||||
"ME writer restored on the same endpoint"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
%addr,
|
||||
attempt = attempt + 1,
|
||||
error = %e,
|
||||
"ME immediate same-endpoint reconnect failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
%addr,
|
||||
"Skipping immediate same-endpoint reconnect because endpoint is quarantined"
|
||||
);
|
||||
}
|
||||
|
||||
let dc_endpoints = self.endpoints_for_same_dc(addr).await;
|
||||
@@ -138,6 +292,9 @@ impl MePool {
|
||||
pub(crate) fn trigger_immediate_refill(self: &Arc<Self>, addr: SocketAddr) {
|
||||
let pool = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
let dc_endpoints = pool.endpoints_for_same_dc(addr).await;
|
||||
let dc_keys = pool.resolve_refill_dc_keys_for_endpoints(&dc_endpoints).await;
|
||||
|
||||
{
|
||||
let mut guard = pool.refill_inflight.lock().await;
|
||||
if !guard.insert(addr) {
|
||||
@@ -145,6 +302,19 @@ impl MePool {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if !dc_keys.is_empty() {
|
||||
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
||||
if dc_keys.iter().any(|key| dc_guard.contains(key)) {
|
||||
pool.stats.increment_me_refill_skipped_inflight_total();
|
||||
drop(dc_guard);
|
||||
let mut guard = pool.refill_inflight.lock().await;
|
||||
guard.remove(&addr);
|
||||
return;
|
||||
}
|
||||
dc_guard.extend(dc_keys.iter().copied());
|
||||
}
|
||||
|
||||
pool.stats.increment_me_refill_triggered_total();
|
||||
|
||||
let restored = pool.refill_writer_after_loss(addr).await;
|
||||
@@ -154,6 +324,13 @@ impl MePool {
|
||||
|
||||
let mut guard = pool.refill_inflight.lock().await;
|
||||
guard.remove(&addr);
|
||||
drop(guard);
|
||||
if !dc_keys.is_empty() {
|
||||
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
||||
for key in &dc_keys {
|
||||
dc_guard.remove(key);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -7,12 +8,58 @@ use std::time::Duration;
|
||||
use rand::Rng;
|
||||
use rand::seq::SliceRandom;
|
||||
use tracing::{debug, info, warn};
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
|
||||
use crate::crypto::SecureRandom;
|
||||
|
||||
use super::pool::MePool;
|
||||
use super::pool::{MePool, WriterContour};
|
||||
|
||||
const ME_HARDSWAP_PENDING_TTL_SECS: u64 = 1800;
|
||||
|
||||
impl MePool {
|
||||
fn desired_map_hash(desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
let mut dcs: Vec<i32> = desired_by_dc.keys().copied().collect();
|
||||
dcs.sort_unstable();
|
||||
for dc in dcs {
|
||||
dc.hash(&mut hasher);
|
||||
let mut endpoints: Vec<SocketAddr> = desired_by_dc
|
||||
.get(&dc)
|
||||
.map(|set| set.iter().copied().collect())
|
||||
.unwrap_or_default();
|
||||
endpoints.sort_unstable();
|
||||
for endpoint in endpoints {
|
||||
endpoint.hash(&mut hasher);
|
||||
}
|
||||
}
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
fn clear_pending_hardswap_state(&self) {
|
||||
self.pending_hardswap_generation.store(0, Ordering::Relaxed);
|
||||
self.pending_hardswap_started_at_epoch_secs
|
||||
.store(0, Ordering::Relaxed);
|
||||
self.pending_hardswap_map_hash.store(0, Ordering::Relaxed);
|
||||
self.warm_generation.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn promote_warm_generation_to_active(&self, generation: u64) {
|
||||
self.active_generation.store(generation, Ordering::Relaxed);
|
||||
self.warm_generation.store(0, Ordering::Relaxed);
|
||||
|
||||
let ws = self.writers.read().await;
|
||||
for writer in ws.iter() {
|
||||
if writer.draining.load(Ordering::Relaxed) {
|
||||
continue;
|
||||
}
|
||||
if writer.generation == generation {
|
||||
writer
|
||||
.contour
|
||||
.store(WriterContour::Active.as_u8(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn coverage_ratio(
|
||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||
active_writer_addrs: &HashSet<SocketAddr>,
|
||||
@@ -101,10 +148,6 @@ impl MePool {
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) fn required_writers_for_dc(endpoint_count: usize) -> usize {
|
||||
endpoint_count.max(3)
|
||||
}
|
||||
|
||||
fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
|
||||
let min_ms = self.me_hardswap_warmup_delay_min_ms.load(Ordering::Relaxed);
|
||||
let max_ms = self.me_hardswap_warmup_delay_max_ms.load(Ordering::Relaxed);
|
||||
@@ -174,7 +217,7 @@ impl MePool {
|
||||
|
||||
let mut endpoint_list: Vec<SocketAddr> = endpoints.iter().copied().collect();
|
||||
endpoint_list.sort_unstable();
|
||||
let required = Self::required_writers_for_dc(endpoint_list.len());
|
||||
let required = self.required_writers_for_dc(endpoint_list.len());
|
||||
let mut completed = false;
|
||||
let mut last_fresh_count = self
|
||||
.fresh_writer_count_for_endpoints(generation, endpoints)
|
||||
@@ -202,7 +245,14 @@ impl MePool {
|
||||
let delay_ms = self.hardswap_warmup_connect_delay_ms();
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
|
||||
let connected = self.connect_endpoints_round_robin(&endpoint_list, rng).await;
|
||||
let connected = self
|
||||
.connect_endpoints_round_robin_with_generation_contour(
|
||||
&endpoint_list,
|
||||
rng,
|
||||
generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
debug!(
|
||||
dc = *dc,
|
||||
pass = pass_idx + 1,
|
||||
@@ -265,11 +315,61 @@ impl MePool {
|
||||
return;
|
||||
}
|
||||
|
||||
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
|
||||
let now_epoch_secs = Self::now_epoch_secs();
|
||||
let previous_generation = self.current_generation();
|
||||
let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let hardswap = self.hardswap.load(Ordering::Relaxed);
|
||||
let generation = if hardswap {
|
||||
let pending_generation = self.pending_hardswap_generation.load(Ordering::Relaxed);
|
||||
let pending_started_at = self
|
||||
.pending_hardswap_started_at_epoch_secs
|
||||
.load(Ordering::Relaxed);
|
||||
let pending_map_hash = self.pending_hardswap_map_hash.load(Ordering::Relaxed);
|
||||
let pending_age_secs = now_epoch_secs.saturating_sub(pending_started_at);
|
||||
let pending_ttl_expired = pending_started_at > 0 && pending_age_secs > ME_HARDSWAP_PENDING_TTL_SECS;
|
||||
let pending_matches_map = pending_map_hash != 0 && pending_map_hash == desired_map_hash;
|
||||
|
||||
if pending_generation != 0
|
||||
&& pending_generation >= previous_generation
|
||||
&& pending_matches_map
|
||||
&& !pending_ttl_expired
|
||||
{
|
||||
self.stats.increment_me_hardswap_pending_reuse_total();
|
||||
debug!(
|
||||
previous_generation,
|
||||
generation = pending_generation,
|
||||
pending_age_secs,
|
||||
"ME hardswap continues with pending generation"
|
||||
);
|
||||
pending_generation
|
||||
} else {
|
||||
if pending_generation != 0 && pending_ttl_expired {
|
||||
self.stats.increment_me_hardswap_pending_ttl_expired_total();
|
||||
warn!(
|
||||
previous_generation,
|
||||
generation = pending_generation,
|
||||
pending_age_secs,
|
||||
pending_ttl_secs = ME_HARDSWAP_PENDING_TTL_SECS,
|
||||
"ME hardswap pending generation expired by TTL; starting fresh generation"
|
||||
);
|
||||
}
|
||||
let next_generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
self.pending_hardswap_generation
|
||||
.store(next_generation, Ordering::Relaxed);
|
||||
self.pending_hardswap_started_at_epoch_secs
|
||||
.store(now_epoch_secs, Ordering::Relaxed);
|
||||
self.pending_hardswap_map_hash
|
||||
.store(desired_map_hash, Ordering::Relaxed);
|
||||
self.warm_generation.store(next_generation, Ordering::Relaxed);
|
||||
next_generation
|
||||
}
|
||||
} else {
|
||||
self.clear_pending_hardswap_state();
|
||||
self.generation.fetch_add(1, Ordering::Relaxed) + 1
|
||||
};
|
||||
|
||||
if hardswap {
|
||||
self.warm_generation.store(generation, Ordering::Relaxed);
|
||||
self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc)
|
||||
.await;
|
||||
} else {
|
||||
@@ -305,7 +405,7 @@ impl MePool {
|
||||
if endpoints.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let required = Self::required_writers_for_dc(endpoints.len());
|
||||
let required = self.required_writers_for_dc(endpoints.len());
|
||||
let fresh_count = writers
|
||||
.iter()
|
||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||
@@ -334,6 +434,10 @@ impl MePool {
|
||||
return;
|
||||
}
|
||||
|
||||
if hardswap {
|
||||
self.promote_warm_generation_to_active(generation).await;
|
||||
}
|
||||
|
||||
let desired_addrs: HashSet<SocketAddr> = desired_by_dc
|
||||
.values()
|
||||
.flat_map(|set| set.iter().copied())
|
||||
@@ -354,6 +458,9 @@ impl MePool {
|
||||
drop(writers);
|
||||
|
||||
if stale_writer_ids.is_empty() {
|
||||
if hardswap {
|
||||
self.clear_pending_hardswap_state();
|
||||
}
|
||||
debug!("ME reinit cycle completed with no stale writers");
|
||||
return;
|
||||
}
|
||||
@@ -375,6 +482,9 @@ impl MePool {
|
||||
self.mark_writer_draining_with_timeout(writer_id, drain_timeout, !hardswap)
|
||||
.await;
|
||||
}
|
||||
if hardswap {
|
||||
self.clear_pending_hardswap_state();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::BytesMut;
|
||||
@@ -9,12 +9,13 @@ use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::MeBindStaleMode;
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::error::{ProxyError, Result};
|
||||
use crate::protocol::constants::RPC_PING_U32;
|
||||
|
||||
use super::codec::{RpcWriter, WriterCommand};
|
||||
use super::pool::{MePool, MeWriter};
|
||||
use super::pool::{MePool, MeWriter, WriterContour};
|
||||
use super::reader::reader_loop;
|
||||
use super::registry::BoundConn;
|
||||
|
||||
@@ -42,7 +43,23 @@ impl MePool {
|
||||
}
|
||||
|
||||
pub(crate) async fn connect_one(self: &Arc<Self>, addr: SocketAddr, rng: &SecureRandom) -> Result<()> {
|
||||
let secret_len = self.proxy_secret.read().await.len();
|
||||
self.connect_one_with_generation_contour(
|
||||
addr,
|
||||
rng,
|
||||
self.current_generation(),
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn connect_one_with_generation_contour(
|
||||
self: &Arc<Self>,
|
||||
addr: SocketAddr,
|
||||
rng: &SecureRandom,
|
||||
generation: u64,
|
||||
contour: WriterContour,
|
||||
) -> Result<()> {
|
||||
let secret_len = self.proxy_secret.read().await.secret.len();
|
||||
if secret_len < 32 {
|
||||
return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into()));
|
||||
}
|
||||
@@ -51,7 +68,7 @@ impl MePool {
|
||||
let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?;
|
||||
|
||||
let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed);
|
||||
let generation = self.current_generation();
|
||||
let contour = Arc::new(AtomicU8::new(contour.as_u8()));
|
||||
let cancel = CancellationToken::new();
|
||||
let degraded = Arc::new(AtomicBool::new(false));
|
||||
let draining = Arc::new(AtomicBool::new(false));
|
||||
@@ -88,6 +105,8 @@ impl MePool {
|
||||
id: writer_id,
|
||||
addr,
|
||||
generation,
|
||||
contour: contour.clone(),
|
||||
created_at: Instant::now(),
|
||||
tx: tx.clone(),
|
||||
cancel: cancel.clone(),
|
||||
degraded: degraded.clone(),
|
||||
@@ -248,6 +267,7 @@ impl MePool {
|
||||
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
|
||||
let mut close_tx: Option<mpsc::Sender<WriterCommand>> = None;
|
||||
let mut removed_addr: Option<SocketAddr> = None;
|
||||
let mut removed_uptime: Option<Duration> = None;
|
||||
let mut trigger_refill = false;
|
||||
{
|
||||
let mut ws = self.writers.write().await;
|
||||
@@ -260,6 +280,7 @@ impl MePool {
|
||||
self.stats.increment_me_writer_removed_total();
|
||||
w.cancel.cancel();
|
||||
removed_addr = Some(w.addr);
|
||||
removed_uptime = Some(w.created_at.elapsed());
|
||||
trigger_refill = !was_draining;
|
||||
if trigger_refill {
|
||||
self.stats.increment_me_writer_removed_unexpected_total();
|
||||
@@ -274,6 +295,9 @@ impl MePool {
|
||||
if trigger_refill
|
||||
&& let Some(addr) = removed_addr
|
||||
{
|
||||
if let Some(uptime) = removed_uptime {
|
||||
self.maybe_quarantine_flapping_endpoint(addr, uptime).await;
|
||||
}
|
||||
self.trigger_immediate_refill(addr);
|
||||
}
|
||||
self.rtt_stats.lock().await.remove(&writer_id);
|
||||
@@ -298,6 +322,8 @@ impl MePool {
|
||||
if !already_draining {
|
||||
self.stats.increment_pool_drain_active();
|
||||
}
|
||||
w.contour
|
||||
.store(WriterContour::Draining.as_u8(), Ordering::Relaxed);
|
||||
w.draining.store(true, Ordering::Relaxed);
|
||||
true
|
||||
} else {
|
||||
@@ -351,16 +377,22 @@ impl MePool {
|
||||
return false;
|
||||
}
|
||||
|
||||
let ttl_secs = self.me_pool_drain_ttl_secs.load(Ordering::Relaxed);
|
||||
if ttl_secs == 0 {
|
||||
return true;
|
||||
}
|
||||
match self.bind_stale_mode() {
|
||||
MeBindStaleMode::Never => false,
|
||||
MeBindStaleMode::Always => true,
|
||||
MeBindStaleMode::Ttl => {
|
||||
let ttl_secs = self.me_bind_stale_ttl_secs.load(Ordering::Relaxed);
|
||||
if ttl_secs == 0 {
|
||||
return true;
|
||||
}
|
||||
|
||||
let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed);
|
||||
if started == 0 {
|
||||
return false;
|
||||
}
|
||||
let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed);
|
||||
if started == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
Self::now_epoch_secs().saturating_sub(started) <= ttl_secs
|
||||
Self::now_epoch_secs().saturating_sub(started) <= ttl_secs
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io::ErrorKind;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Instant;
|
||||
@@ -45,7 +46,11 @@ pub(crate) async fn reader_loop(
|
||||
_ = cancel.cancelled() => return Ok(()),
|
||||
};
|
||||
if n == 0 {
|
||||
return Ok(());
|
||||
stats.increment_me_reader_eof_total();
|
||||
return Err(ProxyError::Io(std::io::Error::new(
|
||||
ErrorKind::UnexpectedEof,
|
||||
"ME socket closed by peer",
|
||||
)));
|
||||
}
|
||||
raw.extend_from_slice(&tmp[..n]);
|
||||
|
||||
|
||||
@@ -1,19 +1,111 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::watch;
|
||||
use tracing::{info, warn};
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::crypto::SecureRandom;
|
||||
|
||||
use super::MePool;
|
||||
|
||||
/// Periodically reinitialize ME generations and swap them after full warmup.
|
||||
pub async fn me_rotation_task(
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MeReinitTrigger {
|
||||
Periodic,
|
||||
MapChanged,
|
||||
}
|
||||
|
||||
impl MeReinitTrigger {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
MeReinitTrigger::Periodic => "periodic",
|
||||
MeReinitTrigger::MapChanged => "map-change",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn enqueue_reinit_trigger(
|
||||
tx: &mpsc::Sender<MeReinitTrigger>,
|
||||
trigger: MeReinitTrigger,
|
||||
) {
|
||||
match tx.try_send(trigger) {
|
||||
Ok(()) => {}
|
||||
Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => {
|
||||
debug!(trigger = trigger.as_str(), "ME reinit trigger dropped (queue full)");
|
||||
}
|
||||
Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {
|
||||
warn!(trigger = trigger.as_str(), "ME reinit trigger dropped (scheduler closed)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn me_reinit_scheduler(
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
|
||||
) {
|
||||
info!("ME reinit scheduler started");
|
||||
loop {
|
||||
let Some(first_trigger) = trigger_rx.recv().await else {
|
||||
warn!("ME reinit scheduler stopped: trigger channel closed");
|
||||
break;
|
||||
};
|
||||
|
||||
let mut map_change_seen = matches!(first_trigger, MeReinitTrigger::MapChanged);
|
||||
let mut periodic_seen = matches!(first_trigger, MeReinitTrigger::Periodic);
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let coalesce_window = Duration::from_millis(cfg.general.me_reinit_coalesce_window_ms);
|
||||
if !coalesce_window.is_zero() {
|
||||
let deadline = tokio::time::Instant::now() + coalesce_window;
|
||||
loop {
|
||||
let now = tokio::time::Instant::now();
|
||||
if now >= deadline {
|
||||
break;
|
||||
}
|
||||
match tokio::time::timeout(deadline - now, trigger_rx.recv()).await {
|
||||
Ok(Some(next)) => {
|
||||
if next == MeReinitTrigger::MapChanged {
|
||||
map_change_seen = true;
|
||||
} else {
|
||||
periodic_seen = true;
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let reason = if map_change_seen && periodic_seen {
|
||||
"map-change+periodic"
|
||||
} else if map_change_seen {
|
||||
"map-change"
|
||||
} else {
|
||||
"periodic"
|
||||
};
|
||||
|
||||
if cfg.general.me_reinit_singleflight {
|
||||
debug!(reason, "ME reinit scheduled (single-flight)");
|
||||
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
|
||||
} else {
|
||||
debug!(reason, "ME reinit scheduled (concurrent mode)");
|
||||
let pool_clone = pool.clone();
|
||||
let rng_clone = rng.clone();
|
||||
tokio::spawn(async move {
|
||||
pool_clone
|
||||
.zero_downtime_reinit_periodic(rng_clone.as_ref())
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/// Periodically enqueue reinitialization triggers for ME generations.
|
||||
pub async fn me_rotation_task(
|
||||
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
reinit_tx: mpsc::Sender<MeReinitTrigger>,
|
||||
) {
|
||||
let mut interval_secs = config_rx
|
||||
.borrow()
|
||||
@@ -31,7 +123,7 @@ pub async fn me_rotation_task(
|
||||
|
||||
tokio::select! {
|
||||
_ = &mut sleep => {
|
||||
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
|
||||
enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic);
|
||||
let refreshed_secs = config_rx
|
||||
.borrow()
|
||||
.general
|
||||
@@ -70,7 +162,7 @@ pub async fn me_rotation_task(
|
||||
);
|
||||
interval_secs = new_secs;
|
||||
interval = Duration::from_secs(interval_secs);
|
||||
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
|
||||
enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic);
|
||||
next_tick = tokio::time::Instant::now() + interval;
|
||||
} else {
|
||||
info!(
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use std::cmp::Reverse;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::mpsc::error::TrySendError;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
@@ -11,11 +13,13 @@ use crate::protocol::constants::RPC_CLOSE_EXT_U32;
|
||||
|
||||
use super::MePool;
|
||||
use super::codec::WriterCommand;
|
||||
use super::pool::WriterContour;
|
||||
use super::wire::build_proxy_req_payload;
|
||||
use rand::seq::SliceRandom;
|
||||
use super::registry::ConnMeta;
|
||||
|
||||
impl MePool {
|
||||
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
|
||||
pub async fn send_proxy_req(
|
||||
self: &Arc<Self>,
|
||||
conn_id: u64,
|
||||
@@ -24,13 +28,15 @@ impl MePool {
|
||||
our_addr: SocketAddr,
|
||||
data: &[u8],
|
||||
proto_flags: u32,
|
||||
tag_override: Option<&[u8]>,
|
||||
) -> Result<()> {
|
||||
let tag = tag_override.or(self.proxy_tag.as_deref());
|
||||
let payload = build_proxy_req_payload(
|
||||
conn_id,
|
||||
client_addr,
|
||||
our_addr,
|
||||
data,
|
||||
self.proxy_tag.as_deref(),
|
||||
tag,
|
||||
proto_flags,
|
||||
);
|
||||
let meta = ConnMeta {
|
||||
@@ -43,15 +49,17 @@ impl MePool {
|
||||
|
||||
loop {
|
||||
if let Some(current) = self.registry.get_writer(conn_id).await {
|
||||
let send_res = {
|
||||
current
|
||||
.tx
|
||||
.send(WriterCommand::Data(payload.clone()))
|
||||
.await
|
||||
};
|
||||
match send_res {
|
||||
match current.tx.try_send(WriterCommand::Data(payload.clone())) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(_) => {
|
||||
Err(TrySendError::Full(cmd)) => {
|
||||
if current.tx.send(cmd).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
warn!(writer_id = current.writer_id, "ME writer channel closed");
|
||||
self.remove_writer_and_close_clients(current.writer_id).await;
|
||||
continue;
|
||||
}
|
||||
Err(TrySendError::Closed(_)) => {
|
||||
warn!(writer_id = current.writer_id, "ME writer channel closed");
|
||||
self.remove_writer_and_close_clients(current.writer_id).await;
|
||||
continue;
|
||||
@@ -94,7 +102,14 @@ impl MePool {
|
||||
ws.clone()
|
||||
};
|
||||
|
||||
let mut candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await;
|
||||
let mut candidate_indices = self
|
||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
||||
.await;
|
||||
if candidate_indices.is_empty() {
|
||||
candidate_indices = self
|
||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
||||
.await;
|
||||
}
|
||||
if candidate_indices.is_empty() {
|
||||
// Emergency connect-on-demand
|
||||
if emergency_attempts >= 3 {
|
||||
@@ -120,7 +135,14 @@ impl MePool {
|
||||
let ws2 = self.writers.read().await;
|
||||
writers_snapshot = ws2.clone();
|
||||
drop(ws2);
|
||||
candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await;
|
||||
candidate_indices = self
|
||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
||||
.await;
|
||||
if candidate_indices.is_empty() {
|
||||
candidate_indices = self
|
||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
||||
.await;
|
||||
}
|
||||
if !candidate_indices.is_empty() {
|
||||
break;
|
||||
}
|
||||
@@ -131,14 +153,44 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
|
||||
candidate_indices.sort_by_key(|idx| {
|
||||
let w = &writers_snapshot[*idx];
|
||||
let degraded = w.degraded.load(Ordering::Relaxed);
|
||||
let stale = (w.generation < self.current_generation()) as usize;
|
||||
(stale, degraded as usize)
|
||||
});
|
||||
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
|
||||
candidate_indices.sort_by(|lhs, rhs| {
|
||||
let left = &writers_snapshot[*lhs];
|
||||
let right = &writers_snapshot[*rhs];
|
||||
let left_key = (
|
||||
self.writer_contour_rank_for_selection(left),
|
||||
(left.generation < self.current_generation()) as usize,
|
||||
left.degraded.load(Ordering::Relaxed) as usize,
|
||||
Reverse(left.tx.capacity()),
|
||||
left.addr,
|
||||
left.id,
|
||||
);
|
||||
let right_key = (
|
||||
self.writer_contour_rank_for_selection(right),
|
||||
(right.generation < self.current_generation()) as usize,
|
||||
right.degraded.load(Ordering::Relaxed) as usize,
|
||||
Reverse(right.tx.capacity()),
|
||||
right.addr,
|
||||
right.id,
|
||||
);
|
||||
left_key.cmp(&right_key)
|
||||
});
|
||||
} else {
|
||||
candidate_indices.sort_by_key(|idx| {
|
||||
let w = &writers_snapshot[*idx];
|
||||
let degraded = w.degraded.load(Ordering::Relaxed);
|
||||
let stale = (w.generation < self.current_generation()) as usize;
|
||||
(
|
||||
self.writer_contour_rank_for_selection(w),
|
||||
stale,
|
||||
degraded as usize,
|
||||
Reverse(w.tx.capacity()),
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len();
|
||||
let mut fallback_blocking_idx: Option<usize> = None;
|
||||
|
||||
for offset in 0..candidate_indices.len() {
|
||||
let idx = candidate_indices[(start + offset) % candidate_indices.len()];
|
||||
@@ -146,29 +198,41 @@ impl MePool {
|
||||
if !self.writer_accepts_new_binding(w) {
|
||||
continue;
|
||||
}
|
||||
if w.tx.send(WriterCommand::Data(payload.clone())).await.is_ok() {
|
||||
self.registry
|
||||
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
||||
.await;
|
||||
if w.generation < self.current_generation() {
|
||||
self.stats.increment_pool_stale_pick_total();
|
||||
debug!(
|
||||
conn_id,
|
||||
writer_id = w.id,
|
||||
writer_generation = w.generation,
|
||||
current_generation = self.current_generation(),
|
||||
"Selected stale ME writer for fallback bind"
|
||||
);
|
||||
match w.tx.try_send(WriterCommand::Data(payload.clone())) {
|
||||
Ok(()) => {
|
||||
self.registry
|
||||
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
||||
.await;
|
||||
if w.generation < self.current_generation() {
|
||||
self.stats.increment_pool_stale_pick_total();
|
||||
debug!(
|
||||
conn_id,
|
||||
writer_id = w.id,
|
||||
writer_generation = w.generation,
|
||||
current_generation = self.current_generation(),
|
||||
"Selected stale ME writer for fallback bind"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Err(TrySendError::Full(_)) => {
|
||||
if fallback_blocking_idx.is_none() {
|
||||
fallback_blocking_idx = Some(idx);
|
||||
}
|
||||
}
|
||||
Err(TrySendError::Closed(_)) => {
|
||||
warn!(writer_id = w.id, "ME writer channel closed");
|
||||
self.remove_writer_and_close_clients(w.id).await;
|
||||
continue;
|
||||
}
|
||||
return Ok(());
|
||||
} else {
|
||||
warn!(writer_id = w.id, "ME writer channel closed");
|
||||
self.remove_writer_and_close_clients(w.id).await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let w = writers_snapshot[candidate_indices[start]].clone();
|
||||
let Some(blocking_idx) = fallback_blocking_idx else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let w = writers_snapshot[blocking_idx].clone();
|
||||
if !self.writer_accepts_new_binding(&w) {
|
||||
continue;
|
||||
}
|
||||
@@ -215,6 +279,7 @@ impl MePool {
|
||||
&self,
|
||||
writers: &[super::pool::MeWriter],
|
||||
target_dc: i16,
|
||||
include_warm: bool,
|
||||
) -> Vec<usize> {
|
||||
let key = target_dc as i32;
|
||||
let mut preferred = Vec::<SocketAddr>::new();
|
||||
@@ -258,13 +323,13 @@ impl MePool {
|
||||
|
||||
if preferred.is_empty() {
|
||||
return (0..writers.len())
|
||||
.filter(|i| self.writer_accepts_new_binding(&writers[*i]))
|
||||
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
|
||||
.collect();
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (idx, w) in writers.iter().enumerate() {
|
||||
if !self.writer_accepts_new_binding(w) {
|
||||
if !self.writer_eligible_for_selection(w, include_warm) {
|
||||
continue;
|
||||
}
|
||||
if preferred.contains(&w.addr) {
|
||||
@@ -273,10 +338,33 @@ impl MePool {
|
||||
}
|
||||
if out.is_empty() {
|
||||
return (0..writers.len())
|
||||
.filter(|i| self.writer_accepts_new_binding(&writers[*i]))
|
||||
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
|
||||
.collect();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn writer_eligible_for_selection(
|
||||
&self,
|
||||
writer: &super::pool::MeWriter,
|
||||
include_warm: bool,
|
||||
) -> bool {
|
||||
if !self.writer_accepts_new_binding(writer) {
|
||||
return false;
|
||||
}
|
||||
|
||||
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
|
||||
WriterContour::Active => true,
|
||||
WriterContour::Warm => include_warm,
|
||||
WriterContour::Draining => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn writer_contour_rank_for_selection(&self, writer: &super::pool::MeWriter) -> usize {
|
||||
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
|
||||
WriterContour::Active => 0,
|
||||
WriterContour::Warm => 1,
|
||||
WriterContour::Draining => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#![allow(deprecated)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::net::{SocketAddr, IpAddr};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -29,6 +29,12 @@ const NUM_DCS: usize = 5;
|
||||
const DC_PING_TIMEOUT_SECS: u64 = 5;
|
||||
/// Timeout for direct TG DC TCP connect readiness.
|
||||
const DIRECT_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
/// Interval between upstream health-check cycles.
|
||||
const HEALTH_CHECK_INTERVAL_SECS: u64 = 30;
|
||||
/// Timeout for a single health-check connect attempt.
|
||||
const HEALTH_CHECK_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
/// Upstream is considered healthy when at least this many DC groups are reachable.
|
||||
const MIN_HEALTHY_DC_GROUPS: usize = 3;
|
||||
|
||||
// ============= RTT Tracking =============
|
||||
|
||||
@@ -167,6 +173,13 @@ pub struct UpstreamEgressInfo {
|
||||
pub socks_proxy_addr: Option<SocketAddr>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct HealthCheckGroup {
|
||||
dc_idx: i16,
|
||||
primary: Vec<SocketAddr>,
|
||||
fallback: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
// ============= Upstream Manager =============
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -987,41 +1000,144 @@ impl UpstreamManager {
|
||||
Ok(start.elapsed().as_secs_f64() * 1000.0)
|
||||
}
|
||||
|
||||
fn required_healthy_group_count(total_groups: usize) -> usize {
|
||||
if total_groups == 0 {
|
||||
0
|
||||
} else {
|
||||
total_groups.min(MIN_HEALTHY_DC_GROUPS)
|
||||
}
|
||||
}
|
||||
|
||||
fn build_health_check_groups(
|
||||
prefer_ipv6: bool,
|
||||
ipv4_enabled: bool,
|
||||
ipv6_enabled: bool,
|
||||
dc_overrides: &HashMap<String, Vec<String>>,
|
||||
) -> Vec<HealthCheckGroup> {
|
||||
let mut v4_by_dc: HashMap<i16, Vec<SocketAddr>> = HashMap::new();
|
||||
let mut v6_by_dc: HashMap<i16, Vec<SocketAddr>> = HashMap::new();
|
||||
|
||||
if ipv4_enabled {
|
||||
for (idx, dc_ip) in TG_DATACENTERS_V4.iter().enumerate() {
|
||||
let dc_idx = (idx + 1) as i16;
|
||||
v4_by_dc
|
||||
.entry(dc_idx)
|
||||
.or_default()
|
||||
.push(SocketAddr::new(*dc_ip, TG_DATACENTER_PORT));
|
||||
}
|
||||
}
|
||||
|
||||
if ipv6_enabled {
|
||||
for (idx, dc_ip) in TG_DATACENTERS_V6.iter().enumerate() {
|
||||
let dc_idx = (idx + 1) as i16;
|
||||
v6_by_dc
|
||||
.entry(dc_idx)
|
||||
.or_default()
|
||||
.push(SocketAddr::new(*dc_ip, TG_DATACENTER_PORT));
|
||||
}
|
||||
}
|
||||
|
||||
for (dc_key, addrs) in dc_overrides {
|
||||
let dc_idx = match dc_key.parse::<i16>() {
|
||||
Ok(v) if v > 0 => v,
|
||||
_ => {
|
||||
warn!(dc = %dc_key, "Invalid dc_overrides key for health-check, skipping");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for addr_str in addrs {
|
||||
match addr_str.parse::<SocketAddr>() {
|
||||
Ok(addr) if addr.is_ipv6() => {
|
||||
if ipv6_enabled {
|
||||
v6_by_dc.entry(dc_idx).or_default().push(addr);
|
||||
}
|
||||
}
|
||||
Ok(addr) => {
|
||||
if ipv4_enabled {
|
||||
v4_by_dc.entry(dc_idx).or_default().push(addr);
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
dc = %dc_idx,
|
||||
addr = %addr_str,
|
||||
"Invalid dc_overrides address for health-check, skipping"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for addrs in v4_by_dc.values_mut() {
|
||||
addrs.sort_unstable();
|
||||
addrs.dedup();
|
||||
}
|
||||
for addrs in v6_by_dc.values_mut() {
|
||||
addrs.sort_unstable();
|
||||
addrs.dedup();
|
||||
}
|
||||
|
||||
let mut all_dcs = BTreeSet::new();
|
||||
all_dcs.extend(v4_by_dc.keys().copied());
|
||||
all_dcs.extend(v6_by_dc.keys().copied());
|
||||
|
||||
let mut groups = Vec::with_capacity(all_dcs.len());
|
||||
for dc_idx in all_dcs {
|
||||
let v4_endpoints = v4_by_dc.remove(&dc_idx).unwrap_or_default();
|
||||
let v6_endpoints = v6_by_dc.remove(&dc_idx).unwrap_or_default();
|
||||
let (primary, fallback) = if prefer_ipv6 {
|
||||
(v6_endpoints, v4_endpoints)
|
||||
} else {
|
||||
(v4_endpoints, v6_endpoints)
|
||||
};
|
||||
|
||||
if primary.is_empty() && fallback.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
groups.push(HealthCheckGroup {
|
||||
dc_idx,
|
||||
primary,
|
||||
fallback,
|
||||
});
|
||||
}
|
||||
|
||||
groups
|
||||
}
|
||||
|
||||
// ============= Health Checks =============
|
||||
|
||||
/// Background health check: rotates through DCs, 30s interval.
|
||||
/// Uses preferred IP version based on config.
|
||||
pub async fn run_health_checks(&self, prefer_ipv6: bool, ipv4_enabled: bool, ipv6_enabled: bool) {
|
||||
let mut dc_rotation = 0usize;
|
||||
/// Background health check based on reachable DC groups through each upstream.
|
||||
/// Upstream stays healthy while at least `MIN_HEALTHY_DC_GROUPS` groups are reachable.
|
||||
pub async fn run_health_checks(
|
||||
&self,
|
||||
prefer_ipv6: bool,
|
||||
ipv4_enabled: bool,
|
||||
ipv6_enabled: bool,
|
||||
dc_overrides: HashMap<String, Vec<String>>,
|
||||
) {
|
||||
let groups = Self::build_health_check_groups(
|
||||
prefer_ipv6,
|
||||
ipv4_enabled,
|
||||
ipv6_enabled,
|
||||
&dc_overrides,
|
||||
);
|
||||
let required_healthy_groups = Self::required_healthy_group_count(groups.len());
|
||||
let mut endpoint_rotation: HashMap<(usize, i16, bool), usize> = HashMap::new();
|
||||
|
||||
if groups.is_empty() {
|
||||
warn!("No DC groups available for upstream health-checks");
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
tokio::time::sleep(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECS)).await;
|
||||
|
||||
let dc_zero_idx = dc_rotation % NUM_DCS;
|
||||
dc_rotation += 1;
|
||||
|
||||
let primary_v6 = SocketAddr::new(TG_DATACENTERS_V6[dc_zero_idx], TG_DATACENTER_PORT);
|
||||
let primary_v4 = SocketAddr::new(TG_DATACENTERS_V4[dc_zero_idx], TG_DATACENTER_PORT);
|
||||
let dc_addr = if prefer_ipv6 && ipv6_enabled {
|
||||
primary_v6
|
||||
} else if ipv4_enabled {
|
||||
primary_v4
|
||||
} else if ipv6_enabled {
|
||||
primary_v6
|
||||
} else {
|
||||
if groups.is_empty() || required_healthy_groups == 0 {
|
||||
continue;
|
||||
};
|
||||
|
||||
let fallback_addr = if dc_addr.is_ipv6() && ipv4_enabled {
|
||||
Some(primary_v4)
|
||||
} else if dc_addr.is_ipv4() && ipv6_enabled {
|
||||
Some(primary_v6)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
}
|
||||
|
||||
let count = self.upstreams.read().await.len();
|
||||
|
||||
for i in 0..count {
|
||||
let (config, bind_rr) = {
|
||||
let guard = self.upstreams.read().await;
|
||||
@@ -1029,104 +1145,123 @@ impl UpstreamManager {
|
||||
(u.config.clone(), u.bind_rr.clone())
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
self.connect_via_upstream(&config, dc_addr, Some(bind_rr.clone()))
|
||||
).await;
|
||||
let mut healthy_groups = 0usize;
|
||||
let mut latency_updates: Vec<(usize, f64)> = Vec::new();
|
||||
|
||||
match result {
|
||||
Ok(Ok(_stream)) => {
|
||||
let rtt_ms = start.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut guard = self.upstreams.write().await;
|
||||
let u = &mut guard[i];
|
||||
u.dc_latency[dc_zero_idx].update(rtt_ms);
|
||||
for group in &groups {
|
||||
let mut group_ok = false;
|
||||
let mut group_rtt_ms = None;
|
||||
|
||||
if !u.healthy {
|
||||
info!(
|
||||
rtt = format!("{:.0} ms", rtt_ms),
|
||||
dc = dc_zero_idx + 1,
|
||||
"Upstream recovered"
|
||||
);
|
||||
}
|
||||
u.healthy = true;
|
||||
u.fails = 0;
|
||||
u.last_check = std::time::Instant::now();
|
||||
}
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
// Try fallback
|
||||
debug!(dc = dc_zero_idx + 1, "Health check failed, trying fallback");
|
||||
|
||||
if let Some(fallback_addr) = fallback_addr {
|
||||
let start2 = Instant::now();
|
||||
let result2 = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
self.connect_via_upstream(&config, fallback_addr, Some(bind_rr.clone()))
|
||||
).await;
|
||||
|
||||
let mut guard = self.upstreams.write().await;
|
||||
let u = &mut guard[i];
|
||||
|
||||
match result2 {
|
||||
Ok(Ok(_stream)) => {
|
||||
let rtt_ms = start2.elapsed().as_secs_f64() * 1000.0;
|
||||
u.dc_latency[dc_zero_idx].update(rtt_ms);
|
||||
|
||||
if !u.healthy {
|
||||
info!(
|
||||
rtt = format!("{:.0} ms", rtt_ms),
|
||||
dc = dc_zero_idx + 1,
|
||||
"Upstream recovered (fallback)"
|
||||
);
|
||||
}
|
||||
u.healthy = true;
|
||||
u.fails = 0;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
u.fails += 1;
|
||||
debug!(dc = dc_zero_idx + 1, fails = u.fails,
|
||||
"Health check failed (both): {}", e);
|
||||
if u.fails >= self.unhealthy_fail_threshold {
|
||||
u.healthy = false;
|
||||
warn!(
|
||||
fails = u.fails,
|
||||
threshold = self.unhealthy_fail_threshold,
|
||||
"Upstream unhealthy (fails)"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
u.fails += 1;
|
||||
debug!(dc = dc_zero_idx + 1, fails = u.fails,
|
||||
"Health check timeout (both)");
|
||||
if u.fails >= self.unhealthy_fail_threshold {
|
||||
u.healthy = false;
|
||||
warn!(
|
||||
fails = u.fails,
|
||||
threshold = self.unhealthy_fail_threshold,
|
||||
"Upstream unhealthy (timeout)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
u.last_check = std::time::Instant::now();
|
||||
for (is_primary, endpoints) in [(true, &group.primary), (false, &group.fallback)] {
|
||||
if endpoints.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut guard = self.upstreams.write().await;
|
||||
let u = &mut guard[i];
|
||||
u.fails += 1;
|
||||
if u.fails >= self.unhealthy_fail_threshold {
|
||||
u.healthy = false;
|
||||
warn!(
|
||||
fails = u.fails,
|
||||
threshold = self.unhealthy_fail_threshold,
|
||||
"Upstream unhealthy (no fallback family)"
|
||||
);
|
||||
let rotation_key = (i, group.dc_idx, is_primary);
|
||||
let start_idx = *endpoint_rotation.entry(rotation_key).or_insert(0) % endpoints.len();
|
||||
let mut next_idx = (start_idx + 1) % endpoints.len();
|
||||
|
||||
for step in 0..endpoints.len() {
|
||||
let endpoint_idx = (start_idx + step) % endpoints.len();
|
||||
let endpoint = endpoints[endpoint_idx];
|
||||
|
||||
let start = Instant::now();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS),
|
||||
self.connect_via_upstream(&config, endpoint, Some(bind_rr.clone())),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Ok(_stream)) => {
|
||||
group_ok = true;
|
||||
group_rtt_ms = Some(start.elapsed().as_secs_f64() * 1000.0);
|
||||
next_idx = (endpoint_idx + 1) % endpoints.len();
|
||||
break;
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
debug!(
|
||||
upstream = i,
|
||||
dc = group.dc_idx,
|
||||
endpoint = %endpoint,
|
||||
primary = is_primary,
|
||||
error = %e,
|
||||
"Health-check endpoint failed"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(
|
||||
upstream = i,
|
||||
dc = group.dc_idx,
|
||||
endpoint = %endpoint,
|
||||
primary = is_primary,
|
||||
"Health-check endpoint timed out"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
endpoint_rotation.insert(rotation_key, next_idx);
|
||||
|
||||
if group_ok {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if group_ok {
|
||||
healthy_groups += 1;
|
||||
if let (Some(dc_array_idx), Some(rtt_ms)) =
|
||||
(UpstreamState::dc_array_idx(group.dc_idx), group_rtt_ms)
|
||||
{
|
||||
latency_updates.push((dc_array_idx, rtt_ms));
|
||||
}
|
||||
u.last_check = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
let mut guard = self.upstreams.write().await;
|
||||
let u = &mut guard[i];
|
||||
|
||||
for (dc_array_idx, rtt_ms) in latency_updates {
|
||||
u.dc_latency[dc_array_idx].update(rtt_ms);
|
||||
}
|
||||
|
||||
if healthy_groups >= required_healthy_groups {
|
||||
if !u.healthy {
|
||||
info!(
|
||||
upstream = i,
|
||||
healthy_groups,
|
||||
total_groups = groups.len(),
|
||||
required_groups = required_healthy_groups,
|
||||
"Upstream recovered by DC-group health threshold"
|
||||
);
|
||||
}
|
||||
u.healthy = true;
|
||||
u.fails = 0;
|
||||
} else {
|
||||
u.fails += 1;
|
||||
debug!(
|
||||
upstream = i,
|
||||
healthy_groups,
|
||||
total_groups = groups.len(),
|
||||
required_groups = required_healthy_groups,
|
||||
fails = u.fails,
|
||||
"Upstream health-check below DC-group threshold"
|
||||
);
|
||||
if u.fails >= self.unhealthy_fail_threshold {
|
||||
u.healthy = false;
|
||||
warn!(
|
||||
upstream = i,
|
||||
healthy_groups,
|
||||
total_groups = groups.len(),
|
||||
required_groups = required_healthy_groups,
|
||||
fails = u.fails,
|
||||
threshold = self.unhealthy_fail_threshold,
|
||||
"Upstream unhealthy (insufficient reachable DC groups)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
u.last_check = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1157,3 +1292,76 @@ impl UpstreamManager {
|
||||
Some(SocketAddr::new(ip, TG_DATACENTER_PORT))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn required_healthy_group_count_applies_three_group_threshold() {
|
||||
assert_eq!(UpstreamManager::required_healthy_group_count(0), 0);
|
||||
assert_eq!(UpstreamManager::required_healthy_group_count(1), 1);
|
||||
assert_eq!(UpstreamManager::required_healthy_group_count(2), 2);
|
||||
assert_eq!(UpstreamManager::required_healthy_group_count(3), 3);
|
||||
assert_eq!(UpstreamManager::required_healthy_group_count(5), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_health_check_groups_merges_family_endpoints_with_preference() {
|
||||
let mut overrides = HashMap::new();
|
||||
overrides.insert(
|
||||
"2".to_string(),
|
||||
vec![
|
||||
"203.0.113.10:443".to_string(),
|
||||
"203.0.113.11:443".to_string(),
|
||||
"[2001:db8::10]:443".to_string(),
|
||||
],
|
||||
);
|
||||
|
||||
let groups = UpstreamManager::build_health_check_groups(true, true, true, &overrides);
|
||||
let dc2 = groups
|
||||
.iter()
|
||||
.find(|g| g.dc_idx == 2)
|
||||
.expect("dc2 must be present");
|
||||
|
||||
assert!(dc2.primary.iter().all(|addr| addr.is_ipv6()));
|
||||
assert!(dc2.fallback.iter().all(|addr| addr.is_ipv4()));
|
||||
assert!(dc2
|
||||
.primary
|
||||
.contains(&"[2001:db8::10]:443".parse::<SocketAddr>().unwrap()));
|
||||
assert!(dc2
|
||||
.fallback
|
||||
.contains(&"203.0.113.10:443".parse::<SocketAddr>().unwrap()));
|
||||
assert!(dc2
|
||||
.fallback
|
||||
.contains(&"203.0.113.11:443".parse::<SocketAddr>().unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_health_check_groups_keeps_multiple_endpoints_per_group() {
|
||||
let mut overrides = HashMap::new();
|
||||
overrides.insert(
|
||||
"9".to_string(),
|
||||
vec![
|
||||
"198.51.100.1:443".to_string(),
|
||||
"198.51.100.2:443".to_string(),
|
||||
"198.51.100.1:443".to_string(),
|
||||
],
|
||||
);
|
||||
|
||||
let groups = UpstreamManager::build_health_check_groups(false, true, false, &overrides);
|
||||
let dc9 = groups
|
||||
.iter()
|
||||
.find(|g| g.dc_idx == 9)
|
||||
.expect("override-only dc group must be present");
|
||||
|
||||
assert_eq!(dc9.primary.len(), 2);
|
||||
assert!(dc9
|
||||
.primary
|
||||
.contains(&"198.51.100.1:443".parse::<SocketAddr>().unwrap()));
|
||||
assert!(dc9
|
||||
.primary
|
||||
.contains(&"198.51.100.2:443".parse::<SocketAddr>().unwrap()));
|
||||
assert!(dc9.fallback.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user