mirror of
https://github.com/telemt/telemt.git
synced 2026-04-17 02:24:10 +03:00
Compare commits
48 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4ea2226dcd | ||
|
|
d752a440e5 | ||
|
|
5ce2ee2dae | ||
|
|
6fd9f0595d | ||
|
|
fcdd8a9796 | ||
|
|
640468d4e7 | ||
|
|
02fe89f7d0 | ||
|
|
24df865503 | ||
|
|
e9f8c79498 | ||
|
|
24ff75701e | ||
|
|
4221230969 | ||
|
|
d87196c105 | ||
|
|
da89415961 | ||
|
|
2d98ebf3c3 | ||
|
|
fb5e9947bd | ||
|
|
2ea85c00d3 | ||
|
|
2a3b6b917f | ||
|
|
83ed9065b0 | ||
|
|
44b825edf5 | ||
|
|
487e95a66e | ||
|
|
c465c200c4 | ||
|
|
d7716ad875 | ||
|
|
edce194948 | ||
|
|
13fdff750d | ||
|
|
bdcf110c87 | ||
|
|
dd12997744 | ||
|
|
fc160913bf | ||
|
|
92c22ef16d | ||
|
|
aff22d0855 | ||
|
|
b3d3bca15a | ||
|
|
92f38392eb | ||
|
|
30ef8df1b3 | ||
|
|
2e174adf16 | ||
|
|
4e803b1412 | ||
|
|
9b174318ce | ||
|
|
99edcbe818 | ||
|
|
ef7dc2b80f | ||
|
|
691607f269 | ||
|
|
55561a23bc | ||
|
|
f32c34f126 | ||
|
|
8f3bdaec2c | ||
|
|
69b02caf77 | ||
|
|
3854955069 | ||
|
|
9b84fc7a5b | ||
|
|
e7cb9238dc | ||
|
|
0e2cbe6178 | ||
|
|
cd076aeeeb | ||
|
|
d683faf922 |
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.0"
|
version = "3.3.5"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
22
README.md
22
README.md
@@ -3,7 +3,7 @@
|
|||||||
***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist***
|
***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist***
|
||||||
|
|
||||||
**Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as:
|
**Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as:
|
||||||
- ME Pool + Reader/Writer + Registry + Refill + Adaptive Floor + Trio-State + Generation Lifecycle
|
- [ME Pool + Reader/Writer + Registry + Refill + Adaptive Floor + Trio-State + Generation Lifecycle](https://github.com/telemt/telemt/blob/main/docs/model/MODEL.en.md)
|
||||||
- [Full-covered API w/ management](https://github.com/telemt/telemt/blob/main/docs/API.md)
|
- [Full-covered API w/ management](https://github.com/telemt/telemt/blob/main/docs/API.md)
|
||||||
- Anti-Replay on Sliding Window
|
- Anti-Replay on Sliding Window
|
||||||
- Prometheus-format Metrics
|
- Prometheus-format Metrics
|
||||||
@@ -19,18 +19,18 @@
|
|||||||
|
|
||||||
### 🇷🇺 RU
|
### 🇷🇺 RU
|
||||||
|
|
||||||
#### Релиз 3.0.15 — 25 февраля
|
#### Релиз 3.3.3 LTS - 6 марта
|
||||||
|
|
||||||
25 февраля мы выпустили версию **3.0.15**
|
6 марта мы выпустили Telemt **3.3.3**
|
||||||
|
|
||||||
Мы предполагаем, что она станет завершающей версией поколения 3.0 и уже сейчас мы рассматриваем её как **LTS-кандидата** для версии **3.1.0**!
|
Это первая версия telemt работающая в комплексных условиях и при этом предоставляющая API
|
||||||
|
|
||||||
После нескольких дней детального анализа особенностей работы Middle-End мы спроектировали и реализовали продуманный режим **ротации ME Writer**. Данный режим позволяет поддерживать стабильно высокую производительность в long-run сценариях без возникновения ошибок, связанных с некорректной конфигурацией прокси
|
В ней используется новый алгоритм - ME NoWait, который вместе с Adaptive Floor и моделью усовершенствованного доступа к KDF Fingerprint на RwLock позволяет достигать максимальную производительность, даже в условиях lossy-сети
|
||||||
|
|
||||||
Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **статистики** и **UX**
|
Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **статистики** и **UX**
|
||||||
|
|
||||||
Релиз:
|
Релиз:
|
||||||
[3.0.15](https://github.com/telemt/telemt/releases/tag/3.0.15)
|
[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -47,18 +47,18 @@
|
|||||||
|
|
||||||
### 🇬🇧 EN
|
### 🇬🇧 EN
|
||||||
|
|
||||||
#### Release 3.0.15 — February 25
|
#### Release 3.3.3 LTS - March 6
|
||||||
|
|
||||||
On February 25, we released version **3.0.15**
|
On March 6, we released Telemt **3.3.3**
|
||||||
|
|
||||||
We expect this to become the final release of the 3.0 generation and at this point, we already see it as a strong **LTS candidate** for the upcoming **3.1.0** release!
|
This is the first telemt's version designed to operate reliably in complex network conditions while also providing a runtime API!
|
||||||
|
|
||||||
After several days of deep analysis of Middle-End behavior, we designed and implemented a well-engineered **ME Writer rotation mode**. This mode enables sustained high throughput in long-run scenarios while preventing proxy misconfiguration errors
|
The release introduces a new algorithm — ME NoWait, which combined with Adaptive Floor and an improved KDF Fingerprint access model based on RwLock, it enables the system to achieve maximum performance even in lossy network environments
|
||||||
|
|
||||||
We are looking forward to your feedback and improvement proposals — especially regarding **statistics** and **UX**
|
We are looking forward to your feedback and improvement proposals — especially regarding **statistics** and **UX**
|
||||||
|
|
||||||
Release:
|
Release:
|
||||||
[3.0.15](https://github.com/telemt/telemt/releases/tag/3.0.15)
|
[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
125
docs/API.md
125
docs/API.md
@@ -16,6 +16,10 @@ API runtime is configured in `[server.api]`.
|
|||||||
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
|
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
|
||||||
| `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
| `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
||||||
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
||||||
|
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
|
||||||
|
| `runtime_edge_cache_ttl_ms` | `u64` | `1000` | Cache TTL for runtime edge summary payloads. `0` disables cache. |
|
||||||
|
| `runtime_edge_top_n` | `usize` | `10` | Top-N rows for runtime edge leaderboard payloads. |
|
||||||
|
| `runtime_edge_events_capacity` | `usize` | `256` | Ring-buffer size for `/v1/runtime/events/recent`. |
|
||||||
| `read_only` | `bool` | `false` | Disables mutating endpoints. |
|
| `read_only` | `bool` | `false` | Disables mutating endpoints. |
|
||||||
|
|
||||||
`server.admin_api` is accepted as an alias for backward compatibility.
|
`server.admin_api` is accepted as an alias for backward compatibility.
|
||||||
@@ -24,6 +28,9 @@ Runtime validation for API config:
|
|||||||
- `server.api.listen` must be a valid `IP:PORT`.
|
- `server.api.listen` must be a valid `IP:PORT`.
|
||||||
- `server.api.request_body_limit_bytes` must be `> 0`.
|
- `server.api.request_body_limit_bytes` must be `> 0`.
|
||||||
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
|
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
|
||||||
|
- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`.
|
||||||
|
- `server.api.runtime_edge_top_n` must be within `[1, 1000]`.
|
||||||
|
- `server.api.runtime_edge_events_capacity` must be within `[16, 4096]`.
|
||||||
|
|
||||||
## Protocol Contract
|
## Protocol Contract
|
||||||
|
|
||||||
@@ -76,12 +83,23 @@ Notes:
|
|||||||
| Method | Path | Body | Success | `data` contract |
|
| Method | Path | Body | Success | `data` contract |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `GET` | `/v1/health` | none | `200` | `HealthData` |
|
| `GET` | `/v1/health` | none | `200` | `HealthData` |
|
||||||
|
| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` |
|
||||||
|
| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` |
|
||||||
|
| `GET` | `/v1/limits/effective` | none | `200` | `EffectiveLimitsData` |
|
||||||
|
| `GET` | `/v1/security/posture` | none | `200` | `SecurityPostureData` |
|
||||||
|
| `GET` | `/v1/security/whitelist` | none | `200` | `SecurityWhitelistData` |
|
||||||
| `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` |
|
| `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` |
|
||||||
| `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` |
|
| `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` |
|
||||||
| `GET` | `/v1/stats/upstreams` | none | `200` | `UpstreamsData` |
|
| `GET` | `/v1/stats/upstreams` | none | `200` | `UpstreamsData` |
|
||||||
| `GET` | `/v1/stats/minimal/all` | none | `200` | `MinimalAllData` |
|
| `GET` | `/v1/stats/minimal/all` | none | `200` | `MinimalAllData` |
|
||||||
| `GET` | `/v1/stats/me-writers` | none | `200` | `MeWritersData` |
|
| `GET` | `/v1/stats/me-writers` | none | `200` | `MeWritersData` |
|
||||||
| `GET` | `/v1/stats/dcs` | none | `200` | `DcStatusData` |
|
| `GET` | `/v1/stats/dcs` | none | `200` | `DcStatusData` |
|
||||||
|
| `GET` | `/v1/runtime/me_pool_state` | none | `200` | `RuntimeMePoolStateData` |
|
||||||
|
| `GET` | `/v1/runtime/me_quality` | none | `200` | `RuntimeMeQualityData` |
|
||||||
|
| `GET` | `/v1/runtime/upstream_quality` | none | `200` | `RuntimeUpstreamQualityData` |
|
||||||
|
| `GET` | `/v1/runtime/nat_stun` | none | `200` | `RuntimeNatStunData` |
|
||||||
|
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
||||||
|
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
||||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||||
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
|
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
|
||||||
| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` |
|
| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` |
|
||||||
@@ -176,6 +194,113 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
|
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
|
||||||
| `configured_users` | `usize` | Number of configured users in config. |
|
| `configured_users` | `usize` | Number of configured users in config. |
|
||||||
|
|
||||||
|
### `SystemInfoData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `version` | `string` | Binary version (`CARGO_PKG_VERSION`). |
|
||||||
|
| `target_arch` | `string` | Target architecture (`std::env::consts::ARCH`). |
|
||||||
|
| `target_os` | `string` | Target OS (`std::env::consts::OS`). |
|
||||||
|
| `build_profile` | `string` | Build profile (`PROFILE` env when available). |
|
||||||
|
| `git_commit` | `string?` | Optional commit hash from build env metadata. |
|
||||||
|
| `build_time_utc` | `string?` | Optional build timestamp from build env metadata. |
|
||||||
|
| `rustc_version` | `string?` | Optional compiler version from build env metadata. |
|
||||||
|
| `process_started_at_epoch_secs` | `u64` | Process start time as Unix epoch seconds. |
|
||||||
|
| `uptime_seconds` | `f64` | Process uptime in seconds. |
|
||||||
|
| `config_path` | `string` | Active config file path used by runtime. |
|
||||||
|
| `config_hash` | `string` | SHA-256 hash of current config content (same value as envelope `revision`). |
|
||||||
|
| `config_reload_count` | `u64` | Number of successfully observed config updates since process start. |
|
||||||
|
| `last_config_reload_epoch_secs` | `u64?` | Unix epoch seconds of the latest observed config reload; null/absent before first reload. |
|
||||||
|
|
||||||
|
### `RuntimeGatesData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `accepting_new_connections` | `bool` | Current admission-gate state for new listener accepts. |
|
||||||
|
| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). |
|
||||||
|
| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. |
|
||||||
|
| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. |
|
||||||
|
| `use_middle_proxy` | `bool` | Current transport mode preference. |
|
||||||
|
|
||||||
|
### `EffectiveLimitsData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `update_every_secs` | `u64` | Effective unified updater interval. |
|
||||||
|
| `me_reinit_every_secs` | `u64` | Effective ME periodic reinit interval. |
|
||||||
|
| `me_pool_force_close_secs` | `u64` | Effective stale-writer force-close timeout. |
|
||||||
|
| `timeouts` | `EffectiveTimeoutLimits` | Effective timeout policy snapshot. |
|
||||||
|
| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. |
|
||||||
|
| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. |
|
||||||
|
| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. |
|
||||||
|
|
||||||
|
#### `EffectiveTimeoutLimits`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `client_handshake_secs` | `u64` | Client handshake timeout. |
|
||||||
|
| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. |
|
||||||
|
| `client_keepalive_secs` | `u64` | Client keepalive interval. |
|
||||||
|
| `client_ack_secs` | `u64` | ACK timeout. |
|
||||||
|
| `me_one_retry` | `u8` | Fast retry count for single-endpoint ME DC. |
|
||||||
|
| `me_one_timeout_ms` | `u64` | Fast retry timeout per attempt for single-endpoint ME DC. |
|
||||||
|
|
||||||
|
#### `EffectiveUpstreamLimits`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `connect_retry_attempts` | `u32` | Upstream connect retry attempts. |
|
||||||
|
| `connect_retry_backoff_ms` | `u64` | Upstream retry backoff delay. |
|
||||||
|
| `connect_budget_ms` | `u64` | Total connect wall-clock budget across retries. |
|
||||||
|
| `unhealthy_fail_threshold` | `u32` | Consecutive fail threshold for unhealthy marking. |
|
||||||
|
| `connect_failfast_hard_errors` | `bool` | Whether hard errors skip additional retries. |
|
||||||
|
|
||||||
|
#### `EffectiveMiddleProxyLimits`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `floor_mode` | `string` | Effective floor mode (`static` or `adaptive`). |
|
||||||
|
| `adaptive_floor_idle_secs` | `u64` | Adaptive floor idle threshold. |
|
||||||
|
| `adaptive_floor_min_writers_single_endpoint` | `u8` | Adaptive floor minimum for single-endpoint DCs. |
|
||||||
|
| `adaptive_floor_recover_grace_secs` | `u64` | Adaptive floor recovery grace period. |
|
||||||
|
| `reconnect_max_concurrent_per_dc` | `u32` | Max concurrent reconnects per DC. |
|
||||||
|
| `reconnect_backoff_base_ms` | `u64` | Reconnect base backoff. |
|
||||||
|
| `reconnect_backoff_cap_ms` | `u64` | Reconnect backoff cap. |
|
||||||
|
| `reconnect_fast_retry_count` | `u32` | Number of fast retries before standard backoff strategy. |
|
||||||
|
| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. |
|
||||||
|
|
||||||
|
#### `EffectiveUserIpPolicyLimits`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). |
|
||||||
|
| `window_secs` | `u64` | Time window length used by unique-IP policy. |
|
||||||
|
|
||||||
|
### `SecurityPostureData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `api_read_only` | `bool` | Current API read-only state. |
|
||||||
|
| `api_whitelist_enabled` | `bool` | Whether whitelist filtering is active. |
|
||||||
|
| `api_whitelist_entries` | `usize` | Number of configured whitelist CIDRs. |
|
||||||
|
| `api_auth_header_enabled` | `bool` | Whether `Authorization` header validation is active. |
|
||||||
|
| `proxy_protocol_enabled` | `bool` | Global PROXY protocol accept setting. |
|
||||||
|
| `log_level` | `string` | Effective log level (`debug`, `verbose`, `normal`, `silent`). |
|
||||||
|
| `telemetry_core_enabled` | `bool` | Core telemetry toggle. |
|
||||||
|
| `telemetry_user_enabled` | `bool` | Per-user telemetry toggle. |
|
||||||
|
| `telemetry_me_level` | `string` | ME telemetry level (`silent`, `normal`, `debug`). |
|
||||||
|
|
||||||
|
### `SecurityWhitelistData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. |
|
||||||
|
| `enabled` | `bool` | `true` when whitelist has at least one CIDR entry. |
|
||||||
|
| `entries_total` | `usize` | Number of whitelist CIDR entries. |
|
||||||
|
| `entries` | `string[]` | Whitelist CIDR entries as strings. |
|
||||||
|
|
||||||
|
### Runtime Min Endpoints
|
||||||
|
- `/v1/runtime/me_pool_state`: generations, hardswap state, writer contour/health counts, refill inflight snapshot.
|
||||||
|
- `/v1/runtime/me_quality`: ME error/drift/reconnect counters and per-DC RTT coverage snapshot.
|
||||||
|
- `/v1/runtime/upstream_quality`: upstream runtime policy, connect counters, health summary and per-upstream DC latency/IP preference.
|
||||||
|
- `/v1/runtime/nat_stun`: NAT/STUN runtime flags, server lists, reflection cache state and backoff remaining.
|
||||||
|
|
||||||
|
### Runtime Edge Endpoints
|
||||||
|
- `/v1/runtime/connections/summary`: cached connection totals (`total/me/direct`), active users and top-N users by connections/traffic.
|
||||||
|
- `/v1/runtime/events/recent?limit=N`: bounded control-plane ring-buffer events (`limit` clamped to `[1, 1000]`).
|
||||||
|
- If `server.api.runtime_edge_enabled=false`, runtime edge endpoints return `enabled=false` with `reason=feature_disabled`.
|
||||||
|
|
||||||
### `ZeroAllData`
|
### `ZeroAllData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
|||||||
@@ -26,6 +26,13 @@ use_middle_proxy = true
|
|||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> У вас не будет отображаться "спонсор прокси" если вы уже подписаны на канал.
|
> У вас не будет отображаться "спонсор прокси" если вы уже подписаны на канал.
|
||||||
|
|
||||||
|
**Также вы можете настроить разные каналы для разных пользователей.**
|
||||||
|
```toml
|
||||||
|
[access.user_ad_tags]
|
||||||
|
hello = "ad_tag"
|
||||||
|
hello2 = "ad_tag2"
|
||||||
|
```
|
||||||
|
|
||||||
## Сколько человек может пользоваться 1 ссылкой
|
## Сколько человек может пользоваться 1 ссылкой
|
||||||
|
|
||||||
По умолчанию 1 ссылкой может пользоваться сколько угодно человек.
|
По умолчанию 1 ссылкой может пользоваться сколько угодно человек.
|
||||||
|
|||||||
285
docs/model/MODEL.en.md
Normal file
285
docs/model/MODEL.en.md
Normal file
@@ -0,0 +1,285 @@
|
|||||||
|
# Telemt Runtime Model
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
This document defines runtime concepts used by the Middle-End (ME) transport pipeline and the orchestration logic around it.
|
||||||
|
|
||||||
|
It focuses on:
|
||||||
|
- `ME Pool / Reader / Writer / Refill / Registry`
|
||||||
|
- `Adaptive Floor`
|
||||||
|
- `Trio-State`
|
||||||
|
- `Generation Lifecycle`
|
||||||
|
|
||||||
|
## Core Entities
|
||||||
|
|
||||||
|
### ME Pool
|
||||||
|
`ME Pool` is the runtime orchestrator for all Middle-End writers.
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
- Holds writer inventory by DC/family/endpoint.
|
||||||
|
- Maintains routing primitives and writer selection policy.
|
||||||
|
- Tracks generation state (`active`, `warm`, `draining` context).
|
||||||
|
- Applies runtime policies (floor mode, refill, reconnect, reinit, fallback behavior).
|
||||||
|
- Exposes readiness gates used by admission logic (for conditional accept/cast behavior).
|
||||||
|
|
||||||
|
Non-goals:
|
||||||
|
- It does not own client protocol decoding.
|
||||||
|
- It does not own per-client business policy (quotas/limits).
|
||||||
|
|
||||||
|
### ME Writer
|
||||||
|
`ME Writer` is a long-lived ME RPC tunnel bound to one concrete ME endpoint (`ip:port`), with:
|
||||||
|
- Outbound command channel (send path).
|
||||||
|
- Associated reader loop (inbound path).
|
||||||
|
- Health/degraded flags.
|
||||||
|
- Contour/state and generation metadata.
|
||||||
|
|
||||||
|
A writer is the actual data plane carrier for client sessions once bound.
|
||||||
|
|
||||||
|
### ME Reader
|
||||||
|
`ME Reader` is the inbound parser/dispatcher for one writer:
|
||||||
|
- Reads/decrypts ME RPC frames.
|
||||||
|
- Validates sequence/checksum.
|
||||||
|
- Routes payloads to client-connection channels via `Registry`.
|
||||||
|
- Emits close/ack/data events and updates telemetry.
|
||||||
|
|
||||||
|
Design intent:
|
||||||
|
- Reader must stay non-blocking as much as possible.
|
||||||
|
- Backpressure on a single client route must not stall the whole writer stream.
|
||||||
|
|
||||||
|
### Refill
|
||||||
|
`Refill` is the recovery mechanism that restores writer coverage when capacity drops:
|
||||||
|
- Per-endpoint restore (same endpoint first).
|
||||||
|
- Per-DC restore to satisfy required floor.
|
||||||
|
- Optional outage-mode/shadow behavior for fragile single-endpoint DCs.
|
||||||
|
|
||||||
|
Refill works asynchronously and should not block hot routing paths.
|
||||||
|
|
||||||
|
### Registry
|
||||||
|
`Registry` is the routing index between ME and client sessions:
|
||||||
|
- `conn_id -> client response channel`
|
||||||
|
- `conn_id <-> writer_id` binding map
|
||||||
|
- writer activity snapshots and idle tracking
|
||||||
|
|
||||||
|
Main invariants:
|
||||||
|
- A `conn_id` routes to at most one active response channel.
|
||||||
|
- Writer loss triggers safe unbind/cleanup and close propagation.
|
||||||
|
- Registry state is the source of truth for active ME-bound session mapping.
|
||||||
|
|
||||||
|
## Adaptive Floor
|
||||||
|
|
||||||
|
### What it is
|
||||||
|
`Adaptive Floor` is a runtime policy that changes target writer count per DC based on observed activity, instead of always holding static peak floor.
|
||||||
|
|
||||||
|
### Why it exists
|
||||||
|
Goals:
|
||||||
|
- Reduce idle writer churn under low traffic.
|
||||||
|
- Keep enough warm capacity to avoid client-visible stalls on burst recovery.
|
||||||
|
- Limit needless reconnect storms on unstable endpoints.
|
||||||
|
|
||||||
|
### Behavioral model
|
||||||
|
- Under activity: floor converges toward configured static requirement.
|
||||||
|
- Under prolonged idle: floor can shrink to a safe minimum.
|
||||||
|
- Recovery/grace windows prevent aggressive oscillation.
|
||||||
|
|
||||||
|
### Safety constraints
|
||||||
|
- Never violate minimal survivability floor for a DC group.
|
||||||
|
- Refill must still restore quickly on demand.
|
||||||
|
- Floor adaptation must not force-drop already bound healthy sessions.
|
||||||
|
|
||||||
|
## Trio-State
|
||||||
|
|
||||||
|
`Trio-State` is writer contouring:
|
||||||
|
- `Warm`
|
||||||
|
- `Active`
|
||||||
|
- `Draining`
|
||||||
|
|
||||||
|
### State semantics
|
||||||
|
- `Warm`: connected and validated, not primary for new binds.
|
||||||
|
- `Active`: preferred for new binds and normal traffic.
|
||||||
|
- `Draining`: no new regular binds; existing sessions continue until graceful retirement rules apply.
|
||||||
|
|
||||||
|
### Transition intent
|
||||||
|
- `Warm -> Active`: when coverage/readiness conditions are satisfied.
|
||||||
|
- `Active -> Draining`: on generation swap, endpoint replacement, or controlled retirement.
|
||||||
|
- `Draining -> removed`: after drain TTL/force-close policy (or when naturally empty).
|
||||||
|
|
||||||
|
This separation reduces SPOF and keeps cutovers predictable.
|
||||||
|
|
||||||
|
## Generation Lifecycle
|
||||||
|
|
||||||
|
Generation isolates pool epochs during reinit/reconfiguration.
|
||||||
|
|
||||||
|
### Lifecycle phases
|
||||||
|
1. `Bootstrap`: initial writers are established.
|
||||||
|
2. `Warmup`: next generation writers are created and validated.
|
||||||
|
3. `Activation`: generation promoted to active when coverage gate passes.
|
||||||
|
4. `Drain`: previous generation becomes draining, existing sessions are allowed to finish.
|
||||||
|
5. `Retire`: old generation writers are removed after graceful rules.
|
||||||
|
|
||||||
|
### Operational guarantees
|
||||||
|
- No partial generation activation without minimum coverage.
|
||||||
|
- Existing healthy client sessions should not be dropped just because a new generation appears.
|
||||||
|
- Draining generation exists to absorb in-flight traffic during swap.
|
||||||
|
|
||||||
|
### Readiness and admission
|
||||||
|
Pool readiness is not equivalent to “all endpoints fully saturated”.
|
||||||
|
Typical gating strategy:
|
||||||
|
- Open admission when per-DC minimal alive coverage exists.
|
||||||
|
- Continue background saturation for multi-endpoint DCs.
|
||||||
|
|
||||||
|
This keeps startup latency low while preserving eventual full capacity.
|
||||||
|
|
||||||
|
## Interactions Between Concepts
|
||||||
|
|
||||||
|
- `Generation` defines pool epochs.
|
||||||
|
- `Trio-State` defines per-writer role inside/around those epochs.
|
||||||
|
- `Adaptive Floor` defines how much capacity should be maintained right now.
|
||||||
|
- `Refill` is the actuator that closes the gap between desired and current capacity.
|
||||||
|
- `Registry` keeps per-session routing correctness while all of the above changes over time.
|
||||||
|
|
||||||
|
## Architectural Approach
|
||||||
|
|
||||||
|
### Layered Design
|
||||||
|
The runtime is intentionally split into two planes:
|
||||||
|
- `Control Plane`: decides desired topology and policy (`floor`, `generation swap`, `refill`, `fallback`).
|
||||||
|
- `Data Plane`: executes packet/session transport (`reader`, `writer`, routing, acks, close propagation).
|
||||||
|
|
||||||
|
Architectural rule:
|
||||||
|
- Control Plane may change writer inventory and policy.
|
||||||
|
- Data Plane must remain stable and low-latency while those changes happen.
|
||||||
|
|
||||||
|
### Ownership Model
|
||||||
|
Ownership is centered around explicit state domains:
|
||||||
|
- `MePool` owns writer lifecycle and policy state.
|
||||||
|
- `Registry` owns per-connection routing bindings.
|
||||||
|
- `Writer task` owns outbound ME socket send progression.
|
||||||
|
- `Reader task` owns inbound ME socket parsing and event dispatch.
|
||||||
|
|
||||||
|
This prevents accidental cross-layer mutation and keeps invariants local.
|
||||||
|
|
||||||
|
### Control Plane Responsibilities
|
||||||
|
Control Plane is event-driven and policy-driven:
|
||||||
|
- Startup initialization and readiness gates.
|
||||||
|
- Runtime reinit (periodic or config-triggered).
|
||||||
|
- Coverage checks per DC/family/endpoint group.
|
||||||
|
- Floor enforcement (static/adaptive).
|
||||||
|
- Refill scheduling and retry orchestration.
|
||||||
|
- Generation transition (`warm -> active`, previous `active -> draining`).
|
||||||
|
|
||||||
|
Control Plane must prioritize determinism over short-term aggressiveness.
|
||||||
|
|
||||||
|
### Data Plane Responsibilities
|
||||||
|
Data Plane is throughput-first and allocation-sensitive:
|
||||||
|
- Session bind to writer.
|
||||||
|
- Per-frame parsing/validation and dispatch.
|
||||||
|
- Ack and close signal propagation.
|
||||||
|
- Route drop behavior under missing connection or closed channel.
|
||||||
|
- Minimal critical logging in hot path.
|
||||||
|
|
||||||
|
Data Plane should avoid waiting on operations that are not strictly required for frame correctness.
|
||||||
|
|
||||||
|
## Concurrency and Synchronization
|
||||||
|
|
||||||
|
### Concurrency Principles
|
||||||
|
- Per-writer isolation: each writer has independent send/read task loops.
|
||||||
|
- Per-connection isolation: client channel state is scoped by `conn_id`.
|
||||||
|
- Asynchronous recovery: refill/reconnect runs outside the packet hot path.
|
||||||
|
|
||||||
|
### Synchronization Strategy
|
||||||
|
- Shared maps use fine-grained, short-lived locking.
|
||||||
|
- Read-mostly paths avoid broad write-lock windows.
|
||||||
|
- Backpressure decisions are localized at route/channel boundary.
|
||||||
|
|
||||||
|
Design target:
|
||||||
|
- A slow consumer should degrade only itself (or its route), not global writer progress.
|
||||||
|
|
||||||
|
### Cancellation and Shutdown
|
||||||
|
Writer and reader loops are cancellation-aware:
|
||||||
|
- explicit cancel token / close command support;
|
||||||
|
- safe unbind and cleanup via registry;
|
||||||
|
- deterministic order: stop admission -> drain/close -> release resources.
|
||||||
|
|
||||||
|
## Consistency Model
|
||||||
|
|
||||||
|
### Session Consistency
|
||||||
|
For one `conn_id`:
|
||||||
|
- exactly one active route target at a time;
|
||||||
|
- close and unbind must be idempotent;
|
||||||
|
- writer loss must not leave dangling bindings.
|
||||||
|
|
||||||
|
### Generation Consistency
|
||||||
|
Generational consistency guarantees:
|
||||||
|
- New generation is not promoted before minimum coverage gate.
|
||||||
|
- Previous generation remains available in `draining` state during handover.
|
||||||
|
- Forced retirement is policy-bound (`drain ttl`, optional force-close), not immediate.
|
||||||
|
|
||||||
|
### Policy Consistency
|
||||||
|
Policy changes (`adaptive/static floor`, fallback mode, retries) should apply without violating established active-session routing invariants.
|
||||||
|
|
||||||
|
## Backpressure and Flow Control
|
||||||
|
|
||||||
|
### Route-Level Backpressure
|
||||||
|
Route channels are bounded by design.
|
||||||
|
When pressure increases:
|
||||||
|
- short burst absorption is allowed;
|
||||||
|
- prolonged congestion triggers controlled drop semantics;
|
||||||
|
- drop accounting is explicit via metrics/counters.
|
||||||
|
|
||||||
|
### Reader Non-Blocking Priority
|
||||||
|
Inbound ME reader path should never be serialized behind one congested client route.
|
||||||
|
Practical implication:
|
||||||
|
- prefer non-blocking route attempt in the parser loop;
|
||||||
|
- move heavy recovery to async side paths.
|
||||||
|
|
||||||
|
## Failure Domain Strategy
|
||||||
|
|
||||||
|
### Endpoint-Level Failure
|
||||||
|
Failure of one endpoint should trigger endpoint-scoped recovery first:
|
||||||
|
- same endpoint reconnect;
|
||||||
|
- endpoint replacement within same DC group if applicable.
|
||||||
|
|
||||||
|
### DC-Level Degradation
|
||||||
|
If a DC group cannot satisfy floor:
|
||||||
|
- keep service via remaining coverage if policy allows;
|
||||||
|
- continue asynchronous refill saturation in background.
|
||||||
|
|
||||||
|
### Whole-Pool Readiness Loss
|
||||||
|
If no sufficient ME coverage exists:
|
||||||
|
- admission gate can hold new accepts (conditional policy);
|
||||||
|
- existing sessions should continue when their path remains healthy.
|
||||||
|
|
||||||
|
## Performance Architecture Notes
|
||||||
|
|
||||||
|
### Hotpath Discipline
|
||||||
|
Allowed in hotpath:
|
||||||
|
- fixed-size parsing and cheap validation;
|
||||||
|
- bounded channel operations;
|
||||||
|
- precomputed or low-allocation access patterns.
|
||||||
|
|
||||||
|
Avoid in hotpath:
|
||||||
|
- repeated expensive decoding;
|
||||||
|
- broad locks with awaits inside critical sections;
|
||||||
|
- verbose high-frequency logging.
|
||||||
|
|
||||||
|
### Throughput Stability Over Peak Spikes
|
||||||
|
Architecture prefers stable throughput and predictable latency over short peak gains that increase churn or long-tail reconnect times.
|
||||||
|
|
||||||
|
## Evolution and Extension Rules
|
||||||
|
|
||||||
|
To evolve this model safely:
|
||||||
|
- Add new policy knobs in Control Plane first.
|
||||||
|
- Keep Data Plane contracts stable (`conn_id`, route semantics, close semantics).
|
||||||
|
- Validate generation and registry invariants before enabling by default.
|
||||||
|
- Introduce new retry/recovery strategies behind explicit config.
|
||||||
|
|
||||||
|
## Failure and Recovery Notes
|
||||||
|
|
||||||
|
- Single-endpoint DC failure is a normal degraded mode case; policy should prioritize fast reconnect and optional shadow/probing strategies.
|
||||||
|
- Idle close by peer should be treated as expected when upstream enforces idle timeout.
|
||||||
|
- Reconnect backoff must protect against synchronized churn while still allowing fast first retries.
|
||||||
|
- Fallback (`ME -> direct DC`) is a policy switch, not a transport bug by itself.
|
||||||
|
|
||||||
|
## Terminology Summary
|
||||||
|
- `Coverage`: enough live writers to satisfy per-DC acceptance policy.
|
||||||
|
- `Floor`: target minimum writer count policy.
|
||||||
|
- `Churn`: frequent writer reconnect/remove cycles.
|
||||||
|
- `Hotpath`: per-packet/per-connection data path where extra waits/allocations are expensive.
|
||||||
285
docs/model/MODEL.ru.md
Normal file
285
docs/model/MODEL.ru.md
Normal file
@@ -0,0 +1,285 @@
|
|||||||
|
# Runtime-модель Telemt
|
||||||
|
|
||||||
|
## Область описания
|
||||||
|
Документ фиксирует ключевые runtime-понятия пайплайна Middle-End (ME) и оркестрации вокруг него.
|
||||||
|
|
||||||
|
Фокус:
|
||||||
|
- `ME Pool / Reader / Writer / Refill / Registry`
|
||||||
|
- `Adaptive Floor`
|
||||||
|
- `Trio-State`
|
||||||
|
- `Generation Lifecycle`
|
||||||
|
|
||||||
|
## Базовые сущности
|
||||||
|
|
||||||
|
### ME Pool
|
||||||
|
`ME Pool` — центральный оркестратор всех Middle-End writer-ов.
|
||||||
|
|
||||||
|
Зона ответственности:
|
||||||
|
- хранит инвентарь writer-ов по DC/family/endpoint;
|
||||||
|
- управляет выбором writer-а и маршрутизацией;
|
||||||
|
- ведёт состояние поколений (`active`, `warm`, `draining` контекст);
|
||||||
|
- применяет runtime-политики (floor, refill, reconnect, reinit, fallback);
|
||||||
|
- отдаёт сигналы готовности для admission-логики (conditional accept/cast).
|
||||||
|
|
||||||
|
Что не делает:
|
||||||
|
- не декодирует клиентский протокол;
|
||||||
|
- не реализует бизнес-политику пользователя (квоты/лимиты).
|
||||||
|
|
||||||
|
### ME Writer
|
||||||
|
`ME Writer` — долгоживущий ME RPC-канал к конкретному endpoint (`ip:port`), у которого есть:
|
||||||
|
- канал команд на отправку;
|
||||||
|
- связанный reader loop для входящего потока;
|
||||||
|
- флаги состояния/деградации;
|
||||||
|
- метаданные contour/state и generation.
|
||||||
|
|
||||||
|
Writer — это фактический data-plane носитель клиентских сессий после бинда.
|
||||||
|
|
||||||
|
### ME Reader
|
||||||
|
`ME Reader` — входной parser/dispatcher одного writer-а:
|
||||||
|
- читает и расшифровывает ME RPC-фреймы;
|
||||||
|
- проверяет sequence/checksum;
|
||||||
|
- маршрутизирует payload в client-каналы через `Registry`;
|
||||||
|
- обрабатывает close/ack/data и обновляет телеметрию.
|
||||||
|
|
||||||
|
Инженерный принцип:
|
||||||
|
- Reader должен оставаться неблокирующим.
|
||||||
|
- Backpressure одной клиентской сессии не должен останавливать весь поток writer-а.
|
||||||
|
|
||||||
|
### Refill
|
||||||
|
`Refill` — механизм восстановления покрытия writer-ов при просадке:
|
||||||
|
- восстановление на том же endpoint в первую очередь;
|
||||||
|
- восстановление по DC до требуемого floor;
|
||||||
|
- опциональные outage/shadow-режимы для хрупких single-endpoint DC.
|
||||||
|
|
||||||
|
Refill работает асинхронно и не должен блокировать hotpath.
|
||||||
|
|
||||||
|
### Registry
|
||||||
|
`Registry` — маршрутизационный индекс между ME и клиентскими сессиями:
|
||||||
|
- `conn_id -> канал ответа клиенту`;
|
||||||
|
- map биндов `conn_id <-> writer_id`;
|
||||||
|
- снимки активности writer-ов и idle-трекинг.
|
||||||
|
|
||||||
|
Ключевые инварианты:
|
||||||
|
- один `conn_id` маршрутизируется максимум в один активный канал ответа;
|
||||||
|
- потеря writer-а приводит к безопасному unbind/cleanup и отправке close;
|
||||||
|
- именно `Registry` является источником истины по активным ME-биндам.
|
||||||
|
|
||||||
|
## Adaptive Floor
|
||||||
|
|
||||||
|
### Что это
|
||||||
|
`Adaptive Floor` — runtime-политика, которая динамически меняет целевое число writer-ов на DC в зависимости от активности, а не держит всегда фиксированный статический floor.
|
||||||
|
|
||||||
|
### Зачем
|
||||||
|
Цели:
|
||||||
|
- уменьшить churn на idle-трафике;
|
||||||
|
- сохранить достаточную прогретую ёмкость для быстрых всплесков;
|
||||||
|
- снизить лишние reconnect-штормы на нестабильных endpoint.
|
||||||
|
|
||||||
|
### Модель поведения
|
||||||
|
- при активности floor стремится к статическому требованию;
|
||||||
|
- при длительном idle floor может снижаться до безопасного минимума;
|
||||||
|
- grace/recovery окна не дают системе "флапать" слишком резко.
|
||||||
|
|
||||||
|
### Ограничения безопасности
|
||||||
|
- нельзя нарушать минимальный floor выживаемости DC-группы;
|
||||||
|
- refill обязан быстро нарастить покрытие по запросу;
|
||||||
|
- адаптация не должна принудительно ронять уже привязанные healthy-сессии.
|
||||||
|
|
||||||
|
## Trio-State
|
||||||
|
|
||||||
|
`Trio-State` — контурная роль writer-а:
|
||||||
|
- `Warm`
|
||||||
|
- `Active`
|
||||||
|
- `Draining`
|
||||||
|
|
||||||
|
### Семантика состояний
|
||||||
|
- `Warm`: writer подключён и валиден, но не основной для новых биндов.
|
||||||
|
- `Active`: приоритетный для новых биндов и обычного трафика.
|
||||||
|
- `Draining`: новые обычные бинды не назначаются; текущие сессии живут до правил graceful-вывода.
|
||||||
|
|
||||||
|
### Логика переходов
|
||||||
|
- `Warm -> Active`: когда достигнуты условия покрытия/готовности.
|
||||||
|
- `Active -> Draining`: при swap поколения, замене endpoint или контролируемом выводе.
|
||||||
|
- `Draining -> removed`: после drain TTL/force-close политики (или естественного опустошения).
|
||||||
|
|
||||||
|
Такое разделение снижает SPOF-риски и делает cutover предсказуемым.
|
||||||
|
|
||||||
|
## Generation Lifecycle
|
||||||
|
|
||||||
|
Generation изолирует эпохи пула при reinit/reconfiguration.
|
||||||
|
|
||||||
|
### Фазы жизненного цикла
|
||||||
|
1. `Bootstrap`: поднимается начальный набор writer-ов.
|
||||||
|
2. `Warmup`: создаётся и валидируется новое поколение.
|
||||||
|
3. `Activation`: новое поколение становится active после прохождения coverage-gate.
|
||||||
|
4. `Drain`: предыдущее поколение переводится в draining, текущим сессиям дают завершиться.
|
||||||
|
5. `Retire`: старое поколение удаляется по graceful-правилам.
|
||||||
|
|
||||||
|
### Операционные гарантии
|
||||||
|
- нельзя активировать поколение частично без минимального покрытия;
|
||||||
|
- healthy-клиенты не должны теряться только из-за появления нового поколения;
|
||||||
|
- draining-поколение служит буфером для in-flight трафика во время swap.
|
||||||
|
|
||||||
|
### Готовность и приём клиентов
|
||||||
|
Готовность пула не равна "все endpoint полностью насыщены".
|
||||||
|
Типичная стратегия:
|
||||||
|
- открыть admission при минимально достаточном alive-покрытии по DC;
|
||||||
|
- параллельно продолжать saturation для multi-endpoint DC.
|
||||||
|
|
||||||
|
Это уменьшает startup latency и сохраняет выход на полную ёмкость.
|
||||||
|
|
||||||
|
## Как понятия связаны между собой
|
||||||
|
|
||||||
|
- `Generation` задаёт эпохи пула.
|
||||||
|
- `Trio-State` задаёт роль каждого writer-а внутри/между эпохами.
|
||||||
|
- `Adaptive Floor` задаёт, сколько ёмкости нужно сейчас.
|
||||||
|
- `Refill` — исполнитель, который закрывает разницу между desired и current capacity.
|
||||||
|
- `Registry` гарантирует корректную маршрутизацию сессий, пока всё выше меняется.
|
||||||
|
|
||||||
|
## Архитектурный подход
|
||||||
|
|
||||||
|
### Слоистая модель
|
||||||
|
Runtime специально разделён на две плоскости:
|
||||||
|
- `Control Plane`: принимает решения о целевой топологии и политиках (`floor`, `generation swap`, `refill`, `fallback`).
|
||||||
|
- `Data Plane`: исполняет транспорт сессий и пакетов (`reader`, `writer`, маршрутизация, ack, close).
|
||||||
|
|
||||||
|
Ключевое правило:
|
||||||
|
- Control Plane может менять состав writer-ов и policy.
|
||||||
|
- Data Plane должен оставаться стабильным и низколатентным в момент этих изменений.
|
||||||
|
|
||||||
|
### Модель владения состоянием
|
||||||
|
Владение разделено по доменам:
|
||||||
|
- `MePool` владеет жизненным циклом writer-ов и policy-state.
|
||||||
|
- `Registry` владеет routing-биндами клиентских сессий.
|
||||||
|
- `Writer task` владеет исходящей прогрессией ME-сокета.
|
||||||
|
- `Reader task` владеет входящим парсингом и dispatch-событиями.
|
||||||
|
|
||||||
|
Это ограничивает побочные мутации и локализует инварианты.
|
||||||
|
|
||||||
|
### Обязанности Control Plane
|
||||||
|
Control Plane работает событийно и policy-ориентированно:
|
||||||
|
- стартовая инициализация и readiness-gate;
|
||||||
|
- runtime reinit (периодический и/или по изменению конфигурации);
|
||||||
|
- проверки покрытия по DC/family/endpoint group;
|
||||||
|
- применение floor-политики (static/adaptive);
|
||||||
|
- планирование refill и orchestration retry;
|
||||||
|
- переходы поколений (`warm -> active`, прежний `active -> draining`).
|
||||||
|
|
||||||
|
Для него важнее детерминизм, чем агрессивная краткосрочная реакция.
|
||||||
|
|
||||||
|
### Обязанности Data Plane
|
||||||
|
Data Plane ориентирован на пропускную способность и предсказуемую задержку:
|
||||||
|
- bind клиентской сессии к writer-у;
|
||||||
|
- per-frame parsing/validation/dispatch;
|
||||||
|
- распространение ack/close;
|
||||||
|
- корректная реакция на missing conn/closed channel;
|
||||||
|
- минимальный лог-шум в hotpath.
|
||||||
|
|
||||||
|
Data Plane не должен ждать операций, не критичных для корректности текущего фрейма.
|
||||||
|
|
||||||
|
## Конкурентность и синхронизация
|
||||||
|
|
||||||
|
### Принципы конкурентности
|
||||||
|
- Изоляция по writer-у: у каждого writer-а независимые send/read loop.
|
||||||
|
- Изоляция по сессии: состояние канала локально для `conn_id`.
|
||||||
|
- Асинхронное восстановление: refill/reconnect выполняются вне пакетного hotpath.
|
||||||
|
|
||||||
|
### Стратегия синхронизации
|
||||||
|
- Для shared map используются короткие и узкие lock-секции.
|
||||||
|
- Read-heavy пути избегают длительных write-lock окон.
|
||||||
|
- Решения по backpressure локализованы на границе route/channel.
|
||||||
|
|
||||||
|
Цель:
|
||||||
|
- медленный consumer должен деградировать локально, не останавливая глобальный прогресс writer-а.
|
||||||
|
|
||||||
|
### Cancellation и shutdown
|
||||||
|
Reader/Writer loop должны быть cancellation-aware:
|
||||||
|
- явные cancel token / close command;
|
||||||
|
- безопасный unbind/cleanup через registry;
|
||||||
|
- детерминированный порядок: stop admission -> drain/close -> release resources.
|
||||||
|
|
||||||
|
## Модель согласованности
|
||||||
|
|
||||||
|
### Согласованность сессии
|
||||||
|
Для одного `conn_id`:
|
||||||
|
- одновременно ровно один активный route-target;
|
||||||
|
- close/unbind операции идемпотентны;
|
||||||
|
- потеря writer-а не оставляет dangling-бинды.
|
||||||
|
|
||||||
|
### Согласованность поколения
|
||||||
|
Гарантии generation:
|
||||||
|
- новое поколение не активируется до прохождения минимального coverage-gate;
|
||||||
|
- предыдущее поколение остаётся в `draining` на время handover;
|
||||||
|
- принудительный вывод writer-ов ограничен policy (`drain ttl`, optional force-close), а не мгновенный.
|
||||||
|
|
||||||
|
### Согласованность политик
|
||||||
|
Изменение policy (`adaptive/static floor`, fallback mode, retries) не должно ломать инварианты маршрутизации уже активных сессий.
|
||||||
|
|
||||||
|
## Backpressure и управление потоком
|
||||||
|
|
||||||
|
### Route-level backpressure
|
||||||
|
Route-каналы намеренно bounded.
|
||||||
|
При росте нагрузки:
|
||||||
|
- кратковременный burst поглощается;
|
||||||
|
- длительная перегрузка переходит в контролируемую drop-семантику;
|
||||||
|
- все drop-сценарии должны быть прозрачно видны в метриках.
|
||||||
|
|
||||||
|
### Приоритет неблокирующего Reader
|
||||||
|
Входящий ME-reader path не должен сериализоваться из-за одной перегруженной клиентской сессии.
|
||||||
|
Практически это означает:
|
||||||
|
- использовать неблокирующую попытку route в parser loop;
|
||||||
|
- выносить тяжёлое восстановление в асинхронные side-path.
|
||||||
|
|
||||||
|
## Стратегия доменов отказа
|
||||||
|
|
||||||
|
### Отказ отдельного endpoint
|
||||||
|
Сначала применяется endpoint-local recovery:
|
||||||
|
- reconnect в тот же endpoint;
|
||||||
|
- затем замена endpoint внутри той же DC-группы (если доступно).
|
||||||
|
|
||||||
|
### Деградация уровня DC
|
||||||
|
Если DC-группа не набирает floor:
|
||||||
|
- сервис сохраняется на остаточном покрытии (если policy разрешает);
|
||||||
|
- saturation refill продолжается асинхронно в фоне.
|
||||||
|
|
||||||
|
### Потеря готовности всего пула
|
||||||
|
Если достаточного ME-покрытия нет:
|
||||||
|
- admission gate может временно закрыть приём новых подключений (conditional policy);
|
||||||
|
- уже активные сессии продолжают работать, пока их маршрут остаётся healthy.
|
||||||
|
|
||||||
|
## Архитектурные заметки по производительности
|
||||||
|
|
||||||
|
### Дисциплина hotpath
|
||||||
|
Допустимо в hotpath:
|
||||||
|
- фиксированный и дешёвый parsing/validation;
|
||||||
|
- bounded channel operations;
|
||||||
|
- precomputed/low-allocation доступ к данным.
|
||||||
|
|
||||||
|
Нежелательно в hotpath:
|
||||||
|
- повторные дорогие decode;
|
||||||
|
- широкие lock-секции с `await` внутри;
|
||||||
|
- высокочастотный подробный logging.
|
||||||
|
|
||||||
|
### Стабильность важнее пиков
|
||||||
|
Архитектура приоритетно выбирает стабильную пропускную способность и предсказуемую latency, а не краткосрочные пики ценой churn и long-tail reconnect.
|
||||||
|
|
||||||
|
## Правила эволюции модели
|
||||||
|
|
||||||
|
Чтобы расширять модель безопасно:
|
||||||
|
- новые policy knobs сначала внедрять в Control Plane;
|
||||||
|
- контракты Data Plane (`conn_id`, route/close семантика) держать стабильными;
|
||||||
|
- перед дефолтным включением проверять generation/registry инварианты;
|
||||||
|
- новые recovery/retry стратегии вводить через явный config-флаг.
|
||||||
|
|
||||||
|
## Нюансы отказов и восстановления
|
||||||
|
|
||||||
|
- падение single-endpoint DC — штатный деградированный сценарий; приоритет: быстрый reconnect и, при необходимости, shadow/probing;
|
||||||
|
- idle-close со стороны peer должен считаться нормальным событием при upstream idle-timeout;
|
||||||
|
- backoff reconnect-логики должен ограничивать синхронный churn, но сохранять быстрые первые попытки;
|
||||||
|
- fallback (`ME -> direct DC`) — это переключаемая policy-ветка, а не автоматический признак бага транспорта.
|
||||||
|
|
||||||
|
## Краткий словарь
|
||||||
|
- `Coverage`: достаточное число живых writer-ов для политики приёма по DC.
|
||||||
|
- `Floor`: целевая минимальная ёмкость writer-ов.
|
||||||
|
- `Churn`: частые циклы reconnect/remove writer-ов.
|
||||||
|
- `Hotpath`: пер-пакетный/пер-коннектный путь, где любые лишние ожидания и аллокации особенно дороги.
|
||||||
90
src/api/events.rs
Normal file
90
src/api/events.rs
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct ApiEventRecord {
|
||||||
|
pub(super) seq: u64,
|
||||||
|
pub(super) ts_epoch_secs: u64,
|
||||||
|
pub(super) event_type: String,
|
||||||
|
pub(super) context: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct ApiEventSnapshot {
|
||||||
|
pub(super) capacity: usize,
|
||||||
|
pub(super) dropped_total: u64,
|
||||||
|
pub(super) events: Vec<ApiEventRecord>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ApiEventsInner {
|
||||||
|
capacity: usize,
|
||||||
|
dropped_total: u64,
|
||||||
|
next_seq: u64,
|
||||||
|
events: VecDeque<ApiEventRecord>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded ring-buffer for control-plane API/runtime events.
|
||||||
|
pub(crate) struct ApiEventStore {
|
||||||
|
inner: Mutex<ApiEventsInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ApiEventStore {
|
||||||
|
pub(super) fn new(capacity: usize) -> Self {
|
||||||
|
let bounded = capacity.max(16);
|
||||||
|
Self {
|
||||||
|
inner: Mutex::new(ApiEventsInner {
|
||||||
|
capacity: bounded,
|
||||||
|
dropped_total: 0,
|
||||||
|
next_seq: 1,
|
||||||
|
events: VecDeque::with_capacity(bounded),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn record(&self, event_type: &str, context: impl Into<String>) {
|
||||||
|
let now_epoch_secs = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs();
|
||||||
|
let mut context = context.into();
|
||||||
|
if context.len() > 256 {
|
||||||
|
context.truncate(256);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut guard = self.inner.lock().expect("api event store mutex poisoned");
|
||||||
|
if guard.events.len() == guard.capacity {
|
||||||
|
guard.events.pop_front();
|
||||||
|
guard.dropped_total = guard.dropped_total.saturating_add(1);
|
||||||
|
}
|
||||||
|
let seq = guard.next_seq;
|
||||||
|
guard.next_seq = guard.next_seq.saturating_add(1);
|
||||||
|
guard.events.push_back(ApiEventRecord {
|
||||||
|
seq,
|
||||||
|
ts_epoch_secs: now_epoch_secs,
|
||||||
|
event_type: event_type.to_string(),
|
||||||
|
context,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn snapshot(&self, limit: usize) -> ApiEventSnapshot {
|
||||||
|
let guard = self.inner.lock().expect("api event store mutex poisoned");
|
||||||
|
let bounded_limit = limit.clamp(1, guard.capacity.max(1));
|
||||||
|
let mut items: Vec<ApiEventRecord> = guard
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.take(bounded_limit)
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
items.reverse();
|
||||||
|
|
||||||
|
ApiEventSnapshot {
|
||||||
|
capacity: guard.capacity,
|
||||||
|
dropped_total: guard.dropped_total,
|
||||||
|
events: items,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
91
src/api/http_utils.rs
Normal file
91
src/api/http_utils.rs
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
use http_body_util::{BodyExt, Full};
|
||||||
|
use hyper::StatusCode;
|
||||||
|
use hyper::body::{Bytes, Incoming};
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde::de::DeserializeOwned;
|
||||||
|
|
||||||
|
use super::model::{ApiFailure, ErrorBody, ErrorResponse, SuccessResponse};
|
||||||
|
|
||||||
|
pub(super) fn success_response<T: Serialize>(
|
||||||
|
status: StatusCode,
|
||||||
|
data: T,
|
||||||
|
revision: String,
|
||||||
|
) -> hyper::Response<Full<Bytes>> {
|
||||||
|
let payload = SuccessResponse {
|
||||||
|
ok: true,
|
||||||
|
data,
|
||||||
|
revision,
|
||||||
|
};
|
||||||
|
let body = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{\"ok\":false}".to_vec());
|
||||||
|
hyper::Response::builder()
|
||||||
|
.status(status)
|
||||||
|
.header("content-type", "application/json; charset=utf-8")
|
||||||
|
.body(Full::new(Bytes::from(body)))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn error_response(
|
||||||
|
request_id: u64,
|
||||||
|
failure: ApiFailure,
|
||||||
|
) -> hyper::Response<Full<Bytes>> {
|
||||||
|
let payload = ErrorResponse {
|
||||||
|
ok: false,
|
||||||
|
error: ErrorBody {
|
||||||
|
code: failure.code,
|
||||||
|
message: failure.message,
|
||||||
|
},
|
||||||
|
request_id,
|
||||||
|
};
|
||||||
|
let body = serde_json::to_vec(&payload).unwrap_or_else(|_| {
|
||||||
|
format!(
|
||||||
|
"{{\"ok\":false,\"error\":{{\"code\":\"internal_error\",\"message\":\"serialization failed\"}},\"request_id\":{}}}",
|
||||||
|
request_id
|
||||||
|
)
|
||||||
|
.into_bytes()
|
||||||
|
});
|
||||||
|
hyper::Response::builder()
|
||||||
|
.status(failure.status)
|
||||||
|
.header("content-type", "application/json; charset=utf-8")
|
||||||
|
.body(Full::new(Bytes::from(body)))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn read_json<T: DeserializeOwned>(
|
||||||
|
body: Incoming,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<T, ApiFailure> {
|
||||||
|
let bytes = read_body_with_limit(body, limit).await?;
|
||||||
|
serde_json::from_slice(&bytes).map_err(|_| ApiFailure::bad_request("Invalid JSON body"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn read_optional_json<T: DeserializeOwned>(
|
||||||
|
body: Incoming,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Option<T>, ApiFailure> {
|
||||||
|
let bytes = read_body_with_limit(body, limit).await?;
|
||||||
|
if bytes.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&bytes)
|
||||||
|
.map(Some)
|
||||||
|
.map_err(|_| ApiFailure::bad_request("Invalid JSON body"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_body_with_limit(body: Incoming, limit: usize) -> Result<Vec<u8>, ApiFailure> {
|
||||||
|
let mut collected = Vec::new();
|
||||||
|
let mut body = body;
|
||||||
|
while let Some(frame_result) = body.frame().await {
|
||||||
|
let frame = frame_result.map_err(|_| ApiFailure::bad_request("Invalid request body"))?;
|
||||||
|
if let Some(chunk) = frame.data_ref() {
|
||||||
|
if collected.len().saturating_add(chunk.len()) > limit {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::PAYLOAD_TOO_LARGE,
|
||||||
|
"payload_too_large",
|
||||||
|
format!("Body exceeds {} bytes", limit),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
collected.extend_from_slice(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(collected)
|
||||||
|
}
|
||||||
270
src/api/mod.rs
270
src/api/mod.rs
@@ -2,16 +2,14 @@ use std::convert::Infallible;
|
|||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
|
||||||
use http_body_util::{BodyExt, Full};
|
use http_body_util::Full;
|
||||||
use hyper::body::{Bytes, Incoming};
|
use hyper::body::{Bytes, Incoming};
|
||||||
use hyper::header::AUTHORIZATION;
|
use hyper::header::AUTHORIZATION;
|
||||||
use hyper::server::conn::http1;
|
use hyper::server::conn::http1;
|
||||||
use hyper::service::service_fn;
|
use hyper::service::service_fn;
|
||||||
use hyper::{Method, Request, Response, StatusCode};
|
use hyper::{Method, Request, Response, StatusCode};
|
||||||
use serde::Serialize;
|
|
||||||
use serde::de::DeserializeOwned;
|
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::sync::{Mutex, watch};
|
use tokio::sync::{Mutex, watch};
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
@@ -23,21 +21,48 @@ use crate::transport::middle_proxy::MePool;
|
|||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
|
|
||||||
mod config_store;
|
mod config_store;
|
||||||
|
mod events;
|
||||||
|
mod http_utils;
|
||||||
mod model;
|
mod model;
|
||||||
|
mod runtime_edge;
|
||||||
|
mod runtime_min;
|
||||||
mod runtime_stats;
|
mod runtime_stats;
|
||||||
|
mod runtime_watch;
|
||||||
|
mod runtime_zero;
|
||||||
mod users;
|
mod users;
|
||||||
|
|
||||||
use config_store::{current_revision, parse_if_match};
|
use config_store::{current_revision, parse_if_match};
|
||||||
|
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||||
|
use events::ApiEventStore;
|
||||||
use model::{
|
use model::{
|
||||||
ApiFailure, CreateUserRequest, ErrorBody, ErrorResponse, HealthData, PatchUserRequest,
|
ApiFailure, CreateUserRequest, HealthData, PatchUserRequest, RotateSecretRequest, SummaryData,
|
||||||
RotateSecretRequest, SuccessResponse, SummaryData,
|
};
|
||||||
|
use runtime_edge::{
|
||||||
|
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||||
|
build_runtime_events_recent_data,
|
||||||
|
};
|
||||||
|
use runtime_min::{
|
||||||
|
build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data,
|
||||||
|
build_runtime_upstream_quality_data, build_security_whitelist_data,
|
||||||
};
|
};
|
||||||
use runtime_stats::{
|
use runtime_stats::{
|
||||||
MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data,
|
MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data,
|
||||||
build_upstreams_data, build_zero_all_data,
|
build_upstreams_data, build_zero_all_data,
|
||||||
};
|
};
|
||||||
|
use runtime_zero::{
|
||||||
|
build_limits_effective_data, build_runtime_gates_data, build_security_posture_data,
|
||||||
|
build_system_info_data,
|
||||||
|
};
|
||||||
|
use runtime_watch::spawn_runtime_watchers;
|
||||||
use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config};
|
use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config};
|
||||||
|
|
||||||
|
pub(super) struct ApiRuntimeState {
|
||||||
|
pub(super) process_started_at_epoch_secs: u64,
|
||||||
|
pub(super) config_reload_count: AtomicU64,
|
||||||
|
pub(super) last_config_reload_epoch_secs: AtomicU64,
|
||||||
|
pub(super) admission_open: AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(super) struct ApiShared {
|
pub(super) struct ApiShared {
|
||||||
pub(super) stats: Arc<Stats>,
|
pub(super) stats: Arc<Stats>,
|
||||||
@@ -49,7 +74,11 @@ pub(super) struct ApiShared {
|
|||||||
pub(super) startup_detected_ip_v6: Option<IpAddr>,
|
pub(super) startup_detected_ip_v6: Option<IpAddr>,
|
||||||
pub(super) mutation_lock: Arc<Mutex<()>>,
|
pub(super) mutation_lock: Arc<Mutex<()>>,
|
||||||
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
||||||
|
pub(super) runtime_edge_connections_cache: Arc<Mutex<Option<EdgeConnectionsCacheEntry>>>,
|
||||||
|
pub(super) runtime_edge_recompute_lock: Arc<Mutex<()>>,
|
||||||
|
pub(super) runtime_events: Arc<ApiEventStore>,
|
||||||
pub(super) request_id: Arc<AtomicU64>,
|
pub(super) request_id: Arc<AtomicU64>,
|
||||||
|
pub(super) runtime_state: Arc<ApiRuntimeState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiShared {
|
impl ApiShared {
|
||||||
@@ -65,9 +94,11 @@ pub async fn serve(
|
|||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
upstream_manager: Arc<UpstreamManager>,
|
upstream_manager: Arc<UpstreamManager>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
admission_rx: watch::Receiver<bool>,
|
||||||
config_path: PathBuf,
|
config_path: PathBuf,
|
||||||
startup_detected_ip_v4: Option<IpAddr>,
|
startup_detected_ip_v4: Option<IpAddr>,
|
||||||
startup_detected_ip_v6: Option<IpAddr>,
|
startup_detected_ip_v6: Option<IpAddr>,
|
||||||
|
process_started_at_epoch_secs: u64,
|
||||||
) {
|
) {
|
||||||
let listener = match TcpListener::bind(listen).await {
|
let listener = match TcpListener::bind(listen).await {
|
||||||
Ok(listener) => listener,
|
Ok(listener) => listener,
|
||||||
@@ -83,6 +114,13 @@ pub async fn serve(
|
|||||||
|
|
||||||
info!("API endpoint: http://{}/v1/*", listen);
|
info!("API endpoint: http://{}/v1/*", listen);
|
||||||
|
|
||||||
|
let runtime_state = Arc::new(ApiRuntimeState {
|
||||||
|
process_started_at_epoch_secs,
|
||||||
|
config_reload_count: AtomicU64::new(0),
|
||||||
|
last_config_reload_epoch_secs: AtomicU64::new(0),
|
||||||
|
admission_open: AtomicBool::new(*admission_rx.borrow()),
|
||||||
|
});
|
||||||
|
|
||||||
let shared = Arc::new(ApiShared {
|
let shared = Arc::new(ApiShared {
|
||||||
stats,
|
stats,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
@@ -93,9 +131,22 @@ pub async fn serve(
|
|||||||
startup_detected_ip_v6,
|
startup_detected_ip_v6,
|
||||||
mutation_lock: Arc::new(Mutex::new(())),
|
mutation_lock: Arc::new(Mutex::new(())),
|
||||||
minimal_cache: Arc::new(Mutex::new(None)),
|
minimal_cache: Arc::new(Mutex::new(None)),
|
||||||
|
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
|
||||||
|
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
|
||||||
|
runtime_events: Arc::new(ApiEventStore::new(
|
||||||
|
config_rx.borrow().server.api.runtime_edge_events_capacity,
|
||||||
|
)),
|
||||||
request_id: Arc::new(AtomicU64::new(1)),
|
request_id: Arc::new(AtomicU64::new(1)),
|
||||||
|
runtime_state: runtime_state.clone(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
spawn_runtime_watchers(
|
||||||
|
config_rx.clone(),
|
||||||
|
admission_rx.clone(),
|
||||||
|
runtime_state.clone(),
|
||||||
|
shared.runtime_events.clone(),
|
||||||
|
);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let (stream, peer) = match listener.accept().await {
|
let (stream, peer) = match listener.accept().await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
@@ -177,6 +228,7 @@ async fn handle(
|
|||||||
|
|
||||||
let method = req.method().clone();
|
let method = req.method().clone();
|
||||||
let path = req.uri().path().to_string();
|
let path = req.uri().path().to_string();
|
||||||
|
let query = req.uri().query().map(str::to_string);
|
||||||
let body_limit = api_cfg.request_body_limit_bytes;
|
let body_limit = api_cfg.request_body_limit_bytes;
|
||||||
|
|
||||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||||
@@ -189,6 +241,31 @@ async fn handle(
|
|||||||
};
|
};
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
}
|
}
|
||||||
|
("GET", "/v1/system/info") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/gates") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/limits/effective") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_limits_effective_data(cfg.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/security/posture") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_security_posture_data(cfg.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/security/whitelist") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_security_whitelist_data(cfg.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
("GET", "/v1/stats/summary") => {
|
("GET", "/v1/stats/summary") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let data = SummaryData {
|
let data = SummaryData {
|
||||||
@@ -225,6 +302,40 @@ async fn handle(
|
|||||||
let data = build_dcs_data(shared.as_ref(), api_cfg).await;
|
let data = build_dcs_data(shared.as_ref(), api_cfg).await;
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
}
|
}
|
||||||
|
("GET", "/v1/runtime/me_pool_state") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_me_pool_state_data(shared.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/me_quality") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_me_quality_data(shared.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/upstream_quality") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_upstream_quality_data(shared.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/nat_stun") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_nat_stun_data(shared.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/connections/summary") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_connections_summary_data(shared.as_ref(), cfg.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/events/recent") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_events_recent_data(
|
||||||
|
shared.as_ref(),
|
||||||
|
cfg.as_ref(),
|
||||||
|
query.as_deref(),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let users = users_from_config(
|
let users = users_from_config(
|
||||||
@@ -250,7 +361,17 @@ async fn handle(
|
|||||||
}
|
}
|
||||||
let expected_revision = parse_if_match(req.headers());
|
let expected_revision = parse_if_match(req.headers());
|
||||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||||
let (data, revision) = create_user(body, expected_revision, &shared).await?;
|
let result = create_user(body, expected_revision, &shared).await;
|
||||||
|
let (data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record("api.user.create.failed", error.code);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.create.ok", format!("username={}", data.user.username));
|
||||||
Ok(success_response(StatusCode::CREATED, data, revision))
|
Ok(success_response(StatusCode::CREATED, data, revision))
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
@@ -290,8 +411,20 @@ async fn handle(
|
|||||||
}
|
}
|
||||||
let expected_revision = parse_if_match(req.headers());
|
let expected_revision = parse_if_match(req.headers());
|
||||||
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||||
let (data, revision) =
|
let result = patch_user(user, body, expected_revision, &shared).await;
|
||||||
patch_user(user, body, expected_revision, &shared).await?;
|
let (data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.patch.failed",
|
||||||
|
format!("username={} code={}", user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||||
return Ok(success_response(StatusCode::OK, data, revision));
|
return Ok(success_response(StatusCode::OK, data, revision));
|
||||||
}
|
}
|
||||||
if method == Method::DELETE {
|
if method == Method::DELETE {
|
||||||
@@ -306,8 +439,21 @@ async fn handle(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
let expected_revision = parse_if_match(req.headers());
|
let expected_revision = parse_if_match(req.headers());
|
||||||
let (deleted_user, revision) =
|
let result = delete_user(user, expected_revision, &shared).await;
|
||||||
delete_user(user, expected_revision, &shared).await?;
|
let (deleted_user, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.delete.failed",
|
||||||
|
format!("username={} code={}", user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.delete.ok",
|
||||||
|
format!("username={}", deleted_user),
|
||||||
|
);
|
||||||
return Ok(success_response(StatusCode::OK, deleted_user, revision));
|
return Ok(success_response(StatusCode::OK, deleted_user, revision));
|
||||||
}
|
}
|
||||||
if method == Method::POST
|
if method == Method::POST
|
||||||
@@ -329,9 +475,27 @@ async fn handle(
|
|||||||
let body =
|
let body =
|
||||||
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
||||||
.await?;
|
.await?;
|
||||||
let (data, revision) =
|
let result = rotate_secret(
|
||||||
rotate_secret(base_user, body.unwrap_or_default(), expected_revision, &shared)
|
base_user,
|
||||||
.await?;
|
body.unwrap_or_default(),
|
||||||
|
expected_revision,
|
||||||
|
&shared,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.failed",
|
||||||
|
format!("username={} code={}", base_user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.ok",
|
||||||
|
format!("username={}", base_user),
|
||||||
|
);
|
||||||
return Ok(success_response(StatusCode::OK, data, revision));
|
return Ok(success_response(StatusCode::OK, data, revision));
|
||||||
}
|
}
|
||||||
if method == Method::POST {
|
if method == Method::POST {
|
||||||
@@ -363,81 +527,3 @@ async fn handle(
|
|||||||
Err(error) => Ok(error_response(request_id, error)),
|
Err(error) => Ok(error_response(request_id, error)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn success_response<T: Serialize>(
|
|
||||||
status: StatusCode,
|
|
||||||
data: T,
|
|
||||||
revision: String,
|
|
||||||
) -> Response<Full<Bytes>> {
|
|
||||||
let payload = SuccessResponse {
|
|
||||||
ok: true,
|
|
||||||
data,
|
|
||||||
revision,
|
|
||||||
};
|
|
||||||
let body = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{\"ok\":false}".to_vec());
|
|
||||||
Response::builder()
|
|
||||||
.status(status)
|
|
||||||
.header("content-type", "application/json; charset=utf-8")
|
|
||||||
.body(Full::new(Bytes::from(body)))
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn error_response(request_id: u64, failure: ApiFailure) -> Response<Full<Bytes>> {
|
|
||||||
let payload = ErrorResponse {
|
|
||||||
ok: false,
|
|
||||||
error: ErrorBody {
|
|
||||||
code: failure.code,
|
|
||||||
message: failure.message,
|
|
||||||
},
|
|
||||||
request_id,
|
|
||||||
};
|
|
||||||
let body = serde_json::to_vec(&payload).unwrap_or_else(|_| {
|
|
||||||
format!(
|
|
||||||
"{{\"ok\":false,\"error\":{{\"code\":\"internal_error\",\"message\":\"serialization failed\"}},\"request_id\":{}}}",
|
|
||||||
request_id
|
|
||||||
)
|
|
||||||
.into_bytes()
|
|
||||||
});
|
|
||||||
Response::builder()
|
|
||||||
.status(failure.status)
|
|
||||||
.header("content-type", "application/json; charset=utf-8")
|
|
||||||
.body(Full::new(Bytes::from(body)))
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_json<T: DeserializeOwned>(body: Incoming, limit: usize) -> Result<T, ApiFailure> {
|
|
||||||
let bytes = read_body_with_limit(body, limit).await?;
|
|
||||||
serde_json::from_slice(&bytes).map_err(|_| ApiFailure::bad_request("Invalid JSON body"))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_optional_json<T: DeserializeOwned>(
|
|
||||||
body: Incoming,
|
|
||||||
limit: usize,
|
|
||||||
) -> Result<Option<T>, ApiFailure> {
|
|
||||||
let bytes = read_body_with_limit(body, limit).await?;
|
|
||||||
if bytes.is_empty() {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
serde_json::from_slice(&bytes)
|
|
||||||
.map(Some)
|
|
||||||
.map_err(|_| ApiFailure::bad_request("Invalid JSON body"))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_body_with_limit(body: Incoming, limit: usize) -> Result<Vec<u8>, ApiFailure> {
|
|
||||||
let mut collected = Vec::new();
|
|
||||||
let mut body = body;
|
|
||||||
while let Some(frame_result) = body.frame().await {
|
|
||||||
let frame = frame_result.map_err(|_| ApiFailure::bad_request("Invalid request body"))?;
|
|
||||||
if let Some(chunk) = frame.data_ref() {
|
|
||||||
if collected.len().saturating_add(chunk.len()) > limit {
|
|
||||||
return Err(ApiFailure::new(
|
|
||||||
StatusCode::PAYLOAD_TOO_LARGE,
|
|
||||||
"payload_too_large",
|
|
||||||
format!("Body exceeds {} bytes", limit),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
collected.extend_from_slice(chunk);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(collected)
|
|
||||||
}
|
|
||||||
|
|||||||
294
src/api/runtime_edge.rs
Normal file
294
src/api/runtime_edge.rs
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
use std::cmp::Reverse;
|
||||||
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
use super::ApiShared;
|
||||||
|
use super::events::ApiEventRecord;
|
||||||
|
|
||||||
|
const FEATURE_DISABLED_REASON: &str = "feature_disabled";
|
||||||
|
const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable";
|
||||||
|
const EVENTS_DEFAULT_LIMIT: usize = 50;
|
||||||
|
const EVENTS_MAX_LIMIT: usize = 1000;
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionUserData {
|
||||||
|
pub(super) username: String,
|
||||||
|
pub(super) current_connections: u64,
|
||||||
|
pub(super) total_octets: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionTotalsData {
|
||||||
|
pub(super) current_connections: u64,
|
||||||
|
pub(super) current_connections_me: u64,
|
||||||
|
pub(super) current_connections_direct: u64,
|
||||||
|
pub(super) active_users: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionTopData {
|
||||||
|
pub(super) limit: usize,
|
||||||
|
pub(super) by_connections: Vec<RuntimeEdgeConnectionUserData>,
|
||||||
|
pub(super) by_throughput: Vec<RuntimeEdgeConnectionUserData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionCacheData {
|
||||||
|
pub(super) ttl_ms: u64,
|
||||||
|
pub(super) served_from_cache: bool,
|
||||||
|
pub(super) stale_cache_used: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionTelemetryData {
|
||||||
|
pub(super) user_enabled: bool,
|
||||||
|
pub(super) throughput_is_cumulative: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionsSummaryPayload {
|
||||||
|
pub(super) cache: RuntimeEdgeConnectionCacheData,
|
||||||
|
pub(super) totals: RuntimeEdgeConnectionTotalsData,
|
||||||
|
pub(super) top: RuntimeEdgeConnectionTopData,
|
||||||
|
pub(super) telemetry: RuntimeEdgeConnectionTelemetryData,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeConnectionsSummaryData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) data: Option<RuntimeEdgeConnectionsSummaryPayload>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct EdgeConnectionsCacheEntry {
|
||||||
|
pub(super) expires_at: Instant,
|
||||||
|
pub(super) payload: RuntimeEdgeConnectionsSummaryPayload,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeEventsPayload {
|
||||||
|
pub(super) capacity: usize,
|
||||||
|
pub(super) dropped_total: u64,
|
||||||
|
pub(super) events: Vec<ApiEventRecord>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeEdgeEventsData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) data: Option<RuntimeEdgeEventsPayload>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_connections_summary_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
) -> RuntimeEdgeConnectionsSummaryData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if !api_cfg.runtime_edge_enabled {
|
||||||
|
return RuntimeEdgeConnectionsSummaryData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(FEATURE_DISABLED_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let (generated_at_epoch_secs, payload) = match get_connections_payload_cached(
|
||||||
|
shared,
|
||||||
|
api_cfg.runtime_edge_cache_ttl_ms,
|
||||||
|
api_cfg.runtime_edge_top_n,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Some(v) => v,
|
||||||
|
None => {
|
||||||
|
return RuntimeEdgeConnectionsSummaryData {
|
||||||
|
enabled: true,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
RuntimeEdgeConnectionsSummaryData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs,
|
||||||
|
data: Some(payload),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_runtime_events_recent_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
query: Option<&str>,
|
||||||
|
) -> RuntimeEdgeEventsData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if !api_cfg.runtime_edge_enabled {
|
||||||
|
return RuntimeEdgeEventsData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(FEATURE_DISABLED_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let limit = parse_recent_events_limit(query, EVENTS_DEFAULT_LIMIT, EVENTS_MAX_LIMIT);
|
||||||
|
let snapshot = shared.runtime_events.snapshot(limit);
|
||||||
|
|
||||||
|
RuntimeEdgeEventsData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: Some(RuntimeEdgeEventsPayload {
|
||||||
|
capacity: snapshot.capacity,
|
||||||
|
dropped_total: snapshot.dropped_total,
|
||||||
|
events: snapshot.events,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_connections_payload_cached(
|
||||||
|
shared: &ApiShared,
|
||||||
|
cache_ttl_ms: u64,
|
||||||
|
top_n: usize,
|
||||||
|
) -> Option<(u64, RuntimeEdgeConnectionsSummaryPayload)> {
|
||||||
|
if cache_ttl_ms > 0 {
|
||||||
|
let now = Instant::now();
|
||||||
|
let cached = shared.runtime_edge_connections_cache.lock().await.clone();
|
||||||
|
if let Some(entry) = cached
|
||||||
|
&& now < entry.expires_at
|
||||||
|
{
|
||||||
|
let mut payload = entry.payload;
|
||||||
|
payload.cache.served_from_cache = true;
|
||||||
|
payload.cache.stale_cache_used = false;
|
||||||
|
return Some((entry.generated_at_epoch_secs, payload));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let Ok(_guard) = shared.runtime_edge_recompute_lock.try_lock() else {
|
||||||
|
let cached = shared.runtime_edge_connections_cache.lock().await.clone();
|
||||||
|
if let Some(entry) = cached {
|
||||||
|
let mut payload = entry.payload;
|
||||||
|
payload.cache.served_from_cache = true;
|
||||||
|
payload.cache.stale_cache_used = true;
|
||||||
|
return Some((entry.generated_at_epoch_secs, payload));
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
|
||||||
|
let generated_at_epoch_secs = now_epoch_secs();
|
||||||
|
let payload = recompute_connections_payload(shared, cache_ttl_ms, top_n).await;
|
||||||
|
|
||||||
|
if cache_ttl_ms > 0 {
|
||||||
|
let entry = EdgeConnectionsCacheEntry {
|
||||||
|
expires_at: Instant::now() + Duration::from_millis(cache_ttl_ms),
|
||||||
|
payload: payload.clone(),
|
||||||
|
generated_at_epoch_secs,
|
||||||
|
};
|
||||||
|
*shared.runtime_edge_connections_cache.lock().await = Some(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
Some((generated_at_epoch_secs, payload))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recompute_connections_payload(
|
||||||
|
shared: &ApiShared,
|
||||||
|
cache_ttl_ms: u64,
|
||||||
|
top_n: usize,
|
||||||
|
) -> RuntimeEdgeConnectionsSummaryPayload {
|
||||||
|
let mut rows = Vec::<RuntimeEdgeConnectionUserData>::new();
|
||||||
|
let mut active_users = 0usize;
|
||||||
|
for entry in shared.stats.iter_user_stats() {
|
||||||
|
let user_stats = entry.value();
|
||||||
|
let current_connections = user_stats
|
||||||
|
.curr_connects
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let total_octets = user_stats
|
||||||
|
.octets_from_client
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed)
|
||||||
|
.saturating_add(
|
||||||
|
user_stats
|
||||||
|
.octets_to_client
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
);
|
||||||
|
if current_connections > 0 {
|
||||||
|
active_users = active_users.saturating_add(1);
|
||||||
|
}
|
||||||
|
rows.push(RuntimeEdgeConnectionUserData {
|
||||||
|
username: entry.key().clone(),
|
||||||
|
current_connections,
|
||||||
|
total_octets,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let limit = top_n.max(1);
|
||||||
|
let mut by_connections = rows.clone();
|
||||||
|
by_connections.sort_by_key(|row| (Reverse(row.current_connections), row.username.clone()));
|
||||||
|
by_connections.truncate(limit);
|
||||||
|
|
||||||
|
let mut by_throughput = rows;
|
||||||
|
by_throughput.sort_by_key(|row| (Reverse(row.total_octets), row.username.clone()));
|
||||||
|
by_throughput.truncate(limit);
|
||||||
|
|
||||||
|
let telemetry = shared.stats.telemetry_policy();
|
||||||
|
RuntimeEdgeConnectionsSummaryPayload {
|
||||||
|
cache: RuntimeEdgeConnectionCacheData {
|
||||||
|
ttl_ms: cache_ttl_ms,
|
||||||
|
served_from_cache: false,
|
||||||
|
stale_cache_used: false,
|
||||||
|
},
|
||||||
|
totals: RuntimeEdgeConnectionTotalsData {
|
||||||
|
current_connections: shared.stats.get_current_connections_total(),
|
||||||
|
current_connections_me: shared.stats.get_current_connections_me(),
|
||||||
|
current_connections_direct: shared.stats.get_current_connections_direct(),
|
||||||
|
active_users,
|
||||||
|
},
|
||||||
|
top: RuntimeEdgeConnectionTopData {
|
||||||
|
limit,
|
||||||
|
by_connections,
|
||||||
|
by_throughput,
|
||||||
|
},
|
||||||
|
telemetry: RuntimeEdgeConnectionTelemetryData {
|
||||||
|
user_enabled: telemetry.user_enabled,
|
||||||
|
throughput_is_cumulative: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_recent_events_limit(query: Option<&str>, default_limit: usize, max_limit: usize) -> usize {
|
||||||
|
let Some(query) = query else {
|
||||||
|
return default_limit;
|
||||||
|
};
|
||||||
|
for pair in query.split('&') {
|
||||||
|
let mut split = pair.splitn(2, '=');
|
||||||
|
if split.next() == Some("limit")
|
||||||
|
&& let Some(raw) = split.next()
|
||||||
|
&& let Ok(parsed) = raw.parse::<usize>()
|
||||||
|
{
|
||||||
|
return parsed.clamp(1, max_limit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default_limit
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
534
src/api/runtime_min.rs
Normal file
534
src/api/runtime_min.rs
Normal file
@@ -0,0 +1,534 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
use super::ApiShared;
|
||||||
|
|
||||||
|
const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable";
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct SecurityWhitelistData {
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
pub(super) entries_total: usize,
|
||||||
|
pub(super) entries: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateGenerationData {
|
||||||
|
pub(super) active_generation: u64,
|
||||||
|
pub(super) warm_generation: u64,
|
||||||
|
pub(super) pending_hardswap_generation: u64,
|
||||||
|
pub(super) pending_hardswap_age_secs: Option<u64>,
|
||||||
|
pub(super) draining_generations: Vec<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateHardswapData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
pub(super) pending: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateWriterContourData {
|
||||||
|
pub(super) warm: usize,
|
||||||
|
pub(super) active: usize,
|
||||||
|
pub(super) draining: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateWriterHealthData {
|
||||||
|
pub(super) healthy: usize,
|
||||||
|
pub(super) degraded: usize,
|
||||||
|
pub(super) draining: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateWriterData {
|
||||||
|
pub(super) total: usize,
|
||||||
|
pub(super) alive_non_draining: usize,
|
||||||
|
pub(super) draining: usize,
|
||||||
|
pub(super) degraded: usize,
|
||||||
|
pub(super) contour: RuntimeMePoolStateWriterContourData,
|
||||||
|
pub(super) health: RuntimeMePoolStateWriterHealthData,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateRefillDcData {
|
||||||
|
pub(super) dc: i16,
|
||||||
|
pub(super) family: &'static str,
|
||||||
|
pub(super) inflight: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateRefillData {
|
||||||
|
pub(super) inflight_endpoints_total: usize,
|
||||||
|
pub(super) inflight_dc_total: usize,
|
||||||
|
pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStatePayload {
|
||||||
|
pub(super) generations: RuntimeMePoolStateGenerationData,
|
||||||
|
pub(super) hardswap: RuntimeMePoolStateHardswapData,
|
||||||
|
pub(super) writers: RuntimeMePoolStateWriterData,
|
||||||
|
pub(super) refill: RuntimeMePoolStateRefillData,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMePoolStateData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) data: Option<RuntimeMePoolStatePayload>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMeQualityCountersData {
|
||||||
|
pub(super) idle_close_by_peer_total: u64,
|
||||||
|
pub(super) reader_eof_total: u64,
|
||||||
|
pub(super) kdf_drift_total: u64,
|
||||||
|
pub(super) kdf_port_only_drift_total: u64,
|
||||||
|
pub(super) reconnect_attempt_total: u64,
|
||||||
|
pub(super) reconnect_success_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMeQualityRouteDropData {
|
||||||
|
pub(super) no_conn_total: u64,
|
||||||
|
pub(super) channel_closed_total: u64,
|
||||||
|
pub(super) queue_full_total: u64,
|
||||||
|
pub(super) queue_full_base_total: u64,
|
||||||
|
pub(super) queue_full_high_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMeQualityDcRttData {
|
||||||
|
pub(super) dc: i16,
|
||||||
|
pub(super) rtt_ema_ms: Option<f64>,
|
||||||
|
pub(super) alive_writers: usize,
|
||||||
|
pub(super) required_writers: usize,
|
||||||
|
pub(super) coverage_pct: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMeQualityPayload {
|
||||||
|
pub(super) counters: RuntimeMeQualityCountersData,
|
||||||
|
pub(super) route_drops: RuntimeMeQualityRouteDropData,
|
||||||
|
pub(super) dc_rtt: Vec<RuntimeMeQualityDcRttData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeMeQualityData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) data: Option<RuntimeMeQualityPayload>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualityPolicyData {
|
||||||
|
pub(super) connect_retry_attempts: u32,
|
||||||
|
pub(super) connect_retry_backoff_ms: u64,
|
||||||
|
pub(super) connect_budget_ms: u64,
|
||||||
|
pub(super) unhealthy_fail_threshold: u32,
|
||||||
|
pub(super) connect_failfast_hard_errors: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualityCountersData {
|
||||||
|
pub(super) connect_attempt_total: u64,
|
||||||
|
pub(super) connect_success_total: u64,
|
||||||
|
pub(super) connect_fail_total: u64,
|
||||||
|
pub(super) connect_failfast_hard_error_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualitySummaryData {
|
||||||
|
pub(super) configured_total: usize,
|
||||||
|
pub(super) healthy_total: usize,
|
||||||
|
pub(super) unhealthy_total: usize,
|
||||||
|
pub(super) direct_total: usize,
|
||||||
|
pub(super) socks4_total: usize,
|
||||||
|
pub(super) socks5_total: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualityDcData {
|
||||||
|
pub(super) dc: i16,
|
||||||
|
pub(super) latency_ema_ms: Option<f64>,
|
||||||
|
pub(super) ip_preference: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualityUpstreamData {
|
||||||
|
pub(super) upstream_id: usize,
|
||||||
|
pub(super) route_kind: &'static str,
|
||||||
|
pub(super) address: String,
|
||||||
|
pub(super) weight: u16,
|
||||||
|
pub(super) scopes: String,
|
||||||
|
pub(super) healthy: bool,
|
||||||
|
pub(super) fails: u32,
|
||||||
|
pub(super) last_check_age_secs: u64,
|
||||||
|
pub(super) effective_latency_ms: Option<f64>,
|
||||||
|
pub(super) dc: Vec<RuntimeUpstreamQualityDcData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeUpstreamQualityData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
pub(super) policy: RuntimeUpstreamQualityPolicyData,
|
||||||
|
pub(super) counters: RuntimeUpstreamQualityCountersData,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) summary: Option<RuntimeUpstreamQualitySummaryData>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) upstreams: Option<Vec<RuntimeUpstreamQualityUpstreamData>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunReflectionData {
|
||||||
|
pub(super) addr: String,
|
||||||
|
pub(super) age_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunFlagsData {
|
||||||
|
pub(super) nat_probe_enabled: bool,
|
||||||
|
pub(super) nat_probe_disabled_runtime: bool,
|
||||||
|
pub(super) nat_probe_attempts: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunServersData {
|
||||||
|
pub(super) configured: Vec<String>,
|
||||||
|
pub(super) live: Vec<String>,
|
||||||
|
pub(super) live_total: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunReflectionBlockData {
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) v4: Option<RuntimeNatStunReflectionData>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) v6: Option<RuntimeNatStunReflectionData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunPayload {
|
||||||
|
pub(super) flags: RuntimeNatStunFlagsData,
|
||||||
|
pub(super) servers: RuntimeNatStunServersData,
|
||||||
|
pub(super) reflection: RuntimeNatStunReflectionBlockData,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) stun_backoff_remaining_ms: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeNatStunData {
|
||||||
|
pub(super) enabled: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) generated_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) data: Option<RuntimeNatStunPayload>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitelistData {
|
||||||
|
let entries = cfg
|
||||||
|
.server
|
||||||
|
.api
|
||||||
|
.whitelist
|
||||||
|
.iter()
|
||||||
|
.map(ToString::to_string)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
SecurityWhitelistData {
|
||||||
|
generated_at_epoch_secs: now_epoch_secs(),
|
||||||
|
enabled: !entries.is_empty(),
|
||||||
|
entries_total: entries.len(),
|
||||||
|
entries,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let Some(pool) = &shared.me_pool else {
|
||||||
|
return RuntimeMePoolStateData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let status = pool.api_status_snapshot().await;
|
||||||
|
let runtime = pool.api_runtime_snapshot().await;
|
||||||
|
let refill = pool.api_refill_snapshot().await;
|
||||||
|
|
||||||
|
let mut draining_generations = BTreeSet::<u64>::new();
|
||||||
|
let mut contour_warm = 0usize;
|
||||||
|
let mut contour_active = 0usize;
|
||||||
|
let mut contour_draining = 0usize;
|
||||||
|
let mut draining = 0usize;
|
||||||
|
let mut degraded = 0usize;
|
||||||
|
let mut healthy = 0usize;
|
||||||
|
|
||||||
|
for writer in &status.writers {
|
||||||
|
if writer.draining {
|
||||||
|
draining_generations.insert(writer.generation);
|
||||||
|
draining += 1;
|
||||||
|
}
|
||||||
|
if writer.degraded && !writer.draining {
|
||||||
|
degraded += 1;
|
||||||
|
}
|
||||||
|
if !writer.degraded && !writer.draining {
|
||||||
|
healthy += 1;
|
||||||
|
}
|
||||||
|
match writer.state {
|
||||||
|
"warm" => contour_warm += 1,
|
||||||
|
"active" => contour_active += 1,
|
||||||
|
_ => contour_draining += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RuntimeMePoolStateData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs: status.generated_at_epoch_secs,
|
||||||
|
data: Some(RuntimeMePoolStatePayload {
|
||||||
|
generations: RuntimeMePoolStateGenerationData {
|
||||||
|
active_generation: runtime.active_generation,
|
||||||
|
warm_generation: runtime.warm_generation,
|
||||||
|
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
||||||
|
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
||||||
|
draining_generations: draining_generations.into_iter().collect(),
|
||||||
|
},
|
||||||
|
hardswap: RuntimeMePoolStateHardswapData {
|
||||||
|
enabled: runtime.hardswap_enabled,
|
||||||
|
pending: runtime.pending_hardswap_generation != 0,
|
||||||
|
},
|
||||||
|
writers: RuntimeMePoolStateWriterData {
|
||||||
|
total: status.writers.len(),
|
||||||
|
alive_non_draining: status.writers.len().saturating_sub(draining),
|
||||||
|
draining,
|
||||||
|
degraded,
|
||||||
|
contour: RuntimeMePoolStateWriterContourData {
|
||||||
|
warm: contour_warm,
|
||||||
|
active: contour_active,
|
||||||
|
draining: contour_draining,
|
||||||
|
},
|
||||||
|
health: RuntimeMePoolStateWriterHealthData {
|
||||||
|
healthy,
|
||||||
|
degraded,
|
||||||
|
draining,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
refill: RuntimeMePoolStateRefillData {
|
||||||
|
inflight_endpoints_total: refill.inflight_endpoints_total,
|
||||||
|
inflight_dc_total: refill.inflight_dc_total,
|
||||||
|
by_dc: refill
|
||||||
|
.by_dc
|
||||||
|
.into_iter()
|
||||||
|
.map(|entry| RuntimeMePoolStateRefillDcData {
|
||||||
|
dc: entry.dc,
|
||||||
|
family: entry.family,
|
||||||
|
inflight: entry.inflight,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let Some(pool) = &shared.me_pool else {
|
||||||
|
return RuntimeMeQualityData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let status = pool.api_status_snapshot().await;
|
||||||
|
RuntimeMeQualityData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs: status.generated_at_epoch_secs,
|
||||||
|
data: Some(RuntimeMeQualityPayload {
|
||||||
|
counters: RuntimeMeQualityCountersData {
|
||||||
|
idle_close_by_peer_total: shared.stats.get_me_idle_close_by_peer_total(),
|
||||||
|
reader_eof_total: shared.stats.get_me_reader_eof_total(),
|
||||||
|
kdf_drift_total: shared.stats.get_me_kdf_drift_total(),
|
||||||
|
kdf_port_only_drift_total: shared.stats.get_me_kdf_port_only_drift_total(),
|
||||||
|
reconnect_attempt_total: shared.stats.get_me_reconnect_attempts(),
|
||||||
|
reconnect_success_total: shared.stats.get_me_reconnect_success(),
|
||||||
|
},
|
||||||
|
route_drops: RuntimeMeQualityRouteDropData {
|
||||||
|
no_conn_total: shared.stats.get_me_route_drop_no_conn(),
|
||||||
|
channel_closed_total: shared.stats.get_me_route_drop_channel_closed(),
|
||||||
|
queue_full_total: shared.stats.get_me_route_drop_queue_full(),
|
||||||
|
queue_full_base_total: shared.stats.get_me_route_drop_queue_full_base(),
|
||||||
|
queue_full_high_total: shared.stats.get_me_route_drop_queue_full_high(),
|
||||||
|
},
|
||||||
|
dc_rtt: status
|
||||||
|
.dcs
|
||||||
|
.into_iter()
|
||||||
|
.map(|dc| RuntimeMeQualityDcRttData {
|
||||||
|
dc: dc.dc,
|
||||||
|
rtt_ema_ms: dc.rtt_ms,
|
||||||
|
alive_writers: dc.alive_writers,
|
||||||
|
required_writers: dc.required_writers,
|
||||||
|
coverage_pct: dc.coverage_pct,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_upstream_quality_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
) -> RuntimeUpstreamQualityData {
|
||||||
|
let generated_at_epoch_secs = now_epoch_secs();
|
||||||
|
let policy = shared.upstream_manager.api_policy_snapshot();
|
||||||
|
let counters = RuntimeUpstreamQualityCountersData {
|
||||||
|
connect_attempt_total: shared.stats.get_upstream_connect_attempt_total(),
|
||||||
|
connect_success_total: shared.stats.get_upstream_connect_success_total(),
|
||||||
|
connect_fail_total: shared.stats.get_upstream_connect_fail_total(),
|
||||||
|
connect_failfast_hard_error_total: shared.stats.get_upstream_connect_failfast_hard_error_total(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(snapshot) = shared.upstream_manager.try_api_snapshot() else {
|
||||||
|
return RuntimeUpstreamQualityData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs,
|
||||||
|
policy: RuntimeUpstreamQualityPolicyData {
|
||||||
|
connect_retry_attempts: policy.connect_retry_attempts,
|
||||||
|
connect_retry_backoff_ms: policy.connect_retry_backoff_ms,
|
||||||
|
connect_budget_ms: policy.connect_budget_ms,
|
||||||
|
unhealthy_fail_threshold: policy.unhealthy_fail_threshold,
|
||||||
|
connect_failfast_hard_errors: policy.connect_failfast_hard_errors,
|
||||||
|
},
|
||||||
|
counters,
|
||||||
|
summary: None,
|
||||||
|
upstreams: None,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
RuntimeUpstreamQualityData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs,
|
||||||
|
policy: RuntimeUpstreamQualityPolicyData {
|
||||||
|
connect_retry_attempts: policy.connect_retry_attempts,
|
||||||
|
connect_retry_backoff_ms: policy.connect_retry_backoff_ms,
|
||||||
|
connect_budget_ms: policy.connect_budget_ms,
|
||||||
|
unhealthy_fail_threshold: policy.unhealthy_fail_threshold,
|
||||||
|
connect_failfast_hard_errors: policy.connect_failfast_hard_errors,
|
||||||
|
},
|
||||||
|
counters,
|
||||||
|
summary: Some(RuntimeUpstreamQualitySummaryData {
|
||||||
|
configured_total: snapshot.summary.configured_total,
|
||||||
|
healthy_total: snapshot.summary.healthy_total,
|
||||||
|
unhealthy_total: snapshot.summary.unhealthy_total,
|
||||||
|
direct_total: snapshot.summary.direct_total,
|
||||||
|
socks4_total: snapshot.summary.socks4_total,
|
||||||
|
socks5_total: snapshot.summary.socks5_total,
|
||||||
|
}),
|
||||||
|
upstreams: Some(
|
||||||
|
snapshot
|
||||||
|
.upstreams
|
||||||
|
.into_iter()
|
||||||
|
.map(|upstream| RuntimeUpstreamQualityUpstreamData {
|
||||||
|
upstream_id: upstream.upstream_id,
|
||||||
|
route_kind: match upstream.route_kind {
|
||||||
|
crate::transport::UpstreamRouteKind::Direct => "direct",
|
||||||
|
crate::transport::UpstreamRouteKind::Socks4 => "socks4",
|
||||||
|
crate::transport::UpstreamRouteKind::Socks5 => "socks5",
|
||||||
|
},
|
||||||
|
address: upstream.address,
|
||||||
|
weight: upstream.weight,
|
||||||
|
scopes: upstream.scopes,
|
||||||
|
healthy: upstream.healthy,
|
||||||
|
fails: upstream.fails,
|
||||||
|
last_check_age_secs: upstream.last_check_age_secs,
|
||||||
|
effective_latency_ms: upstream.effective_latency_ms,
|
||||||
|
dc: upstream
|
||||||
|
.dc
|
||||||
|
.into_iter()
|
||||||
|
.map(|dc| RuntimeUpstreamQualityDcData {
|
||||||
|
dc: dc.dc,
|
||||||
|
latency_ema_ms: dc.latency_ema_ms,
|
||||||
|
ip_preference: match dc.ip_preference {
|
||||||
|
crate::transport::upstream::IpPreference::Unknown => "unknown",
|
||||||
|
crate::transport::upstream::IpPreference::PreferV6 => "prefer_v6",
|
||||||
|
crate::transport::upstream::IpPreference::PreferV4 => "prefer_v4",
|
||||||
|
crate::transport::upstream::IpPreference::BothWork => "both_work",
|
||||||
|
crate::transport::upstream::IpPreference::Unavailable => "unavailable",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let Some(pool) = &shared.me_pool else {
|
||||||
|
return RuntimeNatStunData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||||
|
RuntimeNatStunData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: Some(RuntimeNatStunPayload {
|
||||||
|
flags: RuntimeNatStunFlagsData {
|
||||||
|
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||||
|
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||||
|
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||||
|
},
|
||||||
|
servers: RuntimeNatStunServersData {
|
||||||
|
configured: snapshot.configured_servers,
|
||||||
|
live: snapshot.live_servers.clone(),
|
||||||
|
live_total: snapshot.live_servers.len(),
|
||||||
|
},
|
||||||
|
reflection: RuntimeNatStunReflectionBlockData {
|
||||||
|
v4: snapshot.reflection_v4.map(|entry| RuntimeNatStunReflectionData {
|
||||||
|
addr: entry.addr.to_string(),
|
||||||
|
age_secs: entry.age_secs,
|
||||||
|
}),
|
||||||
|
v6: snapshot.reflection_v6.map(|entry| RuntimeNatStunReflectionData {
|
||||||
|
addr: entry.addr.to_string(),
|
||||||
|
age_secs: entry.age_secs,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
66
src/api/runtime_watch.rs
Normal file
66
src/api/runtime_watch.rs
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use tokio::sync::watch;
|
||||||
|
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
use super::ApiRuntimeState;
|
||||||
|
use super::events::ApiEventStore;
|
||||||
|
|
||||||
|
pub(super) fn spawn_runtime_watchers(
|
||||||
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
admission_rx: watch::Receiver<bool>,
|
||||||
|
runtime_state: Arc<ApiRuntimeState>,
|
||||||
|
runtime_events: Arc<ApiEventStore>,
|
||||||
|
) {
|
||||||
|
let mut config_rx_reload = config_rx;
|
||||||
|
let runtime_state_reload = runtime_state.clone();
|
||||||
|
let runtime_events_reload = runtime_events.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
if config_rx_reload.changed().await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
runtime_state_reload
|
||||||
|
.config_reload_count
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
runtime_state_reload
|
||||||
|
.last_config_reload_epoch_secs
|
||||||
|
.store(now_epoch_secs(), Ordering::Relaxed);
|
||||||
|
runtime_events_reload.record("config.reload.applied", "config receiver updated");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut admission_rx_watch = admission_rx;
|
||||||
|
tokio::spawn(async move {
|
||||||
|
runtime_state
|
||||||
|
.admission_open
|
||||||
|
.store(*admission_rx_watch.borrow(), Ordering::Relaxed);
|
||||||
|
runtime_events.record(
|
||||||
|
"admission.state",
|
||||||
|
format!("accepting_new_connections={}", *admission_rx_watch.borrow()),
|
||||||
|
);
|
||||||
|
loop {
|
||||||
|
if admission_rx_watch.changed().await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let admission_open = *admission_rx_watch.borrow();
|
||||||
|
runtime_state
|
||||||
|
.admission_open
|
||||||
|
.store(admission_open, Ordering::Relaxed);
|
||||||
|
runtime_events.record(
|
||||||
|
"admission.state",
|
||||||
|
format!("accepting_new_connections={}", admission_open),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
227
src/api/runtime_zero.rs
Normal file
227
src/api/runtime_zero.rs
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode};
|
||||||
|
|
||||||
|
use super::ApiShared;
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct SystemInfoData {
|
||||||
|
pub(super) version: String,
|
||||||
|
pub(super) target_arch: String,
|
||||||
|
pub(super) target_os: String,
|
||||||
|
pub(super) build_profile: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) git_commit: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) build_time_utc: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) rustc_version: Option<String>,
|
||||||
|
pub(super) process_started_at_epoch_secs: u64,
|
||||||
|
pub(super) uptime_seconds: f64,
|
||||||
|
pub(super) config_path: String,
|
||||||
|
pub(super) config_hash: String,
|
||||||
|
pub(super) config_reload_count: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) last_config_reload_epoch_secs: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeGatesData {
|
||||||
|
pub(super) accepting_new_connections: bool,
|
||||||
|
pub(super) conditional_cast_enabled: bool,
|
||||||
|
pub(super) me_runtime_ready: bool,
|
||||||
|
pub(super) me2dc_fallback_enabled: bool,
|
||||||
|
pub(super) use_middle_proxy: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct EffectiveTimeoutLimits {
|
||||||
|
pub(super) client_handshake_secs: u64,
|
||||||
|
pub(super) tg_connect_secs: u64,
|
||||||
|
pub(super) client_keepalive_secs: u64,
|
||||||
|
pub(super) client_ack_secs: u64,
|
||||||
|
pub(super) me_one_retry: u8,
|
||||||
|
pub(super) me_one_timeout_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct EffectiveUpstreamLimits {
|
||||||
|
pub(super) connect_retry_attempts: u32,
|
||||||
|
pub(super) connect_retry_backoff_ms: u64,
|
||||||
|
pub(super) connect_budget_ms: u64,
|
||||||
|
pub(super) unhealthy_fail_threshold: u32,
|
||||||
|
pub(super) connect_failfast_hard_errors: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct EffectiveMiddleProxyLimits {
|
||||||
|
pub(super) floor_mode: &'static str,
|
||||||
|
pub(super) adaptive_floor_idle_secs: u64,
|
||||||
|
pub(super) adaptive_floor_min_writers_single_endpoint: u8,
|
||||||
|
pub(super) adaptive_floor_recover_grace_secs: u64,
|
||||||
|
pub(super) reconnect_max_concurrent_per_dc: u32,
|
||||||
|
pub(super) reconnect_backoff_base_ms: u64,
|
||||||
|
pub(super) reconnect_backoff_cap_ms: u64,
|
||||||
|
pub(super) reconnect_fast_retry_count: u32,
|
||||||
|
pub(super) me2dc_fallback: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct EffectiveUserIpPolicyLimits {
|
||||||
|
pub(super) mode: &'static str,
|
||||||
|
pub(super) window_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct EffectiveLimitsData {
|
||||||
|
pub(super) update_every_secs: u64,
|
||||||
|
pub(super) me_reinit_every_secs: u64,
|
||||||
|
pub(super) me_pool_force_close_secs: u64,
|
||||||
|
pub(super) timeouts: EffectiveTimeoutLimits,
|
||||||
|
pub(super) upstream: EffectiveUpstreamLimits,
|
||||||
|
pub(super) middle_proxy: EffectiveMiddleProxyLimits,
|
||||||
|
pub(super) user_ip_policy: EffectiveUserIpPolicyLimits,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct SecurityPostureData {
|
||||||
|
pub(super) api_read_only: bool,
|
||||||
|
pub(super) api_whitelist_enabled: bool,
|
||||||
|
pub(super) api_whitelist_entries: usize,
|
||||||
|
pub(super) api_auth_header_enabled: bool,
|
||||||
|
pub(super) proxy_protocol_enabled: bool,
|
||||||
|
pub(super) log_level: String,
|
||||||
|
pub(super) telemetry_core_enabled: bool,
|
||||||
|
pub(super) telemetry_user_enabled: bool,
|
||||||
|
pub(super) telemetry_me_level: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_system_info_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
_cfg: &ProxyConfig,
|
||||||
|
revision: &str,
|
||||||
|
) -> SystemInfoData {
|
||||||
|
let last_reload_epoch_secs = shared
|
||||||
|
.runtime_state
|
||||||
|
.last_config_reload_epoch_secs
|
||||||
|
.load(Ordering::Relaxed);
|
||||||
|
let last_config_reload_epoch_secs = (last_reload_epoch_secs > 0).then_some(last_reload_epoch_secs);
|
||||||
|
|
||||||
|
let git_commit = option_env!("TELEMT_GIT_COMMIT")
|
||||||
|
.or(option_env!("VERGEN_GIT_SHA"))
|
||||||
|
.or(option_env!("GIT_COMMIT"))
|
||||||
|
.map(ToString::to_string);
|
||||||
|
let build_time_utc = option_env!("BUILD_TIME_UTC")
|
||||||
|
.or(option_env!("VERGEN_BUILD_TIMESTAMP"))
|
||||||
|
.map(ToString::to_string);
|
||||||
|
let rustc_version = option_env!("RUSTC_VERSION")
|
||||||
|
.or(option_env!("VERGEN_RUSTC_SEMVER"))
|
||||||
|
.map(ToString::to_string);
|
||||||
|
|
||||||
|
SystemInfoData {
|
||||||
|
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||||
|
target_arch: std::env::consts::ARCH.to_string(),
|
||||||
|
target_os: std::env::consts::OS.to_string(),
|
||||||
|
build_profile: option_env!("PROFILE").unwrap_or("unknown").to_string(),
|
||||||
|
git_commit,
|
||||||
|
build_time_utc,
|
||||||
|
rustc_version,
|
||||||
|
process_started_at_epoch_secs: shared.runtime_state.process_started_at_epoch_secs,
|
||||||
|
uptime_seconds: shared.stats.uptime_secs(),
|
||||||
|
config_path: shared.config_path.display().to_string(),
|
||||||
|
config_hash: revision.to_string(),
|
||||||
|
config_reload_count: shared.runtime_state.config_reload_count.load(Ordering::Relaxed),
|
||||||
|
last_config_reload_epoch_secs,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) -> RuntimeGatesData {
|
||||||
|
let me_runtime_ready = if !cfg.general.use_middle_proxy {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
shared
|
||||||
|
.me_pool
|
||||||
|
.as_ref()
|
||||||
|
.map(|pool| pool.is_runtime_ready())
|
||||||
|
.unwrap_or(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
RuntimeGatesData {
|
||||||
|
accepting_new_connections: shared.runtime_state.admission_open.load(Ordering::Relaxed),
|
||||||
|
conditional_cast_enabled: cfg.general.use_middle_proxy,
|
||||||
|
me_runtime_ready,
|
||||||
|
me2dc_fallback_enabled: cfg.general.me2dc_fallback,
|
||||||
|
use_middle_proxy: cfg.general.use_middle_proxy,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsData {
|
||||||
|
EffectiveLimitsData {
|
||||||
|
update_every_secs: cfg.general.effective_update_every_secs(),
|
||||||
|
me_reinit_every_secs: cfg.general.effective_me_reinit_every_secs(),
|
||||||
|
me_pool_force_close_secs: cfg.general.effective_me_pool_force_close_secs(),
|
||||||
|
timeouts: EffectiveTimeoutLimits {
|
||||||
|
client_handshake_secs: cfg.timeouts.client_handshake,
|
||||||
|
tg_connect_secs: cfg.timeouts.tg_connect,
|
||||||
|
client_keepalive_secs: cfg.timeouts.client_keepalive,
|
||||||
|
client_ack_secs: cfg.timeouts.client_ack,
|
||||||
|
me_one_retry: cfg.timeouts.me_one_retry,
|
||||||
|
me_one_timeout_ms: cfg.timeouts.me_one_timeout_ms,
|
||||||
|
},
|
||||||
|
upstream: EffectiveUpstreamLimits {
|
||||||
|
connect_retry_attempts: cfg.general.upstream_connect_retry_attempts,
|
||||||
|
connect_retry_backoff_ms: cfg.general.upstream_connect_retry_backoff_ms,
|
||||||
|
connect_budget_ms: cfg.general.upstream_connect_budget_ms,
|
||||||
|
unhealthy_fail_threshold: cfg.general.upstream_unhealthy_fail_threshold,
|
||||||
|
connect_failfast_hard_errors: cfg.general.upstream_connect_failfast_hard_errors,
|
||||||
|
},
|
||||||
|
middle_proxy: EffectiveMiddleProxyLimits {
|
||||||
|
floor_mode: me_floor_mode_label(cfg.general.me_floor_mode),
|
||||||
|
adaptive_floor_idle_secs: cfg.general.me_adaptive_floor_idle_secs,
|
||||||
|
adaptive_floor_min_writers_single_endpoint: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_min_writers_single_endpoint,
|
||||||
|
adaptive_floor_recover_grace_secs: cfg.general.me_adaptive_floor_recover_grace_secs,
|
||||||
|
reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc,
|
||||||
|
reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms,
|
||||||
|
reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms,
|
||||||
|
reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count,
|
||||||
|
me2dc_fallback: cfg.general.me2dc_fallback,
|
||||||
|
},
|
||||||
|
user_ip_policy: EffectiveUserIpPolicyLimits {
|
||||||
|
mode: user_max_unique_ips_mode_label(cfg.access.user_max_unique_ips_mode),
|
||||||
|
window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_security_posture_data(cfg: &ProxyConfig) -> SecurityPostureData {
|
||||||
|
SecurityPostureData {
|
||||||
|
api_read_only: cfg.server.api.read_only,
|
||||||
|
api_whitelist_enabled: !cfg.server.api.whitelist.is_empty(),
|
||||||
|
api_whitelist_entries: cfg.server.api.whitelist.len(),
|
||||||
|
api_auth_header_enabled: !cfg.server.api.auth_header.is_empty(),
|
||||||
|
proxy_protocol_enabled: cfg.server.proxy_protocol,
|
||||||
|
log_level: cfg.general.log_level.to_string(),
|
||||||
|
telemetry_core_enabled: cfg.general.telemetry.core_enabled,
|
||||||
|
telemetry_user_enabled: cfg.general.telemetry.user_enabled,
|
||||||
|
telemetry_me_level: cfg.general.telemetry.me_level.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn user_max_unique_ips_mode_label(mode: UserMaxUniqueIpsMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
UserMaxUniqueIpsMode::ActiveWindow => "active_window",
|
||||||
|
UserMaxUniqueIpsMode::TimeWindow => "time_window",
|
||||||
|
UserMaxUniqueIpsMode::Combined => "combined",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn me_floor_mode_label(mode: MeFloorMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
MeFloorMode::Static => "static",
|
||||||
|
MeFloorMode::Adaptive => "adaptive",
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180;
|
|||||||
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
||||||
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
|
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
|
||||||
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
|
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
|
||||||
|
const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000;
|
||||||
const DEFAULT_LISTEN_ADDR_IPV6: &str = "::";
|
const DEFAULT_LISTEN_ADDR_IPV6: &str = "::";
|
||||||
const DEFAULT_ACCESS_USER: &str = "default";
|
const DEFAULT_ACCESS_USER: &str = "default";
|
||||||
const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000";
|
const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000";
|
||||||
@@ -113,6 +114,15 @@ pub(crate) fn default_api_minimal_runtime_cache_ttl_ms() -> u64 {
|
|||||||
1000
|
1000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_api_runtime_edge_enabled() -> bool { false }
|
||||||
|
pub(crate) fn default_api_runtime_edge_cache_ttl_ms() -> u64 { 1000 }
|
||||||
|
pub(crate) fn default_api_runtime_edge_top_n() -> usize { 10 }
|
||||||
|
pub(crate) fn default_api_runtime_edge_events_capacity() -> usize { 256 }
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_protocol_header_timeout_ms() -> u64 {
|
||||||
|
500
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn default_prefer_4() -> u8 {
|
pub(crate) fn default_prefer_4() -> u8 {
|
||||||
4
|
4
|
||||||
}
|
}
|
||||||
@@ -253,6 +263,10 @@ pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
|||||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||||
|
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -312,6 +312,12 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
|| old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled
|
|| old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled
|
||||||
|| old.server.api.minimal_runtime_cache_ttl_ms
|
|| old.server.api.minimal_runtime_cache_ttl_ms
|
||||||
!= new.server.api.minimal_runtime_cache_ttl_ms
|
!= new.server.api.minimal_runtime_cache_ttl_ms
|
||||||
|
|| old.server.api.runtime_edge_enabled != new.server.api.runtime_edge_enabled
|
||||||
|
|| old.server.api.runtime_edge_cache_ttl_ms
|
||||||
|
!= new.server.api.runtime_edge_cache_ttl_ms
|
||||||
|
|| old.server.api.runtime_edge_top_n != new.server.api.runtime_edge_top_n
|
||||||
|
|| old.server.api.runtime_edge_events_capacity
|
||||||
|
!= new.server.api.runtime_edge_events_capacity
|
||||||
|| old.server.api.read_only != new.server.api.read_only
|
|| old.server.api.read_only != new.server.api.read_only
|
||||||
{
|
{
|
||||||
warned = true;
|
warned = true;
|
||||||
|
|||||||
@@ -265,6 +265,12 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.general.upstream_connect_budget_ms == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.upstream_connect_budget_ms must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.general.upstream_unhealthy_fail_threshold == 0 {
|
if config.general.upstream_unhealthy_fail_threshold == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.upstream_unhealthy_fail_threshold must be > 0".to_string(),
|
"general.upstream_unhealthy_fail_threshold must be > 0".to_string(),
|
||||||
@@ -456,12 +462,36 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.server.api.runtime_edge_cache_ttl_ms > 60_000 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"server.api.runtime_edge_cache_ttl_ms must be within [0, 60000]".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !(1..=1000).contains(&config.server.api.runtime_edge_top_n) {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"server.api.runtime_edge_top_n must be within [1, 1000]".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !(16..=4096).contains(&config.server.api.runtime_edge_events_capacity) {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"server.api.runtime_edge_events_capacity must be within [16, 4096]".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.server.api.listen.parse::<SocketAddr>().is_err() {
|
if config.server.api.listen.parse::<SocketAddr>().is_err() {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"server.api.listen must be in IP:PORT format".to_string(),
|
"server.api.listen must be in IP:PORT format".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.server.proxy_protocol_header_timeout_ms == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"server.proxy_protocol_header_timeout_ms must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.general.effective_me_pool_force_close_secs() > 0
|
if config.general.effective_me_pool_force_close_secs() > 0
|
||||||
&& config.general.effective_me_pool_force_close_secs()
|
&& config.general.effective_me_pool_force_close_secs()
|
||||||
< config.general.me_pool_drain_ttl_secs
|
< config.general.me_pool_drain_ttl_secs
|
||||||
@@ -543,10 +573,11 @@ impl ProxyConfig {
|
|||||||
warn!("prefer_ipv6 is deprecated, use [network].prefer = 6");
|
warn!("prefer_ipv6 is deprecated, use [network].prefer = 6");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto-enable NAT probe when Middle Proxy is requested.
|
if config.general.use_middle_proxy && !config.general.me_secret_atomic_snapshot {
|
||||||
if config.general.use_middle_proxy && !config.general.middle_proxy_nat_probe {
|
config.general.me_secret_atomic_snapshot = true;
|
||||||
config.general.middle_proxy_nat_probe = true;
|
warn!(
|
||||||
warn!("Auto-enabled middle_proxy_nat_probe for middle proxy mode");
|
"Auto-enabled me_secret_atomic_snapshot for middle proxy mode to keep KDF key_selector/secret coherent"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
validate_network_cfg(&mut config.network)?;
|
validate_network_cfg(&mut config.network)?;
|
||||||
@@ -789,6 +820,22 @@ mod tests {
|
|||||||
cfg.server.api.minimal_runtime_cache_ttl_ms,
|
cfg.server.api.minimal_runtime_cache_ttl_ms,
|
||||||
default_api_minimal_runtime_cache_ttl_ms()
|
default_api_minimal_runtime_cache_ttl_ms()
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.api.runtime_edge_enabled,
|
||||||
|
default_api_runtime_edge_enabled()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.api.runtime_edge_cache_ttl_ms,
|
||||||
|
default_api_runtime_edge_cache_ttl_ms()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.api.runtime_edge_top_n,
|
||||||
|
default_api_runtime_edge_top_n()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.api.runtime_edge_events_capacity,
|
||||||
|
default_api_runtime_edge_events_capacity()
|
||||||
|
);
|
||||||
assert_eq!(cfg.access.users, default_access_users());
|
assert_eq!(cfg.access.users, default_access_users());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
cfg.access.user_max_unique_ips_mode,
|
cfg.access.user_max_unique_ips_mode,
|
||||||
@@ -905,6 +952,22 @@ mod tests {
|
|||||||
server.api.minimal_runtime_cache_ttl_ms,
|
server.api.minimal_runtime_cache_ttl_ms,
|
||||||
default_api_minimal_runtime_cache_ttl_ms()
|
default_api_minimal_runtime_cache_ttl_ms()
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
server.api.runtime_edge_enabled,
|
||||||
|
default_api_runtime_edge_enabled()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
server.api.runtime_edge_cache_ttl_ms,
|
||||||
|
default_api_runtime_edge_cache_ttl_ms()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
server.api.runtime_edge_top_n,
|
||||||
|
default_api_runtime_edge_top_n()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
server.api.runtime_edge_events_capacity,
|
||||||
|
default_api_runtime_edge_events_capacity()
|
||||||
|
);
|
||||||
|
|
||||||
let access = AccessConfig::default();
|
let access = AccessConfig::default();
|
||||||
assert_eq!(access.users, default_access_users());
|
assert_eq!(access.users, default_access_users());
|
||||||
@@ -1552,6 +1615,72 @@ mod tests {
|
|||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_runtime_edge_cache_ttl_out_of_range_is_rejected() {
|
||||||
|
let toml = r#"
|
||||||
|
[server.api]
|
||||||
|
enabled = true
|
||||||
|
listen = "127.0.0.1:9091"
|
||||||
|
runtime_edge_cache_ttl_ms = 70000
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#;
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join("telemt_api_runtime_edge_cache_ttl_invalid_test.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
|
assert!(err.contains("server.api.runtime_edge_cache_ttl_ms must be within [0, 60000]"));
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_runtime_edge_top_n_out_of_range_is_rejected() {
|
||||||
|
let toml = r#"
|
||||||
|
[server.api]
|
||||||
|
enabled = true
|
||||||
|
listen = "127.0.0.1:9091"
|
||||||
|
runtime_edge_top_n = 0
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#;
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join("telemt_api_runtime_edge_top_n_invalid_test.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
|
assert!(err.contains("server.api.runtime_edge_top_n must be within [1, 1000]"));
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_runtime_edge_events_capacity_out_of_range_is_rejected() {
|
||||||
|
let toml = r#"
|
||||||
|
[server.api]
|
||||||
|
enabled = true
|
||||||
|
listen = "127.0.0.1:9091"
|
||||||
|
runtime_edge_events_capacity = 8
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#;
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join("telemt_api_runtime_edge_events_capacity_invalid_test.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
|
assert!(err.contains("server.api.runtime_edge_events_capacity must be within [16, 4096]"));
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn force_close_bumped_when_below_drain_ttl() {
|
fn force_close_bumped_when_below_drain_ttl() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
|
|||||||
@@ -187,9 +187,10 @@ impl MeFloorMode {
|
|||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum MeRouteNoWriterMode {
|
pub enum MeRouteNoWriterMode {
|
||||||
#[default]
|
|
||||||
AsyncRecoveryFailfast,
|
AsyncRecoveryFailfast,
|
||||||
InlineRecoveryLegacy,
|
InlineRecoveryLegacy,
|
||||||
|
#[default]
|
||||||
|
HybridAsyncPersistent,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MeRouteNoWriterMode {
|
impl MeRouteNoWriterMode {
|
||||||
@@ -197,13 +198,16 @@ impl MeRouteNoWriterMode {
|
|||||||
match self {
|
match self {
|
||||||
MeRouteNoWriterMode::AsyncRecoveryFailfast => 0,
|
MeRouteNoWriterMode::AsyncRecoveryFailfast => 0,
|
||||||
MeRouteNoWriterMode::InlineRecoveryLegacy => 1,
|
MeRouteNoWriterMode::InlineRecoveryLegacy => 1,
|
||||||
|
MeRouteNoWriterMode::HybridAsyncPersistent => 2,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_u8(raw: u8) -> Self {
|
pub fn from_u8(raw: u8) -> Self {
|
||||||
match raw {
|
match raw {
|
||||||
|
0 => MeRouteNoWriterMode::AsyncRecoveryFailfast,
|
||||||
1 => MeRouteNoWriterMode::InlineRecoveryLegacy,
|
1 => MeRouteNoWriterMode::InlineRecoveryLegacy,
|
||||||
_ => MeRouteNoWriterMode::AsyncRecoveryFailfast,
|
2 => MeRouteNoWriterMode::HybridAsyncPersistent,
|
||||||
|
_ => MeRouteNoWriterMode::HybridAsyncPersistent,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -528,6 +532,10 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_upstream_connect_retry_backoff_ms")]
|
#[serde(default = "default_upstream_connect_retry_backoff_ms")]
|
||||||
pub upstream_connect_retry_backoff_ms: u64,
|
pub upstream_connect_retry_backoff_ms: u64,
|
||||||
|
|
||||||
|
/// Total wall-clock budget in milliseconds for one upstream connect request across retries.
|
||||||
|
#[serde(default = "default_upstream_connect_budget_ms")]
|
||||||
|
pub upstream_connect_budget_ms: u64,
|
||||||
|
|
||||||
/// Consecutive failed requests before upstream is marked unhealthy.
|
/// Consecutive failed requests before upstream is marked unhealthy.
|
||||||
#[serde(default = "default_upstream_unhealthy_fail_threshold")]
|
#[serde(default = "default_upstream_unhealthy_fail_threshold")]
|
||||||
pub upstream_unhealthy_fail_threshold: u32,
|
pub upstream_unhealthy_fail_threshold: u32,
|
||||||
@@ -770,6 +778,7 @@ impl Default for GeneralConfig {
|
|||||||
me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(),
|
me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(),
|
||||||
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
||||||
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
||||||
|
upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
|
||||||
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
|
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
|
||||||
upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(),
|
upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(),
|
||||||
stun_iface_mismatch_ignore: false,
|
stun_iface_mismatch_ignore: false,
|
||||||
@@ -909,6 +918,22 @@ pub struct ApiConfig {
|
|||||||
#[serde(default = "default_api_minimal_runtime_cache_ttl_ms")]
|
#[serde(default = "default_api_minimal_runtime_cache_ttl_ms")]
|
||||||
pub minimal_runtime_cache_ttl_ms: u64,
|
pub minimal_runtime_cache_ttl_ms: u64,
|
||||||
|
|
||||||
|
/// Enables runtime edge endpoints with optional cached aggregation.
|
||||||
|
#[serde(default = "default_api_runtime_edge_enabled")]
|
||||||
|
pub runtime_edge_enabled: bool,
|
||||||
|
|
||||||
|
/// Cache TTL for runtime edge aggregation payloads in milliseconds.
|
||||||
|
#[serde(default = "default_api_runtime_edge_cache_ttl_ms")]
|
||||||
|
pub runtime_edge_cache_ttl_ms: u64,
|
||||||
|
|
||||||
|
/// Top-N limit for edge connection leaderboard payloads.
|
||||||
|
#[serde(default = "default_api_runtime_edge_top_n")]
|
||||||
|
pub runtime_edge_top_n: usize,
|
||||||
|
|
||||||
|
/// Ring-buffer capacity for runtime edge control-plane events.
|
||||||
|
#[serde(default = "default_api_runtime_edge_events_capacity")]
|
||||||
|
pub runtime_edge_events_capacity: usize,
|
||||||
|
|
||||||
/// Read-only mode: mutating endpoints are rejected.
|
/// Read-only mode: mutating endpoints are rejected.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub read_only: bool,
|
pub read_only: bool,
|
||||||
@@ -924,6 +949,10 @@ impl Default for ApiConfig {
|
|||||||
request_body_limit_bytes: default_api_request_body_limit_bytes(),
|
request_body_limit_bytes: default_api_request_body_limit_bytes(),
|
||||||
minimal_runtime_enabled: default_api_minimal_runtime_enabled(),
|
minimal_runtime_enabled: default_api_minimal_runtime_enabled(),
|
||||||
minimal_runtime_cache_ttl_ms: default_api_minimal_runtime_cache_ttl_ms(),
|
minimal_runtime_cache_ttl_ms: default_api_minimal_runtime_cache_ttl_ms(),
|
||||||
|
runtime_edge_enabled: default_api_runtime_edge_enabled(),
|
||||||
|
runtime_edge_cache_ttl_ms: default_api_runtime_edge_cache_ttl_ms(),
|
||||||
|
runtime_edge_top_n: default_api_runtime_edge_top_n(),
|
||||||
|
runtime_edge_events_capacity: default_api_runtime_edge_events_capacity(),
|
||||||
read_only: false,
|
read_only: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -958,6 +987,10 @@ pub struct ServerConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub proxy_protocol: bool,
|
pub proxy_protocol: bool,
|
||||||
|
|
||||||
|
/// Timeout in milliseconds for reading and parsing PROXY protocol headers.
|
||||||
|
#[serde(default = "default_proxy_protocol_header_timeout_ms")]
|
||||||
|
pub proxy_protocol_header_timeout_ms: u64,
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub metrics_port: Option<u16>,
|
pub metrics_port: Option<u16>,
|
||||||
|
|
||||||
@@ -981,6 +1014,7 @@ impl Default for ServerConfig {
|
|||||||
listen_unix_sock_perm: None,
|
listen_unix_sock_perm: None,
|
||||||
listen_tcp: None,
|
listen_tcp: None,
|
||||||
proxy_protocol: false,
|
proxy_protocol: false,
|
||||||
|
proxy_protocol_header_timeout_ms: default_proxy_protocol_header_timeout_ms(),
|
||||||
metrics_port: None,
|
metrics_port: None,
|
||||||
metrics_whitelist: default_metrics_whitelist(),
|
metrics_whitelist: default_metrics_whitelist(),
|
||||||
api: ApiConfig::default(),
|
api: ApiConfig::default(),
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ struct SecureRandomInner {
|
|||||||
rng: StdRng,
|
rng: StdRng,
|
||||||
cipher: AesCtr,
|
cipher: AesCtr,
|
||||||
buffer: Vec<u8>,
|
buffer: Vec<u8>,
|
||||||
|
buffer_start: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for SecureRandomInner {
|
impl Drop for SecureRandomInner {
|
||||||
@@ -48,6 +49,7 @@ impl SecureRandom {
|
|||||||
rng,
|
rng,
|
||||||
cipher,
|
cipher,
|
||||||
buffer: Vec::with_capacity(1024),
|
buffer: Vec::with_capacity(1024),
|
||||||
|
buffer_start: 0,
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -59,16 +61,29 @@ impl SecureRandom {
|
|||||||
|
|
||||||
let mut written = 0usize;
|
let mut written = 0usize;
|
||||||
while written < out.len() {
|
while written < out.len() {
|
||||||
|
if inner.buffer_start >= inner.buffer.len() {
|
||||||
|
inner.buffer.clear();
|
||||||
|
inner.buffer_start = 0;
|
||||||
|
}
|
||||||
|
|
||||||
if inner.buffer.is_empty() {
|
if inner.buffer.is_empty() {
|
||||||
let mut chunk = vec![0u8; CHUNK_SIZE];
|
let mut chunk = vec![0u8; CHUNK_SIZE];
|
||||||
inner.rng.fill_bytes(&mut chunk);
|
inner.rng.fill_bytes(&mut chunk);
|
||||||
inner.cipher.apply(&mut chunk);
|
inner.cipher.apply(&mut chunk);
|
||||||
inner.buffer.extend_from_slice(&chunk);
|
inner.buffer.extend_from_slice(&chunk);
|
||||||
|
inner.buffer_start = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
let take = (out.len() - written).min(inner.buffer.len());
|
let available = inner.buffer.len().saturating_sub(inner.buffer_start);
|
||||||
out[written..written + take].copy_from_slice(&inner.buffer[..take]);
|
let take = (out.len() - written).min(available);
|
||||||
inner.buffer.drain(..take);
|
let start = inner.buffer_start;
|
||||||
|
let end = start + take;
|
||||||
|
out[written..written + take].copy_from_slice(&inner.buffer[start..end]);
|
||||||
|
inner.buffer_start = end;
|
||||||
|
if inner.buffer_start >= inner.buffer.len() {
|
||||||
|
inner.buffer.clear();
|
||||||
|
inner.buffer_start = 0;
|
||||||
|
}
|
||||||
written += take;
|
written += take;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
102
src/main.rs
102
src/main.rs
@@ -4,11 +4,11 @@
|
|||||||
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::signal;
|
use tokio::signal;
|
||||||
use tokio::sync::{Semaphore, mpsc};
|
use tokio::sync::{Semaphore, mpsc, watch};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -241,6 +241,17 @@ fn format_uptime(total_secs: u64) -> String {
|
|||||||
format!("{} / {} seconds", parts.join(", "), total_secs)
|
format!("{} / {} seconds", parts.join(", "), total_secs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn wait_until_admission_open(admission_rx: &mut watch::Receiver<bool>) -> bool {
|
||||||
|
loop {
|
||||||
|
if *admission_rx.borrow() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if admission_rx.changed().await.is_err() {
|
||||||
|
return *admission_rx.borrow();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn load_startup_proxy_config_snapshot(
|
async fn load_startup_proxy_config_snapshot(
|
||||||
url: &str,
|
url: &str,
|
||||||
cache_path: Option<&str>,
|
cache_path: Option<&str>,
|
||||||
@@ -358,6 +369,10 @@ async fn load_startup_proxy_config_snapshot(
|
|||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
let process_started_at = Instant::now();
|
let process_started_at = Instant::now();
|
||||||
|
let process_started_at_epoch_secs = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs();
|
||||||
let (config_path, cli_silent, cli_log_level) = parse_cli();
|
let (config_path, cli_silent, cli_log_level) = parse_cli();
|
||||||
|
|
||||||
let mut config = match ProxyConfig::load(&config_path) {
|
let mut config = match ProxyConfig::load(&config_path) {
|
||||||
@@ -453,6 +468,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
config.upstreams.clone(),
|
config.upstreams.clone(),
|
||||||
config.general.upstream_connect_retry_attempts,
|
config.general.upstream_connect_retry_attempts,
|
||||||
config.general.upstream_connect_retry_backoff_ms,
|
config.general.upstream_connect_retry_backoff_ms,
|
||||||
|
config.general.upstream_connect_budget_ms,
|
||||||
config.general.upstream_unhealthy_fail_threshold,
|
config.general.upstream_unhealthy_fail_threshold,
|
||||||
config.general.upstream_connect_failfast_hard_errors,
|
config.general.upstream_connect_failfast_hard_errors,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -926,22 +942,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
let mut grouped: BTreeMap<i32, Vec<MePingSample>> = BTreeMap::new();
|
let mut grouped: BTreeMap<i32, Vec<MePingSample>> = BTreeMap::new();
|
||||||
for report in me_results {
|
for report in me_results {
|
||||||
for s in report.samples {
|
for s in report.samples {
|
||||||
let key = s.dc.abs();
|
grouped.entry(s.dc).or_default().push(s);
|
||||||
grouped.entry(key).or_default().push(s);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let family_order = if prefer_ipv6 {
|
let family_order = if prefer_ipv6 {
|
||||||
vec![(MePingFamily::V6, true), (MePingFamily::V6, false), (MePingFamily::V4, true), (MePingFamily::V4, false)]
|
vec![MePingFamily::V6, MePingFamily::V4]
|
||||||
} else {
|
} else {
|
||||||
vec![(MePingFamily::V4, true), (MePingFamily::V4, false), (MePingFamily::V6, true), (MePingFamily::V6, false)]
|
vec![MePingFamily::V4, MePingFamily::V6]
|
||||||
};
|
};
|
||||||
|
|
||||||
for (dc_abs, samples) in grouped {
|
for (dc, samples) in grouped {
|
||||||
for (family, is_pos) in &family_order {
|
for family in &family_order {
|
||||||
let fam_samples: Vec<&MePingSample> = samples
|
let fam_samples: Vec<&MePingSample> = samples
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|s| matches!(s.family, f if &f == family) && (s.dc >= 0) == *is_pos)
|
.filter(|s| matches!(s.family, f if &f == family))
|
||||||
.collect();
|
.collect();
|
||||||
if fam_samples.is_empty() {
|
if fam_samples.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
@@ -951,7 +966,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
MePingFamily::V4 => "IPv4",
|
MePingFamily::V4 => "IPv4",
|
||||||
MePingFamily::V6 => "IPv6",
|
MePingFamily::V6 => "IPv6",
|
||||||
};
|
};
|
||||||
info!(" DC{} [{}]", dc_abs, fam_label);
|
info!(" DC{} [{}]", dc, fam_label);
|
||||||
for sample in fam_samples {
|
for sample in fam_samples {
|
||||||
let line = format_sample_line(sample);
|
let line = format_sample_line(sample);
|
||||||
info!("{}", line);
|
info!("{}", line);
|
||||||
@@ -1325,6 +1340,60 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
print_proxy_links(&host, port, &config);
|
print_proxy_links(&host, port, &config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let (admission_tx, admission_rx) = watch::channel(true);
|
||||||
|
if config.general.use_middle_proxy {
|
||||||
|
if let Some(pool) = me_pool.as_ref() {
|
||||||
|
let initial_open = pool.admission_ready_conditional_cast().await;
|
||||||
|
admission_tx.send_replace(initial_open);
|
||||||
|
if initial_open {
|
||||||
|
info!("Conditional-admission gate: open (ME pool ready)");
|
||||||
|
} else {
|
||||||
|
warn!("Conditional-admission gate: closed (ME pool is not ready)");
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool_for_gate = pool.clone();
|
||||||
|
let admission_tx_gate = admission_tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut gate_open = initial_open;
|
||||||
|
let mut open_streak = if initial_open { 1u32 } else { 0u32 };
|
||||||
|
let mut close_streak = if initial_open { 0u32 } else { 1u32 };
|
||||||
|
loop {
|
||||||
|
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
||||||
|
if ready {
|
||||||
|
open_streak = open_streak.saturating_add(1);
|
||||||
|
close_streak = 0;
|
||||||
|
if !gate_open && open_streak >= 2 {
|
||||||
|
gate_open = true;
|
||||||
|
admission_tx_gate.send_replace(true);
|
||||||
|
info!(
|
||||||
|
open_streak,
|
||||||
|
"Conditional-admission gate opened (ME pool recovered)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
close_streak = close_streak.saturating_add(1);
|
||||||
|
open_streak = 0;
|
||||||
|
if gate_open && close_streak >= 2 {
|
||||||
|
gate_open = false;
|
||||||
|
admission_tx_gate.send_replace(false);
|
||||||
|
warn!(
|
||||||
|
close_streak,
|
||||||
|
"Conditional-admission gate closed (ME pool has uncovered DC groups)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
admission_tx.send_replace(false);
|
||||||
|
warn!("Conditional-admission gate: closed (ME pool is unavailable)");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
admission_tx.send_replace(true);
|
||||||
|
}
|
||||||
|
let _admission_tx_hold = admission_tx;
|
||||||
|
|
||||||
// Unix socket setup (before listeners check so unix-only config works)
|
// Unix socket setup (before listeners check so unix-only config works)
|
||||||
let mut has_unix_listener = false;
|
let mut has_unix_listener = false;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -1358,6 +1427,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
has_unix_listener = true;
|
has_unix_listener = true;
|
||||||
|
|
||||||
let mut config_rx_unix: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
let mut config_rx_unix: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
||||||
|
let mut admission_rx_unix = admission_rx.clone();
|
||||||
let stats = stats.clone();
|
let stats = stats.clone();
|
||||||
let upstream_manager = upstream_manager.clone();
|
let upstream_manager = upstream_manager.clone();
|
||||||
let replay_checker = replay_checker.clone();
|
let replay_checker = replay_checker.clone();
|
||||||
@@ -1373,6 +1443,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
let unix_conn_counter = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1));
|
let unix_conn_counter = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1));
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
if !wait_until_admission_open(&mut admission_rx_unix).await {
|
||||||
|
warn!("Conditional-admission gate channel closed for unix listener");
|
||||||
|
break;
|
||||||
|
}
|
||||||
match unix_listener.accept().await {
|
match unix_listener.accept().await {
|
||||||
Ok((stream, _)) => {
|
Ok((stream, _)) => {
|
||||||
let permit = match max_connections_unix.clone().acquire_owned().await {
|
let permit = match max_connections_unix.clone().acquire_owned().await {
|
||||||
@@ -1485,6 +1559,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
let me_pool_api = me_pool.clone();
|
let me_pool_api = me_pool.clone();
|
||||||
let upstream_manager_api = upstream_manager.clone();
|
let upstream_manager_api = upstream_manager.clone();
|
||||||
let config_rx_api = config_rx.clone();
|
let config_rx_api = config_rx.clone();
|
||||||
|
let admission_rx_api = admission_rx.clone();
|
||||||
let config_path_api = std::path::PathBuf::from(&config_path);
|
let config_path_api = std::path::PathBuf::from(&config_path);
|
||||||
let startup_detected_ip_v4 = detected_ip_v4;
|
let startup_detected_ip_v4 = detected_ip_v4;
|
||||||
let startup_detected_ip_v6 = detected_ip_v6;
|
let startup_detected_ip_v6 = detected_ip_v6;
|
||||||
@@ -1496,9 +1571,11 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
me_pool_api,
|
me_pool_api,
|
||||||
upstream_manager_api,
|
upstream_manager_api,
|
||||||
config_rx_api,
|
config_rx_api,
|
||||||
|
admission_rx_api,
|
||||||
config_path_api,
|
config_path_api,
|
||||||
startup_detected_ip_v4,
|
startup_detected_ip_v4,
|
||||||
startup_detected_ip_v6,
|
startup_detected_ip_v6,
|
||||||
|
process_started_at_epoch_secs,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -1507,6 +1584,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
for (listener, listener_proxy_protocol) in listeners {
|
for (listener, listener_proxy_protocol) in listeners {
|
||||||
let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
||||||
|
let mut admission_rx_tcp = admission_rx.clone();
|
||||||
let stats = stats.clone();
|
let stats = stats.clone();
|
||||||
let upstream_manager = upstream_manager.clone();
|
let upstream_manager = upstream_manager.clone();
|
||||||
let replay_checker = replay_checker.clone();
|
let replay_checker = replay_checker.clone();
|
||||||
@@ -1520,6 +1598,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
|
if !wait_until_admission_open(&mut admission_rx_tcp).await {
|
||||||
|
warn!("Conditional-admission gate channel closed for tcp listener");
|
||||||
|
break;
|
||||||
|
}
|
||||||
match listener.accept().await {
|
match listener.accept().await {
|
||||||
Ok((stream, peer_addr)) => {
|
Ok((stream, peer_addr)) => {
|
||||||
let permit = match max_connections_tcp.clone().acquire_owned().await {
|
let permit = match max_connections_tcp.clone().acquire_owned().await {
|
||||||
|
|||||||
@@ -97,8 +97,11 @@ where
|
|||||||
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
|
.unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap());
|
||||||
|
|
||||||
if proxy_protocol_enabled {
|
if proxy_protocol_enabled {
|
||||||
match parse_proxy_protocol(&mut stream, peer).await {
|
let proxy_header_timeout = Duration::from_millis(
|
||||||
Ok(info) => {
|
config.server.proxy_protocol_header_timeout_ms.max(1),
|
||||||
|
);
|
||||||
|
match timeout(proxy_header_timeout, parse_proxy_protocol(&mut stream, peer)).await {
|
||||||
|
Ok(Ok(info)) => {
|
||||||
debug!(
|
debug!(
|
||||||
peer = %peer,
|
peer = %peer,
|
||||||
client = %info.src_addr,
|
client = %info.src_addr,
|
||||||
@@ -110,12 +113,18 @@ where
|
|||||||
local_addr = dst;
|
local_addr = dst;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Ok(Err(e)) => {
|
||||||
stats.increment_connects_bad();
|
stats.increment_connects_bad();
|
||||||
warn!(peer = %peer, error = %e, "Invalid PROXY protocol header");
|
warn!(peer = %peer, error = %e, "Invalid PROXY protocol header");
|
||||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
Err(_) => {
|
||||||
|
stats.increment_connects_bad();
|
||||||
|
warn!(peer = %peer, timeout_ms = proxy_header_timeout.as_millis(), "PROXY protocol header timeout");
|
||||||
|
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||||
|
return Err(ProxyError::InvalidProxyProtocol);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,7 +170,7 @@ where
|
|||||||
|
|
||||||
let (read_half, write_half) = tokio::io::split(stream);
|
let (read_half, write_half) = tokio::io::split(stream);
|
||||||
|
|
||||||
let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake(
|
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake(
|
||||||
&handshake, read_half, write_half, real_peer,
|
&handshake, read_half, write_half, real_peer,
|
||||||
&config, &replay_checker, &rng, tls_cache.clone(),
|
&config, &replay_checker, &rng, tls_cache.clone(),
|
||||||
).await {
|
).await {
|
||||||
@@ -190,7 +199,7 @@ where
|
|||||||
|
|
||||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
|
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
|
||||||
&mtproto_handshake, tls_reader, tls_writer, real_peer,
|
&mtproto_handshake, tls_reader, tls_writer, real_peer,
|
||||||
&config, &replay_checker, true,
|
&config, &replay_checker, true, Some(tls_user.as_str()),
|
||||||
).await {
|
).await {
|
||||||
HandshakeResult::Success(result) => result,
|
HandshakeResult::Success(result) => result,
|
||||||
HandshakeResult::BadClient { reader: _, writer: _ } => {
|
HandshakeResult::BadClient { reader: _, writer: _ } => {
|
||||||
@@ -234,7 +243,7 @@ where
|
|||||||
|
|
||||||
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
|
let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake(
|
||||||
&handshake, read_half, write_half, real_peer,
|
&handshake, read_half, write_half, real_peer,
|
||||||
&config, &replay_checker, false,
|
&config, &replay_checker, false, None,
|
||||||
).await {
|
).await {
|
||||||
HandshakeResult::Success(result) => result,
|
HandshakeResult::Success(result) => result,
|
||||||
HandshakeResult::BadClient { reader, writer } => {
|
HandshakeResult::BadClient { reader, writer } => {
|
||||||
@@ -415,8 +424,16 @@ impl RunningClientHandler {
|
|||||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||||
|
|
||||||
if self.proxy_protocol_enabled {
|
if self.proxy_protocol_enabled {
|
||||||
match parse_proxy_protocol(&mut self.stream, self.peer).await {
|
let proxy_header_timeout = Duration::from_millis(
|
||||||
Ok(info) => {
|
self.config.server.proxy_protocol_header_timeout_ms.max(1),
|
||||||
|
);
|
||||||
|
match timeout(
|
||||||
|
proxy_header_timeout,
|
||||||
|
parse_proxy_protocol(&mut self.stream, self.peer),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(info)) => {
|
||||||
debug!(
|
debug!(
|
||||||
peer = %self.peer,
|
peer = %self.peer,
|
||||||
client = %info.src_addr,
|
client = %info.src_addr,
|
||||||
@@ -428,7 +445,7 @@ impl RunningClientHandler {
|
|||||||
local_addr = dst;
|
local_addr = dst;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Ok(Err(e)) => {
|
||||||
self.stats.increment_connects_bad();
|
self.stats.increment_connects_bad();
|
||||||
warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header");
|
warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header");
|
||||||
record_beobachten_class(
|
record_beobachten_class(
|
||||||
@@ -439,6 +456,21 @@ impl RunningClientHandler {
|
|||||||
);
|
);
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
Err(_) => {
|
||||||
|
self.stats.increment_connects_bad();
|
||||||
|
warn!(
|
||||||
|
peer = %self.peer,
|
||||||
|
timeout_ms = proxy_header_timeout.as_millis(),
|
||||||
|
"PROXY protocol header timeout"
|
||||||
|
);
|
||||||
|
record_beobachten_class(
|
||||||
|
&self.beobachten,
|
||||||
|
&self.config,
|
||||||
|
self.peer.ip(),
|
||||||
|
"other",
|
||||||
|
);
|
||||||
|
return Err(ProxyError::InvalidProxyProtocol);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,7 +526,7 @@ impl RunningClientHandler {
|
|||||||
|
|
||||||
let (read_half, write_half) = self.stream.into_split();
|
let (read_half, write_half) = self.stream.into_split();
|
||||||
|
|
||||||
let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake(
|
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake(
|
||||||
&handshake,
|
&handshake,
|
||||||
read_half,
|
read_half,
|
||||||
write_half,
|
write_half,
|
||||||
@@ -538,6 +570,7 @@ impl RunningClientHandler {
|
|||||||
&config,
|
&config,
|
||||||
&replay_checker,
|
&replay_checker,
|
||||||
true,
|
true,
|
||||||
|
Some(tls_user.as_str()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -611,6 +644,7 @@ impl RunningClientHandler {
|
|||||||
&config,
|
&config,
|
||||||
&replay_checker,
|
&replay_checker,
|
||||||
false,
|
false,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ where
|
|||||||
let user = &success.user;
|
let user = &success.user;
|
||||||
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
let dc_addr = get_dc_addr_static(success.dc_idx, &config)?;
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
user = %user,
|
user = %user,
|
||||||
peer = %success.peer,
|
peer = %success.peer,
|
||||||
dc = success.dc_idx,
|
dc = success.dc_idx,
|
||||||
@@ -57,6 +57,7 @@ where
|
|||||||
|
|
||||||
stats.increment_user_connects(user);
|
stats.increment_user_connects(user);
|
||||||
stats.increment_user_curr_connects(user);
|
stats.increment_user_curr_connects(user);
|
||||||
|
stats.increment_current_connections_direct();
|
||||||
|
|
||||||
let relay_result = relay_bidirectional(
|
let relay_result = relay_bidirectional(
|
||||||
client_reader,
|
client_reader,
|
||||||
@@ -69,6 +70,7 @@ where
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
stats.decrement_current_connections_direct();
|
||||||
stats.decrement_user_curr_connects(user);
|
stats.decrement_user_curr_connects(user);
|
||||||
|
|
||||||
match &relay_result {
|
match &relay_result {
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use std::net::SocketAddr;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
|
||||||
use tracing::{debug, warn, trace, info};
|
use tracing::{debug, warn, trace};
|
||||||
use zeroize::Zeroize;
|
use zeroize::Zeroize;
|
||||||
|
|
||||||
use crate::crypto::{sha256, AesCtr, SecureRandom};
|
use crate::crypto::{sha256, AesCtr, SecureRandom};
|
||||||
@@ -19,6 +19,31 @@ use crate::stats::ReplayChecker;
|
|||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::tls_front::{TlsFrontCache, emulator};
|
use crate::tls_front::{TlsFrontCache, emulator};
|
||||||
|
|
||||||
|
fn decode_user_secrets(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
preferred_user: Option<&str>,
|
||||||
|
) -> Vec<(String, Vec<u8>)> {
|
||||||
|
let mut secrets = Vec::with_capacity(config.access.users.len());
|
||||||
|
|
||||||
|
if let Some(preferred) = preferred_user
|
||||||
|
&& let Some(secret_hex) = config.access.users.get(preferred)
|
||||||
|
&& let Ok(bytes) = hex::decode(secret_hex)
|
||||||
|
{
|
||||||
|
secrets.push((preferred.to_string(), bytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (name, secret_hex) in &config.access.users {
|
||||||
|
if preferred_user.is_some_and(|preferred| preferred == name.as_str()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Ok(bytes) = hex::decode(secret_hex) {
|
||||||
|
secrets.push((name.clone(), bytes));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
secrets
|
||||||
|
}
|
||||||
|
|
||||||
/// Result of successful handshake
|
/// Result of successful handshake
|
||||||
///
|
///
|
||||||
/// Key material (`dec_key`, `dec_iv`, `enc_key`, `enc_iv`) is
|
/// Key material (`dec_key`, `dec_iv`, `enc_key`, `enc_iv`) is
|
||||||
@@ -82,11 +107,7 @@ where
|
|||||||
return HandshakeResult::BadClient { reader, writer };
|
return HandshakeResult::BadClient { reader, writer };
|
||||||
}
|
}
|
||||||
|
|
||||||
let secrets: Vec<(String, Vec<u8>)> = config.access.users.iter()
|
let secrets = decode_user_secrets(config, None);
|
||||||
.filter_map(|(name, hex)| {
|
|
||||||
hex::decode(hex).ok().map(|bytes| (name.clone(), bytes))
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let validation = match tls::validate_tls_handshake(
|
let validation = match tls::validate_tls_handshake(
|
||||||
handshake,
|
handshake,
|
||||||
@@ -201,7 +222,7 @@ where
|
|||||||
return HandshakeResult::Error(ProxyError::Io(e));
|
return HandshakeResult::Error(ProxyError::Io(e));
|
||||||
}
|
}
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
peer = %peer,
|
peer = %peer,
|
||||||
user = %validation.user,
|
user = %validation.user,
|
||||||
"TLS handshake successful"
|
"TLS handshake successful"
|
||||||
@@ -223,6 +244,7 @@ pub async fn handle_mtproto_handshake<R, W>(
|
|||||||
config: &ProxyConfig,
|
config: &ProxyConfig,
|
||||||
replay_checker: &ReplayChecker,
|
replay_checker: &ReplayChecker,
|
||||||
is_tls: bool,
|
is_tls: bool,
|
||||||
|
preferred_user: Option<&str>,
|
||||||
) -> HandshakeResult<(CryptoReader<R>, CryptoWriter<W>, HandshakeSuccess), R, W>
|
) -> HandshakeResult<(CryptoReader<R>, CryptoWriter<W>, HandshakeSuccess), R, W>
|
||||||
where
|
where
|
||||||
R: AsyncRead + Unpin + Send,
|
R: AsyncRead + Unpin + Send,
|
||||||
@@ -239,11 +261,9 @@ where
|
|||||||
|
|
||||||
let enc_prekey_iv: Vec<u8> = dec_prekey_iv.iter().rev().copied().collect();
|
let enc_prekey_iv: Vec<u8> = dec_prekey_iv.iter().rev().copied().collect();
|
||||||
|
|
||||||
for (user, secret_hex) in &config.access.users {
|
let decoded_users = decode_user_secrets(config, preferred_user);
|
||||||
let secret = match hex::decode(secret_hex) {
|
|
||||||
Ok(s) => s,
|
for (user, secret) in decoded_users {
|
||||||
Err(_) => continue,
|
|
||||||
};
|
|
||||||
|
|
||||||
let dec_prekey = &dec_prekey_iv[..PREKEY_LEN];
|
let dec_prekey = &dec_prekey_iv[..PREKEY_LEN];
|
||||||
let dec_iv_bytes = &dec_prekey_iv[PREKEY_LEN..];
|
let dec_iv_bytes = &dec_prekey_iv[PREKEY_LEN..];
|
||||||
@@ -311,7 +331,7 @@ where
|
|||||||
is_tls,
|
is_tls,
|
||||||
};
|
};
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
peer = %peer,
|
peer = %peer,
|
||||||
user = %user,
|
user = %user,
|
||||||
dc = dc_idx,
|
dc = dc_idx,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use std::time::{Duration, Instant};
|
|||||||
|
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||||
use tokio::sync::{mpsc, oneshot};
|
use tokio::sync::{mpsc, oneshot};
|
||||||
use tracing::{debug, info, trace, warn};
|
use tracing::{debug, trace, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
@@ -210,7 +210,7 @@ where
|
|||||||
let proto_tag = success.proto_tag;
|
let proto_tag = success.proto_tag;
|
||||||
let pool_generation = me_pool.current_generation();
|
let pool_generation = me_pool.current_generation();
|
||||||
|
|
||||||
info!(
|
debug!(
|
||||||
user = %user,
|
user = %user,
|
||||||
peer = %peer,
|
peer = %peer,
|
||||||
dc = success.dc_idx,
|
dc = success.dc_idx,
|
||||||
@@ -237,6 +237,7 @@ where
|
|||||||
|
|
||||||
stats.increment_user_connects(&user);
|
stats.increment_user_connects(&user);
|
||||||
stats.increment_user_curr_connects(&user);
|
stats.increment_user_curr_connects(&user);
|
||||||
|
stats.increment_current_connections_me();
|
||||||
|
|
||||||
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
|
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
|
||||||
let user_tag: Option<Vec<u8>> = config
|
let user_tag: Option<Vec<u8>> = config
|
||||||
@@ -466,6 +467,7 @@ where
|
|||||||
"ME relay cleanup"
|
"ME relay cleanup"
|
||||||
);
|
);
|
||||||
me_pool.registry().unregister(conn_id).await;
|
me_pool.registry().unregister(conn_id).await;
|
||||||
|
stats.decrement_current_connections_me();
|
||||||
stats.decrement_user_curr_connects(&user);
|
stats.decrement_user_curr_connects(&user);
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|||||||
108
src/stats/mod.rs
108
src/stats/mod.rs
@@ -25,6 +25,8 @@ use self::telemetry::TelemetryPolicy;
|
|||||||
pub struct Stats {
|
pub struct Stats {
|
||||||
connects_all: AtomicU64,
|
connects_all: AtomicU64,
|
||||||
connects_bad: AtomicU64,
|
connects_bad: AtomicU64,
|
||||||
|
current_connections_direct: AtomicU64,
|
||||||
|
current_connections_me: AtomicU64,
|
||||||
handshake_timeouts: AtomicU64,
|
handshake_timeouts: AtomicU64,
|
||||||
upstream_connect_attempt_total: AtomicU64,
|
upstream_connect_attempt_total: AtomicU64,
|
||||||
upstream_connect_success_total: AtomicU64,
|
upstream_connect_success_total: AtomicU64,
|
||||||
@@ -150,6 +152,24 @@ impl Stats {
|
|||||||
self.telemetry_me_level().allows_debug()
|
self.telemetry_me_level().allows_debug()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decrement_atomic_saturating(counter: &AtomicU64) {
|
||||||
|
let mut current = counter.load(Ordering::Relaxed);
|
||||||
|
loop {
|
||||||
|
if current == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
match counter.compare_exchange_weak(
|
||||||
|
current,
|
||||||
|
current - 1,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
) {
|
||||||
|
Ok(_) => break,
|
||||||
|
Err(actual) => current = actual,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) {
|
pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) {
|
||||||
self.telemetry_core_enabled
|
self.telemetry_core_enabled
|
||||||
.store(policy.core_enabled, Ordering::Relaxed);
|
.store(policy.core_enabled, Ordering::Relaxed);
|
||||||
@@ -177,6 +197,18 @@ impl Stats {
|
|||||||
self.connects_bad.fetch_add(1, Ordering::Relaxed);
|
self.connects_bad.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn increment_current_connections_direct(&self) {
|
||||||
|
self.current_connections_direct.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn decrement_current_connections_direct(&self) {
|
||||||
|
Self::decrement_atomic_saturating(&self.current_connections_direct);
|
||||||
|
}
|
||||||
|
pub fn increment_current_connections_me(&self) {
|
||||||
|
self.current_connections_me.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn decrement_current_connections_me(&self) {
|
||||||
|
Self::decrement_atomic_saturating(&self.current_connections_me);
|
||||||
|
}
|
||||||
pub fn increment_handshake_timeouts(&self) {
|
pub fn increment_handshake_timeouts(&self) {
|
||||||
if self.telemetry_core_enabled() {
|
if self.telemetry_core_enabled() {
|
||||||
self.handshake_timeouts.fetch_add(1, Ordering::Relaxed);
|
self.handshake_timeouts.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -646,6 +678,16 @@ impl Stats {
|
|||||||
}
|
}
|
||||||
pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) }
|
pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) }
|
||||||
pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) }
|
pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) }
|
||||||
|
pub fn get_current_connections_direct(&self) -> u64 {
|
||||||
|
self.current_connections_direct.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_current_connections_me(&self) -> u64 {
|
||||||
|
self.current_connections_me.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_current_connections_total(&self) -> u64 {
|
||||||
|
self.get_current_connections_direct()
|
||||||
|
.saturating_add(self.get_current_connections_me())
|
||||||
|
}
|
||||||
pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) }
|
pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) }
|
||||||
pub fn get_me_keepalive_failed(&self) -> u64 { self.me_keepalive_failed.load(Ordering::Relaxed) }
|
pub fn get_me_keepalive_failed(&self) -> u64 { self.me_keepalive_failed.load(Ordering::Relaxed) }
|
||||||
pub fn get_me_keepalive_pong(&self) -> u64 { self.me_keepalive_pong.load(Ordering::Relaxed) }
|
pub fn get_me_keepalive_pong(&self) -> u64 { self.me_keepalive_pong.load(Ordering::Relaxed) }
|
||||||
@@ -846,16 +888,30 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.connects.fetch_add(1, Ordering::Relaxed);
|
stats.connects.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.connects
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_curr_connects(&self, user: &str) {
|
pub fn increment_user_curr_connects(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.curr_connects.fetch_add(1, Ordering::Relaxed);
|
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.curr_connects
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn decrement_user_curr_connects(&self, user: &str) {
|
pub fn decrement_user_curr_connects(&self, user: &str) {
|
||||||
@@ -889,32 +945,60 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
|
stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.octets_from_client
|
||||||
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn add_user_octets_to(&self, user: &str, bytes: u64) {
|
pub fn add_user_octets_to(&self, user: &str, bytes: u64) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
|
stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.octets_to_client
|
||||||
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_msgs_from(&self, user: &str) {
|
pub fn increment_user_msgs_from(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.msgs_from_client
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_msgs_to(&self, user: &str) {
|
pub fn increment_user_msgs_to(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats.entry(user.to_string()).or_default()
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.user_stats
|
||||||
|
.entry(user.to_string())
|
||||||
|
.or_default()
|
||||||
|
.msgs_to_client
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_user_total_octets(&self, user: &str) -> u64 {
|
pub fn get_user_total_octets(&self, user: &str) -> u64 {
|
||||||
|
|||||||
@@ -84,38 +84,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option<i16> {
|
async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option<i16> {
|
||||||
if addr.is_ipv4() {
|
i16::try_from(self.resolve_dc_for_endpoint(addr).await).ok()
|
||||||
let map = self.proxy_map_v4.read().await;
|
|
||||||
for (dc, addrs) in map.iter() {
|
|
||||||
if addrs
|
|
||||||
.iter()
|
|
||||||
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
|
|
||||||
{
|
|
||||||
let abs_dc = dc.abs();
|
|
||||||
if abs_dc > 0
|
|
||||||
&& let Ok(dc_idx) = i16::try_from(abs_dc)
|
|
||||||
{
|
|
||||||
return Some(dc_idx);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
let map = self.proxy_map_v6.read().await;
|
|
||||||
for (dc, addrs) in map.iter() {
|
|
||||||
if addrs
|
|
||||||
.iter()
|
|
||||||
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
|
|
||||||
{
|
|
||||||
let abs_dc = dc.abs();
|
|
||||||
if abs_dc > 0
|
|
||||||
&& let Ok(dc_idx) = i16::try_from(abs_dc)
|
|
||||||
{
|
|
||||||
return Some(dc_idx);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn direct_bind_ip_for_stun(
|
fn direct_bind_ip_for_stun(
|
||||||
@@ -387,9 +356,11 @@ impl MePool {
|
|||||||
socks_bound_addr.map(|value| value.ip()),
|
socks_bound_addr.map(|value| value.ip()),
|
||||||
client_port_source,
|
client_port_source,
|
||||||
);
|
);
|
||||||
let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await;
|
let previous_kdf_fingerprint = {
|
||||||
if let Some((prev_fingerprint, prev_client_port)) =
|
let kdf_fingerprint_guard = self.kdf_material_fingerprint.read().await;
|
||||||
kdf_fingerprint_guard.get(&peer_addr_nat).copied()
|
kdf_fingerprint_guard.get(&peer_addr_nat).copied()
|
||||||
|
};
|
||||||
|
if let Some((prev_fingerprint, prev_client_port)) = previous_kdf_fingerprint
|
||||||
{
|
{
|
||||||
if prev_fingerprint != kdf_fingerprint {
|
if prev_fingerprint != kdf_fingerprint {
|
||||||
self.stats.increment_me_kdf_drift_total();
|
self.stats.increment_me_kdf_drift_total();
|
||||||
@@ -416,6 +387,9 @@ impl MePool {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Keep fingerprint updates eventually consistent for diagnostics while avoiding
|
||||||
|
// serializing all concurrent handshakes on a single async mutex.
|
||||||
|
let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.write().await;
|
||||||
kdf_fingerprint_guard.insert(peer_addr_nat, (kdf_fingerprint, client_port_for_kdf));
|
kdf_fingerprint_guard.insert(peer_addr_nat, (kdf_fingerprint, client_port_for_kdf));
|
||||||
drop(kdf_fingerprint_guard);
|
drop(kdf_fingerprint_guard);
|
||||||
|
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ async fn check_family(
|
|||||||
|
|
||||||
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
|
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
|
||||||
for (dc, addrs) in map {
|
for (dc, addrs) in map {
|
||||||
let entry = dc_endpoints.entry(dc.abs()).or_default();
|
let entry = dc_endpoints.entry(dc).or_default();
|
||||||
for (ip, port) in addrs {
|
for (ip, port) in addrs {
|
||||||
entry.push(SocketAddr::new(ip, port));
|
entry.push(SocketAddr::new(ip, port));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ mod pool_init;
|
|||||||
mod pool_nat;
|
mod pool_nat;
|
||||||
mod pool_refill;
|
mod pool_refill;
|
||||||
mod pool_reinit;
|
mod pool_reinit;
|
||||||
|
mod pool_runtime_api;
|
||||||
mod pool_writer;
|
mod pool_writer;
|
||||||
mod ping;
|
mod ping;
|
||||||
mod reader;
|
mod reader;
|
||||||
|
|||||||
@@ -119,6 +119,8 @@ pub struct MePool {
|
|||||||
pub(super) ping_tracker: Arc<Mutex<HashMap<i64, (std::time::Instant, u64)>>>,
|
pub(super) ping_tracker: Arc<Mutex<HashMap<i64, (std::time::Instant, u64)>>>,
|
||||||
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
||||||
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
|
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
|
||||||
|
pub(super) nat_reflection_singleflight_v4: Arc<Mutex<()>>,
|
||||||
|
pub(super) nat_reflection_singleflight_v6: Arc<Mutex<()>>,
|
||||||
pub(super) writer_available: Arc<Notify>,
|
pub(super) writer_available: Arc<Notify>,
|
||||||
pub(super) refill_inflight: Arc<Mutex<HashSet<SocketAddr>>>,
|
pub(super) refill_inflight: Arc<Mutex<HashSet<SocketAddr>>>,
|
||||||
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
|
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
|
||||||
@@ -132,7 +134,7 @@ pub struct MePool {
|
|||||||
pub(super) pending_hardswap_map_hash: AtomicU64,
|
pub(super) pending_hardswap_map_hash: AtomicU64,
|
||||||
pub(super) hardswap: AtomicBool,
|
pub(super) hardswap: AtomicBool,
|
||||||
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
|
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
|
||||||
pub(super) kdf_material_fingerprint: Arc<Mutex<HashMap<SocketAddr, (u64, u16)>>>,
|
pub(super) kdf_material_fingerprint: Arc<RwLock<HashMap<SocketAddr, (u64, u16)>>>,
|
||||||
pub(super) me_pool_drain_ttl_secs: AtomicU64,
|
pub(super) me_pool_drain_ttl_secs: AtomicU64,
|
||||||
pub(super) me_pool_force_close_secs: AtomicU64,
|
pub(super) me_pool_force_close_secs: AtomicU64,
|
||||||
pub(super) me_pool_min_fresh_ratio_permille: AtomicU32,
|
pub(super) me_pool_min_fresh_ratio_permille: AtomicU32,
|
||||||
@@ -318,11 +320,13 @@ impl MePool {
|
|||||||
pool_size: 2,
|
pool_size: 2,
|
||||||
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
|
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
|
||||||
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
|
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
|
||||||
default_dc: AtomicI32::new(default_dc.unwrap_or(0)),
|
default_dc: AtomicI32::new(default_dc.unwrap_or(2)),
|
||||||
next_writer_id: AtomicU64::new(1),
|
next_writer_id: AtomicU64::new(1),
|
||||||
ping_tracker: Arc::new(Mutex::new(HashMap::new())),
|
ping_tracker: Arc::new(Mutex::new(HashMap::new())),
|
||||||
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
||||||
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
|
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
|
||||||
|
nat_reflection_singleflight_v4: Arc::new(Mutex::new(())),
|
||||||
|
nat_reflection_singleflight_v6: Arc::new(Mutex::new(())),
|
||||||
writer_available: Arc::new(Notify::new()),
|
writer_available: Arc::new(Notify::new()),
|
||||||
refill_inflight: Arc::new(Mutex::new(HashSet::new())),
|
refill_inflight: Arc::new(Mutex::new(HashSet::new())),
|
||||||
refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())),
|
refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())),
|
||||||
@@ -335,7 +339,7 @@ impl MePool {
|
|||||||
pending_hardswap_map_hash: AtomicU64::new(0),
|
pending_hardswap_map_hash: AtomicU64::new(0),
|
||||||
hardswap: AtomicBool::new(hardswap),
|
hardswap: AtomicBool::new(hardswap),
|
||||||
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
|
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
|
||||||
kdf_material_fingerprint: Arc::new(Mutex::new(HashMap::new())),
|
kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())),
|
||||||
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
|
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
|
||||||
me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs),
|
me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs),
|
||||||
me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille(
|
me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille(
|
||||||
@@ -621,6 +625,58 @@ impl MePool {
|
|||||||
order
|
order
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn default_dc_for_routing(&self) -> i32 {
|
||||||
|
let dc = self.default_dc.load(Ordering::Relaxed);
|
||||||
|
if dc == 0 { 2 } else { dc }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn dc_lookup_chain_for_target(&self, target_dc: i32) -> Vec<i32> {
|
||||||
|
let mut out = Vec::with_capacity(1);
|
||||||
|
if target_dc != 0 {
|
||||||
|
out.push(target_dc);
|
||||||
|
} else {
|
||||||
|
// Use default DC only when target DC is unknown and pinning is not established.
|
||||||
|
let fallback_dc = self.default_dc_for_routing();
|
||||||
|
out.push(fallback_dc);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 {
|
||||||
|
let map_guard = if addr.is_ipv4() {
|
||||||
|
self.proxy_map_v4.read().await
|
||||||
|
} else {
|
||||||
|
self.proxy_map_v6.read().await
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut matched_dc: Option<i32> = None;
|
||||||
|
let mut ambiguous = false;
|
||||||
|
for (dc, addrs) in map_guard.iter() {
|
||||||
|
if addrs
|
||||||
|
.iter()
|
||||||
|
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
|
||||||
|
{
|
||||||
|
match matched_dc {
|
||||||
|
None => matched_dc = Some(*dc),
|
||||||
|
Some(prev_dc) if prev_dc == *dc => {}
|
||||||
|
Some(_) => {
|
||||||
|
ambiguous = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(map_guard);
|
||||||
|
|
||||||
|
if !ambiguous
|
||||||
|
&& let Some(dc) = matched_dc
|
||||||
|
{
|
||||||
|
return dc;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.default_dc_for_routing()
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn proxy_map_for_family(
|
pub(super) async fn proxy_map_for_family(
|
||||||
&self,
|
&self,
|
||||||
family: IpFamily,
|
family: IpFamily,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::HashSet;
|
||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -14,10 +14,12 @@ use super::pool::MePool;
|
|||||||
impl MePool {
|
impl MePool {
|
||||||
pub async fn init(self: &Arc<Self>, pool_size: usize, rng: &Arc<SecureRandom>) -> Result<()> {
|
pub async fn init(self: &Arc<Self>, pool_size: usize, rng: &Arc<SecureRandom>) -> Result<()> {
|
||||||
let family_order = self.family_order();
|
let family_order = self.family_order();
|
||||||
|
let connect_concurrency = self.me_reconnect_max_concurrent_per_dc.max(1) as usize;
|
||||||
let ks = self.key_selector().await;
|
let ks = self.key_selector().await;
|
||||||
info!(
|
info!(
|
||||||
me_servers = self.proxy_map_v4.read().await.len(),
|
me_servers = self.proxy_map_v4.read().await.len(),
|
||||||
pool_size,
|
pool_size,
|
||||||
|
connect_concurrency,
|
||||||
key_selector = format_args!("0x{ks:08x}"),
|
key_selector = format_args!("0x{ks:08x}"),
|
||||||
secret_len = self.proxy_secret.read().await.secret.len(),
|
secret_len = self.proxy_secret.read().await.secret.len(),
|
||||||
"Initializing ME pool"
|
"Initializing ME pool"
|
||||||
@@ -25,39 +27,49 @@ impl MePool {
|
|||||||
|
|
||||||
for family in family_order {
|
for family in family_order {
|
||||||
let map = self.proxy_map_for_family(family).await;
|
let map = self.proxy_map_for_family(family).await;
|
||||||
let mut grouped_dc_addrs: HashMap<i32, Vec<(IpAddr, u16)>> = HashMap::new();
|
let mut dc_addrs: Vec<(i32, Vec<(IpAddr, u16)>)> = map
|
||||||
for (dc, addrs) in map {
|
|
||||||
if addrs.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
grouped_dc_addrs.entry(dc.abs()).or_default().extend(addrs);
|
|
||||||
}
|
|
||||||
let mut dc_addrs: Vec<(i32, Vec<(IpAddr, u16)>)> = grouped_dc_addrs
|
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(dc, mut addrs)| {
|
.map(|(dc, mut addrs)| {
|
||||||
addrs.sort_unstable();
|
addrs.sort_unstable();
|
||||||
addrs.dedup();
|
addrs.dedup();
|
||||||
(dc, addrs)
|
(dc, addrs)
|
||||||
})
|
})
|
||||||
|
.filter(|(_, addrs)| !addrs.is_empty())
|
||||||
.collect();
|
.collect();
|
||||||
dc_addrs.sort_unstable_by_key(|(dc, _)| *dc);
|
dc_addrs.sort_unstable_by_key(|(dc, _)| *dc);
|
||||||
|
dc_addrs.sort_by_key(|(_, addrs)| (addrs.len() != 1, addrs.len()));
|
||||||
|
|
||||||
// Ensure at least one live writer per DC group; run missing DCs in parallel.
|
// Stage 1: build base coverage for conditional-cast.
|
||||||
|
// Single-endpoint DCs are prefilled first; multi-endpoint DCs require one live writer.
|
||||||
let mut join = tokio::task::JoinSet::new();
|
let mut join = tokio::task::JoinSet::new();
|
||||||
for (dc, addrs) in dc_addrs.iter().cloned() {
|
for (dc, addrs) in dc_addrs.iter().cloned() {
|
||||||
if addrs.is_empty() {
|
if addrs.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
let target_writers = if addrs.len() == 1 {
|
||||||
|
self.required_writers_for_dc_with_floor_mode(addrs.len(), false)
|
||||||
|
} else {
|
||||||
|
1usize
|
||||||
|
};
|
||||||
let endpoints: HashSet<SocketAddr> = addrs
|
let endpoints: HashSet<SocketAddr> = addrs
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
.collect();
|
.collect();
|
||||||
if self.active_writer_count_for_endpoints(&endpoints).await > 0 {
|
if self.active_writer_count_for_endpoints(&endpoints).await >= target_writers {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let pool = Arc::clone(self);
|
let pool = Arc::clone(self);
|
||||||
let rng_clone = Arc::clone(rng);
|
let rng_clone = Arc::clone(rng);
|
||||||
join.spawn(async move { pool.connect_primary_for_dc(dc, addrs, rng_clone).await });
|
join.spawn(async move {
|
||||||
|
pool.connect_primary_for_dc(
|
||||||
|
dc,
|
||||||
|
addrs,
|
||||||
|
target_writers,
|
||||||
|
rng_clone,
|
||||||
|
connect_concurrency,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
}
|
}
|
||||||
while join.join_next().await.is_some() {}
|
while join.join_next().await.is_some() {}
|
||||||
|
|
||||||
@@ -77,47 +89,35 @@ impl MePool {
|
|||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Warm reserve writers asynchronously so startup does not block after first working pool is ready.
|
// Stage 2: continue saturating multi-endpoint DC groups in background.
|
||||||
let pool = Arc::clone(self);
|
let pool = Arc::clone(self);
|
||||||
let rng_clone = Arc::clone(rng);
|
let rng_clone = Arc::clone(rng);
|
||||||
let dc_addrs_bg = dc_addrs.clone();
|
let dc_addrs_bg = dc_addrs.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if pool.me_warmup_stagger_enabled {
|
let mut join_bg = tokio::task::JoinSet::new();
|
||||||
for (dc, addrs) in &dc_addrs_bg {
|
for (dc, addrs) in dc_addrs_bg {
|
||||||
for (ip, port) in addrs {
|
if addrs.len() <= 1 {
|
||||||
if pool.connection_count() >= pool_size {
|
continue;
|
||||||
break;
|
|
||||||
}
|
|
||||||
let addr = SocketAddr::new(*ip, *port);
|
|
||||||
let jitter = rand::rng()
|
|
||||||
.random_range(0..=pool.me_warmup_step_jitter.as_millis() as u64);
|
|
||||||
let delay_ms = pool.me_warmup_step_delay.as_millis() as u64 + jitter;
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
|
||||||
if let Err(e) = pool.connect_one(addr, rng_clone.as_ref()).await {
|
|
||||||
debug!(%addr, dc = %dc, error = %e, "Extra ME connect failed (staggered)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
for (dc, addrs) in &dc_addrs_bg {
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
if pool.connection_count() >= pool_size {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let addr = SocketAddr::new(*ip, *port);
|
|
||||||
if let Err(e) = pool.connect_one(addr, rng_clone.as_ref()).await {
|
|
||||||
debug!(%addr, dc = %dc, error = %e, "Extra ME connect failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pool.connection_count() >= pool_size {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
let target_writers = pool.required_writers_for_dc_with_floor_mode(addrs.len(), false);
|
||||||
|
let pool_clone = Arc::clone(&pool);
|
||||||
|
let rng_clone_local = Arc::clone(&rng_clone);
|
||||||
|
join_bg.spawn(async move {
|
||||||
|
pool_clone
|
||||||
|
.connect_primary_for_dc(
|
||||||
|
dc,
|
||||||
|
addrs,
|
||||||
|
target_writers,
|
||||||
|
rng_clone_local,
|
||||||
|
connect_concurrency,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
while join_bg.join_next().await.is_some() {}
|
||||||
debug!(
|
debug!(
|
||||||
target_pool_size = pool_size,
|
|
||||||
current_pool_size = pool.connection_count(),
|
current_pool_size = pool.connection_count(),
|
||||||
"Background ME reserve warmup finished"
|
"Background ME saturation warmup finished"
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -140,62 +140,85 @@ impl MePool {
|
|||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
dc: i32,
|
dc: i32,
|
||||||
mut addrs: Vec<(IpAddr, u16)>,
|
mut addrs: Vec<(IpAddr, u16)>,
|
||||||
|
target_writers: usize,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
|
connect_concurrency: usize,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
if addrs.is_empty() {
|
if addrs.is_empty() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
let target_writers = target_writers.max(1);
|
||||||
addrs.shuffle(&mut rand::rng());
|
addrs.shuffle(&mut rand::rng());
|
||||||
if addrs.len() > 1 {
|
let endpoints: Vec<SocketAddr> = addrs
|
||||||
let concurrency = 2usize;
|
.iter()
|
||||||
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
|
.collect();
|
||||||
|
let endpoint_set: HashSet<SocketAddr> = endpoints.iter().copied().collect();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let alive = self.active_writer_count_for_endpoints(&endpoint_set).await;
|
||||||
|
if alive >= target_writers {
|
||||||
|
info!(
|
||||||
|
dc = %dc,
|
||||||
|
alive,
|
||||||
|
target_writers,
|
||||||
|
"ME connected"
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let missing = target_writers.saturating_sub(alive).max(1);
|
||||||
|
let concurrency = connect_concurrency.max(1).min(missing);
|
||||||
let mut join = tokio::task::JoinSet::new();
|
let mut join = tokio::task::JoinSet::new();
|
||||||
let mut next_idx = 0usize;
|
for _ in 0..concurrency {
|
||||||
|
let pool = Arc::clone(&self);
|
||||||
|
let rng_clone = Arc::clone(&rng);
|
||||||
|
let endpoints_clone = endpoints.clone();
|
||||||
|
join.spawn(async move {
|
||||||
|
pool.connect_endpoints_round_robin(&endpoints_clone, rng_clone.as_ref())
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
while next_idx < addrs.len() || !join.is_empty() {
|
let mut progress = false;
|
||||||
while next_idx < addrs.len() && join.len() < concurrency {
|
while let Some(res) = join.join_next().await {
|
||||||
let (ip, port) = addrs[next_idx];
|
|
||||||
next_idx += 1;
|
|
||||||
let addr = SocketAddr::new(ip, port);
|
|
||||||
let pool = Arc::clone(&self);
|
|
||||||
let rng_clone = Arc::clone(&rng);
|
|
||||||
join.spawn(async move {
|
|
||||||
(addr, pool.connect_one(addr, rng_clone.as_ref()).await)
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let Some(res) = join.join_next().await else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
match res {
|
match res {
|
||||||
Ok((addr, Ok(()))) => {
|
Ok(true) => {
|
||||||
info!(%addr, dc = %dc, "ME connected");
|
progress = true;
|
||||||
join.abort_all();
|
|
||||||
while join.join_next().await.is_some() {}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
Ok((addr, Err(e))) => {
|
|
||||||
warn!(%addr, dc = %dc, error = %e, "ME connect failed, trying next");
|
|
||||||
}
|
}
|
||||||
|
Ok(false) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!(dc = %dc, error = %e, "ME connect task failed");
|
warn!(dc = %dc, error = %e, "ME connect task failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
warn!(dc = %dc, "All ME servers for DC failed at init");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (ip, port) in addrs {
|
let alive_after = self.active_writer_count_for_endpoints(&endpoint_set).await;
|
||||||
let addr = SocketAddr::new(ip, port);
|
if alive_after >= target_writers {
|
||||||
match self.connect_one(addr, rng.as_ref()).await {
|
info!(
|
||||||
Ok(()) => {
|
dc = %dc,
|
||||||
info!(%addr, dc = %dc, "ME connected");
|
alive = alive_after,
|
||||||
return true;
|
target_writers,
|
||||||
}
|
"ME connected"
|
||||||
Err(e) => warn!(%addr, dc = %dc, error = %e, "ME connect failed, trying next"),
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if !progress {
|
||||||
|
warn!(
|
||||||
|
dc = %dc,
|
||||||
|
alive = alive_after,
|
||||||
|
target_writers,
|
||||||
|
"All ME servers for DC failed at init"
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.me_warmup_stagger_enabled {
|
||||||
|
let jitter = rand::rng()
|
||||||
|
.random_range(0..=self.me_warmup_step_jitter.as_millis() as u64);
|
||||||
|
let delay_ms = self.me_warmup_step_delay.as_millis() as u64 + jitter;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
warn!(dc = %dc, "All ME servers for DC failed at init");
|
|
||||||
false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -248,6 +248,43 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let _singleflight_guard = if use_shared_cache {
|
||||||
|
Some(match family {
|
||||||
|
IpFamily::V4 => self.nat_reflection_singleflight_v4.lock().await,
|
||||||
|
IpFamily::V6 => self.nat_reflection_singleflight_v6.lock().await,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
if use_shared_cache
|
||||||
|
&& let Some(until) = *self.stun_backoff_until.read().await
|
||||||
|
&& Instant::now() < until
|
||||||
|
{
|
||||||
|
if let Ok(cache) = self.nat_reflection_cache.try_lock() {
|
||||||
|
let slot = match family {
|
||||||
|
IpFamily::V4 => cache.v4,
|
||||||
|
IpFamily::V6 => cache.v6,
|
||||||
|
};
|
||||||
|
return slot.map(|(_, addr)| addr);
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if use_shared_cache
|
||||||
|
&& let Ok(mut cache) = self.nat_reflection_cache.try_lock()
|
||||||
|
{
|
||||||
|
let slot = match family {
|
||||||
|
IpFamily::V4 => &mut cache.v4,
|
||||||
|
IpFamily::V6 => &mut cache.v6,
|
||||||
|
};
|
||||||
|
if let Some((ts, addr)) = slot
|
||||||
|
&& ts.elapsed() < STUN_CACHE_TTL
|
||||||
|
{
|
||||||
|
return Some(*addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let attempt = if use_shared_cache {
|
let attempt = if use_shared_cache {
|
||||||
self.nat_probe_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
self.nat_probe_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -108,19 +108,10 @@ impl MePool {
|
|||||||
} else {
|
} else {
|
||||||
IpFamily::V6
|
IpFamily::V6
|
||||||
};
|
};
|
||||||
let map = self.proxy_map_for_family(family).await;
|
Some(RefillDcKey {
|
||||||
for (dc, endpoints) in map {
|
dc: self.resolve_dc_for_endpoint(addr).await,
|
||||||
if endpoints
|
family,
|
||||||
.into_iter()
|
})
|
||||||
.any(|(ip, port)| SocketAddr::new(ip, port) == addr)
|
|
||||||
{
|
|
||||||
return Some(RefillDcKey {
|
|
||||||
dc: dc.abs(),
|
|
||||||
family,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn resolve_refill_dc_keys_for_endpoints(
|
async fn resolve_refill_dc_keys_for_endpoints(
|
||||||
@@ -177,47 +168,23 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn endpoints_for_same_dc(&self, addr: SocketAddr) -> Vec<SocketAddr> {
|
async fn endpoints_for_same_dc(&self, addr: SocketAddr) -> Vec<SocketAddr> {
|
||||||
let mut target_dc = HashSet::<i32>::new();
|
|
||||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||||
|
let target_dc = self.resolve_dc_for_endpoint(addr).await;
|
||||||
|
|
||||||
if self.decision.ipv4_me {
|
if self.decision.ipv4_me {
|
||||||
let map = self.proxy_map_v4.read().await.clone();
|
let map = self.proxy_map_v4.read().await.clone();
|
||||||
for (dc, addrs) in &map {
|
if let Some(addrs) = map.get(&target_dc) {
|
||||||
if addrs
|
for (ip, port) in addrs {
|
||||||
.iter()
|
endpoints.insert(SocketAddr::new(*ip, *port));
|
||||||
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
|
|
||||||
{
|
|
||||||
target_dc.insert(dc.abs());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for dc in &target_dc {
|
|
||||||
for key in [*dc, -*dc] {
|
|
||||||
if let Some(addrs) = map.get(&key) {
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
endpoints.insert(SocketAddr::new(*ip, *port));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.decision.ipv6_me {
|
if self.decision.ipv6_me {
|
||||||
let map = self.proxy_map_v6.read().await.clone();
|
let map = self.proxy_map_v6.read().await.clone();
|
||||||
for (dc, addrs) in &map {
|
if let Some(addrs) = map.get(&target_dc) {
|
||||||
if addrs
|
for (ip, port) in addrs {
|
||||||
.iter()
|
endpoints.insert(SocketAddr::new(*ip, *port));
|
||||||
.any(|(ip, port)| SocketAddr::new(*ip, *port) == addr)
|
|
||||||
{
|
|
||||||
target_dc.insert(dc.abs());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for dc in &target_dc {
|
|
||||||
for key in [*dc, -*dc] {
|
|
||||||
if let Some(addrs) = map.get(&key) {
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
endpoints.insert(SocketAddr::new(*ip, *port));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ impl MePool {
|
|||||||
if self.decision.ipv4_me {
|
if self.decision.ipv4_me {
|
||||||
let map_v4 = self.proxy_map_v4.read().await.clone();
|
let map_v4 = self.proxy_map_v4.read().await.clone();
|
||||||
for (dc, addrs) in map_v4 {
|
for (dc, addrs) in map_v4 {
|
||||||
let entry = out.entry(dc.abs()).or_default();
|
let entry = out.entry(dc).or_default();
|
||||||
for (ip, port) in addrs {
|
for (ip, port) in addrs {
|
||||||
entry.insert(SocketAddr::new(ip, port));
|
entry.insert(SocketAddr::new(ip, port));
|
||||||
}
|
}
|
||||||
@@ -138,7 +138,7 @@ impl MePool {
|
|||||||
if self.decision.ipv6_me {
|
if self.decision.ipv6_me {
|
||||||
let map_v6 = self.proxy_map_v6.read().await.clone();
|
let map_v6 = self.proxy_map_v6.read().await.clone();
|
||||||
for (dc, addrs) in map_v6 {
|
for (dc, addrs) in map_v6 {
|
||||||
let entry = out.entry(dc.abs()).or_default();
|
let entry = out.entry(dc).or_default();
|
||||||
for (ip, port) in addrs {
|
for (ip, port) in addrs {
|
||||||
entry.insert(SocketAddr::new(ip, port));
|
entry.insert(SocketAddr::new(ip, port));
|
||||||
}
|
}
|
||||||
|
|||||||
128
src/transport/middle_proxy/pool_runtime_api.rs
Normal file
128
src/transport/middle_proxy/pool_runtime_api.rs
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use super::pool::{MePool, RefillDcKey};
|
||||||
|
use crate::network::IpFamily;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct MeApiRefillDcSnapshot {
|
||||||
|
pub dc: i16,
|
||||||
|
pub family: &'static str,
|
||||||
|
pub inflight: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct MeApiRefillSnapshot {
|
||||||
|
pub inflight_endpoints_total: usize,
|
||||||
|
pub inflight_dc_total: usize,
|
||||||
|
pub by_dc: Vec<MeApiRefillDcSnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct MeApiNatReflectionSnapshot {
|
||||||
|
pub addr: std::net::SocketAddr,
|
||||||
|
pub age_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct MeApiNatStunSnapshot {
|
||||||
|
pub nat_probe_enabled: bool,
|
||||||
|
pub nat_probe_disabled_runtime: bool,
|
||||||
|
pub nat_probe_attempts: u8,
|
||||||
|
pub configured_servers: Vec<String>,
|
||||||
|
pub live_servers: Vec<String>,
|
||||||
|
pub reflection_v4: Option<MeApiNatReflectionSnapshot>,
|
||||||
|
pub reflection_v6: Option<MeApiNatReflectionSnapshot>,
|
||||||
|
pub stun_backoff_remaining_ms: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MePool {
|
||||||
|
pub(crate) async fn api_refill_snapshot(&self) -> MeApiRefillSnapshot {
|
||||||
|
let inflight_endpoints_total = self.refill_inflight.lock().await.len();
|
||||||
|
let inflight_dc_keys = self
|
||||||
|
.refill_inflight_dc
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.collect::<Vec<RefillDcKey>>();
|
||||||
|
|
||||||
|
let mut by_dc_map = HashMap::<(i16, &'static str), usize>::new();
|
||||||
|
for key in inflight_dc_keys {
|
||||||
|
let family = match key.family {
|
||||||
|
IpFamily::V4 => "v4",
|
||||||
|
IpFamily::V6 => "v6",
|
||||||
|
};
|
||||||
|
let dc = key.dc as i16;
|
||||||
|
*by_dc_map.entry((dc, family)).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut by_dc = by_dc_map
|
||||||
|
.into_iter()
|
||||||
|
.map(|((dc, family), inflight)| MeApiRefillDcSnapshot {
|
||||||
|
dc,
|
||||||
|
family,
|
||||||
|
inflight,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
by_dc.sort_by_key(|entry| (entry.dc, entry.family));
|
||||||
|
|
||||||
|
MeApiRefillSnapshot {
|
||||||
|
inflight_endpoints_total,
|
||||||
|
inflight_dc_total: by_dc.len(),
|
||||||
|
by_dc,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn api_nat_stun_snapshot(&self) -> MeApiNatStunSnapshot {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut configured_servers = if !self.nat_stun_servers.is_empty() {
|
||||||
|
self.nat_stun_servers.clone()
|
||||||
|
} else if let Some(stun) = &self.nat_stun {
|
||||||
|
if stun.trim().is_empty() {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
vec![stun.clone()]
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
configured_servers.sort();
|
||||||
|
configured_servers.dedup();
|
||||||
|
|
||||||
|
let mut live_servers = self.nat_stun_live_servers.read().await.clone();
|
||||||
|
live_servers.sort();
|
||||||
|
live_servers.dedup();
|
||||||
|
|
||||||
|
let reflection = self.nat_reflection_cache.lock().await;
|
||||||
|
let reflection_v4 = reflection.v4.map(|(ts, addr)| MeApiNatReflectionSnapshot {
|
||||||
|
addr,
|
||||||
|
age_secs: now.saturating_duration_since(ts).as_secs(),
|
||||||
|
});
|
||||||
|
let reflection_v6 = reflection.v6.map(|(ts, addr)| MeApiNatReflectionSnapshot {
|
||||||
|
addr,
|
||||||
|
age_secs: now.saturating_duration_since(ts).as_secs(),
|
||||||
|
});
|
||||||
|
drop(reflection);
|
||||||
|
|
||||||
|
let backoff_until = *self.stun_backoff_until.read().await;
|
||||||
|
let stun_backoff_remaining_ms = backoff_until.and_then(|until| {
|
||||||
|
(until > now).then_some(until.duration_since(now).as_millis() as u64)
|
||||||
|
});
|
||||||
|
|
||||||
|
MeApiNatStunSnapshot {
|
||||||
|
nat_probe_enabled: self.nat_probe,
|
||||||
|
nat_probe_disabled_runtime: self
|
||||||
|
.nat_probe_disabled
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
nat_probe_attempts: self
|
||||||
|
.nat_probe_attempts
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
configured_servers,
|
||||||
|
live_servers,
|
||||||
|
reflection_v4,
|
||||||
|
reflection_v6,
|
||||||
|
stun_backoff_remaining_ms,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||||
use std::net::SocketAddr;
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
@@ -100,47 +100,103 @@ pub(crate) struct MeApiRuntimeSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
|
pub(crate) async fn admission_ready_conditional_cast(&self) -> bool {
|
||||||
|
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
||||||
|
if self.decision.ipv4_me {
|
||||||
|
let map = self.proxy_map_v4.read().await.clone();
|
||||||
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
|
}
|
||||||
|
if self.decision.ipv6_me {
|
||||||
|
let map = self.proxy_map_v6.read().await.clone();
|
||||||
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
|
}
|
||||||
|
|
||||||
|
if endpoints_by_dc.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let writers = self.writers.read().await.clone();
|
||||||
|
let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
||||||
|
for writer in writers {
|
||||||
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for endpoints in endpoints_by_dc.values() {
|
||||||
|
let alive: usize = endpoints
|
||||||
|
.iter()
|
||||||
|
.map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0))
|
||||||
|
.sum();
|
||||||
|
if alive == 0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub(crate) async fn admission_ready_full_floor(&self) -> bool {
|
||||||
|
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
||||||
|
if self.decision.ipv4_me {
|
||||||
|
let map = self.proxy_map_v4.read().await.clone();
|
||||||
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
|
}
|
||||||
|
if self.decision.ipv6_me {
|
||||||
|
let map = self.proxy_map_v6.read().await.clone();
|
||||||
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
|
}
|
||||||
|
|
||||||
|
if endpoints_by_dc.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let writers = self.writers.read().await.clone();
|
||||||
|
let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
||||||
|
for writer in writers {
|
||||||
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for endpoints in endpoints_by_dc.values() {
|
||||||
|
let endpoint_count = endpoints.len();
|
||||||
|
if endpoint_count == 0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
|
||||||
|
let alive: usize = endpoints
|
||||||
|
.iter()
|
||||||
|
.map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0))
|
||||||
|
.sum();
|
||||||
|
if alive < required {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot {
|
pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot {
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
|
|
||||||
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
||||||
if self.decision.ipv4_me {
|
if self.decision.ipv4_me {
|
||||||
let map = self.proxy_map_v4.read().await.clone();
|
let map = self.proxy_map_v4.read().await.clone();
|
||||||
for (dc, addrs) in map {
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
let abs_dc = dc.abs();
|
|
||||||
if abs_dc == 0 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Ok(dc_idx) = i16::try_from(abs_dc) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let entry = endpoints_by_dc.entry(dc_idx).or_default();
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
entry.insert(SocketAddr::new(ip, port));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if self.decision.ipv6_me {
|
if self.decision.ipv6_me {
|
||||||
let map = self.proxy_map_v6.read().await.clone();
|
let map = self.proxy_map_v6.read().await.clone();
|
||||||
for (dc, addrs) in map {
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
let abs_dc = dc.abs();
|
|
||||||
if abs_dc == 0 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Ok(dc_idx) = i16::try_from(abs_dc) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let entry = endpoints_by_dc.entry(dc_idx).or_default();
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
entry.insert(SocketAddr::new(ip, port));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut endpoint_to_dc = HashMap::<SocketAddr, i16>::new();
|
let mut endpoint_to_dc = HashMap::<SocketAddr, BTreeSet<i16>>::new();
|
||||||
for (dc, endpoints) in &endpoints_by_dc {
|
for (dc, endpoints) in &endpoints_by_dc {
|
||||||
for endpoint in endpoints {
|
for endpoint in endpoints {
|
||||||
endpoint_to_dc.entry(*endpoint).or_insert(*dc);
|
endpoint_to_dc.entry(*endpoint).or_default().insert(*dc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,7 +220,13 @@ impl MePool {
|
|||||||
|
|
||||||
for writer in writers {
|
for writer in writers {
|
||||||
let endpoint = writer.addr;
|
let endpoint = writer.addr;
|
||||||
let dc = endpoint_to_dc.get(&endpoint).copied();
|
let dc = endpoint_to_dc.get(&endpoint).and_then(|dcs| {
|
||||||
|
if dcs.len() == 1 {
|
||||||
|
dcs.iter().next().copied()
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
});
|
||||||
let draining = writer.draining.load(Ordering::Relaxed);
|
let draining = writer.draining.load(Ordering::Relaxed);
|
||||||
let degraded = writer.degraded.load(Ordering::Relaxed);
|
let degraded = writer.degraded.load(Ordering::Relaxed);
|
||||||
let bound_clients = activity
|
let bound_clients = activity
|
||||||
@@ -371,6 +433,24 @@ fn ratio_pct(part: usize, total: usize) -> f64 {
|
|||||||
pct.clamp(0.0, 100.0)
|
pct.clamp(0.0, 100.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn extend_signed_endpoints(
|
||||||
|
endpoints_by_dc: &mut BTreeMap<i16, BTreeSet<SocketAddr>>,
|
||||||
|
map: HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||||
|
) {
|
||||||
|
for (dc, addrs) in map {
|
||||||
|
if dc == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Ok(dc_idx) = i16::try_from(dc) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let entry = endpoints_by_dc.entry(dc_idx).or_default();
|
||||||
|
for (ip, port) in addrs {
|
||||||
|
entry.insert(SocketAddr::new(ip, port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn floor_mode_label(mode: MeFloorMode) -> &'static str {
|
fn floor_mode_label(mode: MeFloorMode) -> &'static str {
|
||||||
match mode {
|
match mode {
|
||||||
MeFloorMode::Static => "static",
|
MeFloorMode::Static => "static",
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ pub(crate) async fn reader_loop(
|
|||||||
let data = Bytes::copy_from_slice(&body[12..]);
|
let data = Bytes::copy_from_slice(&body[12..]);
|
||||||
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
|
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
|
||||||
|
|
||||||
let routed = reg.route(cid, MeResponse::Data { flags, data }).await;
|
let routed = reg.route_nowait(cid, MeResponse::Data { flags, data }).await;
|
||||||
if !matches!(routed, RouteResult::Routed) {
|
if !matches!(routed, RouteResult::Routed) {
|
||||||
match routed {
|
match routed {
|
||||||
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
||||||
@@ -147,7 +147,7 @@ pub(crate) async fn reader_loop(
|
|||||||
let cfm = u32::from_le_bytes(body[8..12].try_into().unwrap());
|
let cfm = u32::from_le_bytes(body[8..12].try_into().unwrap());
|
||||||
trace!(cid, cfm, "RPC_SIMPLE_ACK");
|
trace!(cid, cfm, "RPC_SIMPLE_ACK");
|
||||||
|
|
||||||
let routed = reg.route(cid, MeResponse::Ack(cfm)).await;
|
let routed = reg.route_nowait(cid, MeResponse::Ack(cfm)).await;
|
||||||
if !matches!(routed, RouteResult::Routed) {
|
if !matches!(routed, RouteResult::Routed) {
|
||||||
match routed {
|
match routed {
|
||||||
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
||||||
|
|||||||
@@ -208,6 +208,23 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult {
|
||||||
|
let tx = {
|
||||||
|
let inner = self.inner.read().await;
|
||||||
|
inner.map.get(&id).cloned()
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(tx) = tx else {
|
||||||
|
return RouteResult::NoConn;
|
||||||
|
};
|
||||||
|
|
||||||
|
match tx.try_send(resp) {
|
||||||
|
Ok(()) => RouteResult::Routed,
|
||||||
|
Err(TrySendError::Closed(_)) => RouteResult::ChannelClosed,
|
||||||
|
Err(TrySendError::Full(_)) => RouteResult::QueueFullBase,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn bind_writer(
|
pub async fn bind_writer(
|
||||||
&self,
|
&self,
|
||||||
conn_id: u64,
|
conn_id: u64,
|
||||||
@@ -256,13 +273,12 @@ impl ConnRegistry {
|
|||||||
bound_clients_by_writer.insert(*writer_id, conn_ids.len());
|
bound_clients_by_writer.insert(*writer_id, conn_ids.len());
|
||||||
}
|
}
|
||||||
for conn_meta in inner.meta.values() {
|
for conn_meta in inner.meta.values() {
|
||||||
let dc_u16 = conn_meta.target_dc.unsigned_abs();
|
if conn_meta.target_dc == 0 {
|
||||||
if dc_u16 == 0 {
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if let Ok(dc) = i16::try_from(dc_u16) {
|
*active_sessions_by_target_dc
|
||||||
*active_sessions_by_target_dc.entry(dc).or_insert(0) += 1;
|
.entry(conn_meta.target_dc)
|
||||||
}
|
.or_insert(0) += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
WriterActivitySnapshot {
|
WriterActivitySnapshot {
|
||||||
@@ -385,7 +401,8 @@ mod tests {
|
|||||||
let snapshot = registry.writer_activity_snapshot().await;
|
let snapshot = registry.writer_activity_snapshot().await;
|
||||||
assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&2));
|
assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&2));
|
||||||
assert_eq!(snapshot.bound_clients_by_writer.get(&20), Some(&1));
|
assert_eq!(snapshot.bound_clients_by_writer.get(&20), Some(&1));
|
||||||
assert_eq!(snapshot.active_sessions_by_target_dc.get(&2), Some(&2));
|
assert_eq!(snapshot.active_sessions_by_target_dc.get(&2), Some(&1));
|
||||||
|
assert_eq!(snapshot.active_sessions_by_target_dc.get(&-2), Some(&1));
|
||||||
assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1));
|
assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ use super::registry::ConnMeta;
|
|||||||
|
|
||||||
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
||||||
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
||||||
|
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
|
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
|
||||||
@@ -55,6 +56,9 @@ impl MePool {
|
|||||||
let mut no_writer_deadline: Option<Instant> = None;
|
let mut no_writer_deadline: Option<Instant> = None;
|
||||||
let mut emergency_attempts = 0u32;
|
let mut emergency_attempts = 0u32;
|
||||||
let mut async_recovery_triggered = false;
|
let mut async_recovery_triggered = false;
|
||||||
|
let mut hybrid_recovery_round = 0u32;
|
||||||
|
let mut hybrid_last_recovery_at: Option<Instant> = None;
|
||||||
|
let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50));
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
if let Some(current) = self.registry.get_writer(conn_id).await {
|
if let Some(current) = self.registry.get_writer(conn_id).await {
|
||||||
@@ -138,6 +142,18 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
|
self.maybe_trigger_hybrid_recovery(
|
||||||
|
target_dc,
|
||||||
|
&mut hybrid_recovery_round,
|
||||||
|
&mut hybrid_last_recovery_at,
|
||||||
|
hybrid_wait_step,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let deadline = Instant::now() + hybrid_wait_step;
|
||||||
|
let _ = self.wait_for_writer_until(deadline).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ws.clone()
|
ws.clone()
|
||||||
@@ -179,42 +195,41 @@ impl MePool {
|
|||||||
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
||||||
}
|
}
|
||||||
emergency_attempts += 1;
|
emergency_attempts += 1;
|
||||||
for family in self.family_order() {
|
let mut endpoints = self.endpoint_candidates_for_target_dc(target_dc).await;
|
||||||
let map_guard = match family {
|
endpoints.shuffle(&mut rand::rng());
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await,
|
for addr in endpoints {
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await,
|
if self.connect_one(addr, self.rng.as_ref()).await.is_ok() {
|
||||||
};
|
break;
|
||||||
if let Some(addrs) = map_guard.get(&(target_dc as i32)) {
|
|
||||||
let mut shuffled = addrs.clone();
|
|
||||||
shuffled.shuffle(&mut rand::rng());
|
|
||||||
drop(map_guard);
|
|
||||||
for (ip, port) in shuffled {
|
|
||||||
let addr = SocketAddr::new(ip, port);
|
|
||||||
if self.connect_one(addr, self.rng.as_ref()).await.is_ok() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_millis(100 * emergency_attempts as u64)).await;
|
|
||||||
let ws2 = self.writers.read().await;
|
|
||||||
writers_snapshot = ws2.clone();
|
|
||||||
drop(ws2);
|
|
||||||
candidate_indices = self
|
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
|
||||||
.await;
|
|
||||||
if candidate_indices.is_empty() {
|
|
||||||
candidate_indices = self
|
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
if !candidate_indices.is_empty() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(100 * emergency_attempts as u64)).await;
|
||||||
|
let ws2 = self.writers.read().await;
|
||||||
|
writers_snapshot = ws2.clone();
|
||||||
|
drop(ws2);
|
||||||
|
candidate_indices = self
|
||||||
|
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
||||||
|
.await;
|
||||||
|
if candidate_indices.is_empty() {
|
||||||
|
candidate_indices = self
|
||||||
|
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
|
self.maybe_trigger_hybrid_recovery(
|
||||||
|
target_dc,
|
||||||
|
&mut hybrid_recovery_round,
|
||||||
|
&mut hybrid_last_recovery_at,
|
||||||
|
hybrid_wait_step,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let deadline = Instant::now() + hybrid_wait_step;
|
||||||
|
let _ = self.wait_for_candidate_until(target_dc, deadline).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let writer_idle_since = self.registry.writer_idle_since_snapshot().await;
|
let writer_idle_since = self.registry.writer_idle_since_snapshot().await;
|
||||||
@@ -430,26 +445,28 @@ impl MePool {
|
|||||||
let key = target_dc as i32;
|
let key = target_dc as i32;
|
||||||
let mut preferred = Vec::<SocketAddr>::new();
|
let mut preferred = Vec::<SocketAddr>::new();
|
||||||
let mut seen = HashSet::<SocketAddr>::new();
|
let mut seen = HashSet::<SocketAddr>::new();
|
||||||
|
let lookup_keys = self.dc_lookup_chain_for_target(key);
|
||||||
|
|
||||||
for family in self.family_order() {
|
for family in self.family_order() {
|
||||||
let map = match family {
|
let map = match family {
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await.clone(),
|
IpFamily::V6 => self.proxy_map_v6.read().await.clone(),
|
||||||
};
|
};
|
||||||
let mut lookup_keys = vec![key, key.abs(), -key.abs()];
|
let mut family_selected = Vec::<SocketAddr>::new();
|
||||||
let def = self.default_dc.load(Ordering::Relaxed);
|
for lookup in lookup_keys.iter().copied() {
|
||||||
if def != 0 {
|
|
||||||
lookup_keys.push(def);
|
|
||||||
}
|
|
||||||
for lookup in lookup_keys {
|
|
||||||
if let Some(addrs) = map.get(&lookup) {
|
if let Some(addrs) = map.get(&lookup) {
|
||||||
for (ip, port) in addrs {
|
for (ip, port) in addrs {
|
||||||
let addr = SocketAddr::new(*ip, *port);
|
family_selected.push(SocketAddr::new(*ip, *port));
|
||||||
if seen.insert(addr) {
|
|
||||||
preferred.push(addr);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !family_selected.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for addr in family_selected {
|
||||||
|
if seen.insert(addr) {
|
||||||
|
preferred.push(addr);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !preferred.is_empty() && !self.decision.effective_multipath {
|
if !preferred.is_empty() && !self.decision.effective_multipath {
|
||||||
break;
|
break;
|
||||||
@@ -459,6 +476,28 @@ impl MePool {
|
|||||||
preferred
|
preferred
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn maybe_trigger_hybrid_recovery(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
target_dc: i16,
|
||||||
|
hybrid_recovery_round: &mut u32,
|
||||||
|
hybrid_last_recovery_at: &mut Option<Instant>,
|
||||||
|
hybrid_wait_step: Duration,
|
||||||
|
) {
|
||||||
|
if let Some(last) = *hybrid_last_recovery_at
|
||||||
|
&& last.elapsed() < hybrid_wait_step
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let round = *hybrid_recovery_round;
|
||||||
|
let target_triggered = self.trigger_async_recovery_for_target_dc(target_dc).await;
|
||||||
|
if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 {
|
||||||
|
self.trigger_async_recovery_global().await;
|
||||||
|
}
|
||||||
|
*hybrid_recovery_round = round.saturating_add(1);
|
||||||
|
*hybrid_last_recovery_at = Some(Instant::now());
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
||||||
if let Some(w) = self.registry.get_writer(conn_id).await {
|
if let Some(w) = self.registry.get_writer(conn_id).await {
|
||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
@@ -519,36 +558,23 @@ impl MePool {
|
|||||||
) -> Vec<usize> {
|
) -> Vec<usize> {
|
||||||
let key = target_dc as i32;
|
let key = target_dc as i32;
|
||||||
let mut preferred = Vec::<SocketAddr>::new();
|
let mut preferred = Vec::<SocketAddr>::new();
|
||||||
|
let lookup_keys = self.dc_lookup_chain_for_target(key);
|
||||||
|
|
||||||
for family in self.family_order() {
|
for family in self.family_order() {
|
||||||
let map_guard = match family {
|
let map_guard = match family {
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await,
|
IpFamily::V4 => self.proxy_map_v4.read().await,
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await,
|
IpFamily::V6 => self.proxy_map_v6.read().await,
|
||||||
};
|
};
|
||||||
|
let mut family_selected = Vec::<SocketAddr>::new();
|
||||||
if let Some(v) = map_guard.get(&key) {
|
for lookup in lookup_keys.iter().copied() {
|
||||||
preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
if let Some(v) = map_guard.get(&lookup) {
|
||||||
}
|
family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
||||||
if preferred.is_empty() {
|
}
|
||||||
let abs = key.abs();
|
if !family_selected.is_empty() {
|
||||||
if let Some(v) = map_guard.get(&abs) {
|
break;
|
||||||
preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if preferred.is_empty() {
|
|
||||||
let abs = key.abs();
|
|
||||||
if let Some(v) = map_guard.get(&-abs) {
|
|
||||||
preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if preferred.is_empty() {
|
|
||||||
let def = self.default_dc.load(Ordering::Relaxed);
|
|
||||||
if def != 0
|
|
||||||
&& let Some(v) = map_guard.get(&def)
|
|
||||||
{
|
|
||||||
preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
preferred.extend(family_selected);
|
||||||
|
|
||||||
drop(map_guard);
|
drop(map_guard);
|
||||||
|
|
||||||
@@ -558,9 +584,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if preferred.is_empty() {
|
if preferred.is_empty() {
|
||||||
return (0..writers.len())
|
return Vec::new();
|
||||||
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
|
|
||||||
.collect();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
@@ -572,11 +596,6 @@ impl MePool {
|
|||||||
out.push(idx);
|
out.push(idx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if out.is_empty() {
|
|
||||||
return (0..writers.len())
|
|
||||||
.filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm))
|
|
||||||
.collect();
|
|
||||||
}
|
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -202,6 +202,15 @@ pub struct UpstreamApiSnapshot {
|
|||||||
pub upstreams: Vec<UpstreamApiItemSnapshot>,
|
pub upstreams: Vec<UpstreamApiItemSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct UpstreamApiPolicySnapshot {
|
||||||
|
pub connect_retry_attempts: u32,
|
||||||
|
pub connect_retry_backoff_ms: u64,
|
||||||
|
pub connect_budget_ms: u64,
|
||||||
|
pub unhealthy_fail_threshold: u32,
|
||||||
|
pub connect_failfast_hard_errors: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct UpstreamEgressInfo {
|
pub struct UpstreamEgressInfo {
|
||||||
pub route_kind: UpstreamRouteKind,
|
pub route_kind: UpstreamRouteKind,
|
||||||
@@ -225,6 +234,7 @@ pub struct UpstreamManager {
|
|||||||
upstreams: Arc<RwLock<Vec<UpstreamState>>>,
|
upstreams: Arc<RwLock<Vec<UpstreamState>>>,
|
||||||
connect_retry_attempts: u32,
|
connect_retry_attempts: u32,
|
||||||
connect_retry_backoff: Duration,
|
connect_retry_backoff: Duration,
|
||||||
|
connect_budget: Duration,
|
||||||
unhealthy_fail_threshold: u32,
|
unhealthy_fail_threshold: u32,
|
||||||
connect_failfast_hard_errors: bool,
|
connect_failfast_hard_errors: bool,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
@@ -235,6 +245,7 @@ impl UpstreamManager {
|
|||||||
configs: Vec<UpstreamConfig>,
|
configs: Vec<UpstreamConfig>,
|
||||||
connect_retry_attempts: u32,
|
connect_retry_attempts: u32,
|
||||||
connect_retry_backoff_ms: u64,
|
connect_retry_backoff_ms: u64,
|
||||||
|
connect_budget_ms: u64,
|
||||||
unhealthy_fail_threshold: u32,
|
unhealthy_fail_threshold: u32,
|
||||||
connect_failfast_hard_errors: bool,
|
connect_failfast_hard_errors: bool,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
@@ -248,6 +259,7 @@ impl UpstreamManager {
|
|||||||
upstreams: Arc::new(RwLock::new(states)),
|
upstreams: Arc::new(RwLock::new(states)),
|
||||||
connect_retry_attempts: connect_retry_attempts.max(1),
|
connect_retry_attempts: connect_retry_attempts.max(1),
|
||||||
connect_retry_backoff: Duration::from_millis(connect_retry_backoff_ms),
|
connect_retry_backoff: Duration::from_millis(connect_retry_backoff_ms),
|
||||||
|
connect_budget: Duration::from_millis(connect_budget_ms.max(1)),
|
||||||
unhealthy_fail_threshold: unhealthy_fail_threshold.max(1),
|
unhealthy_fail_threshold: unhealthy_fail_threshold.max(1),
|
||||||
connect_failfast_hard_errors,
|
connect_failfast_hard_errors,
|
||||||
stats,
|
stats,
|
||||||
@@ -312,6 +324,16 @@ impl UpstreamManager {
|
|||||||
Some(UpstreamApiSnapshot { summary, upstreams })
|
Some(UpstreamApiSnapshot { summary, upstreams })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn api_policy_snapshot(&self) -> UpstreamApiPolicySnapshot {
|
||||||
|
UpstreamApiPolicySnapshot {
|
||||||
|
connect_retry_attempts: self.connect_retry_attempts,
|
||||||
|
connect_retry_backoff_ms: self.connect_retry_backoff.as_millis() as u64,
|
||||||
|
connect_budget_ms: self.connect_budget.as_millis() as u64,
|
||||||
|
unhealthy_fail_threshold: self.unhealthy_fail_threshold,
|
||||||
|
connect_failfast_hard_errors: self.connect_failfast_hard_errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
fn resolve_interface_addrs(name: &str, want_ipv6: bool) -> Vec<IpAddr> {
|
fn resolve_interface_addrs(name: &str, want_ipv6: bool) -> Vec<IpAddr> {
|
||||||
use nix::ifaddrs::getifaddrs;
|
use nix::ifaddrs::getifaddrs;
|
||||||
@@ -593,11 +615,27 @@ impl UpstreamManager {
|
|||||||
let mut last_error: Option<ProxyError> = None;
|
let mut last_error: Option<ProxyError> = None;
|
||||||
let mut attempts_used = 0u32;
|
let mut attempts_used = 0u32;
|
||||||
for attempt in 1..=self.connect_retry_attempts {
|
for attempt in 1..=self.connect_retry_attempts {
|
||||||
|
let elapsed = connect_started_at.elapsed();
|
||||||
|
if elapsed >= self.connect_budget {
|
||||||
|
last_error = Some(ProxyError::ConnectionTimeout {
|
||||||
|
addr: target.to_string(),
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let remaining_budget = self.connect_budget.saturating_sub(elapsed);
|
||||||
|
let attempt_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS)
|
||||||
|
.min(remaining_budget);
|
||||||
|
if attempt_timeout.is_zero() {
|
||||||
|
last_error = Some(ProxyError::ConnectionTimeout {
|
||||||
|
addr: target.to_string(),
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
attempts_used = attempt;
|
attempts_used = attempt;
|
||||||
self.stats.increment_upstream_connect_attempt_total();
|
self.stats.increment_upstream_connect_attempt_total();
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
match self
|
match self
|
||||||
.connect_via_upstream(&upstream, target, bind_rr.clone())
|
.connect_via_upstream(&upstream, target, bind_rr.clone(), attempt_timeout)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok((stream, egress)) => {
|
Ok((stream, egress)) => {
|
||||||
@@ -707,6 +745,7 @@ impl UpstreamManager {
|
|||||||
config: &UpstreamConfig,
|
config: &UpstreamConfig,
|
||||||
target: SocketAddr,
|
target: SocketAddr,
|
||||||
bind_rr: Option<Arc<AtomicUsize>>,
|
bind_rr: Option<Arc<AtomicUsize>>,
|
||||||
|
connect_timeout: Duration,
|
||||||
) -> Result<(TcpStream, UpstreamEgressInfo)> {
|
) -> Result<(TcpStream, UpstreamEgressInfo)> {
|
||||||
match &config.upstream_type {
|
match &config.upstream_type {
|
||||||
UpstreamType::Direct { interface, bind_addresses } => {
|
UpstreamType::Direct { interface, bind_addresses } => {
|
||||||
@@ -735,7 +774,6 @@ impl UpstreamManager {
|
|||||||
let std_stream: std::net::TcpStream = socket.into();
|
let std_stream: std::net::TcpStream = socket.into();
|
||||||
let stream = TcpStream::from_std(std_stream)?;
|
let stream = TcpStream::from_std(std_stream)?;
|
||||||
|
|
||||||
let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS);
|
|
||||||
match tokio::time::timeout(connect_timeout, stream.writable()).await {
|
match tokio::time::timeout(connect_timeout, stream.writable()).await {
|
||||||
Ok(Ok(())) => {}
|
Ok(Ok(())) => {}
|
||||||
Ok(Err(e)) => return Err(ProxyError::Io(e)),
|
Ok(Err(e)) => return Err(ProxyError::Io(e)),
|
||||||
@@ -762,7 +800,6 @@ impl UpstreamManager {
|
|||||||
))
|
))
|
||||||
},
|
},
|
||||||
UpstreamType::Socks4 { address, interface, user_id } => {
|
UpstreamType::Socks4 { address, interface, user_id } => {
|
||||||
let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS);
|
|
||||||
// Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port
|
// Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port
|
||||||
let mut stream = if let Ok(proxy_addr) = address.parse::<SocketAddr>() {
|
let mut stream = if let Ok(proxy_addr) = address.parse::<SocketAddr>() {
|
||||||
// IP:port format - use socket with optional interface binding
|
// IP:port format - use socket with optional interface binding
|
||||||
@@ -841,7 +878,6 @@ impl UpstreamManager {
|
|||||||
))
|
))
|
||||||
},
|
},
|
||||||
UpstreamType::Socks5 { address, interface, username, password } => {
|
UpstreamType::Socks5 { address, interface, username, password } => {
|
||||||
let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS);
|
|
||||||
// Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port
|
// Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port
|
||||||
let mut stream = if let Ok(proxy_addr) = address.parse::<SocketAddr>() {
|
let mut stream = if let Ok(proxy_addr) = address.parse::<SocketAddr>() {
|
||||||
// IP:port format - use socket with optional interface binding
|
// IP:port format - use socket with optional interface binding
|
||||||
@@ -1165,7 +1201,14 @@ impl UpstreamManager {
|
|||||||
target: SocketAddr,
|
target: SocketAddr,
|
||||||
) -> Result<f64> {
|
) -> Result<f64> {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let _ = self.connect_via_upstream(config, target, bind_rr).await?;
|
let _ = self
|
||||||
|
.connect_via_upstream(
|
||||||
|
config,
|
||||||
|
target,
|
||||||
|
bind_rr,
|
||||||
|
Duration::from_secs(DC_PING_TIMEOUT_SECS),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
Ok(start.elapsed().as_secs_f64() * 1000.0)
|
Ok(start.elapsed().as_secs_f64() * 1000.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1337,7 +1380,12 @@ impl UpstreamManager {
|
|||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS),
|
Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS),
|
||||||
self.connect_via_upstream(&config, endpoint, Some(bind_rr.clone())),
|
self.connect_via_upstream(
|
||||||
|
&config,
|
||||||
|
endpoint,
|
||||||
|
Some(bind_rr.clone()),
|
||||||
|
Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user