mirror of
https://github.com/telemt/telemt.git
synced 2026-06-19 09:21:10 +03:00
Compare commits
102 Commits
f8e22970c1
...
3.4.3
| Author | SHA1 | Date | |
|---|---|---|---|
| 1b25bada29 | |||
| bde30eaf05 | |||
| b447f60a72 | |||
| 093faed0c2 | |||
| 3ca3e8ff0e | |||
| 6e3b4a1ce5 | |||
| cd0771eee4 | |||
| a858dd799e | |||
| 947ef2beb7 | |||
| 376f9b42fb | |||
| 191ca35076 | |||
| 44485a545e | |||
| 17a966b822 | |||
| 073eacbb37 | |||
| 5c99cd8eb7 | |||
| 7494cb3092 | |||
| d100941426 | |||
| d25aa5a1e9 | |||
| f1b7b9aa08 | |||
| 3bff4fbfcd | |||
| f5b5ea3bbf | |||
| f36f2eae24 | |||
| 497ec6aa84 | |||
| 21ca1014ae | |||
| 982bfd20b9 | |||
| 0bcc3bf935 | |||
| f7913721e2 | |||
| 32d5cee01c | |||
| 3a17901e83 | |||
| 902a4e83cf | |||
| 696316f919 | |||
| d7a0319696 | |||
| 3fefcdd11f | |||
| 57dca639f0 | |||
| 13f86062f4 | |||
| 9303c7854a | |||
| 8267149b53 | |||
| 30fab00bfd | |||
| afc07345f5 | |||
| a965b38bd4 | |||
| f0ebbac338 | |||
| 286662fc51 | |||
| c5390baaf1 | |||
| 1cd1e96079 | |||
| 2b995c31b0 | |||
| 442320302d | |||
| ac0dde567b | |||
| b2fe9b78d8 | |||
| f039ce1827 | |||
| abff2fd7fe | |||
| 0b580eccd3 | |||
| 70b63e4e0b | |||
| 5f5a3e3fa0 | |||
| 3f69b54f5d | |||
| 62a90e05a0 | |||
| f9e54ee739 | |||
| 1b3d2d8bc5 | |||
| d477d6ee29 | |||
| 1383dfcbb1 | |||
| 107a7cc758 | |||
| 4f3193fdaa | |||
| d6be691c67 | |||
| 0b0be07a9c | |||
| 26c40092f3 | |||
| 192a852034 | |||
| 16c7a63fbc | |||
| 69a73d5fec | |||
| 7b1aa46753 | |||
| a728c727bc | |||
| d23ce4a184 | |||
| e48e1b141d | |||
| 82da541f9c | |||
| 6d5a1a29df | |||
| 026ca5cc1d | |||
| b11dec7f91 | |||
| edd1405562 | |||
| 45dd7485a9 | |||
| 901cf11c51 | |||
| 7acc76b422 | |||
| 227a64ef06 | |||
| 6748ed920e | |||
| 303b273c77 | |||
| 3bcc129b8d | |||
| 3ffbd294d2 | |||
| ddeda8d914 | |||
| 17fd01a2c4 | |||
| 8ed43a562c | |||
| fd6243b6cc | |||
| 44127c6f96 | |||
| a0c7a9e62c | |||
| d7af1cc206 | |||
| b246f0ed99 | |||
| 1265234491 | |||
| 07b53785c5 | |||
| 1e3522652c | |||
| a526fee728 | |||
| 970313edcb | |||
| 185e0081d7 | |||
| b6a30c1b51 | |||
| 19f9eb36ac | |||
| 2b8159a65e | |||
| 86be0d53fe |
@@ -0,0 +1,16 @@
|
|||||||
|
# These are supported funding model platforms
|
||||||
|
|
||||||
|
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||||
|
patreon: # Replace with a single Patreon username
|
||||||
|
open_collective: # Replace with a single Open Collective username
|
||||||
|
ko_fi: # Replace with a single Ko-fi username
|
||||||
|
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||||
|
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||||
|
liberapay: # Replace with a single Liberapay username
|
||||||
|
issuehunt: # Replace with a single IssueHunt username
|
||||||
|
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
||||||
|
polar: # Replace with a single Polar username
|
||||||
|
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
||||||
|
thanks_dev: # Replace with a single thanks.dev username
|
||||||
|
custom:
|
||||||
|
- https://nowpayments.io/donation?api_key=2bf1afd2-abc2-49f9-a012-f1e715b37223
|
||||||
+45
-56
@@ -3,50 +3,39 @@
|
|||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
**Telemt exists to solve technical problems.**
|
**Telemt exists to solve technical problems.**
|
||||||
|
- Telemt is open to contributors who want to learn, improve and build meaningful systems together.
|
||||||
|
- It is a place for building, testing, reasoning, documenting, and improving systems.
|
||||||
|
- Discussions that advance this work are in scope, discussions that divert it are not.
|
||||||
|
- Technology has consequences, responsibility is inherent.
|
||||||
|
|
||||||
Telemt is open to contributors who want to learn, improve and build meaningful systems together.
|
> **Absicht bestimmt die Form**
|
||||||
|
|
||||||
It is a place for building, testing, reasoning, documenting, and improving systems.
|
> Design follows intent
|
||||||
|
|
||||||
Discussions that advance this work are in scope. Discussions that divert it are not.
|
|
||||||
|
|
||||||
Technology has consequences. Responsibility is inherent.
|
|
||||||
|
|
||||||
> **Zweck bestimmt die Form.**
|
|
||||||
|
|
||||||
> Purpose defines form.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Principles
|
## Principles
|
||||||
|
|
||||||
* **Technical over emotional**
|
* **Technical over emotional**
|
||||||
|
- Arguments are grounded in data, logs, reproducible cases, or clear reasoning.
|
||||||
Arguments are grounded in data, logs, reproducible cases, or clear reasoning.
|
|
||||||
|
|
||||||
* **Clarity over noise**
|
* **Clarity over noise**
|
||||||
|
- Communication is structured, concise, and relevant.
|
||||||
Communication is structured, concise, and relevant.
|
|
||||||
|
|
||||||
* **Openness with standards**
|
* **Openness with standards**
|
||||||
|
- Participation is open. The work remains disciplined.
|
||||||
Participation is open. The work remains disciplined.
|
|
||||||
|
|
||||||
* **Independence of judgment**
|
* **Independence of judgment**
|
||||||
|
- Claims are evaluated on technical merit, not affiliation or posture.
|
||||||
Claims are evaluated on technical merit, not affiliation or posture.
|
|
||||||
|
|
||||||
* **Responsibility over capability**
|
* **Responsibility over capability**
|
||||||
|
- Capability does not justify careless use.
|
||||||
Capability does not justify careless use.
|
|
||||||
|
|
||||||
* **Cooperation over friction**
|
* **Cooperation over friction**
|
||||||
|
- Progress depends on coordination, mutual support, and honest review.
|
||||||
Progress depends on coordination, mutual support, and honest review.
|
|
||||||
|
|
||||||
* **Good intent, rigorous method**
|
* **Good intent, rigorous method**
|
||||||
|
- Assume good intent, but require rigor.
|
||||||
Assume good intent, but require rigor.
|
|
||||||
|
|
||||||
> **Aussagen gelten nach ihrer Begründung.**
|
> **Aussagen gelten nach ihrer Begründung.**
|
||||||
|
|
||||||
@@ -68,7 +57,9 @@ Participants are expected to:
|
|||||||
|
|
||||||
Precision is learned.
|
Precision is learned.
|
||||||
|
|
||||||
New contributors are welcome. They are expected to grow into these standards. Existing contributors are expected to make that growth possible.
|
- New contributors are welcome
|
||||||
|
- They are expected to grow into these standards
|
||||||
|
- Existing contributors are expected to make that growth possible
|
||||||
|
|
||||||
> **Wer behauptet, belegt.**
|
> **Wer behauptet, belegt.**
|
||||||
|
|
||||||
@@ -112,7 +103,7 @@ Security is both technical and behavioral.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. Openness
|
## Openness
|
||||||
|
|
||||||
Telemt is open to contributors of different backgrounds, experience levels, and working styles.
|
Telemt is open to contributors of different backgrounds, experience levels, and working styles.
|
||||||
|
|
||||||
@@ -148,10 +139,9 @@ Judgment should be exercised with restraint, consistency, and institutional resp
|
|||||||
|
|
||||||
All decisions are expected to serve the durability, clarity, and integrity of Telemt.
|
All decisions are expected to serve the durability, clarity, and integrity of Telemt.
|
||||||
|
|
||||||
> **Ordnung ist Voraussetzung der Funktion.**
|
> **Klarheit vor Zustimmung - Bestand vor Beifall**
|
||||||
|
|
||||||
> Order is the precondition of function.
|
|
||||||
|
|
||||||
|
> Clarity above approval - substantiality before success
|
||||||
---
|
---
|
||||||
|
|
||||||
## Enforcement
|
## Enforcement
|
||||||
@@ -171,42 +161,41 @@ Actions are taken to maintain function, continuity, and signal quality.
|
|||||||
|
|
||||||
## Final
|
## Final
|
||||||
|
|
||||||
Telemt is built on discipline, structure, and shared intent.
|
**Telemt is built on discipline, structure, and shared intent**
|
||||||
- Signal over noise.
|
- Signal over noise
|
||||||
- Facts over opinion.
|
- Facts over opinion
|
||||||
- Systems over rhetoric.
|
- Systems over rhetoric
|
||||||
|
- Work is collective
|
||||||
|
- Outcomes are shared
|
||||||
|
- Responsibility is distributed
|
||||||
|
- Precision is learned
|
||||||
|
- Rigor is expected
|
||||||
|
- Help is part of the work
|
||||||
|
|
||||||
- Work is collective.
|
> **Ordnung ist Voraussetzung der Freiheit**
|
||||||
- Outcomes are shared.
|
|
||||||
- Responsibility is distributed.
|
|
||||||
|
|
||||||
- Precision is learned.
|
- If you contribute — contribute with care
|
||||||
- Rigor is expected.
|
- If you speak — speak with substance
|
||||||
- Help is part of the work.
|
- If you engage — engage constructively
|
||||||
|
|
||||||
> **Ordnung ist Voraussetzung der Freiheit.**
|
|
||||||
|
|
||||||
- If you contribute — contribute with care.
|
|
||||||
- If you speak — speak with substance.
|
|
||||||
- If you engage — engage constructively.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## After All
|
## After All
|
||||||
|
|
||||||
Systems outlive intentions.
|
Systems outlive intentions
|
||||||
- What is built will be used.
|
- What is built will be used
|
||||||
- What is released will propagate.
|
- What is released will propagate
|
||||||
- What is maintained will define the future state.
|
- What is maintained will define the future state
|
||||||
|
|
||||||
There is no neutral infrastructure, only infrastructure shaped well or poorly.
|
There is no neutral infrastructure, only infrastructure shaped well or poorly
|
||||||
|
|
||||||
> **Jedes System trägt Verantwortung.**
|
> **Ordnung → Umsetzung → Ergebnis**
|
||||||
|
|
||||||
> Every system carries responsibility.
|
> Order → Implementation → Result
|
||||||
|
|
||||||
- Stability requires discipline.
|
- Stability requires discipline
|
||||||
- Freedom requires structure.
|
- Freedom requires structure
|
||||||
- Trust requires honesty.
|
- Trust requires honesty
|
||||||
|
|
||||||
|
In the end: the system reflects its contributors
|
||||||
|
|
||||||
In the end: the system reflects its contributors.
|
|
||||||
|
|||||||
Generated
+1
-1
@@ -2780,7 +2780,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.39"
|
version = "3.4.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
|||||||
+1
-2
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.39"
|
version = "3.4.3"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
@@ -98,4 +98,3 @@ harness = false
|
|||||||
[profile.release]
|
[profile.release]
|
||||||
lto = "fat"
|
lto = "fat"
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
|
|
||||||
|
|||||||
+30
@@ -77,6 +77,34 @@ COPY config.toml /app/config.toml
|
|||||||
|
|
||||||
EXPOSE 443 9090 9091
|
EXPOSE 443 9090 9091
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/telemt"]
|
||||||
|
CMD ["config.toml"]
|
||||||
|
|
||||||
|
# ==========================
|
||||||
|
# Production Netfilter Profile
|
||||||
|
# ==========================
|
||||||
|
FROM debian:12-slim AS prod-netfilter
|
||||||
|
|
||||||
|
RUN set -eux; \
|
||||||
|
apt-get update; \
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates \
|
||||||
|
conntrack \
|
||||||
|
nftables \
|
||||||
|
iptables; \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY --from=minimal /telemt /app/telemt
|
||||||
|
COPY config.toml /app/config.toml
|
||||||
|
|
||||||
|
EXPOSE 443 9090 9091
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
|
||||||
|
|
||||||
ENTRYPOINT ["/app/telemt"]
|
ENTRYPOINT ["/app/telemt"]
|
||||||
CMD ["config.toml"]
|
CMD ["config.toml"]
|
||||||
|
|
||||||
@@ -94,5 +122,7 @@ USER nonroot:nonroot
|
|||||||
|
|
||||||
EXPOSE 443 9090 9091
|
EXPOSE 443 9090 9091
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
|
||||||
|
|
||||||
ENTRYPOINT ["/app/telemt"]
|
ENTRYPOINT ["/app/telemt"]
|
||||||
CMD ["config.toml"]
|
CMD ["config.toml"]
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
# Telemt - MTProxy on Rust + Tokio
|
# Telemt - MTProxy on Rust + Tokio
|
||||||
|
|
||||||
   [](https://t.me/telemtrs)
|
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members) [](https://t.me/telemtrs)
|
||||||
|
|
||||||
|
[🇷🇺 README на русском](https://github.com/telemt/telemt/blob/main/README.ru.md)
|
||||||
|
|
||||||
***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist***
|
***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist***
|
||||||
|
|
||||||
@@ -25,6 +27,7 @@ curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
|||||||
- [Quick Start Guide](docs/Quick_start/QUICK_START_GUIDE.en.md)
|
- [Quick Start Guide](docs/Quick_start/QUICK_START_GUIDE.en.md)
|
||||||
- [Инструкция по быстрому запуску](docs/Quick_start/QUICK_START_GUIDE.ru.md)
|
- [Инструкция по быстрому запуску](docs/Quick_start/QUICK_START_GUIDE.ru.md)
|
||||||
|
|
||||||
|
## Features
|
||||||
Our implementation of **TLS-fronting** is one of the most deeply debugged, focused, advanced and *almost* **"behaviorally consistent to real"**: we are confident we have it right - [see evidence on our validation and traces](docs/FAQ.en.md#recognizability-for-dpi-and-crawler)
|
Our implementation of **TLS-fronting** is one of the most deeply debugged, focused, advanced and *almost* **"behaviorally consistent to real"**: we are confident we have it right - [see evidence on our validation and traces](docs/FAQ.en.md#recognizability-for-dpi-and-crawler)
|
||||||
|
|
||||||
Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to the Middle-End Proxy: non dramatically, but usual
|
Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to the Middle-End Proxy: non dramatically, but usual
|
||||||
@@ -77,4 +80,26 @@ telemt config.toml
|
|||||||
- Memory safety and reduced attack surface
|
- Memory safety and reduced attack surface
|
||||||
- Tokio's asynchronous architecture
|
- Tokio's asynchronous architecture
|
||||||
|
|
||||||
|
## Support Telemt
|
||||||
|
|
||||||
|
Telemt is free, open-source, and built in personal time.
|
||||||
|
If it helps you — consider supporting continued development.
|
||||||
|
|
||||||
|
Any cryptocurrency (BTC, ETH, USDT, 350+ coins):
|
||||||
|
|
||||||
|
<p align="center">
|
||||||
|
<a href="https://nowpayments.io/donation?api_key=2bf1afd2-abc2-49f9-a012-f1e715b37223" target="_blank" rel="noreferrer noopener">
|
||||||
|
<img src="https://nowpayments.io/images/embeds/donation-button-white.svg" alt="Cryptocurrency & Bitcoin donation button by NOWPayments" height="80">
|
||||||
|
</a>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
Monero (XMR) directly:
|
||||||
|
|
||||||
|
```
|
||||||
|
8Bk4tZEYPQWSypeD2hrUXG2rKbAKF16GqEN942ZdAP5cFdSqW6h4DwkP5cJMAdszzuPeHeHZPTyjWWFwzeFdjuci3ktfMoB
|
||||||
|
```
|
||||||
|
|
||||||
|
All donations go toward infrastructure, development, and research.
|
||||||
|
|
||||||
|
|
||||||

|

|
||||||
|
|||||||
+22
-3
@@ -1,6 +1,6 @@
|
|||||||
# Telemt — MTProxy на Rust + Tokio
|
# Telemt — MTProxy на Rust + Tokio
|
||||||
|
|
||||||
   [](https://t.me/telemtrs)
|
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members) [](https://t.me/telemtrs)
|
||||||
|
|
||||||
***Решает проблемы раньше, чем другие узнают об их существовании***
|
***Решает проблемы раньше, чем другие узнают об их существовании***
|
||||||
|
|
||||||
@@ -54,7 +54,6 @@ curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
|||||||
- [FAQ EN](docs/FAQ.en.md)
|
- [FAQ EN](docs/FAQ.en.md)
|
||||||
|
|
||||||
## Сборка
|
## Сборка
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Клонируйте репозиторий
|
# Клонируйте репозиторий
|
||||||
git clone https://github.com/telemt/telemt
|
git clone https://github.com/telemt/telemt
|
||||||
@@ -63,7 +62,6 @@ cd telemt
|
|||||||
# Начните процесс сборки
|
# Начните процесс сборки
|
||||||
cargo build --release
|
cargo build --release
|
||||||
|
|
||||||
# Устройства с небольшим объёмом оперативной памяти (1 ГБ, например NanoPi Neo3 / Raspberry Pi Zero 2):
|
|
||||||
# В текущем release-профиле используется lto = "fat" для максимальной оптимизации (см. Cargo.toml).
|
# В текущем release-профиле используется lto = "fat" для максимальной оптимизации (см. Cargo.toml).
|
||||||
# На системах с малым объёмом RAM (~1 ГБ) можно переопределить это значение на "thin".
|
# На системах с малым объёмом RAM (~1 ГБ) можно переопределить это значение на "thin".
|
||||||
|
|
||||||
@@ -87,4 +85,25 @@ telemt config.toml
|
|||||||
- Безопасность памяти;
|
- Безопасность памяти;
|
||||||
- Асинхронная архитектура Tokio.
|
- Асинхронная архитектура Tokio.
|
||||||
|
|
||||||
|
## Поддержать Telemt
|
||||||
|
|
||||||
|
Telemt — это бесплатное программное обеспечение с открытым исходным кодом, разработанное в свободное время.
|
||||||
|
Если оно оказалось вам полезным, вы можете поддержать дальнейшую разработку.
|
||||||
|
|
||||||
|
Принимаемые криптовалюты (BTC, ETH, USDT, 350+ и другие):
|
||||||
|
|
||||||
|
<p align="center">
|
||||||
|
<a href="https://nowpayments.io/donation?api_key=2bf1afd2-abc2-49f9-a012-f1e715b37223" target="_blank" rel="noreferrer noopener">
|
||||||
|
<img src="https://nowpayments.io/images/embeds/donation-button-white.svg" alt="Cryptocurrency & Bitcoin donation button by NOWPayments" height="80">
|
||||||
|
</a>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
Monero (XMR) напрямую:
|
||||||
|
|
||||||
|
```
|
||||||
|
8Bk4tZEYPQWSypeD2hrUXG2rKbAKF16GqEN942ZdAP5cFdSqW6h4DwkP5cJMAdszzuPeHeHZPTyjWWFwzeFdjuci3ktfMoB
|
||||||
|
```
|
||||||
|
|
||||||
|
Все пожертвования пойдут на инфраструктуру, разработку и исследования.
|
||||||
|
|
||||||

|

|
||||||
|
|||||||
+8
-5
@@ -32,13 +32,13 @@ show = "*"
|
|||||||
port = 443
|
port = 443
|
||||||
# proxy_protocol = false # Enable if behind HAProxy/nginx with PROXY protocol
|
# proxy_protocol = false # Enable if behind HAProxy/nginx with PROXY protocol
|
||||||
# metrics_port = 9090
|
# metrics_port = 9090
|
||||||
# metrics_listen = "0.0.0.0:9090" # Listen address for metrics (overrides metrics_port)
|
# metrics_listen = "127.0.0.1:9090" # Listen address for metrics (overrides metrics_port)
|
||||||
# metrics_whitelist = ["127.0.0.1", "::1", "0.0.0.0/0"]
|
# metrics_whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
|
|
||||||
[server.api]
|
[server.api]
|
||||||
enabled = true
|
enabled = true
|
||||||
listen = "0.0.0.0:9091"
|
listen = "127.0.0.1:9091"
|
||||||
whitelist = ["127.0.0.0/8"]
|
whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
minimal_runtime_enabled = false
|
minimal_runtime_enabled = false
|
||||||
minimal_runtime_cache_ttl_ms = 1000
|
minimal_runtime_cache_ttl_ms = 1000
|
||||||
|
|
||||||
@@ -48,9 +48,12 @@ ip = "0.0.0.0"
|
|||||||
|
|
||||||
# === Anti-Censorship & Masking ===
|
# === Anti-Censorship & Masking ===
|
||||||
[censorship]
|
[censorship]
|
||||||
|
# Fake-TLS / SNI masking domain used in generated ee-links.
|
||||||
|
# Changing tls_domain invalidates previously generated TLS links.
|
||||||
tls_domain = "petrovich.ru"
|
tls_domain = "petrovich.ru"
|
||||||
|
|
||||||
mask = true
|
mask = true
|
||||||
tls_emulation = true # Fetch real cert lengths and emulate TLS records
|
tls_emulation = true # Fetch real cert lengths and emulate TLS records
|
||||||
tls_front_dir = "tlsfront" # Cache directory for TLS emulation
|
tls_front_dir = "tlsfront" # Cache directory for TLS emulation
|
||||||
|
|
||||||
[access.users]
|
[access.users]
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
services:
|
||||||
|
telemt:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
target: prod-netfilter
|
||||||
|
network_mode: host
|
||||||
|
ports: []
|
||||||
|
cap_add:
|
||||||
|
- NET_BIND_SERVICE
|
||||||
|
- NET_ADMIN
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
services:
|
||||||
|
telemt:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
target: prod-netfilter
|
||||||
|
cap_add:
|
||||||
|
- NET_BIND_SERVICE
|
||||||
|
- NET_ADMIN
|
||||||
+9
-2
@@ -1,7 +1,9 @@
|
|||||||
services:
|
services:
|
||||||
telemt:
|
telemt:
|
||||||
image: ghcr.io/telemt/telemt:latest
|
image: ghcr.io/telemt/telemt:latest
|
||||||
build: .
|
build:
|
||||||
|
context: .
|
||||||
|
target: prod
|
||||||
container_name: telemt
|
container_name: telemt
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
@@ -16,13 +18,18 @@ services:
|
|||||||
- /etc/telemt:rw,mode=1777,size=4m
|
- /etc/telemt:rw,mode=1777,size=4m
|
||||||
environment:
|
environment:
|
||||||
- RUST_LOG=info
|
- RUST_LOG=info
|
||||||
|
healthcheck:
|
||||||
|
test: [ "CMD", "/app/telemt", "healthcheck", "/etc/telemt/config.toml", "--mode", "liveness" ]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 20s
|
||||||
# Uncomment this line if you want to use host network for IPv6, but bridge is default and usually better
|
# Uncomment this line if you want to use host network for IPv6, but bridge is default and usually better
|
||||||
# network_mode: host
|
# network_mode: host
|
||||||
cap_drop:
|
cap_drop:
|
||||||
- ALL
|
- ALL
|
||||||
cap_add:
|
cap_add:
|
||||||
- NET_BIND_SERVICE
|
- NET_BIND_SERVICE
|
||||||
- NET_ADMIN
|
|
||||||
read_only: true
|
read_only: true
|
||||||
security_opt:
|
security_opt:
|
||||||
- no-new-privileges:true
|
- no-new-privileges:true
|
||||||
|
|||||||
@@ -9,12 +9,12 @@ API runtime is configured in `[server.api]`.
|
|||||||
|
|
||||||
| Field | Type | Default | Description |
|
| Field | Type | Default | Description |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | `bool` | `false` | Enables REST API listener. |
|
| `enabled` | `bool` | `true` | Enables REST API listener. |
|
||||||
| `listen` | `string` (`IP:PORT`) | `127.0.0.1:9091` | API bind address. |
|
| `listen` | `string` (`IP:PORT`) | `0.0.0.0:9091` | API bind address. |
|
||||||
| `whitelist` | `CIDR[]` | `127.0.0.1/32, ::1/128` | Source IP allowlist. Empty list means allow all. |
|
| `whitelist` | `CIDR[]` | `127.0.0.0/8` | Source IP allowlist. Empty list means allow all. |
|
||||||
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
|
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
|
||||||
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
|
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
|
||||||
| `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
| `minimal_runtime_enabled` | `bool` | `true` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
||||||
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
||||||
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
|
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
|
||||||
| `runtime_edge_cache_ttl_ms` | `u64` | `1000` | Cache TTL for runtime edge summary payloads. `0` disables cache. |
|
| `runtime_edge_cache_ttl_ms` | `u64` | `1000` | Cache TTL for runtime edge summary payloads. `0` disables cache. |
|
||||||
|
|||||||
@@ -0,0 +1,225 @@
|
|||||||
|
# TLS Front Profile Fidelity
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This document describes how Telemt reuses captured TLS behavior in the FakeTLS server flight and how to validate the result on a real deployment.
|
||||||
|
|
||||||
|
When TLS front emulation is enabled, Telemt can capture useful server-side TLS behavior from the selected origin and reuse that behavior in the emulated success path. The goal is not to reproduce the origin byte-for-byte, but to reduce stable synthetic traits and make the emitted server flight structurally closer to the captured profile.
|
||||||
|
|
||||||
|
## Why this change exists
|
||||||
|
|
||||||
|
The project already captures useful server-side TLS behavior in the TLS front fetch path:
|
||||||
|
|
||||||
|
- `change_cipher_spec_count`
|
||||||
|
- `app_data_record_sizes`
|
||||||
|
- `ticket_record_sizes`
|
||||||
|
|
||||||
|
Before this change, the emulator used only part of that information. This left a gap between captured origin behavior and emitted FakeTLS server flight.
|
||||||
|
|
||||||
|
## What is implemented
|
||||||
|
|
||||||
|
- The emulator now replays the observed `ChangeCipherSpec` count from the fetched behavior profile.
|
||||||
|
- The emulator now replays observed ticket-like tail ApplicationData record sizes when raw or merged TLS profile data is available.
|
||||||
|
- The emulator now preserves more of the profiled encrypted-flight structure instead of collapsing it into a smaller synthetic shape.
|
||||||
|
- The emulator still falls back to the previous synthetic behavior when the cached profile does not contain raw TLS behavior information.
|
||||||
|
- Operator-configured `tls_new_session_tickets` still works as an additive fallback when the profile does not provide enough tail records.
|
||||||
|
|
||||||
|
## Practical benefit
|
||||||
|
|
||||||
|
- Reduced distinguishability between profiled origin TLS behavior and emulated TLS behavior.
|
||||||
|
- Lower chance of stable server-flight fingerprints caused by fixed CCS count or synthetic-only tail record sizes.
|
||||||
|
- Better reuse of already captured TLS profile data without changing MTProto logic, KDF routing, or transport architecture.
|
||||||
|
|
||||||
|
## Limitations
|
||||||
|
|
||||||
|
This mechanism does not aim to make Telemt byte-identical to the origin server.
|
||||||
|
|
||||||
|
It also does not change:
|
||||||
|
|
||||||
|
- MTProto business logic;
|
||||||
|
- KDF routing behavior;
|
||||||
|
- the overall transport architecture.
|
||||||
|
|
||||||
|
The practical goal is narrower:
|
||||||
|
|
||||||
|
- reuse more captured profile data;
|
||||||
|
- reduce fixed synthetic behavior in the server flight;
|
||||||
|
- preserve a valid FakeTLS success path while changing the emitted shape on the wire.
|
||||||
|
|
||||||
|
## Validation targets
|
||||||
|
|
||||||
|
- Correct count of emulated `ChangeCipherSpec` records.
|
||||||
|
- Correct replay of observed ticket-tail record sizes.
|
||||||
|
- No regression in existing ALPN and payload-placement behavior.
|
||||||
|
|
||||||
|
## How to validate the result
|
||||||
|
|
||||||
|
Recommended validation consists of two layers:
|
||||||
|
|
||||||
|
- focused unit and security tests for CCS-count replay and ticket-tail replay;
|
||||||
|
- real packet-capture comparison for a selected origin and a successful FakeTLS session.
|
||||||
|
|
||||||
|
When testing on the network, the expected result is:
|
||||||
|
|
||||||
|
- a valid FakeTLS and MTProto success path is preserved;
|
||||||
|
- the early encrypted server flight changes shape when richer profile data is available;
|
||||||
|
- the change is visible on the wire without changing MTProto logic or transport architecture.
|
||||||
|
|
||||||
|
This validation is intended to show better reuse of captured TLS profile data.
|
||||||
|
It is not intended to prove byte-level equivalence with the real origin server.
|
||||||
|
|
||||||
|
## How to test on a real deployment
|
||||||
|
|
||||||
|
The strongest practical validation is a side-by-side trace comparison between:
|
||||||
|
|
||||||
|
- a real TLS origin server used as `mask_host`;
|
||||||
|
- a Telemt FakeTLS success-path connection for the same SNI;
|
||||||
|
- optional captures from different Telemt builds or configurations.
|
||||||
|
|
||||||
|
The purpose of the comparison is to inspect the shape of the server flight:
|
||||||
|
|
||||||
|
- record order;
|
||||||
|
- count of `ChangeCipherSpec` records;
|
||||||
|
- count and grouping of early encrypted `ApplicationData` records;
|
||||||
|
- lengths of tail or continuation `ApplicationData` records.
|
||||||
|
|
||||||
|
## Recommended environment
|
||||||
|
|
||||||
|
Use a Linux host or Docker container for the cleanest reproduction.
|
||||||
|
|
||||||
|
Recommended setup:
|
||||||
|
|
||||||
|
1. One Telemt instance.
|
||||||
|
2. One real HTTPS origin as `mask_host`.
|
||||||
|
3. One Telegram client configured with an `ee` proxy link for the Telemt instance.
|
||||||
|
4. `tcpdump` or Wireshark available for capture analysis.
|
||||||
|
|
||||||
|
## Step-by-step test procedure
|
||||||
|
|
||||||
|
### 1. Prepare the origin
|
||||||
|
|
||||||
|
1. Choose a real HTTPS origin.
|
||||||
|
2. Set both `censorship.tls_domain` and `censorship.mask_host` to that hostname.
|
||||||
|
3. Confirm that a direct TLS request works:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Configure Telemt
|
||||||
|
|
||||||
|
Use a configuration that enables:
|
||||||
|
|
||||||
|
- `censorship.mask = true`
|
||||||
|
- `censorship.tls_emulation = true`
|
||||||
|
- `censorship.mask_host`
|
||||||
|
- `censorship.mask_port`
|
||||||
|
|
||||||
|
Recommended for cleaner testing:
|
||||||
|
|
||||||
|
- keep `censorship.tls_new_session_tickets = 0`, so the result depends primarily on fetched profile data rather than operator-forced synthetic tail records;
|
||||||
|
- keep `censorship.tls_fetch.strict_route = true`, if cleaner provenance for captured profile data is important.
|
||||||
|
|
||||||
|
### 3. Refresh TLS profile data
|
||||||
|
|
||||||
|
1. Start Telemt.
|
||||||
|
2. Let it fetch TLS front profile data for the configured domain.
|
||||||
|
3. If `tls_front_dir` is persisted, confirm that the TLS front cache is populated.
|
||||||
|
|
||||||
|
Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result.
|
||||||
|
|
||||||
|
### 4. Capture a direct-origin trace
|
||||||
|
|
||||||
|
From a separate client host, connect directly to the origin:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Capture with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Capture a Telemt FakeTLS success-path trace
|
||||||
|
|
||||||
|
Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance.
|
||||||
|
|
||||||
|
`openssl s_client` is useful for direct-origin capture and fallback sanity checks, but it does not exercise the successful FakeTLS and MTProto path.
|
||||||
|
|
||||||
|
Capture with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. Decode TLS record structure
|
||||||
|
|
||||||
|
Use `tshark` to print record-level structure:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tshark -r origin-direct.pcap -Y "tls.record" -T fields \
|
||||||
|
-e frame.number \
|
||||||
|
-e ip.src \
|
||||||
|
-e ip.dst \
|
||||||
|
-e tls.record.content_type \
|
||||||
|
-e tls.record.length
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tshark -r telemt-emulated.pcap -Y "tls.record" -T fields \
|
||||||
|
-e frame.number \
|
||||||
|
-e ip.src \
|
||||||
|
-e ip.dst \
|
||||||
|
-e tls.record.content_type \
|
||||||
|
-e tls.record.length
|
||||||
|
```
|
||||||
|
|
||||||
|
Focus on the server flight after ClientHello:
|
||||||
|
|
||||||
|
- `22` = Handshake
|
||||||
|
- `20` = ChangeCipherSpec
|
||||||
|
- `23` = ApplicationData
|
||||||
|
|
||||||
|
### 7. Build a comparison table
|
||||||
|
|
||||||
|
A compact table like the following is usually enough:
|
||||||
|
|
||||||
|
| Path | CCS count | AppData count in first encrypted flight | Tail AppData lengths |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Origin | `N` | `M` | `[a, b, ...]` |
|
||||||
|
| Telemt build A | `...` | `...` | `...` |
|
||||||
|
| Telemt build B | `...` | `...` | `...` |
|
||||||
|
|
||||||
|
The comparison should make it easy to see that:
|
||||||
|
|
||||||
|
- the FakeTLS success path remains valid;
|
||||||
|
- the early encrypted server flight changes when richer profile data is reused;
|
||||||
|
- the result is backed by packet evidence.
|
||||||
|
|
||||||
|
## Example capture set
|
||||||
|
|
||||||
|
One practical example of this workflow uses:
|
||||||
|
|
||||||
|
- `origin-direct-nginx.pcap`
|
||||||
|
- `telemt-ee-before-nginx.pcap`
|
||||||
|
- `telemt-ee-after-nginx.pcap`
|
||||||
|
|
||||||
|
Practical notes:
|
||||||
|
|
||||||
|
- `origin` was captured as a direct TLS 1.2 connection to `nginx.org`;
|
||||||
|
- `before` and `after` were captured on the Telemt FakeTLS success path with a real Telegram client;
|
||||||
|
- the first server-side FakeTLS response remains valid in both cases;
|
||||||
|
- the early encrypted server-flight segmentation differs between `before` and `after`, which is consistent with better reuse of captured profile data;
|
||||||
|
- this kind of result shows a wire-visible effect without breaking the success path, but it does not claim full indistinguishability from the origin.
|
||||||
|
|
||||||
|
## Stronger validation
|
||||||
|
|
||||||
|
For broader confidence, repeat the same comparison on:
|
||||||
|
|
||||||
|
1. one CDN-backed origin;
|
||||||
|
2. one regular nginx origin;
|
||||||
|
3. one origin with a multi-record encrypted flight and visible ticket-like tails.
|
||||||
|
|
||||||
|
If the same directional improvement appears across all three, confidence in the result will be much higher than for a single-origin example.
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
# Fidelity TLS Front Profile
|
||||||
|
|
||||||
|
## Обзор
|
||||||
|
|
||||||
|
Этот документ описывает, как Telemt переиспользует захваченное TLS-поведение в FakeTLS server flight и как проверять результат на реальной инсталляции.
|
||||||
|
|
||||||
|
Когда включена TLS front emulation, Telemt может собирать полезное серверное TLS-поведение выбранного origin и использовать его в emulated success path. Цель здесь не в побайтном копировании origin, а в уменьшении устойчивых synthetic признаков и в том, чтобы emitted server flight был структурно ближе к захваченному profile.
|
||||||
|
|
||||||
|
## Зачем нужно это изменение
|
||||||
|
|
||||||
|
Проект уже умеет собирать полезное серверное TLS-поведение в пути TLS front fetch:
|
||||||
|
|
||||||
|
- `change_cipher_spec_count`
|
||||||
|
- `app_data_record_sizes`
|
||||||
|
- `ticket_record_sizes`
|
||||||
|
|
||||||
|
До этого изменения эмулятор использовал только часть этой информации. Из-за этого оставался разрыв между захваченным поведением origin и тем FakeTLS server flight, который реально уходил на провод.
|
||||||
|
|
||||||
|
## Что реализовано
|
||||||
|
|
||||||
|
- Эмулятор теперь воспроизводит наблюдаемое значение `ChangeCipherSpec` из полученного `behavior_profile`.
|
||||||
|
- Эмулятор теперь воспроизводит наблюдаемые размеры ticket-like tail ApplicationData records, когда доступны raw или merged TLS profile data.
|
||||||
|
- Эмулятор теперь сохраняет больше структуры профилированного encrypted flight, а не схлопывает его в более маленькую synthetic форму.
|
||||||
|
- Для профилей без raw TLS behavior по-прежнему сохраняется прежний synthetic fallback.
|
||||||
|
- Операторский `tls_new_session_tickets` по-прежнему работает как дополнительный fallback, если профиль не даёт достаточного количества tail records.
|
||||||
|
|
||||||
|
## Практическая польза
|
||||||
|
|
||||||
|
- Снижается различимость между профилированным origin TLS-поведением и эмулируемым TLS-поведением.
|
||||||
|
- Уменьшается шанс устойчивых server-flight fingerprint, вызванных фиксированным CCS count или полностью synthetic tail record sizes.
|
||||||
|
- Уже собранные TLS profile data используются лучше, без изменения MTProto logic, KDF routing или transport architecture.
|
||||||
|
|
||||||
|
## Ограничения
|
||||||
|
|
||||||
|
Этот механизм не ставит целью сделать Telemt побайтно идентичным origin server.
|
||||||
|
|
||||||
|
Он также не меняет:
|
||||||
|
|
||||||
|
- MTProto business logic;
|
||||||
|
- поведение KDF routing;
|
||||||
|
- общую transport architecture.
|
||||||
|
|
||||||
|
Практическая цель уже:
|
||||||
|
|
||||||
|
- использовать больше уже собранных profile data;
|
||||||
|
- уменьшить fixed synthetic behavior в server flight;
|
||||||
|
- сохранить валидный FakeTLS success path, одновременно меняя форму emitted traffic на проводе.
|
||||||
|
|
||||||
|
## Цели валидации
|
||||||
|
|
||||||
|
- Корректное количество эмулируемых `ChangeCipherSpec` records.
|
||||||
|
- Корректное воспроизведение наблюдаемых ticket-tail record sizes.
|
||||||
|
- Отсутствие регрессии в существующем ALPN и payload-placement behavior.
|
||||||
|
|
||||||
|
## Как проверять результат
|
||||||
|
|
||||||
|
Рекомендуемая валидация состоит из двух слоёв:
|
||||||
|
|
||||||
|
- focused unit и security tests для CCS-count replay и ticket-tail replay;
|
||||||
|
- сравнение реальных packet capture для выбранного origin и успешной FakeTLS session.
|
||||||
|
|
||||||
|
При проверке на сети ожидаемый результат такой:
|
||||||
|
|
||||||
|
- валидный FakeTLS и MTProto success path сохраняется;
|
||||||
|
- форма раннего encrypted server flight меняется, когда доступно более богатое profile data;
|
||||||
|
- изменение видно на проводе без изменения MTProto logic и transport architecture.
|
||||||
|
|
||||||
|
Такая проверка нужна для подтверждения того, что уже собранные TLS profile data используются лучше.
|
||||||
|
Она не предназначена для доказательства побайтной эквивалентности с реальным origin server.
|
||||||
|
|
||||||
|
## Как проверить на реальной инсталляции
|
||||||
|
|
||||||
|
Самая сильная практическая проверка — side-by-side trace comparison между:
|
||||||
|
|
||||||
|
- реальным TLS origin server, используемым как `mask_host`;
|
||||||
|
- Telemt FakeTLS success-path connection для того же SNI;
|
||||||
|
- при необходимости capture от разных Telemt builds или configurations.
|
||||||
|
|
||||||
|
Смысл сравнения состоит в том, чтобы посмотреть на форму server flight:
|
||||||
|
|
||||||
|
- порядок records;
|
||||||
|
- количество `ChangeCipherSpec` records;
|
||||||
|
- количество и группировку ранних encrypted `ApplicationData` records;
|
||||||
|
- размеры tail или continuation `ApplicationData` records.
|
||||||
|
|
||||||
|
## Рекомендуемое окружение
|
||||||
|
|
||||||
|
Для самой чистой проверки лучше использовать Linux host или Docker container.
|
||||||
|
|
||||||
|
Рекомендуемый setup:
|
||||||
|
|
||||||
|
1. Один экземпляр Telemt.
|
||||||
|
2. Один реальный HTTPS origin как `mask_host`.
|
||||||
|
3. Один Telegram client, настроенный на `ee` proxy link для Telemt instance.
|
||||||
|
4. `tcpdump` или Wireshark для анализа capture.
|
||||||
|
|
||||||
|
## Пошаговая процедура проверки
|
||||||
|
|
||||||
|
### 1. Подготовить origin
|
||||||
|
|
||||||
|
1. Выберите реальный HTTPS origin.
|
||||||
|
2. Установите и `censorship.tls_domain`, и `censorship.mask_host` в hostname этого origin.
|
||||||
|
3. Убедитесь, что прямой TLS request работает:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Настроить Telemt
|
||||||
|
|
||||||
|
Используйте config, где включены:
|
||||||
|
|
||||||
|
- `censorship.mask = true`
|
||||||
|
- `censorship.tls_emulation = true`
|
||||||
|
- `censorship.mask_host`
|
||||||
|
- `censorship.mask_port`
|
||||||
|
|
||||||
|
Для более чистой проверки рекомендуется:
|
||||||
|
|
||||||
|
- держать `censorship.tls_new_session_tickets = 0`, чтобы результат в первую очередь зависел от fetched profile data, а не от операторских synthetic tail records;
|
||||||
|
- держать `censorship.tls_fetch.strict_route = true`, если важна более чистая provenance для captured profile data.
|
||||||
|
|
||||||
|
### 3. Обновить TLS profile data
|
||||||
|
|
||||||
|
1. Запустите Telemt.
|
||||||
|
2. Дайте ему получить TLS front profile data для выбранного домена.
|
||||||
|
3. Если `tls_front_dir` хранится persistently, убедитесь, что TLS front cache заполнен.
|
||||||
|
|
||||||
|
Persisted cache artifacts полезны, но не обязательны, если packet capture уже показывают runtime result.
|
||||||
|
|
||||||
|
### 4. Снять direct-origin trace
|
||||||
|
|
||||||
|
С отдельной клиентской машины подключитесь напрямую к origin:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Capture:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Снять Telemt FakeTLS success-path trace
|
||||||
|
|
||||||
|
Теперь подключитесь к Telemt через реальный Telegram client с `ee` proxy link, который указывает на Telemt instance.
|
||||||
|
|
||||||
|
`openssl s_client` полезен для direct-origin capture и для fallback sanity checks, но он не проходит успешный FakeTLS и MTProto path.
|
||||||
|
|
||||||
|
Capture:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. Декодировать структуру TLS records
|
||||||
|
|
||||||
|
Используйте `tshark`, чтобы вывести record-level structure:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tshark -r origin-direct.pcap -Y "tls.record" -T fields \
|
||||||
|
-e frame.number \
|
||||||
|
-e ip.src \
|
||||||
|
-e ip.dst \
|
||||||
|
-e tls.record.content_type \
|
||||||
|
-e tls.record.length
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
tshark -r telemt-emulated.pcap -Y "tls.record" -T fields \
|
||||||
|
-e frame.number \
|
||||||
|
-e ip.src \
|
||||||
|
-e ip.dst \
|
||||||
|
-e tls.record.content_type \
|
||||||
|
-e tls.record.length
|
||||||
|
```
|
||||||
|
|
||||||
|
Смотрите на server flight после ClientHello:
|
||||||
|
|
||||||
|
- `22` = Handshake
|
||||||
|
- `20` = ChangeCipherSpec
|
||||||
|
- `23` = ApplicationData
|
||||||
|
|
||||||
|
### 7. Собрать сравнительную таблицу
|
||||||
|
|
||||||
|
Обычно достаточно короткой таблицы такого вида:
|
||||||
|
|
||||||
|
| Path | CCS count | AppData count in first encrypted flight | Tail AppData lengths |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Origin | `N` | `M` | `[a, b, ...]` |
|
||||||
|
| Telemt build A | `...` | `...` | `...` |
|
||||||
|
| Telemt build B | `...` | `...` | `...` |
|
||||||
|
|
||||||
|
По такой таблице должно быть легко увидеть, что:
|
||||||
|
|
||||||
|
- FakeTLS success path остаётся валидным;
|
||||||
|
- ранний encrypted server flight меняется, когда переиспользуется более богатое profile data;
|
||||||
|
- результат подтверждён packet evidence.
|
||||||
|
|
||||||
|
## Пример набора capture
|
||||||
|
|
||||||
|
Один практический пример такой проверки использует:
|
||||||
|
|
||||||
|
- `origin-direct-nginx.pcap`
|
||||||
|
- `telemt-ee-before-nginx.pcap`
|
||||||
|
- `telemt-ee-after-nginx.pcap`
|
||||||
|
|
||||||
|
Практические замечания:
|
||||||
|
|
||||||
|
- `origin` снимался как прямое TLS 1.2 connection к `nginx.org`;
|
||||||
|
- `before` и `after` снимались на Telemt FakeTLS success path с реальным Telegram client;
|
||||||
|
- первый server-side FakeTLS response остаётся валидным в обоих случаях;
|
||||||
|
- сегментация раннего encrypted server flight отличается между `before` и `after`, что согласуется с лучшим использованием captured profile data;
|
||||||
|
- такой результат показывает заметный эффект на проводе без поломки success path, но не заявляет полной неотличимости от origin.
|
||||||
|
|
||||||
|
## Более сильная валидация
|
||||||
|
|
||||||
|
Для более широкой проверки повторите ту же процедуру ещё на:
|
||||||
|
|
||||||
|
1. одном CDN-backed origin;
|
||||||
|
2. одном regular nginx origin;
|
||||||
|
3. одном origin с multi-record encrypted flight и заметными ticket-like tails.
|
||||||
|
|
||||||
|
Если одно и то же направление улучшения повторится на всех трёх, уверенность в результате будет значительно выше, чем для одного origin example.
|
||||||
File diff suppressed because it is too large
Load Diff
+3094
-3316
File diff suppressed because it is too large
Load Diff
+25
-3
@@ -36,8 +36,11 @@ hello2 = "ad_tag2"
|
|||||||
On April 1, 2026, we became aware of a method for detecting MTProxy Fake-TLS,
|
On April 1, 2026, we became aware of a method for detecting MTProxy Fake-TLS,
|
||||||
based on the ECH extension and the ordering of cipher suites,
|
based on the ECH extension and the ordering of cipher suites,
|
||||||
as well as an overall unique JA3/JA4 fingerprint
|
as well as an overall unique JA3/JA4 fingerprint
|
||||||
that does not occur in modern browsers:
|
that does not occur in modern browsers.
|
||||||
we have already submitted initial changes to the Telegram Desktop developers and are working on updates for other clients.
|
|
||||||
|
> [!IMPORTANT]
|
||||||
|
> TLS fingerprint has been fixed in latest version of clients for Desktop / Android / iOS.
|
||||||
|
> Please update your client for MTProxy Fake-TLS to work correctly.
|
||||||
|
|
||||||
- We consider this a breakthrough aspect, which has no stable analogues today
|
- We consider this a breakthrough aspect, which has no stable analogues today
|
||||||
- Based on this: if `telemt` configured correctly, **TLS mode is completely identical to real-life handshake + communication** with a specified host
|
- Based on this: if `telemt` configured correctly, **TLS mode is completely identical to real-life handshake + communication** with a specified host
|
||||||
@@ -154,6 +157,24 @@ Keep-Alive: timeout=60
|
|||||||
### Why do you need a middle proxy (ME)
|
### Why do you need a middle proxy (ME)
|
||||||
https://github.com/telemt/telemt/discussions/167
|
https://github.com/telemt/telemt/discussions/167
|
||||||
|
|
||||||
|
## How clients interact with Telegram DCs
|
||||||
|
When you register a Telegram account, it gets permanently bound to one of Telegram's data centers (DCs).
|
||||||
|
It is deciced beforehand by Telegram based on the phone number's region.
|
||||||
|
This DC becomes your **home DC**: all content you upload (photos, videos, files, messages) is stored there.
|
||||||
|
Your client authenticates on it with every connection.
|
||||||
|
|
||||||
|
For example, if your account is registered on **DC2**, your client will always connect to DC2 first.
|
||||||
|
When you open a chat with another user whose home DC is **DC5**, your client opens an additional connection to DC5 to download their media.
|
||||||
|
Those cross-DC requests are normal and happen constantly.
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> Because every session is anchored to your home DC, an outage there causes other DCs to be unavaliable.
|
||||||
|
> If your home DC is DC2 and DC2 goes down, you **cannot** reach DC5 even though DC5 itself is perfectly healthy.
|
||||||
|
> The client has no valid session to route the request through.
|
||||||
|
|
||||||
|
This is also why an MTProxy only needs to reach Telegram's DC infrastructure as a whole.
|
||||||
|
The proxy itself doesn't care which DC your account lives on. The client negotiates the correct DC through the proxy after connecting.
|
||||||
|
|
||||||
### How many people can use one link
|
### How many people can use one link
|
||||||
By default, an unlimited number of people can use a single link.
|
By default, an unlimited number of people can use a single link.
|
||||||
However, you can limit the number of unique IP addresses for each user:
|
However, you can limit the number of unique IP addresses for each user:
|
||||||
@@ -161,7 +182,8 @@ However, you can limit the number of unique IP addresses for each user:
|
|||||||
[access.user_max_unique_ips]
|
[access.user_max_unique_ips]
|
||||||
hello = 1
|
hello = 1
|
||||||
```
|
```
|
||||||
This parameter sets the maximum number of unique IP addresses from which a single link can be used simultaneously. If the first user disconnects, a second one can connect. At the same time, multiple users can connect from a single IP address simultaneously (for example, devices on the same Wi-Fi network).
|
This parameter sets the maximum number of unique IP addresses from which a single link can be used simultaneously. If the first user disconnects, a second one can connect.
|
||||||
|
At the same time, multiple users can connect from a single IP address simultaneously (for example, devices on the same Wi-Fi network).
|
||||||
|
|
||||||
### How to create multiple different links
|
### How to create multiple different links
|
||||||
1. Generate the required number of secrets using the command: `openssl rand -hex 16`.
|
1. Generate the required number of secrets using the command: `openssl rand -hex 16`.
|
||||||
|
|||||||
+22
-2
@@ -33,9 +33,12 @@ hello = "ad_tag"
|
|||||||
hello2 = "ad_tag2"
|
hello2 = "ad_tag2"
|
||||||
```
|
```
|
||||||
## Распознаваемость для DPI и сканеров
|
## Распознаваемость для DPI и сканеров
|
||||||
|
1 апреля 2026 года нам стало известно о методе обнаружения MTProxy Fake-TLS, основанном на расширении ECH и порядке набора шифров,
|
||||||
|
а также об общем уникальном отпечатке JA3/JA4, который не встречается в современных браузерах.
|
||||||
|
|
||||||
1 апреля 2026 года нам стало известно о методе обнаружения MTProxy Fake-TLS, основанном на расширении ECH и порядке набора шифров,
|
> [!IMPORTANT]
|
||||||
а также об общем уникальном отпечатке JA3/JA4, который не встречается в современных браузерах: мы уже отправили первоначальные изменения разработчикам Telegram Desktop и работаем над обновлениями для других клиентов.
|
> Проблема с TLS отпечатком исправлена в последних версиях клиентов Telegram для Desktop / Android / iOS.
|
||||||
|
> Обновите свой клиент для корректной работы с MTProxy Fake-TLS!
|
||||||
|
|
||||||
- Мы считаем это прорывом, которому на сегодняшний день нет стабильных аналогов;
|
- Мы считаем это прорывом, которому на сегодняшний день нет стабильных аналогов;
|
||||||
- Исходя из этого: если `telemt` настроен правильно, **режим TLS полностью идентичен реальному «рукопожатию» + обмену данными** с указанным хостом;
|
- Исходя из этого: если `telemt` настроен правильно, **режим TLS полностью идентичен реальному «рукопожатию» + обмену данными** с указанным хостом;
|
||||||
@@ -152,6 +155,23 @@ Keep-Alive: timeout=60
|
|||||||
## Зачем нужен middle proxy (ME)
|
## Зачем нужен middle proxy (ME)
|
||||||
https://github.com/telemt/telemt/discussions/167
|
https://github.com/telemt/telemt/discussions/167
|
||||||
|
|
||||||
|
## Как клиенты взаимодействуют с дата-центрами Telegram
|
||||||
|
При регистрации аккаунта Telegram он навсегда привязывается к одному из дата-центров (DC).
|
||||||
|
Telegram заранее определяет к какому DC привязать аккаунт исходя из региона, к которому относиться номер телефона.
|
||||||
|
Этот DC становится вашим **домашним**: именно там хранится весь контент, который вы загружаете (фото, видео, файлы, сообщения).
|
||||||
|
И именно на нем клиент авторизуется при каждом подключении.
|
||||||
|
|
||||||
|
Например, если ваш аккаунт зарегистрирован на **DC2**, клиент всегда будет подключаться в первую очередь к DC2.
|
||||||
|
Когда вы открываете переписку с пользователем, чей домашний DC — **DC5**, клиент устанавливает доп. соединение с DC5, чтобы загрузить его контент.
|
||||||
|
Такие кросс-запросы к DC — это нормальная часть работы Telegram.
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> Поскольку аккаунт всегда привязан к домашнему DC, при его падении контент с других DC будет недоступен.
|
||||||
|
> Если ваш домашний DC — DC2, и DC2 лежит, вы **не сможете** достучаться и до DC5, даже если сам DC5 полностью исправен.
|
||||||
|
> У клиента просто нет валидной сессии, через которую можно было бы направить запрос.
|
||||||
|
|
||||||
|
По той же причине MTProxy достаточно иметь доступ к инфраструктуре Telegram в целом.
|
||||||
|
Cамому MTProxy всё равно, на каком DC живёт ваш аккаунт. Клиент cам договаривается о нужном DC через прокси уже после подключения.
|
||||||
|
|
||||||
## Что такое dd и ee в контексте MTProxy?
|
## Что такое dd и ee в контексте MTProxy?
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,8 @@ cargo build --release
|
|||||||
./target/release/telemt --version
|
./target/release/telemt --version
|
||||||
```
|
```
|
||||||
|
|
||||||
For low-RAM systems, this repository already uses `lto = "thin"` in release profile.
|
For low-RAM systems, note that this repository currently uses `lto = "fat"` in release profile.
|
||||||
|
On constrained builders, a local override to `lto = "thin"` may be more practical.
|
||||||
|
|
||||||
## 3. Install binary and config
|
## 3. Install binary and config
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,36 @@
|
|||||||
|
# Installation Options
|
||||||
|
There are three options for installing Telemt:
|
||||||
|
- [Automated installation using a script](#very-quick-start).
|
||||||
|
- [Manual installation of Telemt as a service](#telemt-via-systemd).
|
||||||
|
- [Installation using Docker Compose](#telemt-via-docker-compose).
|
||||||
|
|
||||||
# Very quick start
|
# Very quick start
|
||||||
|
|
||||||
### One-command installation / update on re-run
|
### One-command installation / update on re-run
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
After starting, the script will prompt for:
|
||||||
|
- Your language (1 - English, 2 - Russian);
|
||||||
|
- Your TLS domain (press Enter for petrovich.ru).
|
||||||
|
|
||||||
|
The script checks if the port (default **443**) is free. If the port is already in use, installation will fail. You need to free up the port or use the **-p** flag with a different port to retry the installation.
|
||||||
|
|
||||||
|
To modify the script’s startup parameters, you can use the following flags:
|
||||||
|
- **-d, --domain** - TLS domain;
|
||||||
|
- **-p, --port** - server port (1–65535);
|
||||||
|
- **-s, --secret** - 32 hex secret;
|
||||||
|
- **-a, --ad-tag** - ad_tag;
|
||||||
|
- **-l, --lan**g - language (1/en or 2/ru);
|
||||||
|
|
||||||
|
Providing all options skips interactive prompts.
|
||||||
|
|
||||||
|
After completion, the script will provide a link for client connections:
|
||||||
|
```bash
|
||||||
|
tg://proxy?server=IP&port=PORT&secret=SECRET
|
||||||
|
```
|
||||||
|
|
||||||
### Installing a specific version
|
### Installing a specific version
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh -s -- 3.3.39
|
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh -s -- 3.3.39
|
||||||
@@ -110,15 +137,15 @@ show = "*"
|
|||||||
# === Server Binding ===
|
# === Server Binding ===
|
||||||
[server]
|
[server]
|
||||||
port = 443
|
port = 443
|
||||||
# proxy_protocol = false # Enable if behind HAProxy/nginx with PROXY protocol
|
# proxy_protocol = false # Enable if behind HAProxy/nginx with PROXY protocol
|
||||||
# metrics_port = 9090
|
# metrics_port = 9090
|
||||||
# metrics_listen = "0.0.0.0:9090" # Listen address for metrics (overrides metrics_port)
|
# metrics_listen = "127.0.0.1:9090" # Listen address for metrics (overrides metrics_port)
|
||||||
# metrics_whitelist = ["127.0.0.1", "::1", "0.0.0.0/0"]
|
# metrics_whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
|
|
||||||
[server.api]
|
[server.api]
|
||||||
enabled = true
|
enabled = true
|
||||||
listen = "0.0.0.0:9091"
|
listen = "127.0.0.1:9091"
|
||||||
whitelist = ["127.0.0.0/8"]
|
whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
minimal_runtime_enabled = false
|
minimal_runtime_enabled = false
|
||||||
minimal_runtime_cache_ttl_ms = 1000
|
minimal_runtime_cache_ttl_ms = 1000
|
||||||
|
|
||||||
@@ -128,9 +155,9 @@ ip = "0.0.0.0"
|
|||||||
|
|
||||||
# === Anti-Censorship & Masking ===
|
# === Anti-Censorship & Masking ===
|
||||||
[censorship]
|
[censorship]
|
||||||
tls_domain = "petrovich.ru"
|
tls_domain = "petrovich.ru" # Fake-TLS / SNI masking domain used in generated ee-links
|
||||||
mask = true
|
mask = true
|
||||||
tls_emulation = true # Fetch real cert lengths and emulate TLS records
|
tls_emulation = true # Fetch real cert lengths and emulate TLS records
|
||||||
tls_front_dir = "tlsfront" # Cache directory for TLS emulation
|
tls_front_dir = "tlsfront" # Cache directory for TLS emulation
|
||||||
|
|
||||||
[access.users]
|
[access.users]
|
||||||
@@ -141,9 +168,9 @@ hello = "00000000000000000000000000000000"
|
|||||||
then Ctrl+S -> Ctrl+X to save
|
then Ctrl+S -> Ctrl+X to save
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> Replace the value of the hello parameter with the value you obtained in step 0.
|
> Replace the value of the `hello` parameter with the value you obtained in step 0.
|
||||||
> Additionally, change the value of the tls_domain parameter to a different website.
|
> Additionally, change the value of the `tls_domain` parameter to a different website.
|
||||||
> Changing the tls_domain parameter will break all links that use the old domain!
|
> Changing the `tls_domain` parameter will break all links that use the old domain!
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,35 @@
|
|||||||
|
# Варианты установки
|
||||||
|
Имеется три варианта установки Telemt:
|
||||||
|
- [Автоматизированная установка с помощью скрипта](#очень-быстрый-старт).
|
||||||
|
- [Ручная установка Telemt в качестве службы](#telemt-через-systemd-вручную).
|
||||||
|
- [Установка через Docker Compose](#telemt-через-docker-compose).
|
||||||
|
|
||||||
# Очень быстрый старт
|
# Очень быстрый старт
|
||||||
|
|
||||||
### Установка одной командой / обновление при повторном запуске
|
### Установка одной командой / обновление при повторном запуске
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
||||||
```
|
```
|
||||||
|
После запуска скрипт запросит:
|
||||||
|
- ваш язык (1 - English, 2 - Русский);
|
||||||
|
- ваш TLS-домен (нажмите Enter для petrovich.ru).
|
||||||
|
|
||||||
|
Во время установки скрипт проверяет, свободен ли порт (по умолчанию **443**). Если порт занят другим процессом - установка завершится с ошибкой. Для повторной установки необходимо освободить порт или указать другой через флаг **-p**.
|
||||||
|
|
||||||
|
Для изменения параметров запуска скрипта можно использовать следующие флаги:
|
||||||
|
- **-d, --domain** - TLS-домен;
|
||||||
|
- **-p, --port** - порт (1–65535);
|
||||||
|
- **-s, --secret** - секрет (32 hex символа);
|
||||||
|
- **-a, --ad-tag** - ad_tag;
|
||||||
|
- **-l, --lang** - язык (1/en или 2/ru).
|
||||||
|
|
||||||
|
Если заданы флаги для языка и домена, интерактивных вопросов не будет.
|
||||||
|
|
||||||
|
После завершения установки скрипт выдаст ссылку для подключения клиентов:
|
||||||
|
```bash
|
||||||
|
tg://proxy?server=IP&port=PORT&secret=SECRET
|
||||||
|
```
|
||||||
|
|
||||||
### Установка нужной версии
|
### Установка нужной версии
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh -s -- 3.3.39
|
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh -s -- 3.3.39
|
||||||
@@ -103,22 +129,22 @@ tls = true
|
|||||||
[general.links]
|
[general.links]
|
||||||
show = "*"
|
show = "*"
|
||||||
# show = ["alice", "bob"] # Показывать ссылки только для alice и bob
|
# show = ["alice", "bob"] # Показывать ссылки только для alice и bob
|
||||||
# show = "*" # Показывать ссылки для всех пользователей
|
# show = "*" # Показывать ссылки для всех пользователей
|
||||||
# public_host = "proxy.example.com" # Хост (IP-адрес или домен) для ссылок tg://
|
# public_host = "proxy.example.com" # Хост (IP-адрес или домен) для ссылок tg://
|
||||||
# public_port = 443 # Порт для ссылок tg:// (по умолчанию: server.port)
|
# public_port = 443 # Порт для ссылок tg:// (по умолчанию: server.port)
|
||||||
|
|
||||||
# === Привязка сервера ===
|
# === Привязка сервера ===
|
||||||
[server]
|
[server]
|
||||||
port = 443
|
port = 443
|
||||||
# proxy_protocol = false # Включите, если сервер находится за HAProxy/nginx с протоколом PROXY
|
# proxy_protocol = false # Включите, если сервер находится за HAProxy/nginx с протоколом PROXY
|
||||||
# metrics_port = 9090
|
# metrics_port = 9090
|
||||||
# metrics_listen = "0.0.0.0:9090" # Адрес прослушивания для метрик (переопределяет metrics_port)
|
# metrics_listen = "127.0.0.1:9090" # Адрес прослушивания для метрик (переопределяет metrics_port)
|
||||||
# metrics_whitelist = ["127.0.0.1", "::1", "0.0.0.0/0"]
|
# metrics_whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
|
|
||||||
[server.api]
|
[server.api]
|
||||||
enabled = true
|
enabled = true
|
||||||
listen = "0.0.0.0:9091"
|
listen = "127.0.0.1:9091"
|
||||||
whitelist = ["127.0.0.0/8"]
|
whitelist = ["127.0.0.1/32", "::1/128"]
|
||||||
minimal_runtime_enabled = false
|
minimal_runtime_enabled = false
|
||||||
minimal_runtime_cache_ttl_ms = 1000
|
minimal_runtime_cache_ttl_ms = 1000
|
||||||
|
|
||||||
@@ -128,9 +154,9 @@ ip = "0.0.0.0"
|
|||||||
|
|
||||||
# === Обход блокировок и маскировка ===
|
# === Обход блокировок и маскировка ===
|
||||||
[censorship]
|
[censorship]
|
||||||
tls_domain = "petrovich.ru"
|
tls_domain = "petrovich.ru" # Домен Fake-TLS / SNI, который будет использоваться в сгенерированных ee-ссылках
|
||||||
mask = true
|
mask = true
|
||||||
tls_emulation = true # Получить реальную длину сертификата и эмулировать запись TLS
|
tls_emulation = true # Получить реальную длину сертификата и эмулировать запись TLS
|
||||||
tls_front_dir = "tlsfront" # Директория кэша для эмуляции TLS
|
tls_front_dir = "tlsfront" # Директория кэша для эмуляции TLS
|
||||||
|
|
||||||
[access.users]
|
[access.users]
|
||||||
@@ -141,9 +167,9 @@ hello = "00000000000000000000000000000000"
|
|||||||
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
|
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> Замените значение параметра hello на значение, которое вы получили в пункте 0.
|
> Замените значение параметра `hello` на значение, которое вы получили в пункте 0.
|
||||||
> Так же замените значение параметра tls_domain на другой сайт.
|
> Так же замените значение параметра `tls_domain` на другой сайт.
|
||||||
> Изменение параметра tls_domain сделает нерабочими все ссылки, использующие старый домен!
|
> Изменение параметра `tls_domain` сделает нерабочими все ссылки, использующие старый домен!
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -163,7 +163,7 @@ PING 10.10.10.1 (10.10.10.1) 56(84) bytes of data.
|
|||||||
---
|
---
|
||||||
|
|
||||||
## Step 2. Installing telemt on Server B (conditionally Netherlands)
|
## Step 2. Installing telemt on Server B (conditionally Netherlands)
|
||||||
Installation and configuration are described [here](https://github.com/telemt/telemt/blob/main/docs/QUICK_START_GUIDE.ru.md) or [here](https://gitlab.com/An0nX/telemt-docker#-quick-start-docker-compose).\
|
Installation and configuration are described [here](https://github.com/telemt/telemt/blob/main/docs/Quick_start/QUICK_START_GUIDE.en.md) or [here](https://gitlab.com/An0nX/telemt-docker#-quick-start-docker-compose).\
|
||||||
It is assumed that telemt expects connections on port `443\tcp`.
|
It is assumed that telemt expects connections on port `443\tcp`.
|
||||||
|
|
||||||
In the telemt config, you must enable the `Proxy` protocol and restrict connections to it only through the tunnel.
|
In the telemt config, you must enable the `Proxy` protocol and restrict connections to it only through the tunnel.
|
||||||
|
|||||||
@@ -166,7 +166,7 @@ PING 10.10.10.1 (10.10.10.1) 56(84) bytes of data.
|
|||||||
|
|
||||||
## Шаг 2. Установка telemt на Сервере B (_условно Нидерланды_)
|
## Шаг 2. Установка telemt на Сервере B (_условно Нидерланды_)
|
||||||
|
|
||||||
Установка и настройка описаны [здесь](https://github.com/telemt/telemt/blob/main/docs/QUICK_START_GUIDE.ru.md) или [здесь](https://gitlab.com/An0nX/telemt-docker#-quick-start-docker-compose).\
|
Установка и настройка описаны [здесь](https://github.com/telemt/telemt/blob/main/docs/Quick_start/QUICK_START_GUIDE.ru.md) или [здесь](https://gitlab.com/An0nX/telemt-docker#-quick-start-docker-compose).\
|
||||||
Подразумевается что telemt ожидает подключения на порту `443\tcp`.
|
Подразумевается что telemt ожидает подключения на порту `443\tcp`.
|
||||||
|
|
||||||
В конфиге telemt необходимо включить протокол `Proxy` и ограничить подключения к нему только через туннель.
|
В конфиге telemt необходимо включить протокол `Proxy` и ограничить подключения к нему только через туннель.
|
||||||
|
|||||||
@@ -0,0 +1,273 @@
|
|||||||
|
<img src="https://gist.githubusercontent.com/avbor/1f8a128e628f47249aae6e058a57610b/raw/19013276c035e91058e0a9799ab145f8e70e3ff5/scheme.svg">
|
||||||
|
|
||||||
|
## Concept
|
||||||
|
- **Server A** (_e.g., RU_):\
|
||||||
|
Entry point, accepts Telegram proxy user traffic via **Xray** (port `443\tcp`)\
|
||||||
|
and sends it through the tunnel to Server **B**.\
|
||||||
|
Public port for Telegram clients — `443\tcp`
|
||||||
|
- **Server B** (_e.g., NL_):\
|
||||||
|
Exit point, runs the **Xray server** (to terminate the tunnel entry point) and **telemt**.\
|
||||||
|
The server must have unrestricted access to Telegram Data Centers.\
|
||||||
|
Public port for VLESS/REALITY (incoming) — `443\tcp`\
|
||||||
|
Internal telemt port (where decrypted Xray traffic ends up) — `8443\tcp`
|
||||||
|
|
||||||
|
The tunnel works over the `VLESS-XTLS-Reality` (or `VLESS/xhttp/reality`) protocol. The original client IP address is preserved thanks to the PROXYv2 protocol, which Xray on Server A dynamically injects via a local loopback before wrapping the traffic into Reality, transparently delivering the real IPs to telemt on Server B.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Step 1. Setup Xray Tunnel (A <-> B)
|
||||||
|
|
||||||
|
You must install **Xray-core** (version 1.8.4 or newer recommended) on both servers.
|
||||||
|
Official installation script (run on both servers):
|
||||||
|
```bash
|
||||||
|
bash -c "$(curl -L https://github.com/XTLS/Xray-install/raw/main/install-release.sh)" @ install
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key and Parameter Generation (Run Once)
|
||||||
|
For configuration, you need a unique UUID and Xray Reality keys. Run on any server with Xray installed:
|
||||||
|
1. **Client UUID:**
|
||||||
|
```bash
|
||||||
|
xray uuid
|
||||||
|
# Save the output (e.g.: 12345678-abcd-1234-abcd-1234567890ab) — this is <XRAY_UUID>
|
||||||
|
```
|
||||||
|
2. **X25519 Keypair (Private & Public) for Reality:**
|
||||||
|
```bash
|
||||||
|
xray x25519
|
||||||
|
# Save the Private key (<SERVER_B_PRIVATE_KEY>) and Public key (<SERVER_B_PUBLIC_KEY>)
|
||||||
|
```
|
||||||
|
3. **Short ID (Reality identifier):**
|
||||||
|
```bash
|
||||||
|
openssl rand -hex 8
|
||||||
|
# Save the output (e.g.: abc123def456) — this is <SHORT_ID>
|
||||||
|
```
|
||||||
|
4. **Random Path (for xhttp):**
|
||||||
|
```bash
|
||||||
|
openssl rand -hex 16
|
||||||
|
# Save the output (e.g., 0123456789abcdef0123456789abcdef) to replace <YOUR_RANDOM_PATH> in configs
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Configuration for Server B (_EU_):
|
||||||
|
|
||||||
|
Create or edit the file `/usr/local/etc/xray/config.json`.
|
||||||
|
This Xray instance will listen on the public `443` port and proxy valid Reality traffic, while routing "disguised" traffic (e.g., direct web browser scans) to `yahoo.com`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nano /usr/local/etc/xray/config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
File content:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"log": {
|
||||||
|
"loglevel": "error",
|
||||||
|
"access": "none"
|
||||||
|
},
|
||||||
|
"inbounds": [
|
||||||
|
{
|
||||||
|
"tag": "vless-in",
|
||||||
|
"port": 443,
|
||||||
|
"protocol": "vless",
|
||||||
|
"settings": {
|
||||||
|
"clients": [
|
||||||
|
{
|
||||||
|
"id": "<XRAY_UUID>"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"decryption": "none"
|
||||||
|
},
|
||||||
|
"streamSettings": {
|
||||||
|
"network": "xhttp",
|
||||||
|
"security": "reality",
|
||||||
|
"realitySettings": {
|
||||||
|
"dest": "yahoo.com:443",
|
||||||
|
"serverNames": [
|
||||||
|
"yahoo.com"
|
||||||
|
],
|
||||||
|
"privateKey": "<SERVER_B_PRIVATE_KEY>",
|
||||||
|
"shortIds": [
|
||||||
|
"<SHORT_ID>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"xhttpSettings": {
|
||||||
|
"path": "/<YOUR_RANDOM_PATH>",
|
||||||
|
"mode": "auto"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outbounds": [
|
||||||
|
{
|
||||||
|
"tag": "tunnel-to-telemt",
|
||||||
|
"protocol": "freedom",
|
||||||
|
"settings": {
|
||||||
|
"destination": "127.0.0.1:8443"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routing": {
|
||||||
|
"domainStrategy": "AsIs",
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": [
|
||||||
|
"vless-in"
|
||||||
|
],
|
||||||
|
"outboundTag": "tunnel-to-telemt"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Open the firewall port (if enabled):
|
||||||
|
```bash
|
||||||
|
sudo ufw allow 443/tcp
|
||||||
|
```
|
||||||
|
Restart and setup Xray to run at boot:
|
||||||
|
```bash
|
||||||
|
sudo systemctl restart xray
|
||||||
|
sudo systemctl enable xray
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Configuration for Server A (_RU_):
|
||||||
|
|
||||||
|
Similarly, edit `/usr/local/etc/xray/config.json`.
|
||||||
|
Here Xray acts as the public entry point: it listens on `443\tcp`, uses a local loopback (via internal port `10444`) to prepend the `PROXYv2` header, and encapsulates the payload via Reality to Server B, instructing Server B to deliver it to its *local* `127.0.0.1:8443` port (where telemt will listen).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nano /usr/local/etc/xray/config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
File content:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"log": {
|
||||||
|
"loglevel": "error",
|
||||||
|
"access": "none"
|
||||||
|
},
|
||||||
|
"inbounds": [
|
||||||
|
{
|
||||||
|
"tag": "public-in",
|
||||||
|
"port": 443,
|
||||||
|
"listen": "0.0.0.0",
|
||||||
|
"protocol": "dokodemo-door",
|
||||||
|
"settings": {
|
||||||
|
"address": "127.0.0.1",
|
||||||
|
"port": 10444,
|
||||||
|
"network": "tcp"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tag": "tunnel-in",
|
||||||
|
"port": 10444,
|
||||||
|
"listen": "127.0.0.1",
|
||||||
|
"protocol": "dokodemo-door",
|
||||||
|
"settings": {
|
||||||
|
"address": "127.0.0.1",
|
||||||
|
"port": 8443,
|
||||||
|
"network": "tcp"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outbounds": [
|
||||||
|
{
|
||||||
|
"tag": "local-injector",
|
||||||
|
"protocol": "freedom",
|
||||||
|
"settings": {
|
||||||
|
"proxyProtocol": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tag": "vless-out",
|
||||||
|
"protocol": "vless",
|
||||||
|
"settings": {
|
||||||
|
"vnext": [
|
||||||
|
{
|
||||||
|
"address": "<PUBLIC_IP_SERVER_B>",
|
||||||
|
"port": 443,
|
||||||
|
"users": [
|
||||||
|
{
|
||||||
|
"id": "<XRAY_UUID>",
|
||||||
|
"encryption": "none"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"streamSettings": {
|
||||||
|
"network": "xhttp",
|
||||||
|
"security": "reality",
|
||||||
|
"realitySettings": {
|
||||||
|
"serverName": "yahoo.com",
|
||||||
|
"publicKey": "<SERVER_B_PUBLIC_KEY>",
|
||||||
|
"shortId": "<SHORT_ID>",
|
||||||
|
"spiderX": "/",
|
||||||
|
"fingerprint": "chrome"
|
||||||
|
},
|
||||||
|
"xhttpSettings": {
|
||||||
|
"path": "/<YOUR_RANDOM_PATH>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routing": {
|
||||||
|
"domainStrategy": "AsIs",
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": ["public-in"],
|
||||||
|
"outboundTag": "local-injector"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": ["tunnel-in"],
|
||||||
|
"outboundTag": "vless-out"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
*Replace `<PUBLIC_IP_SERVER_B>` with the public IP address of Server B.*
|
||||||
|
|
||||||
|
Open the firewall port for clients (if enabled):
|
||||||
|
```bash
|
||||||
|
sudo ufw allow 443/tcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Restart and setup Xray to run at boot:
|
||||||
|
```bash
|
||||||
|
sudo systemctl restart xray
|
||||||
|
sudo systemctl enable xray
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Step 2. Install telemt on Server B (_EU_)
|
||||||
|
|
||||||
|
telemt installation is heavily covered in the [Quick Start Guide](../Quick_start/QUICK_START_GUIDE.en.md).
|
||||||
|
By contrast to standard setups, telemt must listen strictly _locally_ (since Xray occupies the public `443` interface) and must expect `PROXYv2` packets.
|
||||||
|
|
||||||
|
Edit the configuration file (`config.toml`) on Server B accordingly:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server]
|
||||||
|
port = 8443
|
||||||
|
listen_addr_ipv4 = "127.0.0.1"
|
||||||
|
proxy_protocol = true
|
||||||
|
|
||||||
|
[general.links]
|
||||||
|
show = "*"
|
||||||
|
public_host = "<FQDN_OR_IP_SERVER_A>"
|
||||||
|
public_port = 443
|
||||||
|
```
|
||||||
|
|
||||||
|
- Address `127.0.0.1` and `port = 8443` instructs the core proxy router to process connections unpacked locally via Xray-server.
|
||||||
|
- `proxy_protocol = true` commands telemt to parse the injected PROXY header (from Server A's Xray local loopback) and log genuine end-user IPs.
|
||||||
|
- Under `public_host`, place Server A's public IP address or FQDN to ensure working links are generated for Telegram users.
|
||||||
|
|
||||||
|
Restart `telemt`. Your server is now robust against DPI scanners, passing traffic optimally.
|
||||||
|
|
||||||
@@ -0,0 +1,272 @@
|
|||||||
|
<img src="https://gist.githubusercontent.com/avbor/1f8a128e628f47249aae6e058a57610b/raw/19013276c035e91058e0a9799ab145f8e70e3ff5/scheme.svg">
|
||||||
|
|
||||||
|
## Концепция
|
||||||
|
- **Сервер A** (_РФ_):\
|
||||||
|
Точка входа, принимает трафик пользователей Telegram-прокси напрямую через **Xray** (порт `443\tcp`)\
|
||||||
|
и отправляет его в туннель на Сервер **B**.\
|
||||||
|
Порт для клиентов Telegram — `443\tcp`
|
||||||
|
- **Сервер B** (_условно Нидерланды_):\
|
||||||
|
Точка выхода, на нем работает **Xray-сервер** (принимает подключения точки входа) и **telemt**.\
|
||||||
|
На сервере должен быть неограниченный доступ до серверов Telegram.\
|
||||||
|
Порт для VLESS/REALITY (вход) — `443\tcp`\
|
||||||
|
Внутренний порт telemt (куда пробрасывается трафик) — `8443\tcp`
|
||||||
|
|
||||||
|
Туннель работает по протоколу VLESS-XTLS-Reality (или VLESS/xhttp/reality). Оригинальный IP-адрес клиента сохраняется благодаря протоколу PROXYv2, который Xray на Сервере А добавляет через локальный loopback перед упаковкой в туннель, благодаря чему прозрачно доходит до telemt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Шаг 1. Настройка туннеля Xray (A <-> B)
|
||||||
|
|
||||||
|
На обоих серверах необходимо установить **Xray-core** (рекомендуется версия 1.8.4 или новее).
|
||||||
|
Официальный скрипт установки (выполнить на обоих серверах):
|
||||||
|
```bash
|
||||||
|
bash -c "$(curl -L https://github.com/XTLS/Xray-install/raw/main/install-release.sh)" @ install
|
||||||
|
```
|
||||||
|
|
||||||
|
### Генерация ключей и параметров (выполнить один раз)
|
||||||
|
Для конфигурации потребуются уникальные ID и ключи Xray Reality. Выполните на любом сервере с установленным Xray:
|
||||||
|
1. **UUID клиента:**
|
||||||
|
```bash
|
||||||
|
xray uuid
|
||||||
|
# Сохраните вывод (например: 12345678-abcd-1234-abcd-1234567890ab) — это <XRAY_UUID>
|
||||||
|
```
|
||||||
|
2. **Пара ключей X25519 (Private & Public) для Reality:**
|
||||||
|
```bash
|
||||||
|
xray x25519
|
||||||
|
# Сохраните Private key (<SERVER_B_PRIVATE_KEY>) и Public key (<SERVER_B_PUBLIC_KEY>)
|
||||||
|
```
|
||||||
|
3. **Short ID (идентификатор Reality):**
|
||||||
|
```bash
|
||||||
|
openssl rand -hex 8
|
||||||
|
# Сохраните вывод (например: abc123def456) — это <SHORT_ID>
|
||||||
|
```
|
||||||
|
4. **Random Path (путь для xhttp):**
|
||||||
|
```bash
|
||||||
|
openssl rand -hex 16
|
||||||
|
# Сохраните вывод (например, 0123456789abcdef0123456789abcdef), чтобы заменить <YOUR_RANDOM_PATH> в конфигах
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Конфигурация Сервера B (_Нидерланды_):
|
||||||
|
|
||||||
|
Создаем или редактируем файл `/usr/local/etc/xray/config.json`.
|
||||||
|
Этот Xray-сервер будет слушать порт `443` и прозрачно пропускать валидный Reality трафик дальше, а "замаскированный" трафик (например, если кто-то стучится в лоб веб-браузером) пойдет на `yahoo.com`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nano /usr/local/etc/xray/config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Содержимое файла:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"log": {
|
||||||
|
"loglevel": "error",
|
||||||
|
"access": "none"
|
||||||
|
},
|
||||||
|
"inbounds": [
|
||||||
|
{
|
||||||
|
"tag": "vless-in",
|
||||||
|
"port": 443,
|
||||||
|
"protocol": "vless",
|
||||||
|
"settings": {
|
||||||
|
"clients": [
|
||||||
|
{
|
||||||
|
"id": "<XRAY_UUID>"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"decryption": "none"
|
||||||
|
},
|
||||||
|
"streamSettings": {
|
||||||
|
"network": "xhttp",
|
||||||
|
"security": "reality",
|
||||||
|
"realitySettings": {
|
||||||
|
"dest": "yahoo.com:443",
|
||||||
|
"serverNames": [
|
||||||
|
"yahoo.com"
|
||||||
|
],
|
||||||
|
"privateKey": "<SERVER_B_PRIVATE_KEY>",
|
||||||
|
"shortIds": [
|
||||||
|
"<SHORT_ID>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"xhttpSettings": {
|
||||||
|
"path": "/<YOUR_RANDOM_PATH>",
|
||||||
|
"mode": "auto"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outbounds": [
|
||||||
|
{
|
||||||
|
"tag": "tunnel-to-telemt",
|
||||||
|
"protocol": "freedom",
|
||||||
|
"settings": {
|
||||||
|
"destination": "127.0.0.1:8443"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routing": {
|
||||||
|
"domainStrategy": "AsIs",
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": [
|
||||||
|
"vless-in"
|
||||||
|
],
|
||||||
|
"outboundTag": "tunnel-to-telemt"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Открываем порт на фаерволе (если включен):
|
||||||
|
```bash
|
||||||
|
sudo ufw allow 443/tcp
|
||||||
|
```
|
||||||
|
Перезапускаем Xray:
|
||||||
|
```bash
|
||||||
|
sudo systemctl restart xray
|
||||||
|
sudo systemctl enable xray
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Конфигурация Сервера A (_РФ_):
|
||||||
|
|
||||||
|
Аналогично, редактируем `/usr/local/etc/xray/config.json`.
|
||||||
|
Здесь Xray выступает публичной точкой: он принимает трафик на внешний порт `443\tcp`, пропускает через локальный loopback (порт `10444`) для добавления PROXYv2-заголовка, и упаковывает в Reality до Сервера B, прося тот доставить данные на *свой локальный* порт `127.0.0.1:8443` (именно там будет слушать telemt).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nano /usr/local/etc/xray/config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Содержимое файла:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"log": {
|
||||||
|
"loglevel": "error",
|
||||||
|
"access": "none"
|
||||||
|
},
|
||||||
|
"inbounds": [
|
||||||
|
{
|
||||||
|
"tag": "public-in",
|
||||||
|
"port": 443,
|
||||||
|
"listen": "0.0.0.0",
|
||||||
|
"protocol": "dokodemo-door",
|
||||||
|
"settings": {
|
||||||
|
"address": "127.0.0.1",
|
||||||
|
"port": 10444,
|
||||||
|
"network": "tcp"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tag": "tunnel-in",
|
||||||
|
"port": 10444,
|
||||||
|
"listen": "127.0.0.1",
|
||||||
|
"protocol": "dokodemo-door",
|
||||||
|
"settings": {
|
||||||
|
"address": "127.0.0.1",
|
||||||
|
"port": 8443,
|
||||||
|
"network": "tcp"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outbounds": [
|
||||||
|
{
|
||||||
|
"tag": "local-injector",
|
||||||
|
"protocol": "freedom",
|
||||||
|
"settings": {
|
||||||
|
"proxyProtocol": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tag": "vless-out",
|
||||||
|
"protocol": "vless",
|
||||||
|
"settings": {
|
||||||
|
"vnext": [
|
||||||
|
{
|
||||||
|
"address": "<PUBLIC_IP_SERVER_B>",
|
||||||
|
"port": 443,
|
||||||
|
"users": [
|
||||||
|
{
|
||||||
|
"id": "<XRAY_UUID>",
|
||||||
|
"encryption": "none"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"streamSettings": {
|
||||||
|
"network": "xhttp",
|
||||||
|
"security": "reality",
|
||||||
|
"realitySettings": {
|
||||||
|
"serverName": "yahoo.com",
|
||||||
|
"publicKey": "<SERVER_B_PUBLIC_KEY>",
|
||||||
|
"shortId": "<SHORT_ID>",
|
||||||
|
"spiderX": "/",
|
||||||
|
"fingerprint": "chrome"
|
||||||
|
},
|
||||||
|
"xhttpSettings": {
|
||||||
|
"path": "/<YOUR_RANDOM_PATH>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"routing": {
|
||||||
|
"domainStrategy": "AsIs",
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": ["public-in"],
|
||||||
|
"outboundTag": "local-injector"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "field",
|
||||||
|
"inboundTag": ["tunnel-in"],
|
||||||
|
"outboundTag": "vless-out"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
*Замените `<PUBLIC_IP_SERVER_B>` на внешний IP-адрес Сервера B.*
|
||||||
|
|
||||||
|
Открываем порт на фаерволе для клиентов:
|
||||||
|
```bash
|
||||||
|
sudo ufw allow 443/tcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Перезапускаем Xray:
|
||||||
|
```bash
|
||||||
|
sudo systemctl restart xray
|
||||||
|
sudo systemctl enable xray
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Шаг 2. Установка и настройка telemt на Сервере B (_Нидерланды_)
|
||||||
|
|
||||||
|
Установка telemt описана [в основной инструкции](../Quick_start/QUICK_START_GUIDE.ru.md).
|
||||||
|
Отличие в том, что telemt должен слушать *внутренний* порт (так как 443 занят Xray-сервером), а также ожидать `PROXY` протокол из Xray туннеля.
|
||||||
|
|
||||||
|
В конфиге `config.toml` прокси (на Сервере B) укажите:
|
||||||
|
```toml
|
||||||
|
[server]
|
||||||
|
port = 8443
|
||||||
|
listen_addr_ipv4 = "127.0.0.1"
|
||||||
|
proxy_protocol = true
|
||||||
|
|
||||||
|
[general.links]
|
||||||
|
show = "*"
|
||||||
|
public_host = "<FQDN_OR_IP_SERVER_A>"
|
||||||
|
public_port = 443
|
||||||
|
```
|
||||||
|
|
||||||
|
- `port = 8443` и `listen_addr_ipv4 = "127.0.0.1"` означают, что telemt принимает подключения только изнутри (приходящие от локального Xray-процесса).
|
||||||
|
- `proxy_protocol = true` заставляет telemt парсить PROXYv2-заголовок (который добавил Xray на Сервере A через loopback), восстанавливая IP-адрес конечного пользователя (РФ).
|
||||||
|
- В `public_host` укажите публичный IP-адрес или домен Сервера A, чтобы ссылки на подключение генерировались корректно.
|
||||||
|
|
||||||
|
Перезапустите `telemt`, и клиенты смогут подключаться по выданным ссылкам.
|
||||||
|
|
||||||
+67
-16
@@ -1,6 +1,6 @@
|
|||||||
#![allow(clippy::too_many_arguments)]
|
#![allow(clippy::too_many_arguments)]
|
||||||
|
|
||||||
use std::convert::Infallible;
|
use std::io::{Error as IoError, ErrorKind};
|
||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -16,7 +16,7 @@ use tokio::net::TcpListener;
|
|||||||
use tokio::sync::{Mutex, RwLock, watch};
|
use tokio::sync::{Mutex, RwLock, watch};
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::{ApiGrayAction, ProxyConfig};
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::proxy::route_mode::RouteRuntimeController;
|
use crate::proxy::route_mode::RouteRuntimeController;
|
||||||
use crate::startup::StartupTracker;
|
use crate::startup::StartupTracker;
|
||||||
@@ -41,8 +41,8 @@ use config_store::{current_revision, load_config_from_disk, parse_if_match};
|
|||||||
use events::ApiEventStore;
|
use events::ApiEventStore;
|
||||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||||
use model::{
|
use model::{
|
||||||
ApiFailure, CreateUserRequest, DeleteUserResponse, HealthData, PatchUserRequest,
|
ApiFailure, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
|
||||||
RotateSecretRequest, SummaryData, UserActiveIps,
|
PatchUserRequest, RotateSecretRequest, SummaryData, UserActiveIps,
|
||||||
};
|
};
|
||||||
use runtime_edge::{
|
use runtime_edge::{
|
||||||
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||||
@@ -184,7 +184,9 @@ pub async fn serve(
|
|||||||
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
|
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
debug!(error = %error, "API connection error");
|
if !error.is_user() {
|
||||||
|
debug!(error = %error, "API connection error");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -195,7 +197,7 @@ async fn handle(
|
|||||||
peer: SocketAddr,
|
peer: SocketAddr,
|
||||||
shared: Arc<ApiShared>,
|
shared: Arc<ApiShared>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||||
let request_id = shared.next_request_id();
|
let request_id = shared.next_request_id();
|
||||||
let cfg = config_rx.borrow().clone();
|
let cfg = config_rx.borrow().clone();
|
||||||
let api_cfg = &cfg.server.api;
|
let api_cfg = &cfg.server.api;
|
||||||
@@ -213,14 +215,25 @@ async fn handle(
|
|||||||
|
|
||||||
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||||
{
|
{
|
||||||
return Ok(error_response(
|
return match api_cfg.gray_action {
|
||||||
request_id,
|
ApiGrayAction::Api => Ok(error_response(
|
||||||
ApiFailure::new(
|
request_id,
|
||||||
StatusCode::FORBIDDEN,
|
ApiFailure::new(
|
||||||
"forbidden",
|
StatusCode::FORBIDDEN,
|
||||||
"Source IP is not allowed",
|
"forbidden",
|
||||||
),
|
"Source IP is not allowed",
|
||||||
));
|
),
|
||||||
|
)),
|
||||||
|
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header("content-type", "text/html; charset=utf-8")
|
||||||
|
.body(Full::new(Bytes::new()))
|
||||||
|
.unwrap()),
|
||||||
|
ApiGrayAction::Drop => Err(IoError::new(
|
||||||
|
ErrorKind::ConnectionAborted,
|
||||||
|
"api request dropped by gray_action=drop",
|
||||||
|
)),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if !api_cfg.auth_header.is_empty() {
|
if !api_cfg.auth_header.is_empty() {
|
||||||
@@ -244,11 +257,16 @@ async fn handle(
|
|||||||
|
|
||||||
let method = req.method().clone();
|
let method = req.method().clone();
|
||||||
let path = req.uri().path().to_string();
|
let path = req.uri().path().to_string();
|
||||||
|
let normalized_path = if path.len() > 1 {
|
||||||
|
path.trim_end_matches('/')
|
||||||
|
} else {
|
||||||
|
path.as_str()
|
||||||
|
};
|
||||||
let query = req.uri().query().map(str::to_string);
|
let query = req.uri().query().map(str::to_string);
|
||||||
let body_limit = api_cfg.request_body_limit_bytes;
|
let body_limit = api_cfg.request_body_limit_bytes;
|
||||||
|
|
||||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||||
match (method.as_str(), path.as_str()) {
|
match (method.as_str(), normalized_path) {
|
||||||
("GET", "/v1/health") => {
|
("GET", "/v1/health") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let data = HealthData {
|
let data = HealthData {
|
||||||
@@ -257,6 +275,33 @@ async fn handle(
|
|||||||
};
|
};
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
}
|
}
|
||||||
|
("GET", "/v1/health/ready") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||||
|
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||||
|
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||||
|
let reason = if ready {
|
||||||
|
None
|
||||||
|
} else if !admission_open {
|
||||||
|
Some("admission_closed")
|
||||||
|
} else {
|
||||||
|
Some("no_healthy_upstreams")
|
||||||
|
};
|
||||||
|
let data = HealthReadyData {
|
||||||
|
ready,
|
||||||
|
status: if ready { "ready" } else { "not_ready" },
|
||||||
|
reason,
|
||||||
|
admission_open,
|
||||||
|
healthy_upstreams: upstream_health.healthy_total,
|
||||||
|
total_upstreams: upstream_health.configured_total,
|
||||||
|
};
|
||||||
|
let status_code = if ready {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
|
};
|
||||||
|
Ok(success_response(status_code, data, revision))
|
||||||
|
}
|
||||||
("GET", "/v1/system/info") => {
|
("GET", "/v1/system/info") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
|
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
|
||||||
@@ -431,7 +476,7 @@ async fn handle(
|
|||||||
Ok(success_response(status, data, revision))
|
Ok(success_response(status, data, revision))
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
if let Some(user) = path.strip_prefix("/v1/users/")
|
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||||
&& !user.is_empty()
|
&& !user.is_empty()
|
||||||
&& !user.contains('/')
|
&& !user.contains('/')
|
||||||
{
|
{
|
||||||
@@ -600,6 +645,12 @@ async fn handle(
|
|||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
debug!(
|
||||||
|
method = method.as_str(),
|
||||||
|
path = %path,
|
||||||
|
normalized_path = %normalized_path,
|
||||||
|
"API route not found"
|
||||||
|
);
|
||||||
Ok(error_response(
|
Ok(error_response(
|
||||||
request_id,
|
request_id,
|
||||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||||
|
|||||||
@@ -60,6 +60,17 @@ pub(super) struct HealthData {
|
|||||||
pub(super) read_only: bool,
|
pub(super) read_only: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct HealthReadyData {
|
||||||
|
pub(super) ready: bool,
|
||||||
|
pub(super) status: &'static str,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) reason: Option<&'static str>,
|
||||||
|
pub(super) admission_open: bool,
|
||||||
|
pub(super) healthy_upstreams: usize,
|
||||||
|
pub(super) total_upstreams: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(super) struct SummaryData {
|
pub(super) struct SummaryData {
|
||||||
pub(super) uptime_seconds: f64,
|
pub(super) uptime_seconds: f64,
|
||||||
|
|||||||
+13
-1
@@ -452,7 +452,11 @@ fn build_user_links(
|
|||||||
startup_detected_ip_v6: Option<IpAddr>,
|
startup_detected_ip_v6: Option<IpAddr>,
|
||||||
) -> UserLinks {
|
) -> UserLinks {
|
||||||
let hosts = resolve_link_hosts(cfg, startup_detected_ip_v4, startup_detected_ip_v6);
|
let hosts = resolve_link_hosts(cfg, startup_detected_ip_v4, startup_detected_ip_v6);
|
||||||
let port = cfg.general.links.public_port.unwrap_or(cfg.server.port);
|
let port = cfg
|
||||||
|
.general
|
||||||
|
.links
|
||||||
|
.public_port
|
||||||
|
.unwrap_or(resolve_default_link_port(cfg));
|
||||||
let tls_domains = resolve_tls_domains(cfg);
|
let tls_domains = resolve_tls_domains(cfg);
|
||||||
|
|
||||||
let mut classic = Vec::new();
|
let mut classic = Vec::new();
|
||||||
@@ -490,6 +494,14 @@ fn build_user_links(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_default_link_port(cfg: &ProxyConfig) -> u16 {
|
||||||
|
cfg.server
|
||||||
|
.listeners
|
||||||
|
.first()
|
||||||
|
.and_then(|listener| listener.port)
|
||||||
|
.unwrap_or(cfg.server.port)
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_link_hosts(
|
fn resolve_link_hosts(
|
||||||
cfg: &ProxyConfig,
|
cfg: &ProxyConfig,
|
||||||
startup_detected_ip_v4: Option<IpAddr>,
|
startup_detected_ip_v4: Option<IpAddr>,
|
||||||
|
|||||||
+71
-2
@@ -6,12 +6,15 @@
|
|||||||
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
|
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
|
||||||
//! - `status [--pid-file PATH]` - Check daemon status
|
//! - `status [--pid-file PATH]` - Check daemon status
|
||||||
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
||||||
|
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
|
||||||
|
|
||||||
use rand::RngExt;
|
use rand::RngExt;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
|
use crate::healthcheck::{self, HealthcheckMode};
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
|
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
|
||||||
|
|
||||||
@@ -28,6 +31,8 @@ pub enum Subcommand {
|
|||||||
Reload,
|
Reload,
|
||||||
/// Check daemon status (`status` subcommand).
|
/// Check daemon status (`status` subcommand).
|
||||||
Status,
|
Status,
|
||||||
|
/// Run health probe and exit with status code.
|
||||||
|
Healthcheck,
|
||||||
/// Fire-and-forget setup (`--init`).
|
/// Fire-and-forget setup (`--init`).
|
||||||
Init,
|
Init,
|
||||||
}
|
}
|
||||||
@@ -38,6 +43,8 @@ pub struct ParsedCommand {
|
|||||||
pub subcommand: Subcommand,
|
pub subcommand: Subcommand,
|
||||||
pub pid_file: PathBuf,
|
pub pid_file: PathBuf,
|
||||||
pub config_path: String,
|
pub config_path: String,
|
||||||
|
pub healthcheck_mode: HealthcheckMode,
|
||||||
|
pub healthcheck_mode_invalid: Option<String>,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
pub daemon_opts: DaemonOptions,
|
pub daemon_opts: DaemonOptions,
|
||||||
pub init_opts: Option<InitOptions>,
|
pub init_opts: Option<InitOptions>,
|
||||||
@@ -52,6 +59,8 @@ impl Default for ParsedCommand {
|
|||||||
#[cfg(not(unix))]
|
#[cfg(not(unix))]
|
||||||
pid_file: PathBuf::from("/var/run/telemt.pid"),
|
pid_file: PathBuf::from("/var/run/telemt.pid"),
|
||||||
config_path: "config.toml".to_string(),
|
config_path: "config.toml".to_string(),
|
||||||
|
healthcheck_mode: HealthcheckMode::Liveness,
|
||||||
|
healthcheck_mode_invalid: None,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
daemon_opts: DaemonOptions::default(),
|
daemon_opts: DaemonOptions::default(),
|
||||||
init_opts: None,
|
init_opts: None,
|
||||||
@@ -91,6 +100,9 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
|||||||
"status" => {
|
"status" => {
|
||||||
cmd.subcommand = Subcommand::Status;
|
cmd.subcommand = Subcommand::Status;
|
||||||
}
|
}
|
||||||
|
"healthcheck" => {
|
||||||
|
cmd.subcommand = Subcommand::Healthcheck;
|
||||||
|
}
|
||||||
"run" => {
|
"run" => {
|
||||||
cmd.subcommand = Subcommand::Run;
|
cmd.subcommand = Subcommand::Run;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -113,7 +125,35 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
|||||||
while i < args.len() {
|
while i < args.len() {
|
||||||
match args[i].as_str() {
|
match args[i].as_str() {
|
||||||
// Skip subcommand names
|
// Skip subcommand names
|
||||||
"start" | "stop" | "reload" | "status" | "run" => {}
|
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
|
||||||
|
"--mode" => {
|
||||||
|
i += 1;
|
||||||
|
if i < args.len() {
|
||||||
|
match HealthcheckMode::from_cli_arg(&args[i]) {
|
||||||
|
Some(mode) => {
|
||||||
|
cmd.healthcheck_mode = mode;
|
||||||
|
cmd.healthcheck_mode_invalid = None;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cmd.healthcheck_mode_invalid = Some(args[i].clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cmd.healthcheck_mode_invalid = Some(String::new());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s if s.starts_with("--mode=") => {
|
||||||
|
let raw = s.trim_start_matches("--mode=");
|
||||||
|
match HealthcheckMode::from_cli_arg(raw) {
|
||||||
|
Some(mode) => {
|
||||||
|
cmd.healthcheck_mode = mode;
|
||||||
|
cmd.healthcheck_mode_invalid = None;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cmd.healthcheck_mode_invalid = Some(raw.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// PID file option (for stop/reload/status)
|
// PID file option (for stop/reload/status)
|
||||||
"--pid-file" => {
|
"--pid-file" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
@@ -152,6 +192,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
|||||||
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
|
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
|
||||||
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
|
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
|
||||||
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
|
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
|
||||||
|
Subcommand::Healthcheck => {
|
||||||
|
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
|
||||||
|
if invalid_mode.is_empty() {
|
||||||
|
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
|
||||||
|
} else {
|
||||||
|
eprintln!(
|
||||||
|
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(2)
|
||||||
|
} else {
|
||||||
|
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
|
||||||
|
}
|
||||||
|
}
|
||||||
Subcommand::Init => {
|
Subcommand::Init => {
|
||||||
if let Some(opts) = cmd.init_opts.clone() {
|
if let Some(opts) = cmd.init_opts.clone() {
|
||||||
match run_init(opts) {
|
match run_init(opts) {
|
||||||
@@ -177,6 +231,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
|||||||
eprintln!("[telemt] Subcommand not supported on this platform");
|
eprintln!("[telemt] Subcommand not supported on this platform");
|
||||||
Some(1)
|
Some(1)
|
||||||
}
|
}
|
||||||
|
Subcommand::Healthcheck => {
|
||||||
|
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
|
||||||
|
if invalid_mode.is_empty() {
|
||||||
|
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
|
||||||
|
} else {
|
||||||
|
eprintln!(
|
||||||
|
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(2)
|
||||||
|
} else {
|
||||||
|
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
|
||||||
|
}
|
||||||
|
}
|
||||||
Subcommand::Init => {
|
Subcommand::Init => {
|
||||||
if let Some(opts) = cmd.init_opts.clone() {
|
if let Some(opts) = cmd.init_opts.clone() {
|
||||||
match run_init(opts) {
|
match run_init(opts) {
|
||||||
@@ -598,16 +666,17 @@ secure = false
|
|||||||
tls = true
|
tls = true
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
port = {port}
|
|
||||||
listen_addr_ipv4 = "0.0.0.0"
|
listen_addr_ipv4 = "0.0.0.0"
|
||||||
listen_addr_ipv6 = "::"
|
listen_addr_ipv6 = "::"
|
||||||
|
|
||||||
[[server.listeners]]
|
[[server.listeners]]
|
||||||
ip = "0.0.0.0"
|
ip = "0.0.0.0"
|
||||||
|
port = {port}
|
||||||
# reuse_allow = false # Set true only when intentionally running multiple telemt instances on same port
|
# reuse_allow = false # Set true only when intentionally running multiple telemt instances on same port
|
||||||
|
|
||||||
[[server.listeners]]
|
[[server.listeners]]
|
||||||
ip = "::"
|
ip = "::"
|
||||||
|
port = {port}
|
||||||
|
|
||||||
[timeouts]
|
[timeouts]
|
||||||
client_first_byte_idle_secs = 300
|
client_first_byte_idle_secs = 300
|
||||||
|
|||||||
@@ -615,6 +615,26 @@ pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
|||||||
32 * 1024
|
32 * 1024
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||||
|
60_000
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||||
|
200
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||||
|
5_000
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||||
|
100
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||||
5
|
5
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,8 +17,9 @@
|
|||||||
//! | `network` | `dns_overrides` | Applied immediately |
|
//! | `network` | `dns_overrides` | Applied immediately |
|
||||||
//! | `access` | All user/quota fields | Effective immediately |
|
//! | `access` | All user/quota fields | Effective immediately |
|
||||||
//!
|
//!
|
||||||
//! Fields that require re-binding sockets (`server.port`, `censorship.*`,
|
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||||
//! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted.
|
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||||
|
//! applied; a warning is emitted.
|
||||||
//! Non-hot changes are never mixed into the runtime config snapshot.
|
//! Non-hot changes are never mixed into the runtime config snapshot.
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
@@ -120,6 +121,9 @@ pub struct HotFields {
|
|||||||
pub user_max_tcp_conns_global_each: usize,
|
pub user_max_tcp_conns_global_each: usize,
|
||||||
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
|
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
|
||||||
pub user_data_quota: std::collections::HashMap<String, u64>,
|
pub user_data_quota: std::collections::HashMap<String, u64>,
|
||||||
|
pub user_rate_limits: std::collections::HashMap<String, crate::config::RateLimitBps>,
|
||||||
|
pub cidr_rate_limits:
|
||||||
|
std::collections::HashMap<ipnetwork::IpNetwork, crate::config::RateLimitBps>,
|
||||||
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
|
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
|
||||||
pub user_max_unique_ips_global_each: usize,
|
pub user_max_unique_ips_global_each: usize,
|
||||||
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
||||||
@@ -244,6 +248,8 @@ impl HotFields {
|
|||||||
user_max_tcp_conns_global_each: cfg.access.user_max_tcp_conns_global_each,
|
user_max_tcp_conns_global_each: cfg.access.user_max_tcp_conns_global_each,
|
||||||
user_expirations: cfg.access.user_expirations.clone(),
|
user_expirations: cfg.access.user_expirations.clone(),
|
||||||
user_data_quota: cfg.access.user_data_quota.clone(),
|
user_data_quota: cfg.access.user_data_quota.clone(),
|
||||||
|
user_rate_limits: cfg.access.user_rate_limits.clone(),
|
||||||
|
cidr_rate_limits: cfg.access.cidr_rate_limits.clone(),
|
||||||
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
|
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
|
||||||
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
||||||
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
||||||
@@ -299,6 +305,7 @@ fn listeners_equal(
|
|||||||
}
|
}
|
||||||
lhs.iter().zip(rhs.iter()).all(|(a, b)| {
|
lhs.iter().zip(rhs.iter()).all(|(a, b)| {
|
||||||
a.ip == b.ip
|
a.ip == b.ip
|
||||||
|
&& a.port == b.port
|
||||||
&& a.announce == b.announce
|
&& a.announce == b.announce
|
||||||
&& a.announce_ip == b.announce_ip
|
&& a.announce_ip == b.announce_ip
|
||||||
&& a.proxy_protocol == b.proxy_protocol
|
&& a.proxy_protocol == b.proxy_protocol
|
||||||
@@ -306,6 +313,14 @@ fn listeners_equal(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_default_link_port(cfg: &ProxyConfig) -> u16 {
|
||||||
|
cfg.server
|
||||||
|
.listeners
|
||||||
|
.first()
|
||||||
|
.and_then(|listener| listener.port)
|
||||||
|
.unwrap_or(cfg.server.port)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
struct WatchManifest {
|
struct WatchManifest {
|
||||||
files: BTreeSet<PathBuf>,
|
files: BTreeSet<PathBuf>,
|
||||||
@@ -535,6 +550,8 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
|||||||
cfg.access.user_max_tcp_conns_global_each = new.access.user_max_tcp_conns_global_each;
|
cfg.access.user_max_tcp_conns_global_each = new.access.user_max_tcp_conns_global_each;
|
||||||
cfg.access.user_expirations = new.access.user_expirations.clone();
|
cfg.access.user_expirations = new.access.user_expirations.clone();
|
||||||
cfg.access.user_data_quota = new.access.user_data_quota.clone();
|
cfg.access.user_data_quota = new.access.user_data_quota.clone();
|
||||||
|
cfg.access.user_rate_limits = new.access.user_rate_limits.clone();
|
||||||
|
cfg.access.cidr_rate_limits = new.access.cidr_rate_limits.clone();
|
||||||
cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone();
|
cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone();
|
||||||
cfg.access.user_max_unique_ips_global_each = new.access.user_max_unique_ips_global_each;
|
cfg.access.user_max_unique_ips_global_each = new.access.user_max_unique_ips_global_each;
|
||||||
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
||||||
@@ -560,6 +577,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
if old.server.api.enabled != new.server.api.enabled
|
if old.server.api.enabled != new.server.api.enabled
|
||||||
|| old.server.api.listen != new.server.api.listen
|
|| old.server.api.listen != new.server.api.listen
|
||||||
|| old.server.api.whitelist != new.server.api.whitelist
|
|| old.server.api.whitelist != new.server.api.whitelist
|
||||||
|
|| old.server.api.gray_action != new.server.api.gray_action
|
||||||
|| old.server.api.auth_header != new.server.api.auth_header
|
|| old.server.api.auth_header != new.server.api.auth_header
|
||||||
|| old.server.api.request_body_limit_bytes != new.server.api.request_body_limit_bytes
|
|| old.server.api.request_body_limit_bytes != new.server.api.request_body_limit_bytes
|
||||||
|| old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled
|
|| old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled
|
||||||
@@ -611,6 +629,8 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
|| old.censorship.mask_shape_above_cap_blur_max_bytes
|
|| old.censorship.mask_shape_above_cap_blur_max_bytes
|
||||||
!= new.censorship.mask_shape_above_cap_blur_max_bytes
|
!= new.censorship.mask_shape_above_cap_blur_max_bytes
|
||||||
|| old.censorship.mask_relay_max_bytes != new.censorship.mask_relay_max_bytes
|
|| old.censorship.mask_relay_max_bytes != new.censorship.mask_relay_max_bytes
|
||||||
|
|| old.censorship.mask_relay_timeout_ms != new.censorship.mask_relay_timeout_ms
|
||||||
|
|| old.censorship.mask_relay_idle_timeout_ms != new.censorship.mask_relay_idle_timeout_ms
|
||||||
|| old.censorship.mask_classifier_prefetch_timeout_ms
|
|| old.censorship.mask_classifier_prefetch_timeout_ms
|
||||||
!= new.censorship.mask_classifier_prefetch_timeout_ms
|
!= new.censorship.mask_classifier_prefetch_timeout_ms
|
||||||
|| old.censorship.mask_timing_normalization_enabled
|
|| old.censorship.mask_timing_normalization_enabled
|
||||||
@@ -1117,7 +1137,7 @@ fn log_changes(
|
|||||||
.general
|
.general
|
||||||
.links
|
.links
|
||||||
.public_port
|
.public_port
|
||||||
.unwrap_or(new_cfg.server.port);
|
.unwrap_or(resolve_default_link_port(new_cfg));
|
||||||
for user in &added {
|
for user in &added {
|
||||||
if let Some(secret) = new_hot.users.get(*user) {
|
if let Some(secret) = new_hot.users.get(*user) {
|
||||||
print_user_links(user, secret, &host, port, new_cfg);
|
print_user_links(user, secret, &host, port, new_cfg);
|
||||||
@@ -1170,6 +1190,18 @@ fn log_changes(
|
|||||||
new_hot.user_data_quota.len()
|
new_hot.user_data_quota.len()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if old_hot.user_rate_limits != new_hot.user_rate_limits {
|
||||||
|
info!(
|
||||||
|
"config reload: user_rate_limits updated ({} entries)",
|
||||||
|
new_hot.user_rate_limits.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if old_hot.cidr_rate_limits != new_hot.cidr_rate_limits {
|
||||||
|
info!(
|
||||||
|
"config reload: cidr_rate_limits updated ({} entries)",
|
||||||
|
new_hot.cidr_rate_limits.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
if old_hot.user_max_unique_ips != new_hot.user_max_unique_ips {
|
if old_hot.user_max_unique_ips != new_hot.user_max_unique_ips {
|
||||||
info!(
|
info!(
|
||||||
"config reload: user_max_unique_ips updated ({} entries)",
|
"config reload: user_max_unique_ips updated ({} entries)",
|
||||||
|
|||||||
@@ -253,6 +253,12 @@ fn validate_upstreams(config: &ProxyConfig) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for upstream in &config.upstreams {
|
for upstream in &config.upstreams {
|
||||||
|
if matches!(upstream.ipv4, Some(false)) && matches!(upstream.ipv6, Some(false)) {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"upstream.ipv4 and upstream.ipv6 cannot both be false".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type {
|
if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type {
|
||||||
let parsed = ShadowsocksServerConfig::from_url(url)
|
let parsed = ShadowsocksServerConfig::from_url(url)
|
||||||
.map_err(|error| ProxyError::Config(format!("invalid shadowsocks url: {error}")))?;
|
.map_err(|error| ProxyError::Config(format!("invalid shadowsocks url: {error}")))?;
|
||||||
@@ -337,27 +343,108 @@ impl ProxyConfig {
|
|||||||
let network_table = parsed_toml
|
let network_table = parsed_toml
|
||||||
.get("network")
|
.get("network")
|
||||||
.and_then(|value| value.as_table());
|
.and_then(|value| value.as_table());
|
||||||
|
let server_table = parsed_toml.get("server").and_then(|value| value.as_table());
|
||||||
|
let conntrack_control_table = server_table
|
||||||
|
.and_then(|table| table.get("conntrack_control"))
|
||||||
|
.and_then(|value| value.as_table());
|
||||||
let update_every_is_explicit = general_table
|
let update_every_is_explicit = general_table
|
||||||
.map(|table| table.contains_key("update_every"))
|
.map(|table| table.contains_key("update_every"))
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let beobachten_is_explicit = general_table
|
||||||
|
.map(|table| table.contains_key("beobachten"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
let beobachten_minutes_is_explicit = general_table
|
||||||
|
.map(|table| table.contains_key("beobachten_minutes"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
let beobachten_flush_secs_is_explicit = general_table
|
||||||
|
.map(|table| table.contains_key("beobachten_flush_secs"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
let beobachten_file_is_explicit = general_table
|
||||||
|
.map(|table| table.contains_key("beobachten_file"))
|
||||||
|
.unwrap_or(false);
|
||||||
let legacy_secret_is_explicit = general_table
|
let legacy_secret_is_explicit = general_table
|
||||||
.map(|table| table.contains_key("proxy_secret_auto_reload_secs"))
|
.map(|table| table.contains_key("proxy_secret_auto_reload_secs"))
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
let legacy_config_is_explicit = general_table
|
let legacy_config_is_explicit = general_table
|
||||||
.map(|table| table.contains_key("proxy_config_auto_reload_secs"))
|
.map(|table| table.contains_key("proxy_config_auto_reload_secs"))
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let legacy_top_level_beobachten = parsed_toml.get("beobachten").cloned();
|
||||||
|
let legacy_top_level_beobachten_minutes = parsed_toml.get("beobachten_minutes").cloned();
|
||||||
|
let legacy_top_level_beobachten_flush_secs =
|
||||||
|
parsed_toml.get("beobachten_flush_secs").cloned();
|
||||||
|
let legacy_top_level_beobachten_file = parsed_toml.get("beobachten_file").cloned();
|
||||||
let stun_servers_is_explicit = network_table
|
let stun_servers_is_explicit = network_table
|
||||||
.map(|table| table.contains_key("stun_servers"))
|
.map(|table| table.contains_key("stun_servers"))
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let inline_conntrack_control_is_explicit = conntrack_control_table
|
||||||
|
.map(|table| table.contains_key("inline_conntrack_control"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
let mut config: ProxyConfig = parsed_toml
|
let mut config: ProxyConfig = parsed_toml
|
||||||
.try_into()
|
.try_into()
|
||||||
.map_err(|e| ProxyError::Config(e.to_string()))?;
|
.map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||||
|
config
|
||||||
|
.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit = inline_conntrack_control_is_explicit;
|
||||||
|
|
||||||
if !update_every_is_explicit && (legacy_secret_is_explicit || legacy_config_is_explicit) {
|
if !update_every_is_explicit && (legacy_secret_is_explicit || legacy_config_is_explicit) {
|
||||||
config.general.update_every = None;
|
config.general.update_every = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Backward compatibility: legacy top-level beobachten* keys.
|
||||||
|
// Prefer `[general].*` when both are present.
|
||||||
|
let mut legacy_beobachten_applied = false;
|
||||||
|
if !beobachten_is_explicit && let Some(value) = legacy_top_level_beobachten.as_ref() {
|
||||||
|
let parsed = value.as_bool().ok_or_else(|| {
|
||||||
|
ProxyError::Config("beobachten (top-level) must be a boolean".to_string())
|
||||||
|
})?;
|
||||||
|
config.general.beobachten = parsed;
|
||||||
|
legacy_beobachten_applied = true;
|
||||||
|
}
|
||||||
|
if !beobachten_minutes_is_explicit
|
||||||
|
&& let Some(value) = legacy_top_level_beobachten_minutes.as_ref()
|
||||||
|
{
|
||||||
|
let raw = value.as_integer().ok_or_else(|| {
|
||||||
|
ProxyError::Config("beobachten_minutes (top-level) must be an integer".to_string())
|
||||||
|
})?;
|
||||||
|
let parsed = u64::try_from(raw).map_err(|_| {
|
||||||
|
ProxyError::Config(
|
||||||
|
"beobachten_minutes (top-level) must be within u64 range".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
config.general.beobachten_minutes = parsed;
|
||||||
|
legacy_beobachten_applied = true;
|
||||||
|
}
|
||||||
|
if !beobachten_flush_secs_is_explicit
|
||||||
|
&& let Some(value) = legacy_top_level_beobachten_flush_secs.as_ref()
|
||||||
|
{
|
||||||
|
let raw = value.as_integer().ok_or_else(|| {
|
||||||
|
ProxyError::Config(
|
||||||
|
"beobachten_flush_secs (top-level) must be an integer".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let parsed = u64::try_from(raw).map_err(|_| {
|
||||||
|
ProxyError::Config(
|
||||||
|
"beobachten_flush_secs (top-level) must be within u64 range".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
config.general.beobachten_flush_secs = parsed;
|
||||||
|
legacy_beobachten_applied = true;
|
||||||
|
}
|
||||||
|
if !beobachten_file_is_explicit
|
||||||
|
&& let Some(value) = legacy_top_level_beobachten_file.as_ref()
|
||||||
|
{
|
||||||
|
let parsed = value.as_str().ok_or_else(|| {
|
||||||
|
ProxyError::Config("beobachten_file (top-level) must be a string".to_string())
|
||||||
|
})?;
|
||||||
|
config.general.beobachten_file = parsed.to_string();
|
||||||
|
legacy_beobachten_applied = true;
|
||||||
|
}
|
||||||
|
if legacy_beobachten_applied {
|
||||||
|
warn!("top-level beobachten* keys are deprecated; use general.beobachten* instead");
|
||||||
|
}
|
||||||
|
|
||||||
let legacy_nat_stun = config.general.middle_proxy_nat_stun.take();
|
let legacy_nat_stun = config.general.middle_proxy_nat_stun.take();
|
||||||
let legacy_nat_stun_servers =
|
let legacy_nat_stun_servers =
|
||||||
std::mem::take(&mut config.general.middle_proxy_nat_stun_servers);
|
std::mem::take(&mut config.general.middle_proxy_nat_stun_servers);
|
||||||
@@ -785,6 +872,22 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (user, limit) in &config.access.user_rate_limits {
|
||||||
|
if limit.up_bps == 0 && limit.down_bps == 0 {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"access.user_rate_limits.{user} must set at least one non-zero direction"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (cidr, limit) in &config.access.cidr_rate_limits {
|
||||||
|
if limit.up_bps == 0 && limit.down_bps == 0 {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"access.cidr_rate_limits.{cidr} must set at least one non-zero direction"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if config.general.me_reinit_every_secs == 0 {
|
if config.general.me_reinit_every_secs == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.me_reinit_every_secs must be > 0".to_string(),
|
"general.me_reinit_every_secs must be > 0".to_string(),
|
||||||
@@ -1250,6 +1353,7 @@ impl ProxyConfig {
|
|||||||
if let Ok(ipv4) = ipv4_str.parse::<IpAddr>() {
|
if let Ok(ipv4) = ipv4_str.parse::<IpAddr>() {
|
||||||
config.server.listeners.push(ListenerConfig {
|
config.server.listeners.push(ListenerConfig {
|
||||||
ip: ipv4,
|
ip: ipv4,
|
||||||
|
port: Some(config.server.port),
|
||||||
announce: None,
|
announce: None,
|
||||||
announce_ip: None,
|
announce_ip: None,
|
||||||
proxy_protocol: None,
|
proxy_protocol: None,
|
||||||
@@ -1261,6 +1365,7 @@ impl ProxyConfig {
|
|||||||
{
|
{
|
||||||
config.server.listeners.push(ListenerConfig {
|
config.server.listeners.push(ListenerConfig {
|
||||||
ip: ipv6,
|
ip: ipv6,
|
||||||
|
port: Some(config.server.port),
|
||||||
announce: None,
|
announce: None,
|
||||||
announce_ip: None,
|
announce_ip: None,
|
||||||
proxy_protocol: None,
|
proxy_protocol: None,
|
||||||
@@ -1269,6 +1374,13 @@ impl ProxyConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migration: listeners[].port fallback to legacy server.port.
|
||||||
|
for listener in &mut config.server.listeners {
|
||||||
|
if listener.port.is_none() {
|
||||||
|
listener.port = Some(config.server.port);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Migration: announce_ip → announce for each listener.
|
// Migration: announce_ip → announce for each listener.
|
||||||
for listener in &mut config.server.listeners {
|
for listener in &mut config.server.listeners {
|
||||||
if listener.announce.is_none()
|
if listener.announce.is_none()
|
||||||
@@ -1289,11 +1401,14 @@ impl ProxyConfig {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1385,6 +1500,21 @@ mod tests {
|
|||||||
const TEST_SHADOWSOCKS_URL: &str =
|
const TEST_SHADOWSOCKS_URL: &str =
|
||||||
"ss://2022-blake3-aes-256-gcm:MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=@127.0.0.1:8388";
|
"ss://2022-blake3-aes-256-gcm:MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=@127.0.0.1:8388";
|
||||||
|
|
||||||
|
fn load_config_from_temp_toml(toml: &str) -> ProxyConfig {
|
||||||
|
let nonce = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos();
|
||||||
|
let dir = std::env::temp_dir().join(format!("telemt_load_cfg_{nonce}"));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let path = dir.join("config.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let cfg = ProxyConfig::load(&path).unwrap();
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
let _ = std::fs::remove_dir(dir);
|
||||||
|
cfg
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serde_defaults_remain_unchanged_for_present_sections() {
|
fn serde_defaults_remain_unchanged_for_present_sections() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
@@ -1481,6 +1611,7 @@ mod tests {
|
|||||||
cfg.general.rpc_proxy_req_every,
|
cfg.general.rpc_proxy_req_every,
|
||||||
default_rpc_proxy_req_every()
|
default_rpc_proxy_req_every()
|
||||||
);
|
);
|
||||||
|
assert_eq!(cfg.general.beobachten_file, default_beobachten_file());
|
||||||
assert_eq!(cfg.general.update_every, default_update_every());
|
assert_eq!(cfg.general.update_every, default_update_every());
|
||||||
assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4());
|
assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4());
|
||||||
assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt());
|
assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt());
|
||||||
@@ -1491,6 +1622,7 @@ mod tests {
|
|||||||
assert_eq!(cfg.censorship.unknown_sni_action, UnknownSniAction::Drop);
|
assert_eq!(cfg.censorship.unknown_sni_action, UnknownSniAction::Drop);
|
||||||
assert_eq!(cfg.server.api.listen, default_api_listen());
|
assert_eq!(cfg.server.api.listen, default_api_listen());
|
||||||
assert_eq!(cfg.server.api.whitelist, default_api_whitelist());
|
assert_eq!(cfg.server.api.whitelist, default_api_whitelist());
|
||||||
|
assert_eq!(cfg.server.api.gray_action, ApiGrayAction::Drop);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
cfg.server.api.request_body_limit_bytes,
|
cfg.server.api.request_body_limit_bytes,
|
||||||
default_api_request_body_limit_bytes()
|
default_api_request_body_limit_bytes()
|
||||||
@@ -1647,6 +1779,7 @@ mod tests {
|
|||||||
default_upstream_connect_failfast_hard_errors()
|
default_upstream_connect_failfast_hard_errors()
|
||||||
);
|
);
|
||||||
assert_eq!(general.rpc_proxy_req_every, default_rpc_proxy_req_every());
|
assert_eq!(general.rpc_proxy_req_every, default_rpc_proxy_req_every());
|
||||||
|
assert_eq!(general.beobachten_file, default_beobachten_file());
|
||||||
assert_eq!(general.update_every, default_update_every());
|
assert_eq!(general.update_every, default_update_every());
|
||||||
|
|
||||||
let server = ServerConfig::default();
|
let server = ServerConfig::default();
|
||||||
@@ -1661,6 +1794,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(server.api.listen, default_api_listen());
|
assert_eq!(server.api.listen, default_api_listen());
|
||||||
assert_eq!(server.api.whitelist, default_api_whitelist());
|
assert_eq!(server.api.whitelist, default_api_whitelist());
|
||||||
|
assert_eq!(server.api.gray_action, ApiGrayAction::Drop);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
server.api.request_body_limit_bytes,
|
server.api.request_body_limit_bytes,
|
||||||
default_api_request_body_limit_bytes()
|
default_api_request_body_limit_bytes()
|
||||||
@@ -1758,6 +1892,43 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conntrack_inline_explicit_flag_is_false_when_omitted() {
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[server]
|
||||||
|
[server.conntrack_control]
|
||||||
|
[access]
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!cfg.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conntrack_inline_explicit_flag_is_true_when_present() {
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[server]
|
||||||
|
[server.conntrack_control]
|
||||||
|
inline_conntrack_control = true
|
||||||
|
[access]
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cfg.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unknown_sni_action_parses_and_defaults_to_drop() {
|
fn unknown_sni_action_parses_and_defaults_to_drop() {
|
||||||
let cfg_default: ProxyConfig = toml::from_str(
|
let cfg_default: ProxyConfig = toml::from_str(
|
||||||
@@ -1808,6 +1979,107 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_gray_action_parses_and_defaults_to_drop() {
|
||||||
|
let cfg_default: ProxyConfig = toml::from_str(
|
||||||
|
r#"
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cfg_default.server.api.gray_action, ApiGrayAction::Drop);
|
||||||
|
|
||||||
|
let cfg_api: ProxyConfig = toml::from_str(
|
||||||
|
r#"
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
[server.api]
|
||||||
|
gray_action = "api"
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cfg_api.server.api.gray_action, ApiGrayAction::Api);
|
||||||
|
|
||||||
|
let cfg_200: ProxyConfig = toml::from_str(
|
||||||
|
r#"
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
[server.api]
|
||||||
|
gray_action = "200"
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cfg_200.server.api.gray_action, ApiGrayAction::Ok200);
|
||||||
|
|
||||||
|
let cfg_drop: ProxyConfig = toml::from_str(
|
||||||
|
r#"
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
[server.api]
|
||||||
|
gray_action = "drop"
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cfg_drop.server.api.gray_action, ApiGrayAction::Drop);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn top_level_beobachten_keys_migrate_to_general_when_general_not_explicit() {
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
beobachten = false
|
||||||
|
beobachten_minutes = 7
|
||||||
|
beobachten_flush_secs = 3
|
||||||
|
beobachten_file = "tmp/legacy-beob.txt"
|
||||||
|
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!cfg.general.beobachten);
|
||||||
|
assert_eq!(cfg.general.beobachten_minutes, 7);
|
||||||
|
assert_eq!(cfg.general.beobachten_flush_secs, 3);
|
||||||
|
assert_eq!(cfg.general.beobachten_file, "tmp/legacy-beob.txt");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn general_beobachten_keys_have_priority_over_legacy_top_level() {
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
beobachten = true
|
||||||
|
beobachten_minutes = 30
|
||||||
|
beobachten_flush_secs = 30
|
||||||
|
beobachten_file = "tmp/legacy-beob.txt"
|
||||||
|
|
||||||
|
[server]
|
||||||
|
[general]
|
||||||
|
beobachten = false
|
||||||
|
beobachten_minutes = 5
|
||||||
|
beobachten_flush_secs = 2
|
||||||
|
beobachten_file = "tmp/general-beob.txt"
|
||||||
|
[network]
|
||||||
|
[access]
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!cfg.general.beobachten);
|
||||||
|
assert_eq!(cfg.general.beobachten_minutes, 5);
|
||||||
|
assert_eq!(cfg.general.beobachten_flush_secs, 2);
|
||||||
|
assert_eq!(cfg.general.beobachten_file, "tmp/general-beob.txt");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn dc_overrides_allow_string_and_array() {
|
fn dc_overrides_allow_string_and_array() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
|
|||||||
+109
-1
@@ -159,6 +159,21 @@ impl MeBindStaleMode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// RST-on-close mode for accepted client sockets.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum RstOnCloseMode {
|
||||||
|
/// Normal FIN on all closes (default, no behaviour change).
|
||||||
|
#[default]
|
||||||
|
Off,
|
||||||
|
/// SO_LINGER(0) on accept; cleared after successful auth.
|
||||||
|
/// Pre-handshake failures (scanners, DPI, timeouts) send RST;
|
||||||
|
/// authenticated relay sessions close gracefully with FIN.
|
||||||
|
Errors,
|
||||||
|
/// SO_LINGER(0) on accept, never cleared — all closes send RST.
|
||||||
|
Always,
|
||||||
|
}
|
||||||
|
|
||||||
/// Middle-End writer floor policy mode.
|
/// Middle-End writer floor policy mode.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
@@ -925,6 +940,14 @@ pub struct GeneralConfig {
|
|||||||
/// Minimum unavailable ME DC groups before degrading.
|
/// Minimum unavailable ME DC groups before degrading.
|
||||||
#[serde(default = "default_degradation_min_unavailable_dc_groups")]
|
#[serde(default = "default_degradation_min_unavailable_dc_groups")]
|
||||||
pub degradation_min_unavailable_dc_groups: u8,
|
pub degradation_min_unavailable_dc_groups: u8,
|
||||||
|
|
||||||
|
/// RST-on-close mode for accepted client sockets.
|
||||||
|
/// `off` — normal FIN on all closes (default).
|
||||||
|
/// `errors` — SO_LINGER(0) on accept, cleared after successful auth;
|
||||||
|
/// pre-handshake failures send RST, relayed sessions close gracefully.
|
||||||
|
/// `always` — SO_LINGER(0) on accept, never cleared; all closes send RST.
|
||||||
|
#[serde(default)]
|
||||||
|
pub rst_on_close: RstOnCloseMode,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for GeneralConfig {
|
impl Default for GeneralConfig {
|
||||||
@@ -1086,6 +1109,7 @@ impl Default for GeneralConfig {
|
|||||||
ntp_servers: default_ntp_servers(),
|
ntp_servers: default_ntp_servers(),
|
||||||
auto_degradation_enabled: default_true(),
|
auto_degradation_enabled: default_true(),
|
||||||
degradation_min_unavailable_dc_groups: default_degradation_min_unavailable_dc_groups(),
|
degradation_min_unavailable_dc_groups: default_degradation_min_unavailable_dc_groups(),
|
||||||
|
rst_on_close: RstOnCloseMode::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1129,7 +1153,8 @@ pub struct LinksConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub public_host: Option<String>,
|
pub public_host: Option<String>,
|
||||||
|
|
||||||
/// Public port for tg:// link generation (overrides server.port).
|
/// Public port for tg:// link generation.
|
||||||
|
/// Overrides listener ports and legacy `server.port`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub public_port: Option<u16>,
|
pub public_port: Option<u16>,
|
||||||
}
|
}
|
||||||
@@ -1159,6 +1184,13 @@ pub struct ApiConfig {
|
|||||||
#[serde(default = "default_api_whitelist")]
|
#[serde(default = "default_api_whitelist")]
|
||||||
pub whitelist: Vec<IpNetwork>,
|
pub whitelist: Vec<IpNetwork>,
|
||||||
|
|
||||||
|
/// Behavior for requests from source IPs outside `whitelist`.
|
||||||
|
/// - `api`: return structured API forbidden response.
|
||||||
|
/// - `200`: return `200 OK` with an empty body.
|
||||||
|
/// - `drop`: close the connection without HTTP response.
|
||||||
|
#[serde(default)]
|
||||||
|
pub gray_action: ApiGrayAction,
|
||||||
|
|
||||||
/// Optional static value for `Authorization` header validation.
|
/// Optional static value for `Authorization` header validation.
|
||||||
/// Empty string disables header auth.
|
/// Empty string disables header auth.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -1203,6 +1235,7 @@ impl Default for ApiConfig {
|
|||||||
enabled: default_true(),
|
enabled: default_true(),
|
||||||
listen: default_api_listen(),
|
listen: default_api_listen(),
|
||||||
whitelist: default_api_whitelist(),
|
whitelist: default_api_whitelist(),
|
||||||
|
gray_action: ApiGrayAction::default(),
|
||||||
auth_header: String::new(),
|
auth_header: String::new(),
|
||||||
request_body_limit_bytes: default_api_request_body_limit_bytes(),
|
request_body_limit_bytes: default_api_request_body_limit_bytes(),
|
||||||
minimal_runtime_enabled: default_api_minimal_runtime_enabled(),
|
minimal_runtime_enabled: default_api_minimal_runtime_enabled(),
|
||||||
@@ -1216,6 +1249,19 @@ impl Default for ApiConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum ApiGrayAction {
|
||||||
|
/// Preserve current API behavior for denied source IPs.
|
||||||
|
Api,
|
||||||
|
/// Mimic a plain web endpoint by returning `200 OK` with an empty body.
|
||||||
|
#[serde(rename = "200")]
|
||||||
|
Ok200,
|
||||||
|
/// Drop connection without HTTP response for denied source IPs.
|
||||||
|
#[default]
|
||||||
|
Drop,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
pub enum ConntrackMode {
|
pub enum ConntrackMode {
|
||||||
@@ -1283,6 +1329,10 @@ pub struct ConntrackControlConfig {
|
|||||||
#[serde(default = "default_conntrack_control_enabled")]
|
#[serde(default = "default_conntrack_control_enabled")]
|
||||||
pub inline_conntrack_control: bool,
|
pub inline_conntrack_control: bool,
|
||||||
|
|
||||||
|
/// Tracks whether inline_conntrack_control was explicitly set in config.
|
||||||
|
#[serde(skip)]
|
||||||
|
pub inline_conntrack_control_explicit: bool,
|
||||||
|
|
||||||
/// Conntrack mode for listener ingress traffic.
|
/// Conntrack mode for listener ingress traffic.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub mode: ConntrackMode,
|
pub mode: ConntrackMode,
|
||||||
@@ -1317,6 +1367,7 @@ impl Default for ConntrackControlConfig {
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
inline_conntrack_control: default_conntrack_control_enabled(),
|
inline_conntrack_control: default_conntrack_control_enabled(),
|
||||||
|
inline_conntrack_control_explicit: false,
|
||||||
mode: ConntrackMode::default(),
|
mode: ConntrackMode::default(),
|
||||||
backend: ConntrackBackend::default(),
|
backend: ConntrackBackend::default(),
|
||||||
profile: ConntrackPressureProfile::default(),
|
profile: ConntrackPressureProfile::default(),
|
||||||
@@ -1330,6 +1381,8 @@ impl Default for ConntrackControlConfig {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ServerConfig {
|
pub struct ServerConfig {
|
||||||
|
/// Legacy listener port used for backward compatibility.
|
||||||
|
/// For new configs prefer `[[server.listeners]].port`.
|
||||||
#[serde(default = "default_port")]
|
#[serde(default = "default_port")]
|
||||||
pub port: u16,
|
pub port: u16,
|
||||||
|
|
||||||
@@ -1686,6 +1739,19 @@ pub struct AntiCensorshipConfig {
|
|||||||
#[serde(default = "default_mask_relay_max_bytes")]
|
#[serde(default = "default_mask_relay_max_bytes")]
|
||||||
pub mask_relay_max_bytes: usize,
|
pub mask_relay_max_bytes: usize,
|
||||||
|
|
||||||
|
/// Wall-clock cap for the full masking relay on non-MTProto fallback paths.
|
||||||
|
/// Raise when the mask target is a long-lived service (e.g. WebSocket).
|
||||||
|
/// Default: 60 000 ms (60 s).
|
||||||
|
#[serde(default = "default_mask_relay_timeout_ms")]
|
||||||
|
pub mask_relay_timeout_ms: u64,
|
||||||
|
|
||||||
|
/// Per-read idle timeout on masking relay and drain paths.
|
||||||
|
/// Limits resource consumption by slow-loris attacks and port scanners.
|
||||||
|
/// A read call stalling beyond this is treated as an abandoned connection.
|
||||||
|
/// Default: 5 000 ms (5 s).
|
||||||
|
#[serde(default = "default_mask_relay_idle_timeout_ms")]
|
||||||
|
pub mask_relay_idle_timeout_ms: u64,
|
||||||
|
|
||||||
/// Prefetch timeout (ms) for extending fragmented masking classifier window.
|
/// Prefetch timeout (ms) for extending fragmented masking classifier window.
|
||||||
#[serde(default = "default_mask_classifier_prefetch_timeout_ms")]
|
#[serde(default = "default_mask_classifier_prefetch_timeout_ms")]
|
||||||
pub mask_classifier_prefetch_timeout_ms: u64,
|
pub mask_classifier_prefetch_timeout_ms: u64,
|
||||||
@@ -1731,6 +1797,8 @@ impl Default for AntiCensorshipConfig {
|
|||||||
mask_shape_above_cap_blur: default_mask_shape_above_cap_blur(),
|
mask_shape_above_cap_blur: default_mask_shape_above_cap_blur(),
|
||||||
mask_shape_above_cap_blur_max_bytes: default_mask_shape_above_cap_blur_max_bytes(),
|
mask_shape_above_cap_blur_max_bytes: default_mask_shape_above_cap_blur_max_bytes(),
|
||||||
mask_relay_max_bytes: default_mask_relay_max_bytes(),
|
mask_relay_max_bytes: default_mask_relay_max_bytes(),
|
||||||
|
mask_relay_timeout_ms: default_mask_relay_timeout_ms(),
|
||||||
|
mask_relay_idle_timeout_ms: default_mask_relay_idle_timeout_ms(),
|
||||||
mask_classifier_prefetch_timeout_ms: default_mask_classifier_prefetch_timeout_ms(),
|
mask_classifier_prefetch_timeout_ms: default_mask_classifier_prefetch_timeout_ms(),
|
||||||
mask_timing_normalization_enabled: default_mask_timing_normalization_enabled(),
|
mask_timing_normalization_enabled: default_mask_timing_normalization_enabled(),
|
||||||
mask_timing_normalization_floor_ms: default_mask_timing_normalization_floor_ms(),
|
mask_timing_normalization_floor_ms: default_mask_timing_normalization_floor_ms(),
|
||||||
@@ -1763,6 +1831,21 @@ pub struct AccessConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_data_quota: HashMap<String, u64>,
|
pub user_data_quota: HashMap<String, u64>,
|
||||||
|
|
||||||
|
/// Per-user transport rate limits in bits-per-second.
|
||||||
|
///
|
||||||
|
/// Each entry supports independent upload (`up_bps`) and download
|
||||||
|
/// (`down_bps`) ceilings. A value of `0` in one direction means
|
||||||
|
/// "unlimited" for that direction.
|
||||||
|
#[serde(default)]
|
||||||
|
pub user_rate_limits: HashMap<String, RateLimitBps>,
|
||||||
|
|
||||||
|
/// Per-CIDR aggregate transport rate limits in bits-per-second.
|
||||||
|
///
|
||||||
|
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
|
||||||
|
/// direction means "unlimited" for that direction.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_max_unique_ips: HashMap<String, usize>,
|
pub user_max_unique_ips: HashMap<String, usize>,
|
||||||
|
|
||||||
@@ -1796,6 +1879,8 @@ impl Default for AccessConfig {
|
|||||||
user_max_tcp_conns_global_each: default_user_max_tcp_conns_global_each(),
|
user_max_tcp_conns_global_each: default_user_max_tcp_conns_global_each(),
|
||||||
user_expirations: HashMap::new(),
|
user_expirations: HashMap::new(),
|
||||||
user_data_quota: HashMap::new(),
|
user_data_quota: HashMap::new(),
|
||||||
|
user_rate_limits: HashMap::new(),
|
||||||
|
cidr_rate_limits: HashMap::new(),
|
||||||
user_max_unique_ips: HashMap::new(),
|
user_max_unique_ips: HashMap::new(),
|
||||||
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
|
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
|
||||||
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
|
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
|
||||||
@@ -1807,6 +1892,14 @@ impl Default for AccessConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct RateLimitBps {
|
||||||
|
#[serde(default)]
|
||||||
|
pub up_bps: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub down_bps: u64,
|
||||||
|
}
|
||||||
|
|
||||||
// ============= Aux Structures =============
|
// ============= Aux Structures =============
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
@@ -1817,6 +1910,10 @@ pub enum UpstreamType {
|
|||||||
interface: Option<String>,
|
interface: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
bind_addresses: Option<Vec<String>>,
|
bind_addresses: Option<Vec<String>>,
|
||||||
|
/// Linux-only hard interface pinning via `SO_BINDTODEVICE`.
|
||||||
|
/// Optional alias: `force_bind`.
|
||||||
|
#[serde(default, alias = "force_bind")]
|
||||||
|
bindtodevice: Option<String>,
|
||||||
},
|
},
|
||||||
Socks4 {
|
Socks4 {
|
||||||
address: String,
|
address: String,
|
||||||
@@ -1853,11 +1950,22 @@ pub struct UpstreamConfig {
|
|||||||
pub scopes: String,
|
pub scopes: String,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub selected_scope: String,
|
pub selected_scope: String,
|
||||||
|
/// Allow IPv4 DC targets for this upstream.
|
||||||
|
/// `None` means auto-detect from runtime connectivity state.
|
||||||
|
#[serde(default)]
|
||||||
|
pub ipv4: Option<bool>,
|
||||||
|
/// Allow IPv6 DC targets for this upstream.
|
||||||
|
/// `None` means auto-detect from runtime connectivity state.
|
||||||
|
#[serde(default)]
|
||||||
|
pub ipv6: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ListenerConfig {
|
pub struct ListenerConfig {
|
||||||
pub ip: IpAddr,
|
pub ip: IpAddr,
|
||||||
|
/// Per-listener TCP port. If omitted, falls back to legacy `server.port`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub port: Option<u16>,
|
||||||
/// IP address or hostname to announce in proxy links.
|
/// IP address or hostname to announce in proxy links.
|
||||||
/// Takes precedence over `announce_ip` if both are set.
|
/// Takes precedence over `announce_ip` if both are set.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
|||||||
+125
-54
@@ -24,6 +24,13 @@ enum NetfilterBackend {
|
|||||||
Iptables,
|
Iptables,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct ConntrackRuntimeSupport {
|
||||||
|
netfilter_backend: Option<NetfilterBackend>,
|
||||||
|
has_cap_net_admin: bool,
|
||||||
|
has_conntrack_binary: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct PressureSample {
|
struct PressureSample {
|
||||||
conn_pct: Option<u8>,
|
conn_pct: Option<u8>,
|
||||||
@@ -56,11 +63,8 @@ pub(crate) fn spawn_conntrack_controller(
|
|||||||
shared: Arc<ProxySharedState>,
|
shared: Arc<ProxySharedState>,
|
||||||
) {
|
) {
|
||||||
if !cfg!(target_os = "linux") {
|
if !cfg!(target_os = "linux") {
|
||||||
let enabled = config_rx
|
let cfg = config_rx.borrow();
|
||||||
.borrow()
|
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
|
||||||
.server
|
|
||||||
.conntrack_control
|
|
||||||
.inline_conntrack_control;
|
|
||||||
stats.set_conntrack_control_enabled(enabled);
|
stats.set_conntrack_control_enabled(enabled);
|
||||||
stats.set_conntrack_control_available(false);
|
stats.set_conntrack_control_available(false);
|
||||||
stats.set_conntrack_pressure_active(false);
|
stats.set_conntrack_pressure_active(false);
|
||||||
@@ -68,9 +72,14 @@ pub(crate) fn spawn_conntrack_controller(
|
|||||||
stats.set_conntrack_rule_apply_ok(false);
|
stats.set_conntrack_rule_apply_ok(false);
|
||||||
shared.disable_conntrack_close_sender();
|
shared.disable_conntrack_close_sender();
|
||||||
shared.set_conntrack_pressure_active(false);
|
shared.set_conntrack_pressure_active(false);
|
||||||
if enabled {
|
if enabled
|
||||||
|
&& cfg
|
||||||
|
.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
{
|
||||||
warn!(
|
warn!(
|
||||||
"conntrack control is configured but unsupported on this OS; disabling runtime worker"
|
"conntrack control explicitly enabled but unsupported on this OS; disabling runtime worker"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -92,16 +101,17 @@ async fn run_conntrack_controller(
|
|||||||
let mut cfg = config_rx.borrow().clone();
|
let mut cfg = config_rx.borrow().clone();
|
||||||
let mut pressure_state = PressureState::new(stats.as_ref());
|
let mut pressure_state = PressureState::new(stats.as_ref());
|
||||||
let mut delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
|
let mut delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
|
||||||
let mut backend = pick_backend(cfg.server.conntrack_control.backend);
|
let mut runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
|
||||||
|
let mut effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
|
||||||
|
|
||||||
apply_runtime_state(
|
apply_runtime_state(
|
||||||
stats.as_ref(),
|
stats.as_ref(),
|
||||||
shared.as_ref(),
|
shared.as_ref(),
|
||||||
&cfg,
|
&cfg,
|
||||||
backend.is_some(),
|
runtime_support,
|
||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
reconcile_rules(&cfg, backend, stats.as_ref()).await;
|
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
@@ -110,17 +120,18 @@ async fn run_conntrack_controller(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
cfg = config_rx.borrow_and_update().clone();
|
cfg = config_rx.borrow_and_update().clone();
|
||||||
backend = pick_backend(cfg.server.conntrack_control.backend);
|
runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
|
||||||
|
effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
|
||||||
delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
|
delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
|
||||||
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, backend.is_some(), pressure_state.active);
|
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, runtime_support, pressure_state.active);
|
||||||
reconcile_rules(&cfg, backend, stats.as_ref()).await;
|
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
|
||||||
}
|
}
|
||||||
event = close_rx.recv() => {
|
event = close_rx.recv() => {
|
||||||
let Some(event) = event else {
|
let Some(event) = event else {
|
||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
stats.set_conntrack_event_queue_depth(close_rx.len() as u64);
|
stats.set_conntrack_event_queue_depth(close_rx.len() as u64);
|
||||||
if !cfg.server.conntrack_control.inline_conntrack_control {
|
if !effective_enabled {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !pressure_state.active {
|
if !pressure_state.active {
|
||||||
@@ -156,6 +167,7 @@ async fn run_conntrack_controller(
|
|||||||
stats.as_ref(),
|
stats.as_ref(),
|
||||||
shared.as_ref(),
|
shared.as_ref(),
|
||||||
&cfg,
|
&cfg,
|
||||||
|
effective_enabled,
|
||||||
&sample,
|
&sample,
|
||||||
&mut pressure_state,
|
&mut pressure_state,
|
||||||
);
|
);
|
||||||
@@ -175,20 +187,30 @@ fn apply_runtime_state(
|
|||||||
stats: &Stats,
|
stats: &Stats,
|
||||||
shared: &ProxySharedState,
|
shared: &ProxySharedState,
|
||||||
cfg: &ProxyConfig,
|
cfg: &ProxyConfig,
|
||||||
backend_available: bool,
|
runtime_support: ConntrackRuntimeSupport,
|
||||||
pressure_active: bool,
|
pressure_active: bool,
|
||||||
) {
|
) {
|
||||||
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
|
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
|
||||||
let available = enabled && backend_available && has_cap_net_admin();
|
let available = effective_conntrack_enabled(cfg, runtime_support);
|
||||||
if enabled && !available {
|
if enabled
|
||||||
|
&& !available
|
||||||
|
&& cfg
|
||||||
|
.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
{
|
||||||
warn!(
|
warn!(
|
||||||
"conntrack control enabled but unavailable (missing CAP_NET_ADMIN or backend binaries)"
|
has_cap_net_admin = runtime_support.has_cap_net_admin,
|
||||||
|
backend_available = runtime_support.netfilter_backend.is_some(),
|
||||||
|
conntrack_binary_available = runtime_support.has_conntrack_binary,
|
||||||
|
configured_backend = ?cfg.server.conntrack_control.backend,
|
||||||
|
"conntrack control explicitly enabled but unavailable; disabling runtime features"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
stats.set_conntrack_control_enabled(enabled);
|
stats.set_conntrack_control_enabled(enabled);
|
||||||
stats.set_conntrack_control_available(available);
|
stats.set_conntrack_control_available(available);
|
||||||
shared.set_conntrack_pressure_active(enabled && pressure_active);
|
shared.set_conntrack_pressure_active(available && pressure_active);
|
||||||
stats.set_conntrack_pressure_active(enabled && pressure_active);
|
stats.set_conntrack_pressure_active(available && pressure_active);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn collect_pressure_sample(
|
fn collect_pressure_sample(
|
||||||
@@ -228,10 +250,11 @@ fn update_pressure_state(
|
|||||||
stats: &Stats,
|
stats: &Stats,
|
||||||
shared: &ProxySharedState,
|
shared: &ProxySharedState,
|
||||||
cfg: &ProxyConfig,
|
cfg: &ProxyConfig,
|
||||||
|
effective_enabled: bool,
|
||||||
sample: &PressureSample,
|
sample: &PressureSample,
|
||||||
state: &mut PressureState,
|
state: &mut PressureState,
|
||||||
) {
|
) {
|
||||||
if !cfg.server.conntrack_control.inline_conntrack_control {
|
if !effective_enabled {
|
||||||
if state.active {
|
if state.active {
|
||||||
state.active = false;
|
state.active = false;
|
||||||
state.low_streak = 0;
|
state.low_streak = 0;
|
||||||
@@ -285,22 +308,26 @@ fn update_pressure_state(
|
|||||||
state.low_streak = 0;
|
state.low_streak = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, stats: &Stats) {
|
async fn reconcile_rules(
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
runtime_support: ConntrackRuntimeSupport,
|
||||||
|
stats: &Stats,
|
||||||
|
) {
|
||||||
if !cfg.server.conntrack_control.inline_conntrack_control {
|
if !cfg.server.conntrack_control.inline_conntrack_control {
|
||||||
clear_notrack_rules_all_backends().await;
|
clear_notrack_rules_all_backends().await;
|
||||||
stats.set_conntrack_rule_apply_ok(true);
|
stats.set_conntrack_rule_apply_ok(true);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !has_cap_net_admin() {
|
if !effective_conntrack_enabled(cfg, runtime_support) {
|
||||||
|
clear_notrack_rules_all_backends().await;
|
||||||
stats.set_conntrack_rule_apply_ok(false);
|
stats.set_conntrack_rule_apply_ok(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(backend) = backend else {
|
let backend = runtime_support
|
||||||
stats.set_conntrack_rule_apply_ok(false);
|
.netfilter_backend
|
||||||
return;
|
.expect("netfilter backend must be available for effective conntrack control");
|
||||||
};
|
|
||||||
|
|
||||||
let apply_result = match backend {
|
let apply_result = match backend {
|
||||||
NetfilterBackend::Nftables => apply_nft_rules(cfg).await,
|
NetfilterBackend::Nftables => apply_nft_rules(cfg).await,
|
||||||
@@ -315,6 +342,24 @@ async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn probe_runtime_support(configured_backend: ConntrackBackend) -> ConntrackRuntimeSupport {
|
||||||
|
ConntrackRuntimeSupport {
|
||||||
|
netfilter_backend: pick_backend(configured_backend),
|
||||||
|
has_cap_net_admin: has_cap_net_admin(),
|
||||||
|
has_conntrack_binary: command_exists("conntrack"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn effective_conntrack_enabled(
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
runtime_support: ConntrackRuntimeSupport,
|
||||||
|
) -> bool {
|
||||||
|
cfg.server.conntrack_control.inline_conntrack_control
|
||||||
|
&& runtime_support.has_cap_net_admin
|
||||||
|
&& runtime_support.netfilter_backend.is_some()
|
||||||
|
&& runtime_support.has_conntrack_binary
|
||||||
|
}
|
||||||
|
|
||||||
fn pick_backend(configured: ConntrackBackend) -> Option<NetfilterBackend> {
|
fn pick_backend(configured: ConntrackBackend) -> Option<NetfilterBackend> {
|
||||||
match configured {
|
match configured {
|
||||||
ConntrackBackend::Auto => {
|
ConntrackBackend::Auto => {
|
||||||
@@ -343,15 +388,28 @@ fn command_exists(binary: &str) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn notrack_targets(cfg: &ProxyConfig) -> (Vec<Option<IpAddr>>, Vec<Option<IpAddr>>) {
|
fn listener_port_set(cfg: &ProxyConfig) -> Vec<u16> {
|
||||||
|
let mut ports: BTreeSet<u16> = BTreeSet::new();
|
||||||
|
if cfg.server.listeners.is_empty() {
|
||||||
|
ports.insert(cfg.server.port);
|
||||||
|
} else {
|
||||||
|
for listener in &cfg.server.listeners {
|
||||||
|
ports.insert(listener.port.unwrap_or(cfg.server.port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ports.into_iter().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn notrack_targets(cfg: &ProxyConfig) -> (Vec<(Option<IpAddr>, u16)>, Vec<(Option<IpAddr>, u16)>) {
|
||||||
let mode = cfg.server.conntrack_control.mode;
|
let mode = cfg.server.conntrack_control.mode;
|
||||||
let mut v4_targets: BTreeSet<Option<IpAddr>> = BTreeSet::new();
|
let mut v4_targets: BTreeSet<(Option<IpAddr>, u16)> = BTreeSet::new();
|
||||||
let mut v6_targets: BTreeSet<Option<IpAddr>> = BTreeSet::new();
|
let mut v6_targets: BTreeSet<(Option<IpAddr>, u16)> = BTreeSet::new();
|
||||||
|
|
||||||
match mode {
|
match mode {
|
||||||
ConntrackMode::Tracked => {}
|
ConntrackMode::Tracked => {}
|
||||||
ConntrackMode::Notrack => {
|
ConntrackMode::Notrack => {
|
||||||
if cfg.server.listeners.is_empty() {
|
if cfg.server.listeners.is_empty() {
|
||||||
|
let port = cfg.server.port;
|
||||||
if let Some(ipv4) = cfg
|
if let Some(ipv4) = cfg
|
||||||
.server
|
.server
|
||||||
.listen_addr_ipv4
|
.listen_addr_ipv4
|
||||||
@@ -359,9 +417,9 @@ fn notrack_targets(cfg: &ProxyConfig) -> (Vec<Option<IpAddr>>, Vec<Option<IpAddr
|
|||||||
.and_then(|s| s.parse::<IpAddr>().ok())
|
.and_then(|s| s.parse::<IpAddr>().ok())
|
||||||
{
|
{
|
||||||
if ipv4.is_unspecified() {
|
if ipv4.is_unspecified() {
|
||||||
v4_targets.insert(None);
|
v4_targets.insert((None, port));
|
||||||
} else {
|
} else {
|
||||||
v4_targets.insert(Some(ipv4));
|
v4_targets.insert((Some(ipv4), port));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(ipv6) = cfg
|
if let Some(ipv6) = cfg
|
||||||
@@ -371,33 +429,39 @@ fn notrack_targets(cfg: &ProxyConfig) -> (Vec<Option<IpAddr>>, Vec<Option<IpAddr
|
|||||||
.and_then(|s| s.parse::<IpAddr>().ok())
|
.and_then(|s| s.parse::<IpAddr>().ok())
|
||||||
{
|
{
|
||||||
if ipv6.is_unspecified() {
|
if ipv6.is_unspecified() {
|
||||||
v6_targets.insert(None);
|
v6_targets.insert((None, port));
|
||||||
} else {
|
} else {
|
||||||
v6_targets.insert(Some(ipv6));
|
v6_targets.insert((Some(ipv6), port));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for listener in &cfg.server.listeners {
|
for listener in &cfg.server.listeners {
|
||||||
|
let port = listener.port.unwrap_or(cfg.server.port);
|
||||||
if listener.ip.is_ipv4() {
|
if listener.ip.is_ipv4() {
|
||||||
if listener.ip.is_unspecified() {
|
if listener.ip.is_unspecified() {
|
||||||
v4_targets.insert(None);
|
v4_targets.insert((None, port));
|
||||||
} else {
|
} else {
|
||||||
v4_targets.insert(Some(listener.ip));
|
v4_targets.insert((Some(listener.ip), port));
|
||||||
}
|
}
|
||||||
} else if listener.ip.is_unspecified() {
|
} else if listener.ip.is_unspecified() {
|
||||||
v6_targets.insert(None);
|
v6_targets.insert((None, port));
|
||||||
} else {
|
} else {
|
||||||
v6_targets.insert(Some(listener.ip));
|
v6_targets.insert((Some(listener.ip), port));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ConntrackMode::Hybrid => {
|
ConntrackMode::Hybrid => {
|
||||||
|
let ports = listener_port_set(cfg);
|
||||||
for ip in &cfg.server.conntrack_control.hybrid_listener_ips {
|
for ip in &cfg.server.conntrack_control.hybrid_listener_ips {
|
||||||
if ip.is_ipv4() {
|
if ip.is_ipv4() {
|
||||||
v4_targets.insert(Some(*ip));
|
for port in &ports {
|
||||||
|
v4_targets.insert((Some(*ip), *port));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
v6_targets.insert(Some(*ip));
|
for port in &ports {
|
||||||
|
v6_targets.insert((Some(*ip), *port));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -422,19 +486,19 @@ async fn apply_nft_rules(cfg: &ProxyConfig) -> Result<(), String> {
|
|||||||
|
|
||||||
let (v4_targets, v6_targets) = notrack_targets(cfg);
|
let (v4_targets, v6_targets) = notrack_targets(cfg);
|
||||||
let mut rules = Vec::new();
|
let mut rules = Vec::new();
|
||||||
for ip in v4_targets {
|
for (ip, port) in v4_targets {
|
||||||
let rule = if let Some(ip) = ip {
|
let rule = if let Some(ip) = ip {
|
||||||
format!("tcp dport {} ip daddr {} notrack", cfg.server.port, ip)
|
format!("tcp dport {} ip daddr {} notrack", port, ip)
|
||||||
} else {
|
} else {
|
||||||
format!("tcp dport {} notrack", cfg.server.port)
|
format!("tcp dport {} notrack", port)
|
||||||
};
|
};
|
||||||
rules.push(rule);
|
rules.push(rule);
|
||||||
}
|
}
|
||||||
for ip in v6_targets {
|
for (ip, port) in v6_targets {
|
||||||
let rule = if let Some(ip) = ip {
|
let rule = if let Some(ip) = ip {
|
||||||
format!("tcp dport {} ip6 daddr {} notrack", cfg.server.port, ip)
|
format!("tcp dport {} ip6 daddr {} notrack", port, ip)
|
||||||
} else {
|
} else {
|
||||||
format!("tcp dport {} notrack", cfg.server.port)
|
format!("tcp dport {} notrack", port)
|
||||||
};
|
};
|
||||||
rules.push(rule);
|
rules.push(rule);
|
||||||
}
|
}
|
||||||
@@ -498,7 +562,7 @@ async fn apply_iptables_rules_for_binary(
|
|||||||
|
|
||||||
let (v4_targets, v6_targets) = notrack_targets(cfg);
|
let (v4_targets, v6_targets) = notrack_targets(cfg);
|
||||||
let selected = if ipv4 { v4_targets } else { v6_targets };
|
let selected = if ipv4 { v4_targets } else { v6_targets };
|
||||||
for ip in selected {
|
for (ip, port) in selected {
|
||||||
let mut args = vec![
|
let mut args = vec![
|
||||||
"-t".to_string(),
|
"-t".to_string(),
|
||||||
"raw".to_string(),
|
"raw".to_string(),
|
||||||
@@ -507,7 +571,7 @@ async fn apply_iptables_rules_for_binary(
|
|||||||
"-p".to_string(),
|
"-p".to_string(),
|
||||||
"tcp".to_string(),
|
"tcp".to_string(),
|
||||||
"--dport".to_string(),
|
"--dport".to_string(),
|
||||||
cfg.server.port.to_string(),
|
port.to_string(),
|
||||||
];
|
];
|
||||||
if let Some(ip) = ip {
|
if let Some(ip) = ip {
|
||||||
args.push("-d".to_string());
|
args.push("-d".to_string());
|
||||||
@@ -691,7 +755,7 @@ mod tests {
|
|||||||
me_queue_pressure_delta: 0,
|
me_queue_pressure_delta: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
|
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &sample, &mut state);
|
||||||
|
|
||||||
assert!(state.active);
|
assert!(state.active);
|
||||||
assert!(shared.conntrack_pressure_active());
|
assert!(shared.conntrack_pressure_active());
|
||||||
@@ -712,7 +776,14 @@ mod tests {
|
|||||||
accept_timeout_delta: 0,
|
accept_timeout_delta: 0,
|
||||||
me_queue_pressure_delta: 0,
|
me_queue_pressure_delta: 0,
|
||||||
};
|
};
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &high_sample, &mut state);
|
update_pressure_state(
|
||||||
|
&stats,
|
||||||
|
shared.as_ref(),
|
||||||
|
&cfg,
|
||||||
|
true,
|
||||||
|
&high_sample,
|
||||||
|
&mut state,
|
||||||
|
);
|
||||||
assert!(state.active);
|
assert!(state.active);
|
||||||
|
|
||||||
let low_sample = PressureSample {
|
let low_sample = PressureSample {
|
||||||
@@ -721,11 +792,11 @@ mod tests {
|
|||||||
accept_timeout_delta: 0,
|
accept_timeout_delta: 0,
|
||||||
me_queue_pressure_delta: 0,
|
me_queue_pressure_delta: 0,
|
||||||
};
|
};
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
|
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
|
||||||
assert!(state.active);
|
assert!(state.active);
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
|
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
|
||||||
assert!(state.active);
|
assert!(state.active);
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
|
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
|
||||||
|
|
||||||
assert!(!state.active);
|
assert!(!state.active);
|
||||||
assert!(!shared.conntrack_pressure_active());
|
assert!(!shared.conntrack_pressure_active());
|
||||||
@@ -746,7 +817,7 @@ mod tests {
|
|||||||
me_queue_pressure_delta: 10,
|
me_queue_pressure_delta: 10,
|
||||||
};
|
};
|
||||||
|
|
||||||
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
|
update_pressure_state(&stats, shared.as_ref(), &cfg, false, &sample, &mut state);
|
||||||
|
|
||||||
assert!(!state.active);
|
assert!(!state.active);
|
||||||
assert!(!shared.conntrack_pressure_active());
|
assert!(!shared.conntrack_pressure_active());
|
||||||
|
|||||||
+26
-4
@@ -8,6 +8,7 @@ use std::io::{self, Read, Write};
|
|||||||
use std::os::unix::fs::OpenOptionsExt;
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use nix::errno::Errno;
|
||||||
use nix::fcntl::{Flock, FlockArg};
|
use nix::fcntl::{Flock, FlockArg};
|
||||||
use nix::unistd::{self, ForkResult, Gid, Pid, Uid, chdir, close, fork, getpid, setsid};
|
use nix::unistd::{self, ForkResult, Gid, Pid, Uid, chdir, close, fork, getpid, setsid};
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
@@ -157,15 +158,15 @@ fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
|
|||||||
unsafe {
|
unsafe {
|
||||||
// Redirect stdin (fd 0)
|
// Redirect stdin (fd 0)
|
||||||
if libc::dup2(devnull_fd, 0) < 0 {
|
if libc::dup2(devnull_fd, 0) < 0 {
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
return Err(DaemonError::RedirectFailed(Errno::last()));
|
||||||
}
|
}
|
||||||
// Redirect stdout (fd 1)
|
// Redirect stdout (fd 1)
|
||||||
if libc::dup2(devnull_fd, 1) < 0 {
|
if libc::dup2(devnull_fd, 1) < 0 {
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
return Err(DaemonError::RedirectFailed(Errno::last()));
|
||||||
}
|
}
|
||||||
// Redirect stderr (fd 2)
|
// Redirect stderr (fd 2)
|
||||||
if libc::dup2(devnull_fd, 2) < 0 {
|
if libc::dup2(devnull_fd, 2) < 0 {
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
return Err(DaemonError::RedirectFailed(Errno::last()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,6 +338,27 @@ fn is_process_running(pid: i32) -> bool {
|
|||||||
nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok()
|
nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// macOS gates nix::unistd::setgroups differently in the current dependency set,
|
||||||
|
// so call libc directly there while preserving the original nix path elsewhere.
|
||||||
|
fn set_supplementary_groups(gid: Gid) -> Result<(), nix::Error> {
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
let groups = [gid.as_raw()];
|
||||||
|
let rc = unsafe {
|
||||||
|
libc::setgroups(
|
||||||
|
i32::try_from(groups.len()).expect("single supplementary group must fit in c_int"),
|
||||||
|
groups.as_ptr(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if rc == 0 { Ok(()) } else { Err(Errno::last()) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
{
|
||||||
|
unistd::setgroups(&[gid])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Drops privileges to the specified user and group.
|
/// Drops privileges to the specified user and group.
|
||||||
///
|
///
|
||||||
/// This should be called after binding privileged ports but before entering
|
/// This should be called after binding privileged ports but before entering
|
||||||
@@ -368,7 +390,7 @@ pub fn drop_privileges(
|
|||||||
|
|
||||||
if let Some(gid) = target_gid {
|
if let Some(gid) = target_gid {
|
||||||
unistd::setgid(gid).map_err(DaemonError::PrivilegeDrop)?;
|
unistd::setgid(gid).map_err(DaemonError::PrivilegeDrop)?;
|
||||||
unistd::setgroups(&[gid]).map_err(DaemonError::PrivilegeDrop)?;
|
set_supplementary_groups(gid).map_err(DaemonError::PrivilegeDrop)?;
|
||||||
info!(gid = gid.as_raw(), "Dropped group privileges");
|
info!(gid = gid.as_raw(), "Dropped group privileges");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,211 @@
|
|||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum HealthcheckMode {
|
||||||
|
Liveness,
|
||||||
|
Ready,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealthcheckMode {
|
||||||
|
pub(crate) fn from_cli_arg(value: &str) -> Option<Self> {
|
||||||
|
match value {
|
||||||
|
"liveness" => Some(Self::Liveness),
|
||||||
|
"ready" => Some(Self::Ready),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_path(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Liveness => "/v1/health",
|
||||||
|
Self::Ready => "/v1/health/ready",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
|
||||||
|
match run_inner(config_path, mode) {
|
||||||
|
Ok(()) => 0,
|
||||||
|
Err(error) => {
|
||||||
|
eprintln!("[telemt] healthcheck failed: {error}");
|
||||||
|
1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
||||||
|
let config =
|
||||||
|
ProxyConfig::load(config_path).map_err(|error| format!("config load failed: {error}"))?;
|
||||||
|
let api_cfg = &config.server.api;
|
||||||
|
if !api_cfg.enabled {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let listen: SocketAddr = api_cfg
|
||||||
|
.listen
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| format!("invalid API listen address: {}", api_cfg.listen))?;
|
||||||
|
if listen.port() == 0 {
|
||||||
|
return Err("API listen port is 0".to_string());
|
||||||
|
}
|
||||||
|
let target = probe_target(listen);
|
||||||
|
|
||||||
|
let mut stream = TcpStream::connect_timeout(&target, Duration::from_secs(2))
|
||||||
|
.map_err(|error| format!("connect {target} failed: {error}"))?;
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.map_err(|error| format!("set read timeout failed: {error}"))?;
|
||||||
|
stream
|
||||||
|
.set_write_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.map_err(|error| format!("set write timeout failed: {error}"))?;
|
||||||
|
|
||||||
|
let request = build_request(target, mode.request_path(), &api_cfg.auth_header);
|
||||||
|
stream
|
||||||
|
.write_all(request.as_bytes())
|
||||||
|
.map_err(|error| format!("request write failed: {error}"))?;
|
||||||
|
stream
|
||||||
|
.flush()
|
||||||
|
.map_err(|error| format!("request flush failed: {error}"))?;
|
||||||
|
|
||||||
|
let mut raw_response = Vec::new();
|
||||||
|
stream
|
||||||
|
.read_to_end(&mut raw_response)
|
||||||
|
.map_err(|error| format!("response read failed: {error}"))?;
|
||||||
|
let response =
|
||||||
|
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
|
||||||
|
|
||||||
|
let (status_code, body) = split_response(&response)?;
|
||||||
|
if status_code != 200 {
|
||||||
|
return Err(format!("HTTP status {status_code}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_payload(mode, body)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn probe_target(listen: SocketAddr) -> SocketAddr {
|
||||||
|
match listen {
|
||||||
|
SocketAddr::V4(addr) => {
|
||||||
|
let ip = if addr.ip().is_unspecified() {
|
||||||
|
Ipv4Addr::LOCALHOST
|
||||||
|
} else {
|
||||||
|
*addr.ip()
|
||||||
|
};
|
||||||
|
SocketAddr::from((ip, addr.port()))
|
||||||
|
}
|
||||||
|
SocketAddr::V6(addr) => {
|
||||||
|
let ip = if addr.ip().is_unspecified() {
|
||||||
|
Ipv6Addr::LOCALHOST
|
||||||
|
} else {
|
||||||
|
*addr.ip()
|
||||||
|
};
|
||||||
|
SocketAddr::from((ip, addr.port()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_request(target: SocketAddr, path: &str, auth_header: &str) -> String {
|
||||||
|
let mut request = format!(
|
||||||
|
"GET {path} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\n",
|
||||||
|
target
|
||||||
|
);
|
||||||
|
if !auth_header.is_empty() {
|
||||||
|
request.push_str("Authorization: ");
|
||||||
|
request.push_str(auth_header);
|
||||||
|
request.push_str("\r\n");
|
||||||
|
}
|
||||||
|
request.push_str("\r\n");
|
||||||
|
request
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_response(response: &str) -> Result<(u16, &str), String> {
|
||||||
|
let header_end = response
|
||||||
|
.find("\r\n\r\n")
|
||||||
|
.ok_or_else(|| "invalid HTTP response headers".to_string())?;
|
||||||
|
let header = &response[..header_end];
|
||||||
|
let body = &response[header_end + 4..];
|
||||||
|
let status_line = header
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "missing HTTP status line".to_string())?;
|
||||||
|
let status_code = parse_status_code(status_line)?;
|
||||||
|
Ok((status_code, body))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_status_code(status_line: &str) -> Result<u16, String> {
|
||||||
|
let mut parts = status_line.split_whitespace();
|
||||||
|
let version = parts
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "missing HTTP version".to_string())?;
|
||||||
|
if !version.starts_with("HTTP/") {
|
||||||
|
return Err(format!("invalid HTTP status line: {status_line}"));
|
||||||
|
}
|
||||||
|
let code = parts
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "missing HTTP status code".to_string())?;
|
||||||
|
code.parse::<u16>()
|
||||||
|
.map_err(|_| format!("invalid HTTP status code: {code}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
|
||||||
|
let payload: Value =
|
||||||
|
serde_json::from_str(body).map_err(|_| "response body is not valid JSON".to_string())?;
|
||||||
|
if payload.get("ok").and_then(Value::as_bool) != Some(true) {
|
||||||
|
return Err("response JSON has ok=false".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = payload
|
||||||
|
.get("data")
|
||||||
|
.ok_or_else(|| "response JSON has no data field".to_string())?;
|
||||||
|
match mode {
|
||||||
|
HealthcheckMode::Liveness => {
|
||||||
|
if data.get("status").and_then(Value::as_str) != Some("ok") {
|
||||||
|
return Err("liveness status is not ok".to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HealthcheckMode::Ready => {
|
||||||
|
if data.get("ready").and_then(Value::as_bool) != Some(true) {
|
||||||
|
return Err("readiness flag is false".to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_status_code_reads_http_200() {
|
||||||
|
let status = parse_status_code("HTTP/1.1 200 OK").expect("must parse status");
|
||||||
|
assert_eq!(status, 200);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn split_response_extracts_status_and_body() {
|
||||||
|
let response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"ok\":true}";
|
||||||
|
let (status, body) = split_response(response).expect("must split response");
|
||||||
|
assert_eq!(status, 200);
|
||||||
|
assert_eq!(body, "{\"ok\":true}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_payload_accepts_liveness_contract() {
|
||||||
|
let body = "{\"ok\":true,\"data\":{\"status\":\"ok\"}}";
|
||||||
|
validate_payload(HealthcheckMode::Liveness, body).expect("liveness payload must pass");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_payload_rejects_not_ready() {
|
||||||
|
let body = "{\"ok\":true,\"data\":{\"ready\":false}}";
|
||||||
|
let result = validate_payload(HealthcheckMode::Ready, body);
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,7 +9,7 @@ use tokio::net::UnixListener;
|
|||||||
use tokio::sync::{Semaphore, watch};
|
use tokio::sync::{Semaphore, watch};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::{ProxyConfig, RstOnCloseMode};
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::proxy::ClientHandler;
|
use crate::proxy::ClientHandler;
|
||||||
@@ -21,6 +21,7 @@ use crate::stats::{ReplayChecker, Stats};
|
|||||||
use crate::stream::BufferPool;
|
use crate::stream::BufferPool;
|
||||||
use crate::tls_front::TlsFrontCache;
|
use crate::tls_front::TlsFrontCache;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::transport::socket::set_linger_zero;
|
||||||
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
|
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
|
||||||
|
|
||||||
use super::helpers::{is_expected_handshake_eof, print_proxy_links};
|
use super::helpers::{is_expected_handshake_eof, print_proxy_links};
|
||||||
@@ -30,6 +31,19 @@ pub(crate) struct BoundListeners {
|
|||||||
pub(crate) has_unix_listener: bool,
|
pub(crate) has_unix_listener: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn listener_port_or_legacy(listener: &crate::config::ListenerConfig, config: &ProxyConfig) -> u16 {
|
||||||
|
listener.port.unwrap_or(config.server.port)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_link_port(config: &ProxyConfig) -> u16 {
|
||||||
|
config
|
||||||
|
.server
|
||||||
|
.listeners
|
||||||
|
.first()
|
||||||
|
.and_then(|listener| listener.port)
|
||||||
|
.unwrap_or(config.server.port)
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) async fn bind_listeners(
|
pub(crate) async fn bind_listeners(
|
||||||
config: &Arc<ProxyConfig>,
|
config: &Arc<ProxyConfig>,
|
||||||
@@ -62,7 +76,8 @@ pub(crate) async fn bind_listeners(
|
|||||||
let mut listeners = Vec::new();
|
let mut listeners = Vec::new();
|
||||||
|
|
||||||
for listener_conf in &config.server.listeners {
|
for listener_conf in &config.server.listeners {
|
||||||
let addr = SocketAddr::new(listener_conf.ip, config.server.port);
|
let listener_port = listener_port_or_legacy(listener_conf, config);
|
||||||
|
let addr = SocketAddr::new(listener_conf.ip, listener_port);
|
||||||
if addr.is_ipv4() && !decision_ipv4_dc {
|
if addr.is_ipv4() && !decision_ipv4_dc {
|
||||||
warn!(%addr, "Skipping IPv4 listener: IPv4 disabled by [network]");
|
warn!(%addr, "Skipping IPv4 listener: IPv4 disabled by [network]");
|
||||||
continue;
|
continue;
|
||||||
@@ -105,11 +120,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
if config.general.links.public_host.is_none()
|
if config.general.links.public_host.is_none()
|
||||||
&& !config.general.links.show.is_empty()
|
&& !config.general.links.show.is_empty()
|
||||||
{
|
{
|
||||||
let link_port = config
|
let link_port = config.general.links.public_port.unwrap_or(listener_port);
|
||||||
.general
|
|
||||||
.links
|
|
||||||
.public_port
|
|
||||||
.unwrap_or(config.server.port);
|
|
||||||
print_proxy_links(&public_host, link_port, config);
|
print_proxy_links(&public_host, link_port, config);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,7 +168,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
.general
|
.general
|
||||||
.links
|
.links
|
||||||
.public_port
|
.public_port
|
||||||
.unwrap_or(config.server.port),
|
.unwrap_or(default_link_port(config)),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
let ip = detected_ip_v4.or(detected_ip_v6).map(|ip| ip.to_string());
|
let ip = detected_ip_v4.or(detected_ip_v6).map(|ip| ip.to_string());
|
||||||
@@ -172,7 +183,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
.general
|
.general
|
||||||
.links
|
.links
|
||||||
.public_port
|
.public_port
|
||||||
.unwrap_or(config.server.port),
|
.unwrap_or(default_link_port(config)),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -380,6 +391,15 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
loop {
|
loop {
|
||||||
match listener.accept().await {
|
match listener.accept().await {
|
||||||
Ok((stream, peer_addr)) => {
|
Ok((stream, peer_addr)) => {
|
||||||
|
let rst_mode = config_rx.borrow().general.rst_on_close;
|
||||||
|
#[cfg(unix)]
|
||||||
|
let raw_fd = {
|
||||||
|
use std::os::unix::io::AsRawFd;
|
||||||
|
stream.as_raw_fd()
|
||||||
|
};
|
||||||
|
if matches!(rst_mode, RstOnCloseMode::Errors | RstOnCloseMode::Always) {
|
||||||
|
let _ = set_linger_zero(&stream);
|
||||||
|
}
|
||||||
if !*admission_rx_tcp.borrow() {
|
if !*admission_rx_tcp.borrow() {
|
||||||
debug!(peer = %peer_addr, "Admission gate closed, dropping connection");
|
debug!(peer = %peer_addr, "Admission gate closed, dropping connection");
|
||||||
drop(stream);
|
drop(stream);
|
||||||
@@ -454,6 +474,9 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
shared,
|
shared,
|
||||||
proxy_protocol_enabled,
|
proxy_protocol_enabled,
|
||||||
real_peer_report_for_handler,
|
real_peer_report_for_handler,
|
||||||
|
#[cfg(unix)]
|
||||||
|
raw_fd,
|
||||||
|
rst_mode,
|
||||||
)
|
)
|
||||||
.run()
|
.run()
|
||||||
.await
|
.await
|
||||||
|
|||||||
+48
-28
@@ -81,23 +81,11 @@ pub async fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
||||||
async fn run_inner(
|
// (for privilege drop); it is a no-op on other platforms.
|
||||||
daemon_opts: DaemonOptions,
|
async fn run_telemt_core(
|
||||||
|
drop_after_bind: impl FnOnce(),
|
||||||
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
// Acquire PID file if daemonizing or if explicitly requested
|
|
||||||
// Keep it alive until shutdown (underscore prefix = intentionally kept for RAII cleanup)
|
|
||||||
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
|
||||||
let mut pf = PidFile::new(daemon_opts.pid_file_path());
|
|
||||||
if let Err(e) = pf.acquire() {
|
|
||||||
eprintln!("[telemt] {}", e);
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
Some(pf)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let process_started_at = Instant::now();
|
let process_started_at = Instant::now();
|
||||||
let process_started_at_epoch_secs = SystemTime::now()
|
let process_started_at_epoch_secs = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
@@ -676,6 +664,11 @@ async fn run_inner(
|
|||||||
));
|
));
|
||||||
|
|
||||||
let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096));
|
let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096));
|
||||||
|
let shared_state = ProxySharedState::new();
|
||||||
|
shared_state.traffic_limiter.apply_policy(
|
||||||
|
config.access.user_rate_limits.clone(),
|
||||||
|
config.access.cidr_rate_limits.clone(),
|
||||||
|
);
|
||||||
|
|
||||||
connectivity::run_startup_connectivity(
|
connectivity::run_startup_connectivity(
|
||||||
&config,
|
&config,
|
||||||
@@ -707,6 +700,7 @@ async fn run_inner(
|
|||||||
beobachten.clone(),
|
beobachten.clone(),
|
||||||
api_config_tx.clone(),
|
api_config_tx.clone(),
|
||||||
me_pool.clone(),
|
me_pool.clone(),
|
||||||
|
shared_state.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = runtime_watches.config_rx;
|
let config_rx = runtime_watches.config_rx;
|
||||||
@@ -723,7 +717,6 @@ async fn run_inner(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let _admission_tx_hold = admission_tx;
|
let _admission_tx_hold = admission_tx;
|
||||||
let shared_state = ProxySharedState::new();
|
|
||||||
conntrack_control::spawn_conntrack_controller(
|
conntrack_control::spawn_conntrack_controller(
|
||||||
config_rx.clone(),
|
config_rx.clone(),
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -761,17 +754,8 @@ async fn run_inner(
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop privileges after binding sockets (which may require root for port < 1024)
|
// On Unix, caller supplies privilege drop after bind (may require root for port < 1024).
|
||||||
if daemon_opts.user.is_some() || daemon_opts.group.is_some() {
|
drop_after_bind();
|
||||||
if let Err(e) = drop_privileges(
|
|
||||||
daemon_opts.user.as_deref(),
|
|
||||||
daemon_opts.group.as_deref(),
|
|
||||||
_pid_file.as_ref(),
|
|
||||||
) {
|
|
||||||
error!(error = %e, "Failed to drop privileges");
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runtime_tasks::apply_runtime_log_filter(
|
runtime_tasks::apply_runtime_log_filter(
|
||||||
has_rust_log,
|
has_rust_log,
|
||||||
@@ -819,3 +803,39 @@ async fn run_inner(
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
async fn run_inner(
|
||||||
|
daemon_opts: DaemonOptions,
|
||||||
|
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
|
// Acquire PID file if daemonizing or if explicitly requested
|
||||||
|
// Keep it alive until shutdown (underscore prefix = intentionally kept for RAII cleanup)
|
||||||
|
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
||||||
|
let mut pf = PidFile::new(daemon_opts.pid_file_path());
|
||||||
|
if let Err(e) = pf.acquire() {
|
||||||
|
eprintln!("[telemt] {}", e);
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
Some(pf)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let user = daemon_opts.user.clone();
|
||||||
|
let group = daemon_opts.group.clone();
|
||||||
|
|
||||||
|
run_telemt_core(|| {
|
||||||
|
if user.is_some() || group.is_some() {
|
||||||
|
if let Err(e) = drop_privileges(user.as_deref(), group.as_deref(), _pid_file.as_ref()) {
|
||||||
|
error!(error = %e, "Failed to drop privileges");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
async fn run_inner() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
|
run_telemt_core(|| {}).await
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
|
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
|
||||||
me_pool_for_policy: Option<Arc<MePool>>,
|
me_pool_for_policy: Option<Arc<MePool>>,
|
||||||
|
shared_state: Arc<ProxySharedState>,
|
||||||
) -> RuntimeWatches {
|
) -> RuntimeWatches {
|
||||||
let um_clone = upstream_manager.clone();
|
let um_clone = upstream_manager.clone();
|
||||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||||
@@ -182,6 +183,41 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let limiter = shared_state.traffic_limiter.clone();
|
||||||
|
limiter.apply_policy(
|
||||||
|
config.access.user_rate_limits.clone(),
|
||||||
|
config.access.cidr_rate_limits.clone(),
|
||||||
|
);
|
||||||
|
let mut config_rx_rate_limits = config_rx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut prev_user_limits = config_rx_rate_limits
|
||||||
|
.borrow()
|
||||||
|
.access
|
||||||
|
.user_rate_limits
|
||||||
|
.clone();
|
||||||
|
let mut prev_cidr_limits = config_rx_rate_limits
|
||||||
|
.borrow()
|
||||||
|
.access
|
||||||
|
.cidr_rate_limits
|
||||||
|
.clone();
|
||||||
|
loop {
|
||||||
|
if config_rx_rate_limits.changed().await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let cfg = config_rx_rate_limits.borrow_and_update().clone();
|
||||||
|
if prev_user_limits != cfg.access.user_rate_limits
|
||||||
|
|| prev_cidr_limits != cfg.access.cidr_rate_limits
|
||||||
|
{
|
||||||
|
limiter.apply_policy(
|
||||||
|
cfg.access.user_rate_limits.clone(),
|
||||||
|
cfg.access.cidr_rate_limits.clone(),
|
||||||
|
);
|
||||||
|
prev_user_limits = cfg.access.user_rate_limits.clone();
|
||||||
|
prev_cidr_limits = cfg.access.cidr_rate_limits.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let beobachten_writer = beobachten.clone();
|
let beobachten_writer = beobachten.clone();
|
||||||
let config_rx_beobachten = config_rx.clone();
|
let config_rx_beobachten = config_rx.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ mod crypto;
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
mod daemon;
|
mod daemon;
|
||||||
mod error;
|
mod error;
|
||||||
|
mod healthcheck;
|
||||||
mod ip_tracker;
|
mod ip_tracker;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "tests/ip_tracker_encapsulation_adversarial_tests.rs"]
|
#[path = "tests/ip_tracker_encapsulation_adversarial_tests.rs"]
|
||||||
|
|||||||
+270
@@ -575,6 +575,139 @@ async fn render_metrics(
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_rate_limiter_throttle_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_throttle_up_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_throttle_down_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_throttle_up_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_throttle_down_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_rate_limiter_wait_ms_total Traffic limiter accumulated wait time in milliseconds by scope and direction"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_rate_limiter_wait_ms_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_wait_up_ms_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_wait_down_ms_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_wait_up_ms_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_wait_down_ms_total
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_rate_limiter_active_leases Active relay leases under rate limiting by scope"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_rate_limiter_active_leases gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_active_leases{{scope=\"user\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_active_leases
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_active_leases{{scope=\"cidr\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_active_leases
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_rate_limiter_policy_entries Active rate-limit policy entries by scope"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_rate_limiter_policy_entries gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_policy_entries{{scope=\"user\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.user_policy_entries
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_policy_entries{{scope=\"cidr\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
limiter_metrics.cidr_policy_entries
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_upstream_connect_attempt_total Upstream connect attempts across all requests"
|
"# HELP telemt_upstream_connect_attempt_total Upstream connect attempts across all requests"
|
||||||
@@ -1177,6 +1310,143 @@ async fn render_metrics(
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_fair_pressure_state Worker-local fairness pressure state"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_fair_pressure_state gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_pressure_state {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_pressure_state_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_fair_active_flows Fair-scheduler active flow count"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_fair_active_flows gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_active_flows {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_active_flows_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_fair_queued_bytes Fair-scheduler queued bytes"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_fair_queued_bytes gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_queued_bytes {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_queued_bytes_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_fair_flow_state_gauge Fair-scheduler flow health classes"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_fair_flow_state_gauge gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_flow_state_gauge{{class=\"standing\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_standing_flows_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_flow_state_gauge{{class=\"backpressured\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_backpressured_flows_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_fair_events_total Fair-scheduler event counters"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_fair_events_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"scheduler_round\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_scheduler_rounds_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"deficit_grant\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_deficit_grants_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"deficit_skip\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_deficit_skips_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"enqueue_reject\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_enqueue_rejects_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"shed_drop\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_shed_drops_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"penalty\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_penalties_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_fair_events_total{{event=\"downstream_stall\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_fair_downstream_stalls_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ pub async fn run_probe(
|
|||||||
let UpstreamType::Direct {
|
let UpstreamType::Direct {
|
||||||
interface,
|
interface,
|
||||||
bind_addresses,
|
bind_addresses,
|
||||||
|
..
|
||||||
} = &upstream.upstream_type
|
} = &upstream.upstream_type
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
+28
-1
@@ -804,6 +804,9 @@ pub struct RunningClientHandler {
|
|||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
shared: Arc<ProxySharedState>,
|
shared: Arc<ProxySharedState>,
|
||||||
proxy_protocol_enabled: bool,
|
proxy_protocol_enabled: bool,
|
||||||
|
#[cfg(unix)]
|
||||||
|
raw_fd: std::os::unix::io::RawFd,
|
||||||
|
rst_on_close: crate::config::RstOnCloseMode,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ClientHandler {
|
impl ClientHandler {
|
||||||
@@ -825,6 +828,11 @@ impl ClientHandler {
|
|||||||
proxy_protocol_enabled: bool,
|
proxy_protocol_enabled: bool,
|
||||||
real_peer_report: Arc<std::sync::Mutex<Option<SocketAddr>>>,
|
real_peer_report: Arc<std::sync::Mutex<Option<SocketAddr>>>,
|
||||||
) -> RunningClientHandler {
|
) -> RunningClientHandler {
|
||||||
|
#[cfg(unix)]
|
||||||
|
let raw_fd = {
|
||||||
|
use std::os::unix::io::AsRawFd;
|
||||||
|
stream.as_raw_fd()
|
||||||
|
};
|
||||||
Self::new_with_shared(
|
Self::new_with_shared(
|
||||||
stream,
|
stream,
|
||||||
peer,
|
peer,
|
||||||
@@ -842,6 +850,9 @@ impl ClientHandler {
|
|||||||
ProxySharedState::new(),
|
ProxySharedState::new(),
|
||||||
proxy_protocol_enabled,
|
proxy_protocol_enabled,
|
||||||
real_peer_report,
|
real_peer_report,
|
||||||
|
#[cfg(unix)]
|
||||||
|
raw_fd,
|
||||||
|
crate::config::RstOnCloseMode::Off,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -863,6 +874,8 @@ impl ClientHandler {
|
|||||||
shared: Arc<ProxySharedState>,
|
shared: Arc<ProxySharedState>,
|
||||||
proxy_protocol_enabled: bool,
|
proxy_protocol_enabled: bool,
|
||||||
real_peer_report: Arc<std::sync::Mutex<Option<SocketAddr>>>,
|
real_peer_report: Arc<std::sync::Mutex<Option<SocketAddr>>>,
|
||||||
|
#[cfg(unix)] raw_fd: std::os::unix::io::RawFd,
|
||||||
|
rst_on_close: crate::config::RstOnCloseMode,
|
||||||
) -> RunningClientHandler {
|
) -> RunningClientHandler {
|
||||||
let normalized_peer = normalize_ip(peer);
|
let normalized_peer = normalize_ip(peer);
|
||||||
RunningClientHandler {
|
RunningClientHandler {
|
||||||
@@ -883,6 +896,9 @@ impl ClientHandler {
|
|||||||
beobachten,
|
beobachten,
|
||||||
shared,
|
shared,
|
||||||
proxy_protocol_enabled,
|
proxy_protocol_enabled,
|
||||||
|
#[cfg(unix)]
|
||||||
|
raw_fd,
|
||||||
|
rst_on_close,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -901,6 +917,10 @@ impl RunningClientHandler {
|
|||||||
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
let raw_fd = self.raw_fd;
|
||||||
|
let rst_on_close = self.rst_on_close;
|
||||||
|
|
||||||
let outcome = match self.do_handshake().await? {
|
let outcome = match self.do_handshake().await? {
|
||||||
Some(outcome) => outcome,
|
Some(outcome) => outcome,
|
||||||
None => return Ok(()),
|
None => return Ok(()),
|
||||||
@@ -908,7 +928,14 @@ impl RunningClientHandler {
|
|||||||
|
|
||||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||||
match outcome {
|
match outcome {
|
||||||
HandshakeOutcome::NeedsRelay(fut) | HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
HandshakeOutcome::NeedsRelay(fut) => {
|
||||||
|
#[cfg(unix)]
|
||||||
|
if matches!(rst_on_close, crate::config::RstOnCloseMode::Errors) {
|
||||||
|
let _ = crate::transport::socket::clear_linger_fd(raw_fd);
|
||||||
|
}
|
||||||
|
fut.await
|
||||||
|
}
|
||||||
|
HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -316,6 +316,9 @@ where
|
|||||||
|
|
||||||
stats.increment_user_connects(user);
|
stats.increment_user_connects(user);
|
||||||
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
let _direct_connection_lease = stats.acquire_direct_connection_lease();
|
||||||
|
let traffic_lease = shared
|
||||||
|
.traffic_limiter
|
||||||
|
.acquire_lease(user, success.peer.ip());
|
||||||
|
|
||||||
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
let buffer_pool_trim = Arc::clone(&buffer_pool);
|
||||||
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
let relay_activity_timeout = if shared.conntrack_pressure_active() {
|
||||||
@@ -329,7 +332,7 @@ where
|
|||||||
} else {
|
} else {
|
||||||
Duration::from_secs(1800)
|
Duration::from_secs(1800)
|
||||||
};
|
};
|
||||||
let relay_result = crate::proxy::relay::relay_bidirectional_with_activity_timeout(
|
let relay_result = crate::proxy::relay::relay_bidirectional_with_activity_timeout_and_lease(
|
||||||
client_reader,
|
client_reader,
|
||||||
client_writer,
|
client_writer,
|
||||||
tg_reader,
|
tg_reader,
|
||||||
@@ -340,6 +343,7 @@ where
|
|||||||
Arc::clone(&stats),
|
Arc::clone(&stats),
|
||||||
config.access.user_data_quota.get(user).copied(),
|
config.access.user_data_quota.get(user).copied(),
|
||||||
buffer_pool,
|
buffer_pool,
|
||||||
|
traffic_lease,
|
||||||
relay_activity_timeout,
|
relay_activity_timeout,
|
||||||
);
|
);
|
||||||
tokio::pin!(relay_result);
|
tokio::pin!(relay_result);
|
||||||
|
|||||||
+63
-23
@@ -28,14 +28,10 @@ use tracing::debug;
|
|||||||
const MASK_TIMEOUT: Duration = Duration::from_secs(5);
|
const MASK_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
const MASK_TIMEOUT: Duration = Duration::from_millis(50);
|
const MASK_TIMEOUT: Duration = Duration::from_millis(50);
|
||||||
/// Maximum duration for the entire masking relay.
|
/// Maximum duration for the entire masking relay under test (replaced by config at runtime).
|
||||||
/// Limits resource consumption from slow-loris attacks and port scanners.
|
|
||||||
#[cfg(not(test))]
|
|
||||||
const MASK_RELAY_TIMEOUT: Duration = Duration::from_secs(60);
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
const MASK_RELAY_TIMEOUT: Duration = Duration::from_millis(200);
|
const MASK_RELAY_TIMEOUT: Duration = Duration::from_millis(200);
|
||||||
#[cfg(not(test))]
|
/// Per-read idle timeout for masking relay and drain paths under test (replaced by config at runtime).
|
||||||
const MASK_RELAY_IDLE_TIMEOUT: Duration = Duration::from_secs(5);
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
const MASK_RELAY_IDLE_TIMEOUT: Duration = Duration::from_millis(100);
|
const MASK_RELAY_IDLE_TIMEOUT: Duration = Duration::from_millis(100);
|
||||||
const MASK_BUFFER_SIZE: usize = 8192;
|
const MASK_BUFFER_SIZE: usize = 8192;
|
||||||
@@ -55,6 +51,7 @@ async fn copy_with_idle_timeout<R, W>(
|
|||||||
writer: &mut W,
|
writer: &mut W,
|
||||||
byte_cap: usize,
|
byte_cap: usize,
|
||||||
shutdown_on_eof: bool,
|
shutdown_on_eof: bool,
|
||||||
|
idle_timeout: Duration,
|
||||||
) -> CopyOutcome
|
) -> CopyOutcome
|
||||||
where
|
where
|
||||||
R: AsyncRead + Unpin,
|
R: AsyncRead + Unpin,
|
||||||
@@ -78,7 +75,7 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
|
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
|
||||||
let read_res = timeout(MASK_RELAY_IDLE_TIMEOUT, reader.read(&mut buf[..read_len])).await;
|
let read_res = timeout(idle_timeout, reader.read(&mut buf[..read_len])).await;
|
||||||
let n = match read_res {
|
let n = match read_res {
|
||||||
Ok(Ok(n)) => n,
|
Ok(Ok(n)) => n,
|
||||||
Ok(Err(_)) | Err(_) => break,
|
Ok(Err(_)) | Err(_) => break,
|
||||||
@@ -86,13 +83,13 @@ where
|
|||||||
if n == 0 {
|
if n == 0 {
|
||||||
ended_by_eof = true;
|
ended_by_eof = true;
|
||||||
if shutdown_on_eof {
|
if shutdown_on_eof {
|
||||||
let _ = timeout(MASK_RELAY_IDLE_TIMEOUT, writer.shutdown()).await;
|
let _ = timeout(idle_timeout, writer.shutdown()).await;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
total = total.saturating_add(n);
|
total = total.saturating_add(n);
|
||||||
|
|
||||||
let write_res = timeout(MASK_RELAY_IDLE_TIMEOUT, writer.write_all(&buf[..n])).await;
|
let write_res = timeout(idle_timeout, writer.write_all(&buf[..n])).await;
|
||||||
match write_res {
|
match write_res {
|
||||||
Ok(Ok(())) => {}
|
Ok(Ok(())) => {}
|
||||||
Ok(Err(_)) | Err(_) => break,
|
Ok(Err(_)) | Err(_) => break,
|
||||||
@@ -230,13 +227,20 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn consume_client_data_with_timeout_and_cap<R>(reader: R, byte_cap: usize)
|
async fn consume_client_data_with_timeout_and_cap<R>(
|
||||||
where
|
reader: R,
|
||||||
|
byte_cap: usize,
|
||||||
|
relay_timeout: Duration,
|
||||||
|
idle_timeout: Duration,
|
||||||
|
) where
|
||||||
R: AsyncRead + Unpin,
|
R: AsyncRead + Unpin,
|
||||||
{
|
{
|
||||||
if timeout(MASK_RELAY_TIMEOUT, consume_client_data(reader, byte_cap))
|
if timeout(
|
||||||
.await
|
relay_timeout,
|
||||||
.is_err()
|
consume_client_data(reader, byte_cap, idle_timeout),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
{
|
{
|
||||||
debug!("Timed out while consuming client data on masking fallback path");
|
debug!("Timed out while consuming client data on masking fallback path");
|
||||||
}
|
}
|
||||||
@@ -639,10 +643,18 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
beobachten.record(client_type, peer.ip(), ttl);
|
beobachten.record(client_type, peer.ip(), ttl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let relay_timeout = Duration::from_millis(config.censorship.mask_relay_timeout_ms);
|
||||||
|
let idle_timeout = Duration::from_millis(config.censorship.mask_relay_idle_timeout_ms);
|
||||||
|
|
||||||
if !config.censorship.mask {
|
if !config.censorship.mask {
|
||||||
// Masking disabled, just consume data
|
// Masking disabled, just consume data
|
||||||
consume_client_data_with_timeout_and_cap(reader, config.censorship.mask_relay_max_bytes)
|
consume_client_data_with_timeout_and_cap(
|
||||||
.await;
|
reader,
|
||||||
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -674,7 +686,7 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if timeout(
|
if timeout(
|
||||||
MASK_RELAY_TIMEOUT,
|
relay_timeout,
|
||||||
relay_to_mask(
|
relay_to_mask(
|
||||||
reader,
|
reader,
|
||||||
writer,
|
writer,
|
||||||
@@ -688,6 +700,7 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
idle_timeout,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -703,6 +716,8 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
consume_client_data_with_timeout_and_cap(
|
consume_client_data_with_timeout_and_cap(
|
||||||
reader,
|
reader,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
wait_mask_outcome_budget(outcome_started, config).await;
|
wait_mask_outcome_budget(outcome_started, config).await;
|
||||||
@@ -712,6 +727,8 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
consume_client_data_with_timeout_and_cap(
|
consume_client_data_with_timeout_and_cap(
|
||||||
reader,
|
reader,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
wait_mask_outcome_budget(outcome_started, config).await;
|
wait_mask_outcome_budget(outcome_started, config).await;
|
||||||
@@ -742,8 +759,13 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
local = %local_addr,
|
local = %local_addr,
|
||||||
"Mask target resolves to local listener; refusing self-referential masking fallback"
|
"Mask target resolves to local listener; refusing self-referential masking fallback"
|
||||||
);
|
);
|
||||||
consume_client_data_with_timeout_and_cap(reader, config.censorship.mask_relay_max_bytes)
|
consume_client_data_with_timeout_and_cap(
|
||||||
.await;
|
reader,
|
||||||
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
wait_mask_outcome_budget(outcome_started, config).await;
|
wait_mask_outcome_budget(outcome_started, config).await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -777,7 +799,7 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if timeout(
|
if timeout(
|
||||||
MASK_RELAY_TIMEOUT,
|
relay_timeout,
|
||||||
relay_to_mask(
|
relay_to_mask(
|
||||||
reader,
|
reader,
|
||||||
writer,
|
writer,
|
||||||
@@ -791,6 +813,7 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
config.censorship.mask_shape_above_cap_blur_max_bytes,
|
||||||
config.censorship.mask_shape_hardening_aggressive_mode,
|
config.censorship.mask_shape_hardening_aggressive_mode,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
idle_timeout,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -806,6 +829,8 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
consume_client_data_with_timeout_and_cap(
|
consume_client_data_with_timeout_and_cap(
|
||||||
reader,
|
reader,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
wait_mask_outcome_budget(outcome_started, config).await;
|
wait_mask_outcome_budget(outcome_started, config).await;
|
||||||
@@ -815,6 +840,8 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
consume_client_data_with_timeout_and_cap(
|
consume_client_data_with_timeout_and_cap(
|
||||||
reader,
|
reader,
|
||||||
config.censorship.mask_relay_max_bytes,
|
config.censorship.mask_relay_max_bytes,
|
||||||
|
relay_timeout,
|
||||||
|
idle_timeout,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
wait_mask_outcome_budget(outcome_started, config).await;
|
wait_mask_outcome_budget(outcome_started, config).await;
|
||||||
@@ -836,6 +863,7 @@ async fn relay_to_mask<R, W, MR, MW>(
|
|||||||
shape_above_cap_blur_max_bytes: usize,
|
shape_above_cap_blur_max_bytes: usize,
|
||||||
shape_hardening_aggressive_mode: bool,
|
shape_hardening_aggressive_mode: bool,
|
||||||
mask_relay_max_bytes: usize,
|
mask_relay_max_bytes: usize,
|
||||||
|
idle_timeout: Duration,
|
||||||
) where
|
) where
|
||||||
R: AsyncRead + Unpin + Send + 'static,
|
R: AsyncRead + Unpin + Send + 'static,
|
||||||
W: AsyncWrite + Unpin + Send + 'static,
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
@@ -857,11 +885,19 @@ async fn relay_to_mask<R, W, MR, MW>(
|
|||||||
&mut mask_write,
|
&mut mask_write,
|
||||||
mask_relay_max_bytes,
|
mask_relay_max_bytes,
|
||||||
!shape_hardening_enabled,
|
!shape_hardening_enabled,
|
||||||
|
idle_timeout,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
},
|
},
|
||||||
async {
|
async {
|
||||||
copy_with_idle_timeout(&mut mask_read, &mut writer, mask_relay_max_bytes, true).await
|
copy_with_idle_timeout(
|
||||||
|
&mut mask_read,
|
||||||
|
&mut writer,
|
||||||
|
mask_relay_max_bytes,
|
||||||
|
true,
|
||||||
|
idle_timeout,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -889,7 +925,11 @@ async fn relay_to_mask<R, W, MR, MW>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Just consume all data from client without responding.
|
/// Just consume all data from client without responding.
|
||||||
async fn consume_client_data<R: AsyncRead + Unpin>(mut reader: R, byte_cap: usize) {
|
async fn consume_client_data<R: AsyncRead + Unpin>(
|
||||||
|
mut reader: R,
|
||||||
|
byte_cap: usize,
|
||||||
|
idle_timeout: Duration,
|
||||||
|
) {
|
||||||
if byte_cap == 0 {
|
if byte_cap == 0 {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -905,7 +945,7 @@ async fn consume_client_data<R: AsyncRead + Unpin>(mut reader: R, byte_cap: usiz
|
|||||||
}
|
}
|
||||||
|
|
||||||
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
|
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
|
||||||
let n = match timeout(MASK_RELAY_IDLE_TIMEOUT, reader.read(&mut buf[..read_len])).await {
|
let n = match timeout(idle_timeout, reader.read(&mut buf[..read_len])).await {
|
||||||
Ok(Ok(n)) => n,
|
Ok(Ok(n)) => n,
|
||||||
Ok(Err(_)) | Err(_) => break,
|
Ok(Err(_)) | Err(_) => break,
|
||||||
};
|
};
|
||||||
|
|||||||
+138
-39
@@ -28,6 +28,7 @@ use crate::proxy::route_mode::{
|
|||||||
use crate::proxy::shared_state::{
|
use crate::proxy::shared_state::{
|
||||||
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
||||||
};
|
};
|
||||||
|
use crate::proxy::traffic_limiter::{RateDirection, TrafficLease, next_refill_delay};
|
||||||
use crate::stats::{
|
use crate::stats::{
|
||||||
MeD2cFlushReason, MeD2cQuotaRejectStage, MeD2cWriteMode, QuotaReserveError, Stats, UserStats,
|
MeD2cFlushReason, MeD2cQuotaRejectStage, MeD2cWriteMode, QuotaReserveError, Stats, UserStats,
|
||||||
};
|
};
|
||||||
@@ -286,6 +287,10 @@ impl RelayClientIdleState {
|
|||||||
self.last_client_frame_at = now;
|
self.last_client_frame_at = now;
|
||||||
self.soft_idle_marked = false;
|
self.soft_idle_marked = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn on_client_tiny_frame(&mut self, now: Instant) {
|
||||||
|
self.last_client_frame_at = now;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MeD2cFlushPolicy {
|
impl MeD2cFlushPolicy {
|
||||||
@@ -595,6 +600,41 @@ async fn reserve_user_quota_with_yield(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn wait_for_traffic_budget(
|
||||||
|
lease: Option<&Arc<TrafficLease>>,
|
||||||
|
direction: RateDirection,
|
||||||
|
bytes: u64,
|
||||||
|
) {
|
||||||
|
if bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(lease) = lease else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut remaining = bytes;
|
||||||
|
while remaining > 0 {
|
||||||
|
let consume = lease.try_consume(direction, remaining);
|
||||||
|
if consume.granted > 0 {
|
||||||
|
remaining = remaining.saturating_sub(consume.granted);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let wait_started_at = Instant::now();
|
||||||
|
tokio::time::sleep(next_refill_delay()).await;
|
||||||
|
let wait_ms = wait_started_at
|
||||||
|
.elapsed()
|
||||||
|
.as_millis()
|
||||||
|
.min(u128::from(u64::MAX)) as u64;
|
||||||
|
lease.observe_wait_ms(
|
||||||
|
direction,
|
||||||
|
consume.blocked_user,
|
||||||
|
consume.blocked_cidr,
|
||||||
|
wait_ms,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn classify_me_d2c_flush_reason(
|
fn classify_me_d2c_flush_reason(
|
||||||
flush_immediately: bool,
|
flush_immediately: bool,
|
||||||
batch_frames: usize,
|
batch_frames: usize,
|
||||||
@@ -985,6 +1025,7 @@ where
|
|||||||
let quota_limit = config.access.user_data_quota.get(&user).copied();
|
let quota_limit = config.access.user_data_quota.get(&user).copied();
|
||||||
let quota_user_stats = quota_limit.map(|_| stats.get_or_create_user_stats_handle(&user));
|
let quota_user_stats = quota_limit.map(|_| stats.get_or_create_user_stats_handle(&user));
|
||||||
let peer = success.peer;
|
let peer = success.peer;
|
||||||
|
let traffic_lease = shared.traffic_limiter.acquire_lease(&user, peer.ip());
|
||||||
let proto_tag = success.proto_tag;
|
let proto_tag = success.proto_tag;
|
||||||
let pool_generation = me_pool.current_generation();
|
let pool_generation = me_pool.current_generation();
|
||||||
|
|
||||||
@@ -1120,6 +1161,7 @@ where
|
|||||||
let rng_clone = rng.clone();
|
let rng_clone = rng.clone();
|
||||||
let user_clone = user.clone();
|
let user_clone = user.clone();
|
||||||
let quota_user_stats_me_writer = quota_user_stats.clone();
|
let quota_user_stats_me_writer = quota_user_stats.clone();
|
||||||
|
let traffic_lease_me_writer = traffic_lease.clone();
|
||||||
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
||||||
let bytes_me2c_clone = bytes_me2c.clone();
|
let bytes_me2c_clone = bytes_me2c.clone();
|
||||||
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
||||||
@@ -1153,7 +1195,7 @@ where
|
|||||||
|
|
||||||
let first_is_downstream_activity =
|
let first_is_downstream_activity =
|
||||||
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
matches!(&first, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||||
match process_me_writer_response(
|
match process_me_writer_response_with_traffic_lease(
|
||||||
first,
|
first,
|
||||||
&mut writer,
|
&mut writer,
|
||||||
proto_tag,
|
proto_tag,
|
||||||
@@ -1164,6 +1206,7 @@ where
|
|||||||
quota_user_stats_me_writer.as_deref(),
|
quota_user_stats_me_writer.as_deref(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
|
traffic_lease_me_writer.as_ref(),
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1213,7 +1256,7 @@ where
|
|||||||
|
|
||||||
let next_is_downstream_activity =
|
let next_is_downstream_activity =
|
||||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||||
match process_me_writer_response(
|
match process_me_writer_response_with_traffic_lease(
|
||||||
next,
|
next,
|
||||||
&mut writer,
|
&mut writer,
|
||||||
proto_tag,
|
proto_tag,
|
||||||
@@ -1224,6 +1267,7 @@ where
|
|||||||
quota_user_stats_me_writer.as_deref(),
|
quota_user_stats_me_writer.as_deref(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
|
traffic_lease_me_writer.as_ref(),
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1276,7 +1320,7 @@ where
|
|||||||
Ok(Some(next)) => {
|
Ok(Some(next)) => {
|
||||||
let next_is_downstream_activity =
|
let next_is_downstream_activity =
|
||||||
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
matches!(&next, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||||
match process_me_writer_response(
|
match process_me_writer_response_with_traffic_lease(
|
||||||
next,
|
next,
|
||||||
&mut writer,
|
&mut writer,
|
||||||
proto_tag,
|
proto_tag,
|
||||||
@@ -1287,6 +1331,7 @@ where
|
|||||||
quota_user_stats_me_writer.as_deref(),
|
quota_user_stats_me_writer.as_deref(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
|
traffic_lease_me_writer.as_ref(),
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1341,7 +1386,7 @@ where
|
|||||||
|
|
||||||
let extra_is_downstream_activity =
|
let extra_is_downstream_activity =
|
||||||
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
matches!(&extra, MeResponse::Data { .. } | MeResponse::Ack(_));
|
||||||
match process_me_writer_response(
|
match process_me_writer_response_with_traffic_lease(
|
||||||
extra,
|
extra,
|
||||||
&mut writer,
|
&mut writer,
|
||||||
proto_tag,
|
proto_tag,
|
||||||
@@ -1352,6 +1397,7 @@ where
|
|||||||
quota_user_stats_me_writer.as_deref(),
|
quota_user_stats_me_writer.as_deref(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
|
traffic_lease_me_writer.as_ref(),
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1542,6 +1588,12 @@ where
|
|||||||
match payload_result {
|
match payload_result {
|
||||||
Ok(Some((payload, quickack))) => {
|
Ok(Some((payload, quickack))) => {
|
||||||
trace!(conn_id, bytes = payload.len(), "C->ME frame");
|
trace!(conn_id, bytes = payload.len(), "C->ME frame");
|
||||||
|
wait_for_traffic_budget(
|
||||||
|
traffic_lease.as_ref(),
|
||||||
|
RateDirection::Up,
|
||||||
|
payload.len() as u64,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
forensics.bytes_c2me = forensics
|
forensics.bytes_c2me = forensics
|
||||||
.bytes_c2me
|
.bytes_c2me
|
||||||
.saturating_add(payload.len() as u64);
|
.saturating_add(payload.len() as u64);
|
||||||
@@ -1762,40 +1814,6 @@ where
|
|||||||
let downstream_ms = last_downstream_activity_ms.load(Ordering::Relaxed);
|
let downstream_ms = last_downstream_activity_ms.load(Ordering::Relaxed);
|
||||||
let hard_deadline =
|
let hard_deadline =
|
||||||
hard_deadline(idle_policy, idle_state, session_started_at, downstream_ms);
|
hard_deadline(idle_policy, idle_state, session_started_at, downstream_ms);
|
||||||
if now >= hard_deadline {
|
|
||||||
clear_relay_idle_candidate_in(shared, forensics.conn_id);
|
|
||||||
stats.increment_relay_idle_hard_close_total();
|
|
||||||
let client_idle_secs = now
|
|
||||||
.saturating_duration_since(idle_state.last_client_frame_at)
|
|
||||||
.as_secs();
|
|
||||||
let downstream_idle_secs = now
|
|
||||||
.saturating_duration_since(
|
|
||||||
session_started_at + Duration::from_millis(downstream_ms),
|
|
||||||
)
|
|
||||||
.as_secs();
|
|
||||||
warn!(
|
|
||||||
trace_id = format_args!("0x{:016x}", forensics.trace_id),
|
|
||||||
conn_id = forensics.conn_id,
|
|
||||||
user = %forensics.user,
|
|
||||||
read_label,
|
|
||||||
client_idle_secs,
|
|
||||||
downstream_idle_secs,
|
|
||||||
soft_idle_secs = idle_policy.soft_idle.as_secs(),
|
|
||||||
hard_idle_secs = idle_policy.hard_idle.as_secs(),
|
|
||||||
grace_secs = idle_policy.grace_after_downstream_activity.as_secs(),
|
|
||||||
"Middle-relay hard idle close"
|
|
||||||
);
|
|
||||||
return Err(ProxyError::Io(std::io::Error::new(
|
|
||||||
std::io::ErrorKind::TimedOut,
|
|
||||||
format!(
|
|
||||||
"middle-relay hard idle timeout while reading {read_label}: client_idle_secs={client_idle_secs}, downstream_idle_secs={downstream_idle_secs}, soft_idle_secs={}, hard_idle_secs={}, grace_secs={}",
|
|
||||||
idle_policy.soft_idle.as_secs(),
|
|
||||||
idle_policy.hard_idle.as_secs(),
|
|
||||||
idle_policy.grace_after_downstream_activity.as_secs(),
|
|
||||||
),
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
if !idle_state.soft_idle_marked
|
if !idle_state.soft_idle_marked
|
||||||
&& now.saturating_duration_since(idle_state.last_client_frame_at)
|
&& now.saturating_duration_since(idle_state.last_client_frame_at)
|
||||||
>= idle_policy.soft_idle
|
>= idle_policy.soft_idle
|
||||||
@@ -1850,7 +1868,45 @@ where
|
|||||||
),
|
),
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
Err(_) => {}
|
Err(_) => {
|
||||||
|
let now = Instant::now();
|
||||||
|
let downstream_ms = last_downstream_activity_ms.load(Ordering::Relaxed);
|
||||||
|
let hard_deadline =
|
||||||
|
hard_deadline(idle_policy, idle_state, session_started_at, downstream_ms);
|
||||||
|
if now >= hard_deadline {
|
||||||
|
clear_relay_idle_candidate_in(shared, forensics.conn_id);
|
||||||
|
stats.increment_relay_idle_hard_close_total();
|
||||||
|
let client_idle_secs = now
|
||||||
|
.saturating_duration_since(idle_state.last_client_frame_at)
|
||||||
|
.as_secs();
|
||||||
|
let downstream_idle_secs = now
|
||||||
|
.saturating_duration_since(
|
||||||
|
session_started_at + Duration::from_millis(downstream_ms),
|
||||||
|
)
|
||||||
|
.as_secs();
|
||||||
|
warn!(
|
||||||
|
trace_id = format_args!("0x{:016x}", forensics.trace_id),
|
||||||
|
conn_id = forensics.conn_id,
|
||||||
|
user = %forensics.user,
|
||||||
|
read_label,
|
||||||
|
client_idle_secs,
|
||||||
|
downstream_idle_secs,
|
||||||
|
soft_idle_secs = idle_policy.soft_idle.as_secs(),
|
||||||
|
hard_idle_secs = idle_policy.hard_idle.as_secs(),
|
||||||
|
grace_secs = idle_policy.grace_after_downstream_activity.as_secs(),
|
||||||
|
"Middle-relay hard idle close"
|
||||||
|
);
|
||||||
|
return Err(ProxyError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::TimedOut,
|
||||||
|
format!(
|
||||||
|
"middle-relay hard idle timeout while reading {read_label}: client_idle_secs={client_idle_secs}, downstream_idle_secs={downstream_idle_secs}, soft_idle_secs={}, hard_idle_secs={}, grace_secs={}",
|
||||||
|
idle_policy.soft_idle.as_secs(),
|
||||||
|
idle_policy.hard_idle.as_secs(),
|
||||||
|
idle_policy.grace_after_downstream_activity.as_secs(),
|
||||||
|
),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1941,6 +1997,7 @@ where
|
|||||||
};
|
};
|
||||||
|
|
||||||
if len == 0 {
|
if len == 0 {
|
||||||
|
idle_state.on_client_tiny_frame(Instant::now());
|
||||||
idle_state.tiny_frame_debt = idle_state
|
idle_state.tiny_frame_debt = idle_state
|
||||||
.tiny_frame_debt
|
.tiny_frame_debt
|
||||||
.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
.saturating_add(TINY_FRAME_DEBT_PER_TINY);
|
||||||
@@ -2160,6 +2217,46 @@ async fn process_me_writer_response<W>(
|
|||||||
ack_flush_immediate: bool,
|
ack_flush_immediate: bool,
|
||||||
batched: bool,
|
batched: bool,
|
||||||
) -> Result<MeWriterResponseOutcome>
|
) -> Result<MeWriterResponseOutcome>
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
process_me_writer_response_with_traffic_lease(
|
||||||
|
response,
|
||||||
|
client_writer,
|
||||||
|
proto_tag,
|
||||||
|
rng,
|
||||||
|
frame_buf,
|
||||||
|
stats,
|
||||||
|
user,
|
||||||
|
quota_user_stats,
|
||||||
|
quota_limit,
|
||||||
|
quota_soft_overshoot_bytes,
|
||||||
|
None,
|
||||||
|
bytes_me2c,
|
||||||
|
conn_id,
|
||||||
|
ack_flush_immediate,
|
||||||
|
batched,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_me_writer_response_with_traffic_lease<W>(
|
||||||
|
response: MeResponse,
|
||||||
|
client_writer: &mut CryptoWriter<W>,
|
||||||
|
proto_tag: ProtoTag,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
frame_buf: &mut Vec<u8>,
|
||||||
|
stats: &Stats,
|
||||||
|
user: &str,
|
||||||
|
quota_user_stats: Option<&UserStats>,
|
||||||
|
quota_limit: Option<u64>,
|
||||||
|
quota_soft_overshoot_bytes: u64,
|
||||||
|
traffic_lease: Option<&Arc<TrafficLease>>,
|
||||||
|
bytes_me2c: &AtomicU64,
|
||||||
|
conn_id: u64,
|
||||||
|
ack_flush_immediate: bool,
|
||||||
|
batched: bool,
|
||||||
|
) -> Result<MeWriterResponseOutcome>
|
||||||
where
|
where
|
||||||
W: AsyncWrite + Unpin + Send + 'static,
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
{
|
{
|
||||||
@@ -2183,6 +2280,7 @@ where
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
wait_for_traffic_budget(traffic_lease, RateDirection::Down, data_len).await;
|
||||||
|
|
||||||
let write_mode =
|
let write_mode =
|
||||||
match write_client_payload(client_writer, proto_tag, flags, &data, rng, frame_buf)
|
match write_client_payload(client_writer, proto_tag, flags, &data, rng, frame_buf)
|
||||||
@@ -2220,6 +2318,7 @@ where
|
|||||||
} else {
|
} else {
|
||||||
trace!(conn_id, confirm, "ME->C quickack");
|
trace!(conn_id, confirm, "ME->C quickack");
|
||||||
}
|
}
|
||||||
|
wait_for_traffic_budget(traffic_lease, RateDirection::Down, 4).await;
|
||||||
write_client_ack(client_writer, proto_tag, confirm).await?;
|
write_client_ack(client_writer, proto_tag, confirm).await?;
|
||||||
stats.increment_me_d2c_ack_frames_total();
|
stats.increment_me_d2c_ack_frames_total();
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,7 @@ pub mod relay;
|
|||||||
pub mod route_mode;
|
pub mod route_mode;
|
||||||
pub mod session_eviction;
|
pub mod session_eviction;
|
||||||
pub mod shared_state;
|
pub mod shared_state;
|
||||||
|
pub mod traffic_limiter;
|
||||||
|
|
||||||
pub use client::ClientHandler;
|
pub use client::ClientHandler;
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
|
|||||||
+250
-47
@@ -52,6 +52,7 @@
|
|||||||
//! - `SharedCounters` (atomics) let the watchdog read stats without locking
|
//! - `SharedCounters` (atomics) let the watchdog read stats without locking
|
||||||
|
|
||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
|
use crate::proxy::traffic_limiter::{RateDirection, TrafficLease, next_refill_delay};
|
||||||
use crate::stats::{Stats, UserStats};
|
use crate::stats::{Stats, UserStats};
|
||||||
use crate::stream::BufferPool;
|
use crate::stream::BufferPool;
|
||||||
use std::io;
|
use std::io;
|
||||||
@@ -61,7 +62,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_bidirectional_with_sizes};
|
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_bidirectional_with_sizes};
|
||||||
use tokio::time::Instant;
|
use tokio::time::{Instant, Sleep};
|
||||||
use tracing::{debug, trace, warn};
|
use tracing::{debug, trace, warn};
|
||||||
|
|
||||||
// ============= Constants =============
|
// ============= Constants =============
|
||||||
@@ -210,12 +211,24 @@ struct StatsIo<S> {
|
|||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
user: String,
|
user: String,
|
||||||
user_stats: Arc<UserStats>,
|
user_stats: Arc<UserStats>,
|
||||||
|
traffic_lease: Option<Arc<TrafficLease>>,
|
||||||
|
c2s_rate_debt_bytes: u64,
|
||||||
|
c2s_wait: RateWaitState,
|
||||||
|
s2c_wait: RateWaitState,
|
||||||
quota_limit: Option<u64>,
|
quota_limit: Option<u64>,
|
||||||
quota_exceeded: Arc<AtomicBool>,
|
quota_exceeded: Arc<AtomicBool>,
|
||||||
quota_bytes_since_check: u64,
|
quota_bytes_since_check: u64,
|
||||||
epoch: Instant,
|
epoch: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct RateWaitState {
|
||||||
|
sleep: Option<Pin<Box<Sleep>>>,
|
||||||
|
started_at: Option<Instant>,
|
||||||
|
blocked_user: bool,
|
||||||
|
blocked_cidr: bool,
|
||||||
|
}
|
||||||
|
|
||||||
impl<S> StatsIo<S> {
|
impl<S> StatsIo<S> {
|
||||||
fn new(
|
fn new(
|
||||||
inner: S,
|
inner: S,
|
||||||
@@ -225,6 +238,28 @@ impl<S> StatsIo<S> {
|
|||||||
quota_limit: Option<u64>,
|
quota_limit: Option<u64>,
|
||||||
quota_exceeded: Arc<AtomicBool>,
|
quota_exceeded: Arc<AtomicBool>,
|
||||||
epoch: Instant,
|
epoch: Instant,
|
||||||
|
) -> Self {
|
||||||
|
Self::new_with_traffic_lease(
|
||||||
|
inner,
|
||||||
|
counters,
|
||||||
|
stats,
|
||||||
|
user,
|
||||||
|
None,
|
||||||
|
quota_limit,
|
||||||
|
quota_exceeded,
|
||||||
|
epoch,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new_with_traffic_lease(
|
||||||
|
inner: S,
|
||||||
|
counters: Arc<SharedCounters>,
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
user: String,
|
||||||
|
traffic_lease: Option<Arc<TrafficLease>>,
|
||||||
|
quota_limit: Option<u64>,
|
||||||
|
quota_exceeded: Arc<AtomicBool>,
|
||||||
|
epoch: Instant,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
// Mark initial activity so the watchdog doesn't fire before data flows
|
// Mark initial activity so the watchdog doesn't fire before data flows
|
||||||
counters.touch(Instant::now(), epoch);
|
counters.touch(Instant::now(), epoch);
|
||||||
@@ -235,12 +270,88 @@ impl<S> StatsIo<S> {
|
|||||||
stats,
|
stats,
|
||||||
user,
|
user,
|
||||||
user_stats,
|
user_stats,
|
||||||
|
traffic_lease,
|
||||||
|
c2s_rate_debt_bytes: 0,
|
||||||
|
c2s_wait: RateWaitState::default(),
|
||||||
|
s2c_wait: RateWaitState::default(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
quota_exceeded,
|
quota_exceeded,
|
||||||
quota_bytes_since_check: 0,
|
quota_bytes_since_check: 0,
|
||||||
epoch,
|
epoch,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn record_wait(
|
||||||
|
wait: &mut RateWaitState,
|
||||||
|
lease: Option<&Arc<TrafficLease>>,
|
||||||
|
direction: RateDirection,
|
||||||
|
) {
|
||||||
|
let Some(started_at) = wait.started_at.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let wait_ms = started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
|
||||||
|
if let Some(lease) = lease {
|
||||||
|
lease.observe_wait_ms(direction, wait.blocked_user, wait.blocked_cidr, wait_ms);
|
||||||
|
}
|
||||||
|
wait.blocked_user = false;
|
||||||
|
wait.blocked_cidr = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn arm_wait(wait: &mut RateWaitState, blocked_user: bool, blocked_cidr: bool) {
|
||||||
|
if wait.sleep.is_none() {
|
||||||
|
wait.sleep = Some(Box::pin(tokio::time::sleep(next_refill_delay())));
|
||||||
|
wait.started_at = Some(Instant::now());
|
||||||
|
}
|
||||||
|
wait.blocked_user |= blocked_user;
|
||||||
|
wait.blocked_cidr |= blocked_cidr;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_wait(
|
||||||
|
wait: &mut RateWaitState,
|
||||||
|
cx: &mut Context<'_>,
|
||||||
|
lease: Option<&Arc<TrafficLease>>,
|
||||||
|
direction: RateDirection,
|
||||||
|
) -> Poll<()> {
|
||||||
|
let Some(sleep) = wait.sleep.as_mut() else {
|
||||||
|
return Poll::Ready(());
|
||||||
|
};
|
||||||
|
if sleep.as_mut().poll(cx).is_pending() {
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
wait.sleep = None;
|
||||||
|
Self::record_wait(wait, lease, direction);
|
||||||
|
Poll::Ready(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn settle_c2s_rate_debt(&mut self, cx: &mut Context<'_>) -> Poll<()> {
|
||||||
|
let Some(lease) = self.traffic_lease.as_ref() else {
|
||||||
|
self.c2s_rate_debt_bytes = 0;
|
||||||
|
return Poll::Ready(());
|
||||||
|
};
|
||||||
|
|
||||||
|
while self.c2s_rate_debt_bytes > 0 {
|
||||||
|
let consume = lease.try_consume(RateDirection::Up, self.c2s_rate_debt_bytes);
|
||||||
|
if consume.granted > 0 {
|
||||||
|
self.c2s_rate_debt_bytes = self.c2s_rate_debt_bytes.saturating_sub(consume.granted);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Self::arm_wait(
|
||||||
|
&mut self.c2s_wait,
|
||||||
|
consume.blocked_user,
|
||||||
|
consume.blocked_cidr,
|
||||||
|
);
|
||||||
|
if Self::poll_wait(&mut self.c2s_wait, cx, Some(lease), RateDirection::Up).is_pending()
|
||||||
|
{
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if Self::poll_wait(&mut self.c2s_wait, cx, Some(lease), RateDirection::Up).is_pending() {
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
|
||||||
|
Poll::Ready(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -286,6 +397,25 @@ fn should_immediate_quota_check(remaining_before: u64, charge_bytes: u64) -> boo
|
|||||||
remaining_before <= QUOTA_NEAR_LIMIT_BYTES || charge_bytes >= QUOTA_LARGE_CHARGE_BYTES
|
remaining_before <= QUOTA_NEAR_LIMIT_BYTES || charge_bytes >= QUOTA_LARGE_CHARGE_BYTES
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn refund_reserved_quota_bytes(user_stats: &UserStats, reserved_bytes: u64) {
|
||||||
|
if reserved_bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut current = user_stats.quota_used.load(Ordering::Relaxed);
|
||||||
|
loop {
|
||||||
|
let next = current.saturating_sub(reserved_bytes);
|
||||||
|
match user_stats.quota_used.compare_exchange_weak(
|
||||||
|
current,
|
||||||
|
next,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
) {
|
||||||
|
Ok(_) => return,
|
||||||
|
Err(observed) => current = observed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
||||||
fn poll_read(
|
fn poll_read(
|
||||||
self: Pin<&mut Self>,
|
self: Pin<&mut Self>,
|
||||||
@@ -296,6 +426,9 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
if this.quota_exceeded.load(Ordering::Acquire) {
|
if this.quota_exceeded.load(Ordering::Acquire) {
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
|
if this.settle_c2s_rate_debt(cx).is_pending() {
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
|
||||||
let mut remaining_before = None;
|
let mut remaining_before = None;
|
||||||
if let Some(limit) = this.quota_limit {
|
if let Some(limit) = this.quota_limit {
|
||||||
@@ -377,6 +510,11 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
.add_user_octets_from_handle(this.user_stats.as_ref(), n_to_charge);
|
.add_user_octets_from_handle(this.user_stats.as_ref(), n_to_charge);
|
||||||
this.stats
|
this.stats
|
||||||
.increment_user_msgs_from_handle(this.user_stats.as_ref());
|
.increment_user_msgs_from_handle(this.user_stats.as_ref());
|
||||||
|
if this.traffic_lease.is_some() {
|
||||||
|
this.c2s_rate_debt_bytes =
|
||||||
|
this.c2s_rate_debt_bytes.saturating_add(n_to_charge);
|
||||||
|
let _ = this.settle_c2s_rate_debt(cx);
|
||||||
|
}
|
||||||
|
|
||||||
trace!(user = %this.user, bytes = n, "C->S");
|
trace!(user = %this.user, bytes = n, "C->S");
|
||||||
}
|
}
|
||||||
@@ -398,28 +536,66 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut shaper_reserved_bytes = 0u64;
|
||||||
|
let mut write_buf = buf;
|
||||||
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
|
if !buf.is_empty() {
|
||||||
|
loop {
|
||||||
|
let consume = lease.try_consume(RateDirection::Down, buf.len() as u64);
|
||||||
|
if consume.granted > 0 {
|
||||||
|
shaper_reserved_bytes = consume.granted;
|
||||||
|
if consume.granted < buf.len() as u64 {
|
||||||
|
write_buf = &buf[..consume.granted as usize];
|
||||||
|
}
|
||||||
|
let _ = Self::poll_wait(
|
||||||
|
&mut this.s2c_wait,
|
||||||
|
cx,
|
||||||
|
Some(lease),
|
||||||
|
RateDirection::Down,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
Self::arm_wait(
|
||||||
|
&mut this.s2c_wait,
|
||||||
|
consume.blocked_user,
|
||||||
|
consume.blocked_cidr,
|
||||||
|
);
|
||||||
|
if Self::poll_wait(&mut this.s2c_wait, cx, Some(lease), RateDirection::Down)
|
||||||
|
.is_pending()
|
||||||
|
{
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let _ = Self::poll_wait(&mut this.s2c_wait, cx, Some(lease), RateDirection::Down);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut remaining_before = None;
|
let mut remaining_before = None;
|
||||||
let mut reserved_bytes = 0u64;
|
let mut reserved_bytes = 0u64;
|
||||||
let mut write_buf = buf;
|
|
||||||
if let Some(limit) = this.quota_limit {
|
if let Some(limit) = this.quota_limit {
|
||||||
if !buf.is_empty() {
|
if !write_buf.is_empty() {
|
||||||
let mut reserve_rounds = 0usize;
|
let mut reserve_rounds = 0usize;
|
||||||
while reserved_bytes == 0 {
|
while reserved_bytes == 0 {
|
||||||
let used_before = this.user_stats.quota_used();
|
let used_before = this.user_stats.quota_used();
|
||||||
let remaining = limit.saturating_sub(used_before);
|
let remaining = limit.saturating_sub(used_before);
|
||||||
if remaining == 0 {
|
if remaining == 0 {
|
||||||
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
|
}
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
remaining_before = Some(remaining);
|
remaining_before = Some(remaining);
|
||||||
|
|
||||||
let desired = remaining.min(buf.len() as u64);
|
let desired = remaining.min(write_buf.len() as u64);
|
||||||
let mut saw_contention = false;
|
let mut saw_contention = false;
|
||||||
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
||||||
match this.user_stats.quota_try_reserve(desired, limit) {
|
match this.user_stats.quota_try_reserve(desired, limit) {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
reserved_bytes = desired;
|
reserved_bytes = desired;
|
||||||
write_buf = &buf[..desired as usize];
|
write_buf = &write_buf[..desired as usize];
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
|
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
|
||||||
@@ -434,6 +610,9 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
if reserved_bytes == 0 {
|
if reserved_bytes == 0 {
|
||||||
reserve_rounds = reserve_rounds.saturating_add(1);
|
reserve_rounds = reserve_rounds.saturating_add(1);
|
||||||
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
||||||
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
|
}
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
@@ -446,6 +625,9 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
let used_before = this.user_stats.quota_used();
|
let used_before = this.user_stats.quota_used();
|
||||||
let remaining = limit.saturating_sub(used_before);
|
let remaining = limit.saturating_sub(used_before);
|
||||||
if remaining == 0 {
|
if remaining == 0 {
|
||||||
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
|
}
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
@@ -456,23 +638,20 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
|
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
|
||||||
Poll::Ready(Ok(n)) => {
|
Poll::Ready(Ok(n)) => {
|
||||||
if reserved_bytes > n as u64 {
|
if reserved_bytes > n as u64 {
|
||||||
let refund = reserved_bytes - n as u64;
|
refund_reserved_quota_bytes(
|
||||||
let mut current = this.user_stats.quota_used.load(Ordering::Relaxed);
|
this.user_stats.as_ref(),
|
||||||
loop {
|
reserved_bytes - n as u64,
|
||||||
let next = current.saturating_sub(refund);
|
);
|
||||||
match this.user_stats.quota_used.compare_exchange_weak(
|
}
|
||||||
current,
|
if shaper_reserved_bytes > n as u64
|
||||||
next,
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
Ordering::Relaxed,
|
{
|
||||||
Ordering::Relaxed,
|
lease.refund(RateDirection::Down, shaper_reserved_bytes - n as u64);
|
||||||
) {
|
|
||||||
Ok(_) => break,
|
|
||||||
Err(observed) => current = observed,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
|
Self::record_wait(&mut this.s2c_wait, Some(lease), RateDirection::Down);
|
||||||
|
}
|
||||||
let n_to_charge = n as u64;
|
let n_to_charge = n as u64;
|
||||||
|
|
||||||
// S→C: data written to client
|
// S→C: data written to client
|
||||||
@@ -512,37 +691,23 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
}
|
}
|
||||||
Poll::Ready(Err(err)) => {
|
Poll::Ready(Err(err)) => {
|
||||||
if reserved_bytes > 0 {
|
if reserved_bytes > 0 {
|
||||||
let mut current = this.user_stats.quota_used.load(Ordering::Relaxed);
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
||||||
loop {
|
}
|
||||||
let next = current.saturating_sub(reserved_bytes);
|
if shaper_reserved_bytes > 0
|
||||||
match this.user_stats.quota_used.compare_exchange_weak(
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
current,
|
{
|
||||||
next,
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
Ordering::Relaxed,
|
|
||||||
Ordering::Relaxed,
|
|
||||||
) {
|
|
||||||
Ok(_) => break,
|
|
||||||
Err(observed) => current = observed,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Poll::Ready(Err(err))
|
Poll::Ready(Err(err))
|
||||||
}
|
}
|
||||||
Poll::Pending => {
|
Poll::Pending => {
|
||||||
if reserved_bytes > 0 {
|
if reserved_bytes > 0 {
|
||||||
let mut current = this.user_stats.quota_used.load(Ordering::Relaxed);
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
||||||
loop {
|
}
|
||||||
let next = current.saturating_sub(reserved_bytes);
|
if shaper_reserved_bytes > 0
|
||||||
match this.user_stats.quota_used.compare_exchange_weak(
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
current,
|
{
|
||||||
next,
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
Ordering::Relaxed,
|
|
||||||
Ordering::Relaxed,
|
|
||||||
) {
|
|
||||||
Ok(_) => break,
|
|
||||||
Err(observed) => current = observed,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Poll::Pending
|
Poll::Pending
|
||||||
}
|
}
|
||||||
@@ -627,6 +792,43 @@ pub async fn relay_bidirectional_with_activity_timeout<CR, CW, SR, SW>(
|
|||||||
_buffer_pool: Arc<BufferPool>,
|
_buffer_pool: Arc<BufferPool>,
|
||||||
activity_timeout: Duration,
|
activity_timeout: Duration,
|
||||||
) -> Result<()>
|
) -> Result<()>
|
||||||
|
where
|
||||||
|
CR: AsyncRead + Unpin + Send + 'static,
|
||||||
|
CW: AsyncWrite + Unpin + Send + 'static,
|
||||||
|
SR: AsyncRead + Unpin + Send + 'static,
|
||||||
|
SW: AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
relay_bidirectional_with_activity_timeout_and_lease(
|
||||||
|
client_reader,
|
||||||
|
client_writer,
|
||||||
|
server_reader,
|
||||||
|
server_writer,
|
||||||
|
c2s_buf_size,
|
||||||
|
s2c_buf_size,
|
||||||
|
user,
|
||||||
|
stats,
|
||||||
|
quota_limit,
|
||||||
|
_buffer_pool,
|
||||||
|
None,
|
||||||
|
activity_timeout,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn relay_bidirectional_with_activity_timeout_and_lease<CR, CW, SR, SW>(
|
||||||
|
client_reader: CR,
|
||||||
|
client_writer: CW,
|
||||||
|
server_reader: SR,
|
||||||
|
server_writer: SW,
|
||||||
|
c2s_buf_size: usize,
|
||||||
|
s2c_buf_size: usize,
|
||||||
|
user: &str,
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
quota_limit: Option<u64>,
|
||||||
|
_buffer_pool: Arc<BufferPool>,
|
||||||
|
traffic_lease: Option<Arc<TrafficLease>>,
|
||||||
|
activity_timeout: Duration,
|
||||||
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
CR: AsyncRead + Unpin + Send + 'static,
|
CR: AsyncRead + Unpin + Send + 'static,
|
||||||
CW: AsyncWrite + Unpin + Send + 'static,
|
CW: AsyncWrite + Unpin + Send + 'static,
|
||||||
@@ -644,11 +846,12 @@ where
|
|||||||
let mut server = CombinedStream::new(server_reader, server_writer);
|
let mut server = CombinedStream::new(server_reader, server_writer);
|
||||||
|
|
||||||
// Wrap client with stats/activity tracking
|
// Wrap client with stats/activity tracking
|
||||||
let mut client = StatsIo::new(
|
let mut client = StatsIo::new_with_traffic_lease(
|
||||||
client_combined,
|
client_combined,
|
||||||
Arc::clone(&counters),
|
Arc::clone(&counters),
|
||||||
Arc::clone(&stats),
|
Arc::clone(&stats),
|
||||||
user_owned.clone(),
|
user_owned.clone(),
|
||||||
|
traffic_lease,
|
||||||
quota_limit,
|
quota_limit,
|
||||||
Arc::clone("a_exceeded),
|
Arc::clone("a_exceeded),
|
||||||
epoch,
|
epoch,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use tokio::sync::mpsc;
|
|||||||
|
|
||||||
use crate::proxy::handshake::{AuthProbeSaturationState, AuthProbeState};
|
use crate::proxy::handshake::{AuthProbeSaturationState, AuthProbeState};
|
||||||
use crate::proxy::middle_relay::{DesyncDedupRotationState, RelayIdleCandidateRegistry};
|
use crate::proxy::middle_relay::{DesyncDedupRotationState, RelayIdleCandidateRegistry};
|
||||||
|
use crate::proxy::traffic_limiter::TrafficLimiter;
|
||||||
|
|
||||||
const HANDSHAKE_RECENT_USER_RING_LEN: usize = 64;
|
const HANDSHAKE_RECENT_USER_RING_LEN: usize = 64;
|
||||||
|
|
||||||
@@ -65,6 +66,7 @@ pub(crate) struct MiddleRelaySharedState {
|
|||||||
pub(crate) struct ProxySharedState {
|
pub(crate) struct ProxySharedState {
|
||||||
pub(crate) handshake: HandshakeSharedState,
|
pub(crate) handshake: HandshakeSharedState,
|
||||||
pub(crate) middle_relay: MiddleRelaySharedState,
|
pub(crate) middle_relay: MiddleRelaySharedState,
|
||||||
|
pub(crate) traffic_limiter: Arc<TrafficLimiter>,
|
||||||
pub(crate) conntrack_pressure_active: AtomicBool,
|
pub(crate) conntrack_pressure_active: AtomicBool,
|
||||||
pub(crate) conntrack_close_tx: Mutex<Option<mpsc::Sender<ConntrackCloseEvent>>>,
|
pub(crate) conntrack_close_tx: Mutex<Option<mpsc::Sender<ConntrackCloseEvent>>>,
|
||||||
}
|
}
|
||||||
@@ -98,6 +100,7 @@ impl ProxySharedState {
|
|||||||
relay_idle_registry: Mutex::new(RelayIdleCandidateRegistry::default()),
|
relay_idle_registry: Mutex::new(RelayIdleCandidateRegistry::default()),
|
||||||
relay_idle_mark_seq: AtomicU64::new(0),
|
relay_idle_mark_seq: AtomicU64::new(0),
|
||||||
},
|
},
|
||||||
|
traffic_limiter: TrafficLimiter::new(),
|
||||||
conntrack_pressure_active: AtomicBool::new(false),
|
conntrack_pressure_active: AtomicBool::new(false),
|
||||||
conntrack_close_tx: Mutex::new(None),
|
conntrack_close_tx: Mutex::new(None),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -31,11 +31,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -27,11 +27,14 @@ fn build_harness(config: ProxyConfig) -> PipelineHarness {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -25,11 +25,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -38,11 +38,14 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -16,11 +16,14 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -39,11 +39,14 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> RedTeamHarness {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -229,11 +232,14 @@ async fn redteam_03_masking_duration_must_be_less_than_1ms_when_backend_down() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -470,11 +476,14 @@ async fn measure_invalid_probe_duration_ms(delay_ms: u64, tls_len: u16, body_sen
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -544,11 +553,14 @@ async fn capture_forwarded_probe_len(tls_len: u16, body_sent: usize) -> usize {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -13,11 +13,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -11,11 +11,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -25,11 +25,14 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -332,11 +332,14 @@ async fn relay_task_abort_releases_user_gate_and_ip_reservation() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -446,11 +449,14 @@ async fn relay_cutover_releases_user_gate_and_ip_reservation() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -570,11 +576,14 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -740,11 +749,14 @@ async fn proxy_protocol_header_is_rejected_when_trust_list_is_empty() {
|
|||||||
upstream_type: crate::config::UpstreamType::Direct {
|
upstream_type: crate::config::UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -817,11 +829,14 @@ async fn proxy_protocol_header_from_untrusted_peer_range_is_rejected_under_load(
|
|||||||
upstream_type: crate::config::UpstreamType::Direct {
|
upstream_type: crate::config::UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -977,11 +992,14 @@ async fn short_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1065,11 +1083,14 @@ async fn tls12_record_probe_is_masked_through_client_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1151,11 +1172,14 @@ async fn handle_client_stream_increments_connects_all_exactly_once() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1244,11 +1268,14 @@ async fn running_client_handler_increments_connects_all_exactly_once() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1334,11 +1361,14 @@ async fn idle_pooled_connection_closes_cleanly_in_generic_stream_path() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1405,11 +1435,14 @@ async fn idle_pooled_connection_closes_cleanly_in_client_handler_path() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1491,11 +1524,14 @@ async fn partial_tls_header_stall_triggers_handshake_timeout() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1816,11 +1852,14 @@ async fn valid_tls_path_does_not_fall_back_to_mask_backend() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1925,11 +1964,14 @@ async fn valid_tls_with_invalid_mtproto_falls_back_to_mask_backend() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -2032,11 +2074,14 @@ async fn client_handler_tls_bad_mtproto_is_forwarded_to_mask_backend() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -2154,11 +2199,14 @@ async fn alpn_mismatch_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -2247,11 +2295,14 @@ async fn invalid_hmac_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -2346,11 +2397,14 @@ async fn burst_invalid_tls_probes_are_masked_verbatim() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -3251,11 +3305,14 @@ async fn relay_connect_error_releases_user_and_ip_before_return() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -3812,11 +3869,14 @@ async fn untrusted_proxy_header_source_is_rejected() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -3882,11 +3942,14 @@ async fn empty_proxy_trusted_cidrs_rejects_proxy_header_by_default() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -3979,11 +4042,14 @@ async fn oversized_tls_record_is_masked_in_generic_stream_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -4082,11 +4148,14 @@ async fn oversized_tls_record_is_masked_in_client_handler_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -4199,11 +4268,14 @@ async fn tls_record_len_min_minus_1_is_rejected_in_generic_stream_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -4302,11 +4374,14 @@ async fn tls_record_len_min_minus_1_is_rejected_in_client_handler_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -4408,11 +4483,14 @@ async fn tls_record_len_16384_is_accepted_in_generic_stream_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -4509,11 +4587,14 @@ async fn tls_record_len_16384_is_accepted_in_client_handler_pipeline() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -24,11 +24,14 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -26,11 +26,14 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -27,11 +27,14 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -41,11 +41,14 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -1293,11 +1293,14 @@ async fn direct_relay_abort_midflight_releases_route_gauge() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1400,11 +1403,14 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1522,11 +1528,14 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
@@ -1758,8 +1767,11 @@ async fn negative_direct_relay_dc_connection_refused_fails_fast() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
100,
|
100,
|
||||||
@@ -1849,8 +1861,11 @@ async fn adversarial_direct_relay_cutover_integrity() {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
100,
|
100,
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ async fn consume_client_data_stops_after_byte_cap_without_eof() {
|
|||||||
};
|
};
|
||||||
let cap = 10_000usize;
|
let cap = 10_000usize;
|
||||||
|
|
||||||
consume_client_data(reader, cap).await;
|
consume_client_data(reader, cap, MASK_RELAY_IDLE_TIMEOUT).await;
|
||||||
|
|
||||||
let total = produced.load(Ordering::Relaxed);
|
let total = produced.load(Ordering::Relaxed);
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ async fn stalling_client_terminates_at_idle_not_relay_timeout() {
|
|||||||
|
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
MASK_RELAY_TIMEOUT,
|
MASK_RELAY_TIMEOUT,
|
||||||
consume_client_data(reader, MASK_BUFFER_SIZE * 4),
|
consume_client_data(reader, MASK_BUFFER_SIZE * 4, MASK_RELAY_IDLE_TIMEOUT),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -57,9 +57,12 @@ async fn fast_reader_drains_to_eof() {
|
|||||||
let data = vec![0xAAu8; 32 * 1024];
|
let data = vec![0xAAu8; 32 * 1024];
|
||||||
let reader = std::io::Cursor::new(data);
|
let reader = std::io::Cursor::new(data);
|
||||||
|
|
||||||
tokio::time::timeout(MASK_RELAY_TIMEOUT, consume_client_data(reader, usize::MAX))
|
tokio::time::timeout(
|
||||||
.await
|
MASK_RELAY_TIMEOUT,
|
||||||
.expect("consume_client_data did not complete for fast EOF reader");
|
consume_client_data(reader, usize::MAX, MASK_RELAY_IDLE_TIMEOUT),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("consume_client_data did not complete for fast EOF reader");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -81,7 +84,7 @@ async fn io_error_terminates_cleanly() {
|
|||||||
|
|
||||||
tokio::time::timeout(
|
tokio::time::timeout(
|
||||||
MASK_RELAY_TIMEOUT,
|
MASK_RELAY_TIMEOUT,
|
||||||
consume_client_data(ErrReader, usize::MAX),
|
consume_client_data(ErrReader, usize::MAX, MASK_RELAY_IDLE_TIMEOUT),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.expect("consume_client_data did not return on I/O error");
|
.expect("consume_client_data did not return on I/O error");
|
||||||
|
|||||||
@@ -34,7 +34,11 @@ async fn consume_stall_stress_finishes_within_idle_budget() {
|
|||||||
set.spawn(async {
|
set.spawn(async {
|
||||||
tokio::time::timeout(
|
tokio::time::timeout(
|
||||||
MASK_RELAY_TIMEOUT,
|
MASK_RELAY_TIMEOUT,
|
||||||
consume_client_data(OneByteThenStall { sent: false }, usize::MAX),
|
consume_client_data(
|
||||||
|
OneByteThenStall { sent: false },
|
||||||
|
usize::MAX,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.expect("consume_client_data exceeded relay timeout under stall load");
|
.expect("consume_client_data exceeded relay timeout under stall load");
|
||||||
@@ -56,7 +60,7 @@ async fn consume_stall_stress_finishes_within_idle_budget() {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn consume_zero_cap_returns_immediately() {
|
async fn consume_zero_cap_returns_immediately() {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
consume_client_data(tokio::io::empty(), 0).await;
|
consume_client_data(tokio::io::empty(), 0, MASK_RELAY_IDLE_TIMEOUT).await;
|
||||||
assert!(
|
assert!(
|
||||||
started.elapsed() < MASK_RELAY_IDLE_TIMEOUT,
|
started.elapsed() < MASK_RELAY_IDLE_TIMEOUT,
|
||||||
"zero byte cap must return immediately"
|
"zero byte cap must return immediately"
|
||||||
|
|||||||
@@ -127,7 +127,14 @@ async fn positive_copy_with_production_cap_stops_exactly_at_budget() {
|
|||||||
let mut reader = FinitePatternReader::new(PROD_CAP_BYTES + (256 * 1024), 4096, read_calls);
|
let mut reader = FinitePatternReader::new(PROD_CAP_BYTES + (256 * 1024), 4096, read_calls);
|
||||||
let mut writer = CountingWriter::default();
|
let mut writer = CountingWriter::default();
|
||||||
|
|
||||||
let outcome = copy_with_idle_timeout(&mut reader, &mut writer, PROD_CAP_BYTES, true).await;
|
let outcome = copy_with_idle_timeout(
|
||||||
|
&mut reader,
|
||||||
|
&mut writer,
|
||||||
|
PROD_CAP_BYTES,
|
||||||
|
true,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
outcome.total, PROD_CAP_BYTES,
|
outcome.total, PROD_CAP_BYTES,
|
||||||
@@ -145,7 +152,13 @@ async fn negative_consume_with_zero_cap_performs_no_reads() {
|
|||||||
let read_calls = Arc::new(AtomicUsize::new(0));
|
let read_calls = Arc::new(AtomicUsize::new(0));
|
||||||
let reader = FinitePatternReader::new(1024, 64, Arc::clone(&read_calls));
|
let reader = FinitePatternReader::new(1024, 64, Arc::clone(&read_calls));
|
||||||
|
|
||||||
consume_client_data_with_timeout_and_cap(reader, 0).await;
|
consume_client_data_with_timeout_and_cap(
|
||||||
|
reader,
|
||||||
|
0,
|
||||||
|
MASK_RELAY_TIMEOUT,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
read_calls.load(Ordering::Relaxed),
|
read_calls.load(Ordering::Relaxed),
|
||||||
@@ -161,7 +174,14 @@ async fn edge_copy_below_cap_reports_eof_without_overread() {
|
|||||||
let mut reader = FinitePatternReader::new(payload, 3072, read_calls);
|
let mut reader = FinitePatternReader::new(payload, 3072, read_calls);
|
||||||
let mut writer = CountingWriter::default();
|
let mut writer = CountingWriter::default();
|
||||||
|
|
||||||
let outcome = copy_with_idle_timeout(&mut reader, &mut writer, PROD_CAP_BYTES, true).await;
|
let outcome = copy_with_idle_timeout(
|
||||||
|
&mut reader,
|
||||||
|
&mut writer,
|
||||||
|
PROD_CAP_BYTES,
|
||||||
|
true,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_eq!(outcome.total, payload);
|
assert_eq!(outcome.total, payload);
|
||||||
assert_eq!(writer.written, payload);
|
assert_eq!(writer.written, payload);
|
||||||
@@ -175,7 +195,13 @@ async fn edge_copy_below_cap_reports_eof_without_overread() {
|
|||||||
async fn adversarial_blackhat_never_ready_reader_is_bounded_by_timeout_guards() {
|
async fn adversarial_blackhat_never_ready_reader_is_bounded_by_timeout_guards() {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
|
|
||||||
consume_client_data_with_timeout_and_cap(NeverReadyReader, PROD_CAP_BYTES).await;
|
consume_client_data_with_timeout_and_cap(
|
||||||
|
NeverReadyReader,
|
||||||
|
PROD_CAP_BYTES,
|
||||||
|
MASK_RELAY_TIMEOUT,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
started.elapsed() < Duration::from_millis(350),
|
started.elapsed() < Duration::from_millis(350),
|
||||||
@@ -190,7 +216,12 @@ async fn integration_consume_path_honors_production_cap_for_large_payload() {
|
|||||||
|
|
||||||
let bounded = timeout(
|
let bounded = timeout(
|
||||||
Duration::from_millis(350),
|
Duration::from_millis(350),
|
||||||
consume_client_data_with_timeout_and_cap(reader, PROD_CAP_BYTES),
|
consume_client_data_with_timeout_and_cap(
|
||||||
|
reader,
|
||||||
|
PROD_CAP_BYTES,
|
||||||
|
MASK_RELAY_TIMEOUT,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -206,7 +237,13 @@ async fn adversarial_consume_path_never_reads_beyond_declared_byte_cap() {
|
|||||||
let total_read = Arc::new(AtomicUsize::new(0));
|
let total_read = Arc::new(AtomicUsize::new(0));
|
||||||
let reader = BudgetProbeReader::new(256 * 1024, Arc::clone(&total_read));
|
let reader = BudgetProbeReader::new(256 * 1024, Arc::clone(&total_read));
|
||||||
|
|
||||||
consume_client_data_with_timeout_and_cap(reader, byte_cap).await;
|
consume_client_data_with_timeout_and_cap(
|
||||||
|
reader,
|
||||||
|
byte_cap,
|
||||||
|
MASK_RELAY_TIMEOUT,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
total_read.load(Ordering::Relaxed) <= byte_cap,
|
total_read.load(Ordering::Relaxed) <= byte_cap,
|
||||||
@@ -231,7 +268,9 @@ async fn light_fuzz_cap_and_payload_matrix_preserves_min_budget_invariant() {
|
|||||||
let mut reader = FinitePatternReader::new(payload, chunk, read_calls);
|
let mut reader = FinitePatternReader::new(payload, chunk, read_calls);
|
||||||
let mut writer = CountingWriter::default();
|
let mut writer = CountingWriter::default();
|
||||||
|
|
||||||
let outcome = copy_with_idle_timeout(&mut reader, &mut writer, cap, true).await;
|
let outcome =
|
||||||
|
copy_with_idle_timeout(&mut reader, &mut writer, cap, true, MASK_RELAY_IDLE_TIMEOUT)
|
||||||
|
.await;
|
||||||
let expected = payload.min(cap);
|
let expected = payload.min(cap);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -261,7 +300,14 @@ async fn stress_parallel_copy_tasks_with_production_cap_complete_without_leaks()
|
|||||||
read_calls,
|
read_calls,
|
||||||
);
|
);
|
||||||
let mut writer = CountingWriter::default();
|
let mut writer = CountingWriter::default();
|
||||||
copy_with_idle_timeout(&mut reader, &mut writer, PROD_CAP_BYTES, true).await
|
copy_with_idle_timeout(
|
||||||
|
&mut reader,
|
||||||
|
&mut writer,
|
||||||
|
PROD_CAP_BYTES,
|
||||||
|
true,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ async fn relay_to_mask_enforces_masking_session_byte_cap() {
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
32 * 1024,
|
32 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -81,6 +82,7 @@ async fn relay_to_mask_propagates_client_half_close_without_waiting_for_other_di
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
32 * 1024,
|
32 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1377,6 +1377,7 @@ async fn relay_to_mask_keeps_backend_to_client_flow_when_client_to_backend_stall
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
5 * 1024 * 1024,
|
5 * 1024 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -1508,6 +1509,7 @@ async fn relay_to_mask_timeout_cancels_and_drops_all_io_endpoints() {
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
5 * 1024 * 1024,
|
5 * 1024 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -228,6 +228,7 @@ async fn relay_path_idle_timeout_eviction_remains_effective() {
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
5 * 1024 * 1024,
|
5 * 1024 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ async fn run_relay_case(
|
|||||||
above_cap_blur_max_bytes,
|
above_cap_blur_max_bytes,
|
||||||
false,
|
false,
|
||||||
5 * 1024 * 1024,
|
5 * 1024 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ async fn relay_to_mask_applies_cap_clamped_padding_for_non_power_of_two_cap() {
|
|||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
5 * 1024 * 1024,
|
5 * 1024 * 1024,
|
||||||
|
MASK_RELAY_IDLE_TIMEOUT,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -53,11 +53,14 @@ fn new_client_harness() -> ClientHarness {
|
|||||||
upstream_type: UpstreamType::Direct {
|
upstream_type: UpstreamType::Direct {
|
||||||
interface: None,
|
interface: None,
|
||||||
bind_addresses: None,
|
bind_addresses: None,
|
||||||
|
bindtodevice: None,
|
||||||
},
|
},
|
||||||
weight: 1,
|
weight: 1,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
scopes: String::new(),
|
scopes: String::new(),
|
||||||
selected_scope: String::new(),
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
}],
|
}],
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
|
|||||||
@@ -0,0 +1,853 @@
|
|||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::hash::{Hash, Hasher};
|
||||||
|
use std::net::IpAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use dashmap::DashMap;
|
||||||
|
use ipnetwork::IpNetwork;
|
||||||
|
|
||||||
|
use crate::config::RateLimitBps;
|
||||||
|
|
||||||
|
const REGISTRY_SHARDS: usize = 64;
|
||||||
|
const FAIR_EPOCH_MS: u64 = 20;
|
||||||
|
const MAX_BORROW_CHUNK_BYTES: u64 = 32 * 1024;
|
||||||
|
const CLEANUP_INTERVAL_SECS: u64 = 60;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RateDirection {
|
||||||
|
Up,
|
||||||
|
Down,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct TrafficConsumeResult {
|
||||||
|
pub granted: u64,
|
||||||
|
pub blocked_user: bool,
|
||||||
|
pub blocked_cidr: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct TrafficLimiterMetricsSnapshot {
|
||||||
|
pub user_throttle_up_total: u64,
|
||||||
|
pub user_throttle_down_total: u64,
|
||||||
|
pub cidr_throttle_up_total: u64,
|
||||||
|
pub cidr_throttle_down_total: u64,
|
||||||
|
pub user_wait_up_ms_total: u64,
|
||||||
|
pub user_wait_down_ms_total: u64,
|
||||||
|
pub cidr_wait_up_ms_total: u64,
|
||||||
|
pub cidr_wait_down_ms_total: u64,
|
||||||
|
pub user_active_leases: u64,
|
||||||
|
pub cidr_active_leases: u64,
|
||||||
|
pub user_policy_entries: u64,
|
||||||
|
pub cidr_policy_entries: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ScopeMetrics {
|
||||||
|
throttle_up_total: AtomicU64,
|
||||||
|
throttle_down_total: AtomicU64,
|
||||||
|
wait_up_ms_total: AtomicU64,
|
||||||
|
wait_down_ms_total: AtomicU64,
|
||||||
|
active_leases: AtomicU64,
|
||||||
|
policy_entries: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeMetrics {
|
||||||
|
fn throttle(&self, direction: RateDirection) {
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => {
|
||||||
|
self.throttle_up_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
RateDirection::Down => {
|
||||||
|
self.throttle_down_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_ms(&self, direction: RateDirection, wait_ms: u64) {
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => {
|
||||||
|
self.wait_up_ms_total.fetch_add(wait_ms, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
RateDirection::Down => {
|
||||||
|
self.wait_down_ms_total
|
||||||
|
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct AtomicRatePair {
|
||||||
|
up_bps: AtomicU64,
|
||||||
|
down_bps: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AtomicRatePair {
|
||||||
|
fn set(&self, limits: RateLimitBps) {
|
||||||
|
self.up_bps.store(limits.up_bps, Ordering::Relaxed);
|
||||||
|
self.down_bps.store(limits.down_bps, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get(&self, direction: RateDirection) -> u64 {
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => self.up_bps.load(Ordering::Relaxed),
|
||||||
|
RateDirection::Down => self.down_bps.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct DirectionBucket {
|
||||||
|
epoch: AtomicU64,
|
||||||
|
used: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DirectionBucket {
|
||||||
|
fn sync_epoch(&self, epoch: u64) {
|
||||||
|
let current = self.epoch.load(Ordering::Relaxed);
|
||||||
|
if current == epoch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if current < epoch
|
||||||
|
&& self
|
||||||
|
.epoch
|
||||||
|
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
self.used.store(0, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_consume(&self, cap_bps: u64, requested: u64) -> u64 {
|
||||||
|
if requested == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if cap_bps == 0 {
|
||||||
|
return requested;
|
||||||
|
}
|
||||||
|
|
||||||
|
let epoch = current_epoch();
|
||||||
|
self.sync_epoch(epoch);
|
||||||
|
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let used = self.used.load(Ordering::Relaxed);
|
||||||
|
if used >= cap_epoch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let remaining = cap_epoch.saturating_sub(used);
|
||||||
|
let grant = requested.min(remaining);
|
||||||
|
if grant == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let next = used.saturating_add(grant);
|
||||||
|
if self
|
||||||
|
.used
|
||||||
|
.compare_exchange_weak(used, next, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
return grant;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refund(&self, bytes: u64) {
|
||||||
|
if bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
decrement_atomic_saturating(&self.used, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct UserBucket {
|
||||||
|
rates: AtomicRatePair,
|
||||||
|
up: DirectionBucket,
|
||||||
|
down: DirectionBucket,
|
||||||
|
active_leases: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UserBucket {
|
||||||
|
fn new(limits: RateLimitBps) -> Self {
|
||||||
|
let rates = AtomicRatePair::default();
|
||||||
|
rates.set(limits);
|
||||||
|
Self {
|
||||||
|
rates,
|
||||||
|
up: DirectionBucket::default(),
|
||||||
|
down: DirectionBucket::default(),
|
||||||
|
active_leases: AtomicU64::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_rates(&self, limits: RateLimitBps) {
|
||||||
|
self.rates.set(limits);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_consume(&self, direction: RateDirection, requested: u64) -> u64 {
|
||||||
|
let cap_bps = self.rates.get(direction);
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => self.up.try_consume(cap_bps, requested),
|
||||||
|
RateDirection::Down => self.down.try_consume(cap_bps, requested),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => self.up.refund(bytes),
|
||||||
|
RateDirection::Down => self.down.refund(bytes),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct CidrDirectionBucket {
|
||||||
|
epoch: AtomicU64,
|
||||||
|
used: AtomicU64,
|
||||||
|
active_users: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CidrDirectionBucket {
|
||||||
|
fn sync_epoch(&self, epoch: u64) {
|
||||||
|
let current = self.epoch.load(Ordering::Relaxed);
|
||||||
|
if current == epoch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if current < epoch
|
||||||
|
&& self
|
||||||
|
.epoch
|
||||||
|
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
self.used.store(0, Ordering::Relaxed);
|
||||||
|
self.active_users.store(0, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_consume(
|
||||||
|
&self,
|
||||||
|
user_state: &CidrUserDirectionState,
|
||||||
|
cap_epoch: u64,
|
||||||
|
requested: u64,
|
||||||
|
) -> u64 {
|
||||||
|
if requested == 0 || cap_epoch == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let epoch = current_epoch();
|
||||||
|
self.sync_epoch(epoch);
|
||||||
|
user_state.sync_epoch_and_mark_active(epoch, &self.active_users);
|
||||||
|
let active_users = self.active_users.load(Ordering::Relaxed).max(1);
|
||||||
|
let fair_share = cap_epoch.saturating_div(active_users).max(1);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let total_used = self.used.load(Ordering::Relaxed);
|
||||||
|
if total_used >= cap_epoch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let total_remaining = cap_epoch.saturating_sub(total_used);
|
||||||
|
let user_used = user_state.used.load(Ordering::Relaxed);
|
||||||
|
let guaranteed_remaining = fair_share.saturating_sub(user_used);
|
||||||
|
|
||||||
|
let grant = if guaranteed_remaining > 0 {
|
||||||
|
requested.min(guaranteed_remaining).min(total_remaining)
|
||||||
|
} else {
|
||||||
|
requested.min(total_remaining).min(MAX_BORROW_CHUNK_BYTES)
|
||||||
|
};
|
||||||
|
|
||||||
|
if grant == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let next_total = total_used.saturating_add(grant);
|
||||||
|
if self
|
||||||
|
.used
|
||||||
|
.compare_exchange_weak(total_used, next_total, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
user_state.used.fetch_add(grant, Ordering::Relaxed);
|
||||||
|
return grant;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refund(&self, bytes: u64) {
|
||||||
|
if bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
decrement_atomic_saturating(&self.used, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct CidrUserDirectionState {
|
||||||
|
epoch: AtomicU64,
|
||||||
|
used: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CidrUserDirectionState {
|
||||||
|
fn sync_epoch_and_mark_active(&self, epoch: u64, active_users: &AtomicU64) {
|
||||||
|
let current = self.epoch.load(Ordering::Relaxed);
|
||||||
|
if current == epoch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if current < epoch
|
||||||
|
&& self
|
||||||
|
.epoch
|
||||||
|
.compare_exchange(current, epoch, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
self.used.store(0, Ordering::Relaxed);
|
||||||
|
active_users.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refund(&self, bytes: u64) {
|
||||||
|
if bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
decrement_atomic_saturating(&self.used, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CidrUserShare {
|
||||||
|
active_conns: AtomicU64,
|
||||||
|
up: CidrUserDirectionState,
|
||||||
|
down: CidrUserDirectionState,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CidrUserShare {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
active_conns: AtomicU64::new(0),
|
||||||
|
up: CidrUserDirectionState::default(),
|
||||||
|
down: CidrUserDirectionState::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CidrBucket {
|
||||||
|
rates: AtomicRatePair,
|
||||||
|
up: CidrDirectionBucket,
|
||||||
|
down: CidrDirectionBucket,
|
||||||
|
users: ShardedRegistry<CidrUserShare>,
|
||||||
|
active_leases: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CidrBucket {
|
||||||
|
fn new(limits: RateLimitBps) -> Self {
|
||||||
|
let rates = AtomicRatePair::default();
|
||||||
|
rates.set(limits);
|
||||||
|
Self {
|
||||||
|
rates,
|
||||||
|
up: CidrDirectionBucket::default(),
|
||||||
|
down: CidrDirectionBucket::default(),
|
||||||
|
users: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||||
|
active_leases: AtomicU64::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_rates(&self, limits: RateLimitBps) {
|
||||||
|
self.rates.set(limits);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn acquire_user_share(&self, user: &str) -> Arc<CidrUserShare> {
|
||||||
|
let share = self.users.get_or_insert_with(user, CidrUserShare::new);
|
||||||
|
share.active_conns.fetch_add(1, Ordering::Relaxed);
|
||||||
|
share
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_user_share(&self, user: &str, share: &Arc<CidrUserShare>) {
|
||||||
|
decrement_atomic_saturating(&share.active_conns, 1);
|
||||||
|
let share_for_remove = Arc::clone(share);
|
||||||
|
let _ = self.users.remove_if(user, |candidate| {
|
||||||
|
Arc::ptr_eq(candidate, &share_for_remove)
|
||||||
|
&& candidate.active_conns.load(Ordering::Relaxed) == 0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_consume_for_user(
|
||||||
|
&self,
|
||||||
|
direction: RateDirection,
|
||||||
|
share: &CidrUserShare,
|
||||||
|
requested: u64,
|
||||||
|
) -> u64 {
|
||||||
|
let cap_bps = self.rates.get(direction);
|
||||||
|
if cap_bps == 0 {
|
||||||
|
return requested;
|
||||||
|
}
|
||||||
|
let cap_epoch = bytes_per_epoch(cap_bps);
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => self.up.try_consume(&share.up, cap_epoch, requested),
|
||||||
|
RateDirection::Down => self.down.try_consume(&share.down, cap_epoch, requested),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refund_for_user(&self, direction: RateDirection, share: &CidrUserShare, bytes: u64) {
|
||||||
|
match direction {
|
||||||
|
RateDirection::Up => {
|
||||||
|
self.up.refund(bytes);
|
||||||
|
share.up.refund(bytes);
|
||||||
|
}
|
||||||
|
RateDirection::Down => {
|
||||||
|
self.down.refund(bytes);
|
||||||
|
share.down.refund(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_idle_users(&self) {
|
||||||
|
self.users
|
||||||
|
.retain(|_, share| share.active_conns.load(Ordering::Relaxed) > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct CidrRule {
|
||||||
|
key: String,
|
||||||
|
cidr: IpNetwork,
|
||||||
|
limits: RateLimitBps,
|
||||||
|
prefix_len: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct PolicySnapshot {
|
||||||
|
user_limits: HashMap<String, RateLimitBps>,
|
||||||
|
cidr_rules_v4: Vec<CidrRule>,
|
||||||
|
cidr_rules_v6: Vec<CidrRule>,
|
||||||
|
cidr_rule_keys: HashSet<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PolicySnapshot {
|
||||||
|
fn match_cidr(&self, ip: IpAddr) -> Option<&CidrRule> {
|
||||||
|
match ip {
|
||||||
|
IpAddr::V4(_) => self
|
||||||
|
.cidr_rules_v4
|
||||||
|
.iter()
|
||||||
|
.find(|rule| rule.cidr.contains(ip)),
|
||||||
|
IpAddr::V6(_) => self
|
||||||
|
.cidr_rules_v6
|
||||||
|
.iter()
|
||||||
|
.find(|rule| rule.cidr.contains(ip)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ShardedRegistry<T> {
|
||||||
|
shards: Box<[DashMap<String, Arc<T>>]>,
|
||||||
|
mask: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> ShardedRegistry<T> {
|
||||||
|
fn new(shards: usize) -> Self {
|
||||||
|
let shard_count = shards.max(1).next_power_of_two();
|
||||||
|
let mut items = Vec::with_capacity(shard_count);
|
||||||
|
for _ in 0..shard_count {
|
||||||
|
items.push(DashMap::<String, Arc<T>>::new());
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
shards: items.into_boxed_slice(),
|
||||||
|
mask: shard_count.saturating_sub(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn shard_index(&self, key: &str) -> usize {
|
||||||
|
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||||
|
key.hash(&mut hasher);
|
||||||
|
(hasher.finish() as usize) & self.mask
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_or_insert_with<F>(&self, key: &str, make: F) -> Arc<T>
|
||||||
|
where
|
||||||
|
F: FnOnce() -> T,
|
||||||
|
{
|
||||||
|
let shard = &self.shards[self.shard_index(key)];
|
||||||
|
match shard.entry(key.to_string()) {
|
||||||
|
dashmap::mapref::entry::Entry::Occupied(entry) => Arc::clone(entry.get()),
|
||||||
|
dashmap::mapref::entry::Entry::Vacant(slot) => {
|
||||||
|
let value = Arc::new(make());
|
||||||
|
slot.insert(Arc::clone(&value));
|
||||||
|
value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retain<F>(&self, predicate: F)
|
||||||
|
where
|
||||||
|
F: Fn(&String, &Arc<T>) -> bool + Copy,
|
||||||
|
{
|
||||||
|
for shard in &*self.shards {
|
||||||
|
shard.retain(|key, value| predicate(key, value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_if<F>(&self, key: &str, predicate: F) -> bool
|
||||||
|
where
|
||||||
|
F: Fn(&Arc<T>) -> bool,
|
||||||
|
{
|
||||||
|
let shard = &self.shards[self.shard_index(key)];
|
||||||
|
let should_remove = match shard.get(key) {
|
||||||
|
Some(entry) => predicate(entry.value()),
|
||||||
|
None => false,
|
||||||
|
};
|
||||||
|
if !should_remove {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
shard.remove(key).is_some()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct TrafficLease {
|
||||||
|
limiter: Arc<TrafficLimiter>,
|
||||||
|
user_bucket: Option<Arc<UserBucket>>,
|
||||||
|
cidr_bucket: Option<Arc<CidrBucket>>,
|
||||||
|
cidr_user_key: Option<String>,
|
||||||
|
cidr_user_share: Option<Arc<CidrUserShare>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TrafficLease {
|
||||||
|
pub fn try_consume(&self, direction: RateDirection, requested: u64) -> TrafficConsumeResult {
|
||||||
|
if requested == 0 {
|
||||||
|
return TrafficConsumeResult {
|
||||||
|
granted: 0,
|
||||||
|
blocked_user: false,
|
||||||
|
blocked_cidr: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut granted = requested;
|
||||||
|
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||||
|
let user_granted = user_bucket.try_consume(direction, granted);
|
||||||
|
if user_granted == 0 {
|
||||||
|
self.limiter.observe_throttle(direction, true, false);
|
||||||
|
return TrafficConsumeResult {
|
||||||
|
granted: 0,
|
||||||
|
blocked_user: true,
|
||||||
|
blocked_cidr: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
granted = user_granted;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||||
|
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||||
|
{
|
||||||
|
let cidr_granted =
|
||||||
|
cidr_bucket.try_consume_for_user(direction, cidr_user_share, granted);
|
||||||
|
if cidr_granted < granted
|
||||||
|
&& let Some(user_bucket) = self.user_bucket.as_ref()
|
||||||
|
{
|
||||||
|
user_bucket.refund(direction, granted.saturating_sub(cidr_granted));
|
||||||
|
}
|
||||||
|
if cidr_granted == 0 {
|
||||||
|
self.limiter.observe_throttle(direction, false, true);
|
||||||
|
return TrafficConsumeResult {
|
||||||
|
granted: 0,
|
||||||
|
blocked_user: false,
|
||||||
|
blocked_cidr: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
granted = cidr_granted;
|
||||||
|
}
|
||||||
|
|
||||||
|
TrafficConsumeResult {
|
||||||
|
granted,
|
||||||
|
blocked_user: false,
|
||||||
|
blocked_cidr: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn refund(&self, direction: RateDirection, bytes: u64) {
|
||||||
|
if bytes == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(user_bucket) = self.user_bucket.as_ref() {
|
||||||
|
user_bucket.refund(direction, bytes);
|
||||||
|
}
|
||||||
|
if let (Some(cidr_bucket), Some(cidr_user_share)) =
|
||||||
|
(self.cidr_bucket.as_ref(), self.cidr_user_share.as_ref())
|
||||||
|
{
|
||||||
|
cidr_bucket.refund_for_user(direction, cidr_user_share, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn observe_wait_ms(
|
||||||
|
&self,
|
||||||
|
direction: RateDirection,
|
||||||
|
blocked_user: bool,
|
||||||
|
blocked_cidr: bool,
|
||||||
|
wait_ms: u64,
|
||||||
|
) {
|
||||||
|
if wait_ms == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.limiter
|
||||||
|
.observe_wait(direction, blocked_user, blocked_cidr, wait_ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for TrafficLease {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(bucket) = self.user_bucket.as_ref() {
|
||||||
|
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||||
|
decrement_atomic_saturating(&self.limiter.user_scope.active_leases, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(bucket) = self.cidr_bucket.as_ref() {
|
||||||
|
if let (Some(user_key), Some(share)) =
|
||||||
|
(self.cidr_user_key.as_ref(), self.cidr_user_share.as_ref())
|
||||||
|
{
|
||||||
|
bucket.release_user_share(user_key, share);
|
||||||
|
}
|
||||||
|
decrement_atomic_saturating(&bucket.active_leases, 1);
|
||||||
|
decrement_atomic_saturating(&self.limiter.cidr_scope.active_leases, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct TrafficLimiter {
|
||||||
|
policy: ArcSwap<PolicySnapshot>,
|
||||||
|
user_buckets: ShardedRegistry<UserBucket>,
|
||||||
|
cidr_buckets: ShardedRegistry<CidrBucket>,
|
||||||
|
user_scope: ScopeMetrics,
|
||||||
|
cidr_scope: ScopeMetrics,
|
||||||
|
last_cleanup_epoch_secs: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TrafficLimiter {
|
||||||
|
pub fn new() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
policy: ArcSwap::from_pointee(PolicySnapshot::default()),
|
||||||
|
user_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||||
|
cidr_buckets: ShardedRegistry::new(REGISTRY_SHARDS),
|
||||||
|
user_scope: ScopeMetrics::default(),
|
||||||
|
cidr_scope: ScopeMetrics::default(),
|
||||||
|
last_cleanup_epoch_secs: AtomicU64::new(0),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn apply_policy(
|
||||||
|
&self,
|
||||||
|
user_limits: HashMap<String, RateLimitBps>,
|
||||||
|
cidr_limits: HashMap<IpNetwork, RateLimitBps>,
|
||||||
|
) {
|
||||||
|
let filtered_users = user_limits
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(_, limit)| limit.up_bps > 0 || limit.down_bps > 0)
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
|
||||||
|
let mut cidr_rules_v4 = Vec::new();
|
||||||
|
let mut cidr_rules_v6 = Vec::new();
|
||||||
|
let mut cidr_rule_keys = HashSet::new();
|
||||||
|
for (cidr, limits) in cidr_limits {
|
||||||
|
if limits.up_bps == 0 && limits.down_bps == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let key = cidr.to_string();
|
||||||
|
let rule = CidrRule {
|
||||||
|
key: key.clone(),
|
||||||
|
cidr,
|
||||||
|
limits,
|
||||||
|
prefix_len: cidr.prefix(),
|
||||||
|
};
|
||||||
|
cidr_rule_keys.insert(key);
|
||||||
|
match rule.cidr {
|
||||||
|
IpNetwork::V4(_) => cidr_rules_v4.push(rule),
|
||||||
|
IpNetwork::V6(_) => cidr_rules_v6.push(rule),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cidr_rules_v4.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||||
|
cidr_rules_v6.sort_by(|a, b| b.prefix_len.cmp(&a.prefix_len));
|
||||||
|
|
||||||
|
self.user_scope
|
||||||
|
.policy_entries
|
||||||
|
.store(filtered_users.len() as u64, Ordering::Relaxed);
|
||||||
|
self.cidr_scope
|
||||||
|
.policy_entries
|
||||||
|
.store(cidr_rule_keys.len() as u64, Ordering::Relaxed);
|
||||||
|
|
||||||
|
self.policy.store(Arc::new(PolicySnapshot {
|
||||||
|
user_limits: filtered_users,
|
||||||
|
cidr_rules_v4,
|
||||||
|
cidr_rules_v6,
|
||||||
|
cidr_rule_keys,
|
||||||
|
}));
|
||||||
|
|
||||||
|
self.maybe_cleanup();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn acquire_lease(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
user: &str,
|
||||||
|
client_ip: IpAddr,
|
||||||
|
) -> Option<Arc<TrafficLease>> {
|
||||||
|
let policy = self.policy.load_full();
|
||||||
|
let mut user_bucket = None;
|
||||||
|
if let Some(limit) = policy.user_limits.get(user).copied() {
|
||||||
|
let bucket = self
|
||||||
|
.user_buckets
|
||||||
|
.get_or_insert_with(user, || UserBucket::new(limit));
|
||||||
|
bucket.set_rates(limit);
|
||||||
|
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.user_scope
|
||||||
|
.active_leases
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
user_bucket = Some(bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut cidr_bucket = None;
|
||||||
|
let mut cidr_user_key = None;
|
||||||
|
let mut cidr_user_share = None;
|
||||||
|
if let Some(rule) = policy.match_cidr(client_ip) {
|
||||||
|
let bucket = self
|
||||||
|
.cidr_buckets
|
||||||
|
.get_or_insert_with(rule.key.as_str(), || CidrBucket::new(rule.limits));
|
||||||
|
bucket.set_rates(rule.limits);
|
||||||
|
bucket.active_leases.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.cidr_scope
|
||||||
|
.active_leases
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let share = bucket.acquire_user_share(user);
|
||||||
|
cidr_user_key = Some(user.to_string());
|
||||||
|
cidr_user_share = Some(share);
|
||||||
|
cidr_bucket = Some(bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
if user_bucket.is_none() && cidr_bucket.is_none() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.maybe_cleanup();
|
||||||
|
Some(Arc::new(TrafficLease {
|
||||||
|
limiter: Arc::clone(self),
|
||||||
|
user_bucket,
|
||||||
|
cidr_bucket,
|
||||||
|
cidr_user_key,
|
||||||
|
cidr_user_share,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn metrics_snapshot(&self) -> TrafficLimiterMetricsSnapshot {
|
||||||
|
TrafficLimiterMetricsSnapshot {
|
||||||
|
user_throttle_up_total: self.user_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||||
|
user_throttle_down_total: self.user_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||||
|
cidr_throttle_up_total: self.cidr_scope.throttle_up_total.load(Ordering::Relaxed),
|
||||||
|
cidr_throttle_down_total: self.cidr_scope.throttle_down_total.load(Ordering::Relaxed),
|
||||||
|
user_wait_up_ms_total: self.user_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||||
|
user_wait_down_ms_total: self.user_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||||
|
cidr_wait_up_ms_total: self.cidr_scope.wait_up_ms_total.load(Ordering::Relaxed),
|
||||||
|
cidr_wait_down_ms_total: self.cidr_scope.wait_down_ms_total.load(Ordering::Relaxed),
|
||||||
|
user_active_leases: self.user_scope.active_leases.load(Ordering::Relaxed),
|
||||||
|
cidr_active_leases: self.cidr_scope.active_leases.load(Ordering::Relaxed),
|
||||||
|
user_policy_entries: self.user_scope.policy_entries.load(Ordering::Relaxed),
|
||||||
|
cidr_policy_entries: self.cidr_scope.policy_entries.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_throttle(&self, direction: RateDirection, blocked_user: bool, blocked_cidr: bool) {
|
||||||
|
if blocked_user {
|
||||||
|
self.user_scope.throttle(direction);
|
||||||
|
}
|
||||||
|
if blocked_cidr {
|
||||||
|
self.cidr_scope.throttle(direction);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_wait(
|
||||||
|
&self,
|
||||||
|
direction: RateDirection,
|
||||||
|
blocked_user: bool,
|
||||||
|
blocked_cidr: bool,
|
||||||
|
wait_ms: u64,
|
||||||
|
) {
|
||||||
|
if blocked_user {
|
||||||
|
self.user_scope.wait_ms(direction, wait_ms);
|
||||||
|
}
|
||||||
|
if blocked_cidr {
|
||||||
|
self.cidr_scope.wait_ms(direction, wait_ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn maybe_cleanup(&self) {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let last = self.last_cleanup_epoch_secs.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_secs.saturating_sub(last) < CLEANUP_INTERVAL_SECS {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.last_cleanup_epoch_secs
|
||||||
|
.compare_exchange(last, now_epoch_secs, Ordering::Relaxed, Ordering::Relaxed)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy = self.policy.load_full();
|
||||||
|
self.user_buckets.retain(|user, bucket| {
|
||||||
|
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||||
|
|| policy.user_limits.contains_key(user)
|
||||||
|
});
|
||||||
|
self.cidr_buckets.retain(|cidr_key, bucket| {
|
||||||
|
bucket.cleanup_idle_users();
|
||||||
|
bucket.active_leases.load(Ordering::Relaxed) > 0
|
||||||
|
|| policy.cidr_rule_keys.contains(cidr_key)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn next_refill_delay() -> Duration {
|
||||||
|
let start = limiter_epoch_start();
|
||||||
|
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||||
|
let epoch_pos = elapsed_ms % FAIR_EPOCH_MS;
|
||||||
|
let wait_ms = FAIR_EPOCH_MS.saturating_sub(epoch_pos).max(1);
|
||||||
|
Duration::from_millis(wait_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decrement_atomic_saturating(counter: &AtomicU64, by: u64) {
|
||||||
|
if by == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut current = counter.load(Ordering::Relaxed);
|
||||||
|
loop {
|
||||||
|
if current == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let next = current.saturating_sub(by);
|
||||||
|
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||||
|
Ok(_) => return,
|
||||||
|
Err(actual) => current = actual,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bytes_per_epoch(bps: u64) -> u64 {
|
||||||
|
if bps == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let numerator = bps.saturating_mul(FAIR_EPOCH_MS);
|
||||||
|
let bytes = numerator.saturating_div(8_000);
|
||||||
|
bytes.max(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_epoch() -> u64 {
|
||||||
|
let start = limiter_epoch_start();
|
||||||
|
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||||
|
elapsed_ms / FAIR_EPOCH_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
fn limiter_epoch_start() -> &'static Instant {
|
||||||
|
static START: OnceLock<Instant> = OnceLock::new();
|
||||||
|
START.get_or_init(Instant::now)
|
||||||
|
}
|
||||||
@@ -175,6 +175,18 @@ pub struct Stats {
|
|||||||
me_route_drop_queue_full: AtomicU64,
|
me_route_drop_queue_full: AtomicU64,
|
||||||
me_route_drop_queue_full_base: AtomicU64,
|
me_route_drop_queue_full_base: AtomicU64,
|
||||||
me_route_drop_queue_full_high: AtomicU64,
|
me_route_drop_queue_full_high: AtomicU64,
|
||||||
|
me_fair_pressure_state_gauge: AtomicU64,
|
||||||
|
me_fair_active_flows_gauge: AtomicU64,
|
||||||
|
me_fair_queued_bytes_gauge: AtomicU64,
|
||||||
|
me_fair_standing_flows_gauge: AtomicU64,
|
||||||
|
me_fair_backpressured_flows_gauge: AtomicU64,
|
||||||
|
me_fair_scheduler_rounds_total: AtomicU64,
|
||||||
|
me_fair_deficit_grants_total: AtomicU64,
|
||||||
|
me_fair_deficit_skips_total: AtomicU64,
|
||||||
|
me_fair_enqueue_rejects_total: AtomicU64,
|
||||||
|
me_fair_shed_drops_total: AtomicU64,
|
||||||
|
me_fair_penalties_total: AtomicU64,
|
||||||
|
me_fair_downstream_stalls_total: AtomicU64,
|
||||||
me_d2c_batches_total: AtomicU64,
|
me_d2c_batches_total: AtomicU64,
|
||||||
me_d2c_batch_frames_total: AtomicU64,
|
me_d2c_batch_frames_total: AtomicU64,
|
||||||
me_d2c_batch_bytes_total: AtomicU64,
|
me_d2c_batch_bytes_total: AtomicU64,
|
||||||
@@ -856,6 +868,78 @@ impl Stats {
|
|||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn set_me_fair_pressure_state_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_fair_pressure_state_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_fair_active_flows_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_fair_active_flows_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_fair_queued_bytes_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_fair_queued_bytes_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_fair_standing_flows_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_fair_standing_flows_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_fair_backpressured_flows_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_fair_backpressured_flows_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_scheduler_rounds_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_scheduler_rounds_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_deficit_grants_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_deficit_grants_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_deficit_skips_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_deficit_skips_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_enqueue_rejects_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_enqueue_rejects_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_shed_drops_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_shed_drops_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_penalties_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_penalties_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn add_me_fair_downstream_stalls_total(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() && value > 0 {
|
||||||
|
self.me_fair_downstream_stalls_total
|
||||||
|
.fetch_add(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
pub fn increment_me_d2c_batches_total(&self) {
|
pub fn increment_me_d2c_batches_total(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_d2c_batches_total.fetch_add(1, Ordering::Relaxed);
|
self.me_d2c_batches_total.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -1806,6 +1890,43 @@ impl Stats {
|
|||||||
pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
|
pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
|
||||||
self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
|
self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
pub fn get_me_fair_pressure_state_gauge(&self) -> u64 {
|
||||||
|
self.me_fair_pressure_state_gauge.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_active_flows_gauge(&self) -> u64 {
|
||||||
|
self.me_fair_active_flows_gauge.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_queued_bytes_gauge(&self) -> u64 {
|
||||||
|
self.me_fair_queued_bytes_gauge.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_standing_flows_gauge(&self) -> u64 {
|
||||||
|
self.me_fair_standing_flows_gauge.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_backpressured_flows_gauge(&self) -> u64 {
|
||||||
|
self.me_fair_backpressured_flows_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_scheduler_rounds_total(&self) -> u64 {
|
||||||
|
self.me_fair_scheduler_rounds_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_deficit_grants_total(&self) -> u64 {
|
||||||
|
self.me_fair_deficit_grants_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_deficit_skips_total(&self) -> u64 {
|
||||||
|
self.me_fair_deficit_skips_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_enqueue_rejects_total(&self) -> u64 {
|
||||||
|
self.me_fair_enqueue_rejects_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_shed_drops_total(&self) -> u64 {
|
||||||
|
self.me_fair_shed_drops_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_penalties_total(&self) -> u64 {
|
||||||
|
self.me_fair_penalties_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_fair_downstream_stalls_total(&self) -> u64 {
|
||||||
|
self.me_fair_downstream_stalls_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
pub fn get_me_d2c_batches_total(&self) -> u64 {
|
pub fn get_me_d2c_batches_total(&self) -> u64 {
|
||||||
self.me_d2c_batches_total.load(Ordering::Relaxed)
|
self.me_d2c_batches_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|||||||
+93
-41
@@ -11,6 +11,7 @@ use crc32fast::Hasher;
|
|||||||
|
|
||||||
const MIN_APP_DATA: usize = 64;
|
const MIN_APP_DATA: usize = 64;
|
||||||
const MAX_APP_DATA: usize = MAX_TLS_CIPHERTEXT_SIZE;
|
const MAX_APP_DATA: usize = MAX_TLS_CIPHERTEXT_SIZE;
|
||||||
|
const MAX_TICKET_RECORDS: usize = 4;
|
||||||
|
|
||||||
fn jitter_and_clamp_sizes(sizes: &[usize], rng: &SecureRandom) -> Vec<usize> {
|
fn jitter_and_clamp_sizes(sizes: &[usize], rng: &SecureRandom) -> Vec<usize> {
|
||||||
sizes
|
sizes
|
||||||
@@ -62,6 +63,64 @@ fn ensure_payload_capacity(mut sizes: Vec<usize>, payload_len: usize) -> Vec<usi
|
|||||||
sizes
|
sizes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn emulated_app_data_sizes(cached: &CachedTlsData) -> Vec<usize> {
|
||||||
|
match cached.behavior_profile.source {
|
||||||
|
TlsProfileSource::Raw | TlsProfileSource::Merged => {
|
||||||
|
return cached
|
||||||
|
.app_data_records_sizes
|
||||||
|
.first()
|
||||||
|
.copied()
|
||||||
|
.or_else(|| {
|
||||||
|
cached
|
||||||
|
.behavior_profile
|
||||||
|
.app_data_record_sizes
|
||||||
|
.first()
|
||||||
|
.copied()
|
||||||
|
})
|
||||||
|
.map(|size| vec![size])
|
||||||
|
.unwrap_or_else(|| vec![cached.total_app_data_len.max(1024)]);
|
||||||
|
}
|
||||||
|
TlsProfileSource::Default | TlsProfileSource::Rustls => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sizes = cached.app_data_records_sizes.clone();
|
||||||
|
if sizes.is_empty() {
|
||||||
|
sizes.push(cached.total_app_data_len.max(1024));
|
||||||
|
}
|
||||||
|
sizes
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emulated_change_cipher_spec_count(_cached: &CachedTlsData) -> usize {
|
||||||
|
1
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emulated_ticket_record_sizes(
|
||||||
|
cached: &CachedTlsData,
|
||||||
|
new_session_tickets: u8,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
) -> Vec<usize> {
|
||||||
|
let target_count = usize::from(new_session_tickets.min(MAX_TICKET_RECORDS as u8));
|
||||||
|
if target_count == 0 {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let profiled_sizes = match cached.behavior_profile.source {
|
||||||
|
TlsProfileSource::Raw | TlsProfileSource::Merged => {
|
||||||
|
cached.behavior_profile.ticket_record_sizes.as_slice()
|
||||||
|
}
|
||||||
|
TlsProfileSource::Default | TlsProfileSource::Rustls => &[],
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut sizes = Vec::with_capacity(target_count);
|
||||||
|
sizes.extend(profiled_sizes.iter().copied().take(target_count));
|
||||||
|
|
||||||
|
while sizes.len() < target_count {
|
||||||
|
sizes.push(rng.range(48) + 48);
|
||||||
|
}
|
||||||
|
|
||||||
|
sizes
|
||||||
|
}
|
||||||
|
|
||||||
fn build_compact_cert_info_payload(cert_info: &ParsedCertificateInfo) -> Option<Vec<u8>> {
|
fn build_compact_cert_info_payload(cert_info: &ParsedCertificateInfo) -> Option<Vec<u8>> {
|
||||||
let mut fields = Vec::new();
|
let mut fields = Vec::new();
|
||||||
|
|
||||||
@@ -180,39 +239,32 @@ pub fn build_emulated_server_hello(
|
|||||||
server_hello.extend_from_slice(&message);
|
server_hello.extend_from_slice(&message);
|
||||||
|
|
||||||
// --- ChangeCipherSpec ---
|
// --- ChangeCipherSpec ---
|
||||||
let change_cipher_spec = [
|
let change_cipher_spec_count = emulated_change_cipher_spec_count(cached);
|
||||||
TLS_RECORD_CHANGE_CIPHER,
|
let mut change_cipher_spec = Vec::with_capacity(change_cipher_spec_count * 6);
|
||||||
TLS_VERSION[0],
|
for _ in 0..change_cipher_spec_count {
|
||||||
TLS_VERSION[1],
|
change_cipher_spec.extend_from_slice(&[
|
||||||
0x00,
|
TLS_RECORD_CHANGE_CIPHER,
|
||||||
0x01,
|
TLS_VERSION[0],
|
||||||
0x01,
|
TLS_VERSION[1],
|
||||||
];
|
0x00,
|
||||||
|
0x01,
|
||||||
|
0x01,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
// --- ApplicationData (fake encrypted records) ---
|
// --- ApplicationData (fake encrypted records) ---
|
||||||
let sizes = match cached.behavior_profile.source {
|
let mut sizes = {
|
||||||
TlsProfileSource::Raw | TlsProfileSource::Merged => cached
|
let base_sizes = emulated_app_data_sizes(cached);
|
||||||
.app_data_records_sizes
|
match cached.behavior_profile.source {
|
||||||
.first()
|
TlsProfileSource::Raw | TlsProfileSource::Merged => base_sizes
|
||||||
.copied()
|
.into_iter()
|
||||||
.or_else(|| {
|
.map(|size| size.clamp(MIN_APP_DATA, MAX_APP_DATA))
|
||||||
cached
|
.collect(),
|
||||||
.behavior_profile
|
TlsProfileSource::Default | TlsProfileSource::Rustls => {
|
||||||
.app_data_record_sizes
|
jitter_and_clamp_sizes(&base_sizes, rng)
|
||||||
.first()
|
|
||||||
.copied()
|
|
||||||
})
|
|
||||||
.map(|size| vec![size])
|
|
||||||
.unwrap_or_else(|| vec![cached.total_app_data_len.max(1024)]),
|
|
||||||
_ => {
|
|
||||||
let mut sizes = cached.app_data_records_sizes.clone();
|
|
||||||
if sizes.is_empty() {
|
|
||||||
sizes.push(cached.total_app_data_len.max(1024));
|
|
||||||
}
|
}
|
||||||
sizes
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let mut sizes = jitter_and_clamp_sizes(&sizes, rng);
|
|
||||||
let compact_payload = cached
|
let compact_payload = cached
|
||||||
.cert_info
|
.cert_info
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -299,17 +351,13 @@ pub fn build_emulated_server_hello(
|
|||||||
// --- Combine ---
|
// --- Combine ---
|
||||||
// Optional NewSessionTicket mimic records (opaque ApplicationData for fingerprint).
|
// Optional NewSessionTicket mimic records (opaque ApplicationData for fingerprint).
|
||||||
let mut tickets = Vec::new();
|
let mut tickets = Vec::new();
|
||||||
let ticket_count = new_session_tickets.min(4);
|
for ticket_len in emulated_ticket_record_sizes(cached, new_session_tickets, rng) {
|
||||||
if ticket_count > 0 {
|
let mut rec = Vec::with_capacity(5 + ticket_len);
|
||||||
for _ in 0..ticket_count {
|
rec.push(TLS_RECORD_APPLICATION);
|
||||||
let ticket_len: usize = rng.range(48) + 48;
|
rec.extend_from_slice(&TLS_VERSION);
|
||||||
let mut rec = Vec::with_capacity(5 + ticket_len);
|
rec.extend_from_slice(&(ticket_len as u16).to_be_bytes());
|
||||||
rec.push(TLS_RECORD_APPLICATION);
|
rec.extend_from_slice(&rng.bytes(ticket_len));
|
||||||
rec.extend_from_slice(&TLS_VERSION);
|
tickets.extend_from_slice(&rec);
|
||||||
rec.extend_from_slice(&(ticket_len as u16).to_be_bytes());
|
|
||||||
rec.extend_from_slice(&rng.bytes(ticket_len));
|
|
||||||
tickets.extend_from_slice(&rec);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut response = Vec::with_capacity(
|
let mut response = Vec::with_capacity(
|
||||||
@@ -334,6 +382,10 @@ pub fn build_emulated_server_hello(
|
|||||||
#[path = "tests/emulator_security_tests.rs"]
|
#[path = "tests/emulator_security_tests.rs"]
|
||||||
mod security_tests;
|
mod security_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "tests/emulator_profile_fidelity_security_tests.rs"]
|
||||||
|
mod emulator_profile_fidelity_security_tests;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
@@ -478,7 +530,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_build_emulated_server_hello_ignores_tail_records_for_raw_profile() {
|
fn test_build_emulated_server_hello_ignores_tail_records_for_profiled_tls() {
|
||||||
let mut cached = make_cached(None);
|
let mut cached = make_cached(None);
|
||||||
cached.app_data_records_sizes = vec![27, 3905, 537, 69];
|
cached.app_data_records_sizes = vec![27, 3905, 537, 69];
|
||||||
cached.total_app_data_len = 4538;
|
cached.total_app_data_len = 4538;
|
||||||
@@ -503,8 +555,8 @@ mod tests {
|
|||||||
let app_start = ccs_start + 6;
|
let app_start = ccs_start + 6;
|
||||||
let app_len =
|
let app_len =
|
||||||
u16::from_be_bytes([response[app_start + 3], response[app_start + 4]]) as usize;
|
u16::from_be_bytes([response[app_start + 3], response[app_start + 4]]) as usize;
|
||||||
|
|
||||||
assert_eq!(response[app_start], TLS_RECORD_APPLICATION);
|
assert_eq!(response[app_start], TLS_RECORD_APPLICATION);
|
||||||
|
assert_eq!(app_len, 64);
|
||||||
assert_eq!(app_start + 5 + app_len, response.len());
|
assert_eq!(app_start + 5 + app_len, response.len());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+108
-18
@@ -1,6 +1,7 @@
|
|||||||
#![allow(clippy::too_many_arguments)]
|
#![allow(clippy::too_many_arguments)]
|
||||||
|
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
@@ -793,6 +794,51 @@ async fn connect_tcp_with_upstream(
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn socket_addrs_from_upstream_stream(
|
||||||
|
stream: &UpstreamStream,
|
||||||
|
) -> (Option<SocketAddr>, Option<SocketAddr>) {
|
||||||
|
match stream {
|
||||||
|
UpstreamStream::Tcp(tcp) => (tcp.local_addr().ok(), tcp.peer_addr().ok()),
|
||||||
|
UpstreamStream::Shadowsocks(_) => (None, None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_tls_fetch_proxy_header(
|
||||||
|
proxy_protocol: u8,
|
||||||
|
src_addr: Option<SocketAddr>,
|
||||||
|
dst_addr: Option<SocketAddr>,
|
||||||
|
) -> Option<Vec<u8>> {
|
||||||
|
match proxy_protocol {
|
||||||
|
0 => None,
|
||||||
|
2 => {
|
||||||
|
let header = match (src_addr, dst_addr) {
|
||||||
|
(Some(src @ SocketAddr::V4(_)), Some(dst @ SocketAddr::V4(_)))
|
||||||
|
| (Some(src @ SocketAddr::V6(_)), Some(dst @ SocketAddr::V6(_))) => {
|
||||||
|
ProxyProtocolV2Builder::new().with_addrs(src, dst).build()
|
||||||
|
}
|
||||||
|
_ => ProxyProtocolV2Builder::new().build(),
|
||||||
|
};
|
||||||
|
Some(header)
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
let header = match (src_addr, dst_addr) {
|
||||||
|
(Some(SocketAddr::V4(src)), Some(SocketAddr::V4(dst))) => {
|
||||||
|
ProxyProtocolV1Builder::new()
|
||||||
|
.tcp4(src.into(), dst.into())
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
(Some(SocketAddr::V6(src)), Some(SocketAddr::V6(dst))) => {
|
||||||
|
ProxyProtocolV1Builder::new()
|
||||||
|
.tcp6(src.into(), dst.into())
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
_ => ProxyProtocolV1Builder::new().build(),
|
||||||
|
};
|
||||||
|
Some(header)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn encode_tls13_certificate_message(cert_chain_der: &[Vec<u8>]) -> Option<Vec<u8>> {
|
fn encode_tls13_certificate_message(cert_chain_der: &[Vec<u8>]) -> Option<Vec<u8>> {
|
||||||
if cert_chain_der.is_empty() {
|
if cert_chain_der.is_empty() {
|
||||||
return None;
|
return None;
|
||||||
@@ -824,7 +870,7 @@ async fn fetch_via_raw_tls_stream<S>(
|
|||||||
mut stream: S,
|
mut stream: S,
|
||||||
sni: &str,
|
sni: &str,
|
||||||
connect_timeout: Duration,
|
connect_timeout: Duration,
|
||||||
proxy_protocol: u8,
|
proxy_header: Option<Vec<u8>>,
|
||||||
profile: TlsFetchProfile,
|
profile: TlsFetchProfile,
|
||||||
grease_enabled: bool,
|
grease_enabled: bool,
|
||||||
deterministic: bool,
|
deterministic: bool,
|
||||||
@@ -835,11 +881,7 @@ where
|
|||||||
let rng = SecureRandom::new();
|
let rng = SecureRandom::new();
|
||||||
let client_hello = build_client_hello(sni, &rng, profile, grease_enabled, deterministic);
|
let client_hello = build_client_hello(sni, &rng, profile, grease_enabled, deterministic);
|
||||||
timeout(connect_timeout, async {
|
timeout(connect_timeout, async {
|
||||||
if proxy_protocol > 0 {
|
if let Some(header) = proxy_header.as_ref() {
|
||||||
let header = match proxy_protocol {
|
|
||||||
2 => ProxyProtocolV2Builder::new().build(),
|
|
||||||
_ => ProxyProtocolV1Builder::new().build(),
|
|
||||||
};
|
|
||||||
stream.write_all(&header).await?;
|
stream.write_all(&header).await?;
|
||||||
}
|
}
|
||||||
stream.write_all(&client_hello).await?;
|
stream.write_all(&client_hello).await?;
|
||||||
@@ -921,11 +963,12 @@ async fn fetch_via_raw_tls(
|
|||||||
sock = %sock_path,
|
sock = %sock_path,
|
||||||
"Raw TLS fetch using mask unix socket"
|
"Raw TLS fetch using mask unix socket"
|
||||||
);
|
);
|
||||||
|
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, None, None);
|
||||||
return fetch_via_raw_tls_stream(
|
return fetch_via_raw_tls_stream(
|
||||||
stream,
|
stream,
|
||||||
sni,
|
sni,
|
||||||
connect_timeout,
|
connect_timeout,
|
||||||
proxy_protocol,
|
proxy_header,
|
||||||
profile,
|
profile,
|
||||||
grease_enabled,
|
grease_enabled,
|
||||||
deterministic,
|
deterministic,
|
||||||
@@ -956,11 +999,13 @@ async fn fetch_via_raw_tls(
|
|||||||
let stream =
|
let stream =
|
||||||
connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope, strict_route)
|
connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope, strict_route)
|
||||||
.await?;
|
.await?;
|
||||||
|
let (src_addr, dst_addr) = socket_addrs_from_upstream_stream(&stream);
|
||||||
|
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, src_addr, dst_addr);
|
||||||
fetch_via_raw_tls_stream(
|
fetch_via_raw_tls_stream(
|
||||||
stream,
|
stream,
|
||||||
sni,
|
sni,
|
||||||
connect_timeout,
|
connect_timeout,
|
||||||
proxy_protocol,
|
proxy_header,
|
||||||
profile,
|
profile,
|
||||||
grease_enabled,
|
grease_enabled,
|
||||||
deterministic,
|
deterministic,
|
||||||
@@ -972,17 +1017,13 @@ async fn fetch_via_rustls_stream<S>(
|
|||||||
mut stream: S,
|
mut stream: S,
|
||||||
host: &str,
|
host: &str,
|
||||||
sni: &str,
|
sni: &str,
|
||||||
proxy_protocol: u8,
|
proxy_header: Option<Vec<u8>>,
|
||||||
) -> Result<TlsFetchResult>
|
) -> Result<TlsFetchResult>
|
||||||
where
|
where
|
||||||
S: AsyncRead + AsyncWrite + Unpin,
|
S: AsyncRead + AsyncWrite + Unpin,
|
||||||
{
|
{
|
||||||
// rustls handshake path for certificate and basic negotiated metadata.
|
// rustls handshake path for certificate and basic negotiated metadata.
|
||||||
if proxy_protocol > 0 {
|
if let Some(header) = proxy_header.as_ref() {
|
||||||
let header = match proxy_protocol {
|
|
||||||
2 => ProxyProtocolV2Builder::new().build(),
|
|
||||||
_ => ProxyProtocolV1Builder::new().build(),
|
|
||||||
};
|
|
||||||
stream.write_all(&header).await?;
|
stream.write_all(&header).await?;
|
||||||
stream.flush().await?;
|
stream.flush().await?;
|
||||||
}
|
}
|
||||||
@@ -1082,7 +1123,8 @@ async fn fetch_via_rustls(
|
|||||||
sock = %sock_path,
|
sock = %sock_path,
|
||||||
"Rustls fetch using mask unix socket"
|
"Rustls fetch using mask unix socket"
|
||||||
);
|
);
|
||||||
return fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await;
|
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, None, None);
|
||||||
|
return fetch_via_rustls_stream(stream, host, sni, proxy_header).await;
|
||||||
}
|
}
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -1108,7 +1150,9 @@ async fn fetch_via_rustls(
|
|||||||
let stream =
|
let stream =
|
||||||
connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope, strict_route)
|
connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope, strict_route)
|
||||||
.await?;
|
.await?;
|
||||||
fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await
|
let (src_addr, dst_addr) = socket_addrs_from_upstream_stream(&stream);
|
||||||
|
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, src_addr, dst_addr);
|
||||||
|
fetch_via_rustls_stream(stream, host, sni, proxy_header).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch real TLS metadata with an adaptive multi-profile strategy.
|
/// Fetch real TLS metadata with an adaptive multi-profile strategy.
|
||||||
@@ -1278,11 +1322,13 @@ pub async fn fetch_real_tls(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::net::SocketAddr;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
ProfileCacheValue, TlsFetchStrategy, build_client_hello, derive_behavior_profile,
|
ProfileCacheValue, TlsFetchStrategy, build_client_hello, build_tls_fetch_proxy_header,
|
||||||
encode_tls13_certificate_message, order_profiles, profile_cache, profile_cache_key,
|
derive_behavior_profile, encode_tls13_certificate_message, order_profiles, profile_cache,
|
||||||
|
profile_cache_key,
|
||||||
};
|
};
|
||||||
use crate::config::TlsFetchProfile;
|
use crate::config::TlsFetchProfile;
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
@@ -1423,4 +1469,48 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(first, second);
|
assert_eq!(first, second);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_tls_fetch_proxy_header_v2_with_tcp_addrs() {
|
||||||
|
let src: SocketAddr = "198.51.100.10:42000".parse().expect("valid src");
|
||||||
|
let dst: SocketAddr = "203.0.113.20:443".parse().expect("valid dst");
|
||||||
|
let header = build_tls_fetch_proxy_header(2, Some(src), Some(dst)).expect("header");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
&header[..12],
|
||||||
|
&[
|
||||||
|
0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(header[12], 0x21);
|
||||||
|
assert_eq!(header[13], 0x11);
|
||||||
|
assert_eq!(u16::from_be_bytes([header[14], header[15]]), 12);
|
||||||
|
assert_eq!(&header[16..20], &[198, 51, 100, 10]);
|
||||||
|
assert_eq!(&header[20..24], &[203, 0, 113, 20]);
|
||||||
|
assert_eq!(u16::from_be_bytes([header[24], header[25]]), 42000);
|
||||||
|
assert_eq!(u16::from_be_bytes([header[26], header[27]]), 443);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_tls_fetch_proxy_header_v2_mixed_family_falls_back_to_local_command() {
|
||||||
|
let src: SocketAddr = "198.51.100.10:42000".parse().expect("valid src");
|
||||||
|
let dst: SocketAddr = "[2001:db8::20]:443".parse().expect("valid dst");
|
||||||
|
let header = build_tls_fetch_proxy_header(2, Some(src), Some(dst)).expect("header");
|
||||||
|
|
||||||
|
assert_eq!(header[12], 0x20);
|
||||||
|
assert_eq!(header[13], 0x00);
|
||||||
|
assert_eq!(u16::from_be_bytes([header[14], header[15]]), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_tls_fetch_proxy_header_v1_with_tcp_addrs() {
|
||||||
|
let src: SocketAddr = "198.51.100.10:42000".parse().expect("valid src");
|
||||||
|
let dst: SocketAddr = "203.0.113.20:443".parse().expect("valid dst");
|
||||||
|
let header = build_tls_fetch_proxy_header(1, Some(src), Some(dst)).expect("header");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
header,
|
||||||
|
b"PROXY TCP4 198.51.100.10 203.0.113.20 42000 443\r\n"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
use crate::crypto::SecureRandom;
|
||||||
|
use crate::protocol::constants::{
|
||||||
|
TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE,
|
||||||
|
};
|
||||||
|
use crate::tls_front::emulator::build_emulated_server_hello;
|
||||||
|
use crate::tls_front::types::{
|
||||||
|
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsProfileSource,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn make_cached() -> CachedTlsData {
|
||||||
|
CachedTlsData {
|
||||||
|
server_hello_template: ParsedServerHello {
|
||||||
|
version: [0x03, 0x03],
|
||||||
|
random: [0u8; 32],
|
||||||
|
session_id: Vec::new(),
|
||||||
|
cipher_suite: [0x13, 0x01],
|
||||||
|
compression: 0,
|
||||||
|
extensions: Vec::new(),
|
||||||
|
},
|
||||||
|
cert_info: None,
|
||||||
|
cert_payload: None,
|
||||||
|
app_data_records_sizes: vec![1200, 900, 220, 180],
|
||||||
|
total_app_data_len: 2500,
|
||||||
|
behavior_profile: TlsBehaviorProfile {
|
||||||
|
change_cipher_spec_count: 2,
|
||||||
|
app_data_record_sizes: vec![1200, 900],
|
||||||
|
ticket_record_sizes: vec![220, 180],
|
||||||
|
source: TlsProfileSource::Merged,
|
||||||
|
},
|
||||||
|
fetched_at: SystemTime::now(),
|
||||||
|
domain: "example.com".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_lengths_by_type(response: &[u8], wanted_type: u8) -> Vec<usize> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut pos = 0usize;
|
||||||
|
while pos + 5 <= response.len() {
|
||||||
|
let record_type = response[pos];
|
||||||
|
let record_len = u16::from_be_bytes([response[pos + 3], response[pos + 4]]) as usize;
|
||||||
|
if pos + 5 + record_len > response.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if record_type == wanted_type {
|
||||||
|
out.push(record_len);
|
||||||
|
}
|
||||||
|
pos += 5 + record_len;
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn emulated_server_hello_keeps_single_change_cipher_spec_for_client_compatibility() {
|
||||||
|
let cached = make_cached();
|
||||||
|
let rng = SecureRandom::new();
|
||||||
|
|
||||||
|
let response = build_emulated_server_hello(
|
||||||
|
b"secret",
|
||||||
|
&[0x71; 32],
|
||||||
|
&[0x72; 16],
|
||||||
|
&cached,
|
||||||
|
false,
|
||||||
|
&rng,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(response[0], TLS_RECORD_HANDSHAKE);
|
||||||
|
let ccs_records = record_lengths_by_type(&response, TLS_RECORD_CHANGE_CIPHER);
|
||||||
|
assert_eq!(ccs_records.len(), 1);
|
||||||
|
assert!(ccs_records.iter().all(|len| *len == 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn emulated_server_hello_does_not_emit_profile_ticket_tail_when_disabled() {
|
||||||
|
let cached = make_cached();
|
||||||
|
let rng = SecureRandom::new();
|
||||||
|
|
||||||
|
let response = build_emulated_server_hello(
|
||||||
|
b"secret",
|
||||||
|
&[0x81; 32],
|
||||||
|
&[0x82; 16],
|
||||||
|
&cached,
|
||||||
|
false,
|
||||||
|
&rng,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
|
||||||
|
assert_eq!(app_records, vec![1200]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn emulated_server_hello_uses_profile_ticket_lengths_when_enabled() {
|
||||||
|
let cached = make_cached();
|
||||||
|
let rng = SecureRandom::new();
|
||||||
|
|
||||||
|
let response = build_emulated_server_hello(
|
||||||
|
b"secret",
|
||||||
|
&[0x91; 32],
|
||||||
|
&[0x92; 16],
|
||||||
|
&cached,
|
||||||
|
false,
|
||||||
|
&rng,
|
||||||
|
None,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
|
||||||
|
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
|
||||||
|
assert_eq!(app_records, vec![1200, 220, 180]);
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
//! Backpressure-driven fairness control for ME reader routing.
|
||||||
|
//!
|
||||||
|
//! This module keeps fairness decisions worker-local:
|
||||||
|
//! each reader loop owns one scheduler instance and mutates it without locks.
|
||||||
|
|
||||||
|
mod model;
|
||||||
|
mod pressure;
|
||||||
|
mod scheduler;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use model::PressureState;
|
||||||
|
pub(crate) use model::{AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision};
|
||||||
|
pub(crate) use scheduler::{WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState};
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
|
#[repr(u8)]
|
||||||
|
pub(crate) enum PressureState {
|
||||||
|
Normal = 0,
|
||||||
|
Pressured = 1,
|
||||||
|
Shedding = 2,
|
||||||
|
Saturated = 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PressureState {
|
||||||
|
pub(crate) fn as_u8(self) -> u8 {
|
||||||
|
self as u8
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PressureState {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::Normal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum FlowPressureClass {
|
||||||
|
Healthy,
|
||||||
|
Bursty,
|
||||||
|
Backpressured,
|
||||||
|
Standing,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum StandingQueueState {
|
||||||
|
Transient,
|
||||||
|
Standing,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum FlowSchedulerState {
|
||||||
|
Idle,
|
||||||
|
Active,
|
||||||
|
Backpressured,
|
||||||
|
Penalized,
|
||||||
|
SheddingCandidate,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct QueuedFrame {
|
||||||
|
pub(crate) conn_id: u64,
|
||||||
|
pub(crate) flags: u32,
|
||||||
|
pub(crate) data: Bytes,
|
||||||
|
pub(crate) enqueued_at: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl QueuedFrame {
|
||||||
|
#[inline]
|
||||||
|
pub(crate) fn queued_bytes(&self) -> u64 {
|
||||||
|
self.data.len() as u64
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct FlowFairnessState {
|
||||||
|
pub(crate) _flow_id: u64,
|
||||||
|
pub(crate) _worker_id: u16,
|
||||||
|
pub(crate) pending_bytes: u64,
|
||||||
|
pub(crate) deficit_bytes: i64,
|
||||||
|
pub(crate) queue_started_at: Option<Instant>,
|
||||||
|
pub(crate) last_drain_at: Option<Instant>,
|
||||||
|
pub(crate) recent_drain_bytes: u64,
|
||||||
|
pub(crate) consecutive_stalls: u8,
|
||||||
|
pub(crate) consecutive_skips: u8,
|
||||||
|
pub(crate) penalty_score: u16,
|
||||||
|
pub(crate) pressure_class: FlowPressureClass,
|
||||||
|
pub(crate) standing_state: StandingQueueState,
|
||||||
|
pub(crate) scheduler_state: FlowSchedulerState,
|
||||||
|
pub(crate) bucket_id: usize,
|
||||||
|
pub(crate) in_active_ring: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FlowFairnessState {
|
||||||
|
pub(crate) fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
_flow_id: flow_id,
|
||||||
|
_worker_id: worker_id,
|
||||||
|
pending_bytes: 0,
|
||||||
|
deficit_bytes: 0,
|
||||||
|
queue_started_at: None,
|
||||||
|
last_drain_at: None,
|
||||||
|
recent_drain_bytes: 0,
|
||||||
|
consecutive_stalls: 0,
|
||||||
|
consecutive_skips: 0,
|
||||||
|
penalty_score: 0,
|
||||||
|
pressure_class: FlowPressureClass::Healthy,
|
||||||
|
standing_state: StandingQueueState::Transient,
|
||||||
|
scheduler_state: FlowSchedulerState::Idle,
|
||||||
|
bucket_id,
|
||||||
|
in_active_ring: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum AdmissionDecision {
|
||||||
|
Admit,
|
||||||
|
RejectWorkerCap,
|
||||||
|
RejectFlowCap,
|
||||||
|
RejectBucketCap,
|
||||||
|
RejectSaturated,
|
||||||
|
RejectStandingFlow,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) enum SchedulerDecision {
|
||||||
|
Idle,
|
||||||
|
Dispatch(DispatchCandidate),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct DispatchCandidate {
|
||||||
|
pub(crate) frame: QueuedFrame,
|
||||||
|
pub(crate) pressure_state: PressureState,
|
||||||
|
pub(crate) flow_class: FlowPressureClass,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum DispatchFeedback {
|
||||||
|
Routed,
|
||||||
|
QueueFull,
|
||||||
|
ChannelClosed,
|
||||||
|
NoConn,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum DispatchAction {
|
||||||
|
Continue,
|
||||||
|
CloseFlow,
|
||||||
|
}
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use super::model::PressureState;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub(crate) struct PressureSignals {
|
||||||
|
pub(crate) active_flows: usize,
|
||||||
|
pub(crate) total_queued_bytes: u64,
|
||||||
|
pub(crate) standing_flows: usize,
|
||||||
|
pub(crate) backpressured_flows: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct PressureConfig {
|
||||||
|
pub(crate) evaluate_every_rounds: u32,
|
||||||
|
pub(crate) transition_hysteresis_rounds: u8,
|
||||||
|
pub(crate) standing_ratio_pressured_pct: u8,
|
||||||
|
pub(crate) standing_ratio_shedding_pct: u8,
|
||||||
|
pub(crate) standing_ratio_saturated_pct: u8,
|
||||||
|
pub(crate) queue_ratio_pressured_pct: u8,
|
||||||
|
pub(crate) queue_ratio_shedding_pct: u8,
|
||||||
|
pub(crate) queue_ratio_saturated_pct: u8,
|
||||||
|
pub(crate) reject_window: Duration,
|
||||||
|
pub(crate) rejects_pressured: u32,
|
||||||
|
pub(crate) rejects_shedding: u32,
|
||||||
|
pub(crate) rejects_saturated: u32,
|
||||||
|
pub(crate) stalls_pressured: u32,
|
||||||
|
pub(crate) stalls_shedding: u32,
|
||||||
|
pub(crate) stalls_saturated: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PressureConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
evaluate_every_rounds: 8,
|
||||||
|
transition_hysteresis_rounds: 3,
|
||||||
|
standing_ratio_pressured_pct: 20,
|
||||||
|
standing_ratio_shedding_pct: 35,
|
||||||
|
standing_ratio_saturated_pct: 50,
|
||||||
|
queue_ratio_pressured_pct: 65,
|
||||||
|
queue_ratio_shedding_pct: 82,
|
||||||
|
queue_ratio_saturated_pct: 94,
|
||||||
|
reject_window: Duration::from_secs(2),
|
||||||
|
rejects_pressured: 32,
|
||||||
|
rejects_shedding: 96,
|
||||||
|
rejects_saturated: 256,
|
||||||
|
stalls_pressured: 32,
|
||||||
|
stalls_shedding: 96,
|
||||||
|
stalls_saturated: 256,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct PressureEvaluator {
|
||||||
|
state: PressureState,
|
||||||
|
candidate_state: PressureState,
|
||||||
|
candidate_hits: u8,
|
||||||
|
rounds_since_eval: u32,
|
||||||
|
window_started_at: Instant,
|
||||||
|
admission_rejects_window: u32,
|
||||||
|
route_stalls_window: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PressureEvaluator {
|
||||||
|
pub(crate) fn new(now: Instant) -> Self {
|
||||||
|
Self {
|
||||||
|
state: PressureState::Normal,
|
||||||
|
candidate_state: PressureState::Normal,
|
||||||
|
candidate_hits: 0,
|
||||||
|
rounds_since_eval: 0,
|
||||||
|
window_started_at: now,
|
||||||
|
admission_rejects_window: 0,
|
||||||
|
route_stalls_window: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub(crate) fn state(&self) -> PressureState {
|
||||||
|
self.state
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn note_admission_reject(&mut self, now: Instant, cfg: &PressureConfig) {
|
||||||
|
self.rotate_window_if_needed(now, cfg);
|
||||||
|
self.admission_rejects_window = self.admission_rejects_window.saturating_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn note_route_stall(&mut self, now: Instant, cfg: &PressureConfig) {
|
||||||
|
self.rotate_window_if_needed(now, cfg);
|
||||||
|
self.route_stalls_window = self.route_stalls_window.saturating_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn maybe_evaluate(
|
||||||
|
&mut self,
|
||||||
|
now: Instant,
|
||||||
|
cfg: &PressureConfig,
|
||||||
|
max_total_queued_bytes: u64,
|
||||||
|
signals: PressureSignals,
|
||||||
|
force: bool,
|
||||||
|
) -> PressureState {
|
||||||
|
self.rotate_window_if_needed(now, cfg);
|
||||||
|
self.rounds_since_eval = self.rounds_since_eval.saturating_add(1);
|
||||||
|
if !force && self.rounds_since_eval < cfg.evaluate_every_rounds.max(1) {
|
||||||
|
return self.state;
|
||||||
|
}
|
||||||
|
self.rounds_since_eval = 0;
|
||||||
|
|
||||||
|
let target = self.derive_target_state(cfg, max_total_queued_bytes, signals);
|
||||||
|
if target == self.state {
|
||||||
|
self.candidate_state = target;
|
||||||
|
self.candidate_hits = 0;
|
||||||
|
return self.state;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.candidate_state == target {
|
||||||
|
self.candidate_hits = self.candidate_hits.saturating_add(1);
|
||||||
|
} else {
|
||||||
|
self.candidate_state = target;
|
||||||
|
self.candidate_hits = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.candidate_hits >= cfg.transition_hysteresis_rounds.max(1) {
|
||||||
|
self.state = target;
|
||||||
|
self.candidate_hits = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.state
|
||||||
|
}
|
||||||
|
|
||||||
|
fn derive_target_state(
|
||||||
|
&self,
|
||||||
|
cfg: &PressureConfig,
|
||||||
|
max_total_queued_bytes: u64,
|
||||||
|
signals: PressureSignals,
|
||||||
|
) -> PressureState {
|
||||||
|
let queue_ratio_pct = if max_total_queued_bytes == 0 {
|
||||||
|
100
|
||||||
|
} else {
|
||||||
|
((signals.total_queued_bytes.saturating_mul(100)) / max_total_queued_bytes).min(100)
|
||||||
|
as u8
|
||||||
|
};
|
||||||
|
|
||||||
|
let standing_ratio_pct = if signals.active_flows == 0 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
((signals.standing_flows.saturating_mul(100)) / signals.active_flows).min(100) as u8
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut pressure_score = 0u8;
|
||||||
|
|
||||||
|
if queue_ratio_pct >= cfg.queue_ratio_pressured_pct {
|
||||||
|
pressure_score = pressure_score.max(1);
|
||||||
|
}
|
||||||
|
if queue_ratio_pct >= cfg.queue_ratio_shedding_pct {
|
||||||
|
pressure_score = pressure_score.max(2);
|
||||||
|
}
|
||||||
|
if queue_ratio_pct >= cfg.queue_ratio_saturated_pct {
|
||||||
|
pressure_score = pressure_score.max(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
if standing_ratio_pct >= cfg.standing_ratio_pressured_pct {
|
||||||
|
pressure_score = pressure_score.max(1);
|
||||||
|
}
|
||||||
|
if standing_ratio_pct >= cfg.standing_ratio_shedding_pct {
|
||||||
|
pressure_score = pressure_score.max(2);
|
||||||
|
}
|
||||||
|
if standing_ratio_pct >= cfg.standing_ratio_saturated_pct {
|
||||||
|
pressure_score = pressure_score.max(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.admission_rejects_window >= cfg.rejects_pressured {
|
||||||
|
pressure_score = pressure_score.max(1);
|
||||||
|
}
|
||||||
|
if self.admission_rejects_window >= cfg.rejects_shedding {
|
||||||
|
pressure_score = pressure_score.max(2);
|
||||||
|
}
|
||||||
|
if self.admission_rejects_window >= cfg.rejects_saturated {
|
||||||
|
pressure_score = pressure_score.max(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.route_stalls_window >= cfg.stalls_pressured {
|
||||||
|
pressure_score = pressure_score.max(1);
|
||||||
|
}
|
||||||
|
if self.route_stalls_window >= cfg.stalls_shedding {
|
||||||
|
pressure_score = pressure_score.max(2);
|
||||||
|
}
|
||||||
|
if self.route_stalls_window >= cfg.stalls_saturated {
|
||||||
|
pressure_score = pressure_score.max(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
if signals.backpressured_flows > signals.active_flows.saturating_div(2)
|
||||||
|
&& signals.active_flows > 0
|
||||||
|
{
|
||||||
|
pressure_score = pressure_score.max(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
match pressure_score {
|
||||||
|
0 => PressureState::Normal,
|
||||||
|
1 => PressureState::Pressured,
|
||||||
|
2 => PressureState::Shedding,
|
||||||
|
_ => PressureState::Saturated,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rotate_window_if_needed(&mut self, now: Instant, cfg: &PressureConfig) {
|
||||||
|
if now.saturating_duration_since(self.window_started_at) < cfg.reject_window {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.window_started_at = now;
|
||||||
|
self.admission_rejects_window = 0;
|
||||||
|
self.route_stalls_window = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,556 @@
|
|||||||
|
use std::collections::{HashMap, VecDeque};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
use super::model::{
|
||||||
|
AdmissionDecision, DispatchAction, DispatchCandidate, DispatchFeedback, FlowFairnessState,
|
||||||
|
FlowPressureClass, FlowSchedulerState, PressureState, QueuedFrame, SchedulerDecision,
|
||||||
|
StandingQueueState,
|
||||||
|
};
|
||||||
|
use super::pressure::{PressureConfig, PressureEvaluator, PressureSignals};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct WorkerFairnessConfig {
|
||||||
|
pub(crate) worker_id: u16,
|
||||||
|
pub(crate) max_active_flows: usize,
|
||||||
|
pub(crate) max_total_queued_bytes: u64,
|
||||||
|
pub(crate) max_flow_queued_bytes: u64,
|
||||||
|
pub(crate) base_quantum_bytes: u32,
|
||||||
|
pub(crate) pressured_quantum_bytes: u32,
|
||||||
|
pub(crate) penalized_quantum_bytes: u32,
|
||||||
|
pub(crate) standing_queue_min_age: Duration,
|
||||||
|
pub(crate) standing_queue_min_backlog_bytes: u64,
|
||||||
|
pub(crate) standing_stall_threshold: u8,
|
||||||
|
pub(crate) max_consecutive_stalls_before_shed: u8,
|
||||||
|
pub(crate) max_consecutive_stalls_before_close: u8,
|
||||||
|
pub(crate) soft_bucket_count: usize,
|
||||||
|
pub(crate) soft_bucket_share_pct: u8,
|
||||||
|
pub(crate) pressure: PressureConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WorkerFairnessConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
worker_id: 0,
|
||||||
|
max_active_flows: 4096,
|
||||||
|
max_total_queued_bytes: 16 * 1024 * 1024,
|
||||||
|
max_flow_queued_bytes: 512 * 1024,
|
||||||
|
base_quantum_bytes: 32 * 1024,
|
||||||
|
pressured_quantum_bytes: 16 * 1024,
|
||||||
|
penalized_quantum_bytes: 8 * 1024,
|
||||||
|
standing_queue_min_age: Duration::from_millis(250),
|
||||||
|
standing_queue_min_backlog_bytes: 64 * 1024,
|
||||||
|
standing_stall_threshold: 3,
|
||||||
|
max_consecutive_stalls_before_shed: 4,
|
||||||
|
max_consecutive_stalls_before_close: 16,
|
||||||
|
soft_bucket_count: 64,
|
||||||
|
soft_bucket_share_pct: 25,
|
||||||
|
pressure: PressureConfig::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FlowEntry {
|
||||||
|
fairness: FlowFairnessState,
|
||||||
|
queue: VecDeque<QueuedFrame>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FlowEntry {
|
||||||
|
fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
fairness: FlowFairnessState::new(flow_id, worker_id, bucket_id),
|
||||||
|
queue: VecDeque::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
|
pub(crate) struct WorkerFairnessSnapshot {
|
||||||
|
pub(crate) pressure_state: PressureState,
|
||||||
|
pub(crate) active_flows: usize,
|
||||||
|
pub(crate) total_queued_bytes: u64,
|
||||||
|
pub(crate) standing_flows: usize,
|
||||||
|
pub(crate) backpressured_flows: usize,
|
||||||
|
pub(crate) scheduler_rounds: u64,
|
||||||
|
pub(crate) deficit_grants: u64,
|
||||||
|
pub(crate) deficit_skips: u64,
|
||||||
|
pub(crate) enqueue_rejects: u64,
|
||||||
|
pub(crate) shed_drops: u64,
|
||||||
|
pub(crate) fairness_penalties: u64,
|
||||||
|
pub(crate) downstream_stalls: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct WorkerFairnessState {
|
||||||
|
config: WorkerFairnessConfig,
|
||||||
|
pressure: PressureEvaluator,
|
||||||
|
flows: HashMap<u64, FlowEntry>,
|
||||||
|
active_ring: VecDeque<u64>,
|
||||||
|
total_queued_bytes: u64,
|
||||||
|
bucket_queued_bytes: Vec<u64>,
|
||||||
|
bucket_active_flows: Vec<usize>,
|
||||||
|
standing_flow_count: usize,
|
||||||
|
backpressured_flow_count: usize,
|
||||||
|
scheduler_rounds: u64,
|
||||||
|
deficit_grants: u64,
|
||||||
|
deficit_skips: u64,
|
||||||
|
enqueue_rejects: u64,
|
||||||
|
shed_drops: u64,
|
||||||
|
fairness_penalties: u64,
|
||||||
|
downstream_stalls: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerFairnessState {
|
||||||
|
pub(crate) fn new(config: WorkerFairnessConfig, now: Instant) -> Self {
|
||||||
|
let bucket_count = config.soft_bucket_count.max(1);
|
||||||
|
Self {
|
||||||
|
config,
|
||||||
|
pressure: PressureEvaluator::new(now),
|
||||||
|
flows: HashMap::new(),
|
||||||
|
active_ring: VecDeque::new(),
|
||||||
|
total_queued_bytes: 0,
|
||||||
|
bucket_queued_bytes: vec![0; bucket_count],
|
||||||
|
bucket_active_flows: vec![0; bucket_count],
|
||||||
|
standing_flow_count: 0,
|
||||||
|
backpressured_flow_count: 0,
|
||||||
|
scheduler_rounds: 0,
|
||||||
|
deficit_grants: 0,
|
||||||
|
deficit_skips: 0,
|
||||||
|
enqueue_rejects: 0,
|
||||||
|
shed_drops: 0,
|
||||||
|
fairness_penalties: 0,
|
||||||
|
downstream_stalls: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pressure_state(&self) -> PressureState {
|
||||||
|
self.pressure.state()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn snapshot(&self) -> WorkerFairnessSnapshot {
|
||||||
|
WorkerFairnessSnapshot {
|
||||||
|
pressure_state: self.pressure.state(),
|
||||||
|
active_flows: self.flows.len(),
|
||||||
|
total_queued_bytes: self.total_queued_bytes,
|
||||||
|
standing_flows: self.standing_flow_count,
|
||||||
|
backpressured_flows: self.backpressured_flow_count,
|
||||||
|
scheduler_rounds: self.scheduler_rounds,
|
||||||
|
deficit_grants: self.deficit_grants,
|
||||||
|
deficit_skips: self.deficit_skips,
|
||||||
|
enqueue_rejects: self.enqueue_rejects,
|
||||||
|
shed_drops: self.shed_drops,
|
||||||
|
fairness_penalties: self.fairness_penalties,
|
||||||
|
downstream_stalls: self.downstream_stalls,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn enqueue_data(
|
||||||
|
&mut self,
|
||||||
|
conn_id: u64,
|
||||||
|
flags: u32,
|
||||||
|
data: Bytes,
|
||||||
|
now: Instant,
|
||||||
|
) -> AdmissionDecision {
|
||||||
|
let frame = QueuedFrame {
|
||||||
|
conn_id,
|
||||||
|
flags,
|
||||||
|
data,
|
||||||
|
enqueued_at: now,
|
||||||
|
};
|
||||||
|
let frame_bytes = frame.queued_bytes();
|
||||||
|
|
||||||
|
if self.pressure.state() == PressureState::Saturated {
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
return AdmissionDecision::RejectSaturated;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.total_queued_bytes.saturating_add(frame_bytes) > self.config.max_total_queued_bytes
|
||||||
|
{
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return AdmissionDecision::RejectWorkerCap;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !self.flows.contains_key(&conn_id) && self.flows.len() >= self.config.max_active_flows {
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return AdmissionDecision::RejectWorkerCap;
|
||||||
|
}
|
||||||
|
|
||||||
|
let bucket_id = self.bucket_for(conn_id);
|
||||||
|
let bucket_cap = self
|
||||||
|
.config
|
||||||
|
.max_total_queued_bytes
|
||||||
|
.saturating_mul(self.config.soft_bucket_share_pct.max(1) as u64)
|
||||||
|
.saturating_div(100)
|
||||||
|
.max(self.config.max_flow_queued_bytes);
|
||||||
|
if self.bucket_queued_bytes[bucket_id].saturating_add(frame_bytes) > bucket_cap {
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return AdmissionDecision::RejectBucketCap;
|
||||||
|
}
|
||||||
|
|
||||||
|
let entry = if let Some(flow) = self.flows.get_mut(&conn_id) {
|
||||||
|
flow
|
||||||
|
} else {
|
||||||
|
self.bucket_active_flows[bucket_id] =
|
||||||
|
self.bucket_active_flows[bucket_id].saturating_add(1);
|
||||||
|
self.flows.insert(
|
||||||
|
conn_id,
|
||||||
|
FlowEntry::new(conn_id, self.config.worker_id, bucket_id),
|
||||||
|
);
|
||||||
|
self.flows
|
||||||
|
.get_mut(&conn_id)
|
||||||
|
.expect("flow inserted must be retrievable")
|
||||||
|
};
|
||||||
|
|
||||||
|
if entry.fairness.pending_bytes.saturating_add(frame_bytes)
|
||||||
|
> self.config.max_flow_queued_bytes
|
||||||
|
{
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return AdmissionDecision::RejectFlowCap;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.pressure.state() >= PressureState::Shedding
|
||||||
|
&& entry.fairness.standing_state == StandingQueueState::Standing
|
||||||
|
{
|
||||||
|
self.pressure
|
||||||
|
.note_admission_reject(now, &self.config.pressure);
|
||||||
|
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return AdmissionDecision::RejectStandingFlow;
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.fairness.pending_bytes = entry.fairness.pending_bytes.saturating_add(frame_bytes);
|
||||||
|
if entry.fairness.queue_started_at.is_none() {
|
||||||
|
entry.fairness.queue_started_at = Some(now);
|
||||||
|
}
|
||||||
|
entry.queue.push_back(frame);
|
||||||
|
|
||||||
|
self.total_queued_bytes = self.total_queued_bytes.saturating_add(frame_bytes);
|
||||||
|
self.bucket_queued_bytes[bucket_id] =
|
||||||
|
self.bucket_queued_bytes[bucket_id].saturating_add(frame_bytes);
|
||||||
|
|
||||||
|
if !entry.fairness.in_active_ring {
|
||||||
|
entry.fairness.in_active_ring = true;
|
||||||
|
self.active_ring.push_back(conn_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn next_decision(&mut self, now: Instant) -> SchedulerDecision {
|
||||||
|
self.scheduler_rounds = self.scheduler_rounds.saturating_add(1);
|
||||||
|
self.evaluate_pressure(now, false);
|
||||||
|
|
||||||
|
let active_len = self.active_ring.len();
|
||||||
|
for _ in 0..active_len {
|
||||||
|
let Some(conn_id) = self.active_ring.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut candidate = None;
|
||||||
|
let mut requeue_active = false;
|
||||||
|
let mut drained_bytes = 0u64;
|
||||||
|
let mut bucket_id = 0usize;
|
||||||
|
let pressure_state = self.pressure.state();
|
||||||
|
|
||||||
|
if let Some(flow) = self.flows.get_mut(&conn_id) {
|
||||||
|
bucket_id = flow.fairness.bucket_id;
|
||||||
|
|
||||||
|
if flow.queue.is_empty() {
|
||||||
|
flow.fairness.in_active_ring = false;
|
||||||
|
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
|
||||||
|
flow.fairness.pending_bytes = 0;
|
||||||
|
flow.fairness.queue_started_at = None;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Self::classify_flow(&self.config, pressure_state, now, &mut flow.fairness);
|
||||||
|
|
||||||
|
let quantum =
|
||||||
|
Self::effective_quantum_bytes(&self.config, pressure_state, &flow.fairness);
|
||||||
|
flow.fairness.deficit_bytes = flow
|
||||||
|
.fairness
|
||||||
|
.deficit_bytes
|
||||||
|
.saturating_add(i64::from(quantum));
|
||||||
|
self.deficit_grants = self.deficit_grants.saturating_add(1);
|
||||||
|
|
||||||
|
let front_len = flow.queue.front().map_or(0, |front| front.queued_bytes());
|
||||||
|
if flow.fairness.deficit_bytes < front_len as i64 {
|
||||||
|
flow.fairness.consecutive_skips =
|
||||||
|
flow.fairness.consecutive_skips.saturating_add(1);
|
||||||
|
self.deficit_skips = self.deficit_skips.saturating_add(1);
|
||||||
|
requeue_active = true;
|
||||||
|
} else if let Some(frame) = flow.queue.pop_front() {
|
||||||
|
drained_bytes = frame.queued_bytes();
|
||||||
|
flow.fairness.pending_bytes =
|
||||||
|
flow.fairness.pending_bytes.saturating_sub(drained_bytes);
|
||||||
|
flow.fairness.deficit_bytes = flow
|
||||||
|
.fairness
|
||||||
|
.deficit_bytes
|
||||||
|
.saturating_sub(drained_bytes as i64);
|
||||||
|
flow.fairness.consecutive_skips = 0;
|
||||||
|
flow.fairness.queue_started_at =
|
||||||
|
flow.queue.front().map(|front| front.enqueued_at);
|
||||||
|
requeue_active = !flow.queue.is_empty();
|
||||||
|
if !requeue_active {
|
||||||
|
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
|
||||||
|
flow.fairness.in_active_ring = false;
|
||||||
|
}
|
||||||
|
candidate = Some(DispatchCandidate {
|
||||||
|
pressure_state,
|
||||||
|
flow_class: flow.fairness.pressure_class,
|
||||||
|
frame,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if drained_bytes > 0 {
|
||||||
|
self.total_queued_bytes = self.total_queued_bytes.saturating_sub(drained_bytes);
|
||||||
|
self.bucket_queued_bytes[bucket_id] =
|
||||||
|
self.bucket_queued_bytes[bucket_id].saturating_sub(drained_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
if requeue_active {
|
||||||
|
if let Some(flow) = self.flows.get_mut(&conn_id) {
|
||||||
|
flow.fairness.in_active_ring = true;
|
||||||
|
}
|
||||||
|
self.active_ring.push_back(conn_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(candidate) = candidate {
|
||||||
|
return SchedulerDecision::Dispatch(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SchedulerDecision::Idle
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn apply_dispatch_feedback(
|
||||||
|
&mut self,
|
||||||
|
conn_id: u64,
|
||||||
|
candidate: DispatchCandidate,
|
||||||
|
feedback: DispatchFeedback,
|
||||||
|
now: Instant,
|
||||||
|
) -> DispatchAction {
|
||||||
|
match feedback {
|
||||||
|
DispatchFeedback::Routed => {
|
||||||
|
if let Some(flow) = self.flows.get_mut(&conn_id) {
|
||||||
|
flow.fairness.last_drain_at = Some(now);
|
||||||
|
flow.fairness.recent_drain_bytes = flow
|
||||||
|
.fairness
|
||||||
|
.recent_drain_bytes
|
||||||
|
.saturating_add(candidate.frame.queued_bytes());
|
||||||
|
flow.fairness.consecutive_stalls = 0;
|
||||||
|
if flow.fairness.scheduler_state != FlowSchedulerState::Idle {
|
||||||
|
flow.fairness.scheduler_state = FlowSchedulerState::Active;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.evaluate_pressure(now, false);
|
||||||
|
DispatchAction::Continue
|
||||||
|
}
|
||||||
|
DispatchFeedback::QueueFull => {
|
||||||
|
self.pressure.note_route_stall(now, &self.config.pressure);
|
||||||
|
self.downstream_stalls = self.downstream_stalls.saturating_add(1);
|
||||||
|
let Some(flow) = self.flows.get_mut(&conn_id) else {
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return DispatchAction::Continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
flow.fairness.consecutive_stalls =
|
||||||
|
flow.fairness.consecutive_stalls.saturating_add(1);
|
||||||
|
flow.fairness.scheduler_state = FlowSchedulerState::Backpressured;
|
||||||
|
flow.fairness.pressure_class = FlowPressureClass::Backpressured;
|
||||||
|
|
||||||
|
let state = self.pressure.state();
|
||||||
|
let should_shed_frame = matches!(state, PressureState::Saturated)
|
||||||
|
|| (matches!(state, PressureState::Shedding)
|
||||||
|
&& flow.fairness.standing_state == StandingQueueState::Standing
|
||||||
|
&& flow.fairness.consecutive_stalls
|
||||||
|
>= self.config.max_consecutive_stalls_before_shed);
|
||||||
|
|
||||||
|
if should_shed_frame {
|
||||||
|
self.shed_drops = self.shed_drops.saturating_add(1);
|
||||||
|
self.fairness_penalties = self.fairness_penalties.saturating_add(1);
|
||||||
|
} else {
|
||||||
|
let frame_bytes = candidate.frame.queued_bytes();
|
||||||
|
flow.queue.push_front(candidate.frame);
|
||||||
|
flow.fairness.pending_bytes =
|
||||||
|
flow.fairness.pending_bytes.saturating_add(frame_bytes);
|
||||||
|
flow.fairness.queue_started_at =
|
||||||
|
flow.queue.front().map(|front| front.enqueued_at);
|
||||||
|
self.total_queued_bytes = self.total_queued_bytes.saturating_add(frame_bytes);
|
||||||
|
self.bucket_queued_bytes[flow.fairness.bucket_id] = self.bucket_queued_bytes
|
||||||
|
[flow.fairness.bucket_id]
|
||||||
|
.saturating_add(frame_bytes);
|
||||||
|
if !flow.fairness.in_active_ring {
|
||||||
|
flow.fairness.in_active_ring = true;
|
||||||
|
self.active_ring.push_back(conn_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if flow.fairness.consecutive_stalls
|
||||||
|
>= self.config.max_consecutive_stalls_before_close
|
||||||
|
&& self.pressure.state() == PressureState::Saturated
|
||||||
|
{
|
||||||
|
self.remove_flow(conn_id);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
return DispatchAction::CloseFlow;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
DispatchAction::Continue
|
||||||
|
}
|
||||||
|
DispatchFeedback::ChannelClosed | DispatchFeedback::NoConn => {
|
||||||
|
self.remove_flow(conn_id);
|
||||||
|
self.evaluate_pressure(now, true);
|
||||||
|
DispatchAction::CloseFlow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn remove_flow(&mut self, conn_id: u64) {
|
||||||
|
let Some(entry) = self.flows.remove(&conn_id) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
self.bucket_active_flows[entry.fairness.bucket_id] =
|
||||||
|
self.bucket_active_flows[entry.fairness.bucket_id].saturating_sub(1);
|
||||||
|
|
||||||
|
let mut reclaimed = 0u64;
|
||||||
|
for frame in entry.queue {
|
||||||
|
reclaimed = reclaimed.saturating_add(frame.queued_bytes());
|
||||||
|
}
|
||||||
|
self.total_queued_bytes = self.total_queued_bytes.saturating_sub(reclaimed);
|
||||||
|
self.bucket_queued_bytes[entry.fairness.bucket_id] =
|
||||||
|
self.bucket_queued_bytes[entry.fairness.bucket_id].saturating_sub(reclaimed);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evaluate_pressure(&mut self, now: Instant, force: bool) {
|
||||||
|
let mut standing = 0usize;
|
||||||
|
let mut backpressured = 0usize;
|
||||||
|
|
||||||
|
for flow in self.flows.values_mut() {
|
||||||
|
Self::classify_flow(&self.config, self.pressure.state(), now, &mut flow.fairness);
|
||||||
|
if flow.fairness.standing_state == StandingQueueState::Standing {
|
||||||
|
standing = standing.saturating_add(1);
|
||||||
|
}
|
||||||
|
if matches!(
|
||||||
|
flow.fairness.scheduler_state,
|
||||||
|
FlowSchedulerState::Backpressured
|
||||||
|
| FlowSchedulerState::Penalized
|
||||||
|
| FlowSchedulerState::SheddingCandidate
|
||||||
|
) {
|
||||||
|
backpressured = backpressured.saturating_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.standing_flow_count = standing;
|
||||||
|
self.backpressured_flow_count = backpressured;
|
||||||
|
|
||||||
|
let _ = self.pressure.maybe_evaluate(
|
||||||
|
now,
|
||||||
|
&self.config.pressure,
|
||||||
|
self.config.max_total_queued_bytes,
|
||||||
|
PressureSignals {
|
||||||
|
active_flows: self.flows.len(),
|
||||||
|
total_queued_bytes: self.total_queued_bytes,
|
||||||
|
standing_flows: standing,
|
||||||
|
backpressured_flows: backpressured,
|
||||||
|
},
|
||||||
|
force,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_flow(
|
||||||
|
config: &WorkerFairnessConfig,
|
||||||
|
pressure_state: PressureState,
|
||||||
|
now: Instant,
|
||||||
|
fairness: &mut FlowFairnessState,
|
||||||
|
) {
|
||||||
|
if fairness.pending_bytes == 0 {
|
||||||
|
fairness.pressure_class = FlowPressureClass::Healthy;
|
||||||
|
fairness.standing_state = StandingQueueState::Transient;
|
||||||
|
fairness.scheduler_state = FlowSchedulerState::Idle;
|
||||||
|
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let queue_age = fairness
|
||||||
|
.queue_started_at
|
||||||
|
.map(|ts| now.saturating_duration_since(ts))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let drain_stalled = fairness
|
||||||
|
.last_drain_at
|
||||||
|
.map(|ts| now.saturating_duration_since(ts) >= config.standing_queue_min_age)
|
||||||
|
.unwrap_or(true);
|
||||||
|
|
||||||
|
let standing = fairness.pending_bytes >= config.standing_queue_min_backlog_bytes
|
||||||
|
&& queue_age >= config.standing_queue_min_age
|
||||||
|
&& (fairness.consecutive_stalls >= config.standing_stall_threshold || drain_stalled);
|
||||||
|
|
||||||
|
if standing {
|
||||||
|
fairness.standing_state = StandingQueueState::Standing;
|
||||||
|
fairness.pressure_class = FlowPressureClass::Standing;
|
||||||
|
fairness.penalty_score = fairness.penalty_score.saturating_add(1);
|
||||||
|
fairness.scheduler_state = if pressure_state >= PressureState::Shedding {
|
||||||
|
FlowSchedulerState::SheddingCandidate
|
||||||
|
} else {
|
||||||
|
FlowSchedulerState::Penalized
|
||||||
|
};
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
fairness.standing_state = StandingQueueState::Transient;
|
||||||
|
if fairness.consecutive_stalls > 0 {
|
||||||
|
fairness.pressure_class = FlowPressureClass::Backpressured;
|
||||||
|
fairness.scheduler_state = FlowSchedulerState::Backpressured;
|
||||||
|
} else if fairness.pending_bytes >= config.standing_queue_min_backlog_bytes {
|
||||||
|
fairness.pressure_class = FlowPressureClass::Bursty;
|
||||||
|
fairness.scheduler_state = FlowSchedulerState::Active;
|
||||||
|
} else {
|
||||||
|
fairness.pressure_class = FlowPressureClass::Healthy;
|
||||||
|
fairness.scheduler_state = FlowSchedulerState::Active;
|
||||||
|
}
|
||||||
|
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn effective_quantum_bytes(
|
||||||
|
config: &WorkerFairnessConfig,
|
||||||
|
pressure_state: PressureState,
|
||||||
|
fairness: &FlowFairnessState,
|
||||||
|
) -> u32 {
|
||||||
|
let penalized = matches!(
|
||||||
|
fairness.scheduler_state,
|
||||||
|
FlowSchedulerState::Penalized | FlowSchedulerState::SheddingCandidate
|
||||||
|
);
|
||||||
|
|
||||||
|
if penalized {
|
||||||
|
return config.penalized_quantum_bytes.max(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
match pressure_state {
|
||||||
|
PressureState::Normal => config.base_quantum_bytes.max(1),
|
||||||
|
PressureState::Pressured => config.pressured_quantum_bytes.max(1),
|
||||||
|
PressureState::Shedding => config.pressured_quantum_bytes.max(1),
|
||||||
|
PressureState::Saturated => config.penalized_quantum_bytes.max(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bucket_for(&self, conn_id: u64) -> usize {
|
||||||
|
(conn_id as usize) % self.bucket_queued_bytes.len().max(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -67,10 +67,8 @@ struct FamilyReconnectOutcome {
|
|||||||
key: (i32, IpFamily),
|
key: (i32, IpFamily),
|
||||||
dc: i32,
|
dc: i32,
|
||||||
family: IpFamily,
|
family: IpFamily,
|
||||||
alive: usize,
|
|
||||||
required: usize,
|
required: usize,
|
||||||
endpoint_count: usize,
|
endpoint_count: usize,
|
||||||
restored: usize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
||||||
@@ -82,8 +80,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new();
|
let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new();
|
||||||
let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
|
||||||
let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
|
||||||
let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
let mut drain_warn_next_allowed: HashMap<u64, Instant> = HashMap::new();
|
let mut drain_warn_next_allowed: HashMap<u64, Instant> = HashMap::new();
|
||||||
let mut degraded_interval = true;
|
let mut degraded_interval = true;
|
||||||
@@ -109,8 +105,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
&mut single_endpoint_outage,
|
&mut single_endpoint_outage,
|
||||||
&mut shadow_rotate_deadline,
|
&mut shadow_rotate_deadline,
|
||||||
&mut idle_refresh_next_attempt,
|
&mut idle_refresh_next_attempt,
|
||||||
&mut adaptive_idle_since,
|
|
||||||
&mut adaptive_recover_until,
|
|
||||||
&mut floor_warn_next_allowed,
|
&mut floor_warn_next_allowed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -126,8 +120,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
&mut single_endpoint_outage,
|
&mut single_endpoint_outage,
|
||||||
&mut shadow_rotate_deadline,
|
&mut shadow_rotate_deadline,
|
||||||
&mut idle_refresh_next_attempt,
|
&mut idle_refresh_next_attempt,
|
||||||
&mut adaptive_idle_since,
|
|
||||||
&mut adaptive_recover_until,
|
|
||||||
&mut floor_warn_next_allowed,
|
&mut floor_warn_next_allowed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -360,8 +352,6 @@ async fn check_family(
|
|||||||
single_endpoint_outage: &mut HashSet<(i32, IpFamily)>,
|
single_endpoint_outage: &mut HashSet<(i32, IpFamily)>,
|
||||||
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let enabled = match family {
|
let enabled = match family {
|
||||||
@@ -393,10 +383,7 @@ async fn check_family(
|
|||||||
let reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len());
|
let reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len());
|
||||||
let reconnect_sem = Arc::new(Semaphore::new(reconnect_budget));
|
let reconnect_sem = Arc::new(Semaphore::new(reconnect_budget));
|
||||||
|
|
||||||
if pool.floor_mode() == MeFloorMode::Static {
|
if pool.floor_mode() == MeFloorMode::Static {}
|
||||||
adaptive_idle_since.clear();
|
|
||||||
adaptive_recover_until.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut live_addr_counts = HashMap::<(i32, SocketAddr), usize>::new();
|
let mut live_addr_counts = HashMap::<(i32, SocketAddr), usize>::new();
|
||||||
let mut live_writer_ids_by_addr = HashMap::<(i32, SocketAddr), Vec<u64>>::new();
|
let mut live_writer_ids_by_addr = HashMap::<(i32, SocketAddr), Vec<u64>>::new();
|
||||||
@@ -435,8 +422,6 @@ async fn check_family(
|
|||||||
&live_addr_counts,
|
&live_addr_counts,
|
||||||
&live_writer_ids_by_addr,
|
&live_writer_ids_by_addr,
|
||||||
&bound_clients_by_writer,
|
&bound_clients_by_writer,
|
||||||
adaptive_idle_since,
|
|
||||||
adaptive_recover_until,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
pool.set_adaptive_floor_runtime_caps(
|
pool.set_adaptive_floor_runtime_caps(
|
||||||
@@ -503,8 +488,6 @@ async fn check_family(
|
|||||||
outage_next_attempt.remove(&key);
|
outage_next_attempt.remove(&key);
|
||||||
shadow_rotate_deadline.remove(&key);
|
shadow_rotate_deadline.remove(&key);
|
||||||
idle_refresh_next_attempt.remove(&key);
|
idle_refresh_next_attempt.remove(&key);
|
||||||
adaptive_idle_since.remove(&key);
|
|
||||||
adaptive_recover_until.remove(&key);
|
|
||||||
info!(
|
info!(
|
||||||
dc = %dc,
|
dc = %dc,
|
||||||
?family,
|
?family,
|
||||||
@@ -632,22 +615,28 @@ async fn check_family(
|
|||||||
restored += 1;
|
restored += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
let base_req = pool_for_reconnect
|
||||||
.increment_me_floor_cap_block_total();
|
.required_writers_for_dc_with_floor_mode(endpoints_for_dc.len(), false);
|
||||||
pool_for_reconnect
|
if alive + restored >= base_req {
|
||||||
.stats
|
pool_for_reconnect
|
||||||
.increment_me_floor_swap_idle_failed_total();
|
.stats
|
||||||
debug!(
|
.increment_me_floor_cap_block_total();
|
||||||
dc = %dc,
|
pool_for_reconnect
|
||||||
?family,
|
.stats
|
||||||
alive,
|
.increment_me_floor_swap_idle_failed_total();
|
||||||
required,
|
debug!(
|
||||||
active_cap_effective_total,
|
dc = %dc,
|
||||||
"Adaptive floor cap reached, reconnect attempt blocked"
|
?family,
|
||||||
);
|
alive,
|
||||||
break;
|
required,
|
||||||
|
active_cap_effective_total,
|
||||||
|
"Adaptive floor cap reached, reconnect attempt blocked"
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
||||||
let res = tokio::time::timeout(
|
let res = tokio::time::timeout(
|
||||||
pool_for_reconnect.reconnect_runtime.me_one_timeout,
|
pool_for_reconnect.reconnect_runtime.me_one_timeout,
|
||||||
pool_for_reconnect.connect_endpoints_round_robin(
|
pool_for_reconnect.connect_endpoints_round_robin(
|
||||||
@@ -663,11 +652,9 @@ async fn check_family(
|
|||||||
pool_for_reconnect.stats.increment_me_reconnect_success();
|
pool_for_reconnect.stats.increment_me_reconnect_success();
|
||||||
}
|
}
|
||||||
Ok(false) => {
|
Ok(false) => {
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
|
||||||
debug!(dc = %dc, ?family, "ME round-robin reconnect failed")
|
debug!(dc = %dc, ?family, "ME round-robin reconnect failed")
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
|
||||||
debug!(dc = %dc, ?family, "ME reconnect timed out");
|
debug!(dc = %dc, ?family, "ME reconnect timed out");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -678,10 +665,8 @@ async fn check_family(
|
|||||||
key,
|
key,
|
||||||
dc,
|
dc,
|
||||||
family,
|
family,
|
||||||
alive,
|
|
||||||
required,
|
required,
|
||||||
endpoint_count: endpoints_for_dc.len(),
|
endpoint_count: endpoints_for_dc.len(),
|
||||||
restored,
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -695,7 +680,7 @@ async fn check_family(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let now_alive = outcome.alive + outcome.restored;
|
let now_alive = live_active_writers_for_dc_family(pool, outcome.dc, outcome.family).await;
|
||||||
if now_alive >= outcome.required {
|
if now_alive >= outcome.required {
|
||||||
info!(
|
info!(
|
||||||
dc = %outcome.dc,
|
dc = %outcome.dc,
|
||||||
@@ -851,6 +836,33 @@ fn should_emit_rate_limited_warn(
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn live_active_writers_for_dc_family(pool: &Arc<MePool>, dc: i32, family: IpFamily) -> usize {
|
||||||
|
let writers = pool.writers.read().await;
|
||||||
|
writers
|
||||||
|
.iter()
|
||||||
|
.filter(|writer| {
|
||||||
|
if writer.draining.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if writer.writer_dc != dc {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
super::pool::WriterContour::from_u8(
|
||||||
|
writer.contour.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
),
|
||||||
|
super::pool::WriterContour::Active
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
match family {
|
||||||
|
IpFamily::V4 => writer.addr.is_ipv4(),
|
||||||
|
IpFamily::V6 => writer.addr.is_ipv6(),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.count()
|
||||||
|
}
|
||||||
|
|
||||||
fn adaptive_floor_class_min(
|
fn adaptive_floor_class_min(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
endpoint_count: usize,
|
endpoint_count: usize,
|
||||||
@@ -904,8 +916,6 @@ async fn build_family_floor_plan(
|
|||||||
live_addr_counts: &HashMap<(i32, SocketAddr), usize>,
|
live_addr_counts: &HashMap<(i32, SocketAddr), usize>,
|
||||||
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
bound_clients_by_writer: &HashMap<u64, usize>,
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
) -> FamilyFloorPlan {
|
) -> FamilyFloorPlan {
|
||||||
let mut entries = Vec::<DcFloorPlanEntry>::new();
|
let mut entries = Vec::<DcFloorPlanEntry>::new();
|
||||||
let mut by_dc = HashMap::<i32, DcFloorPlanEntry>::new();
|
let mut by_dc = HashMap::<i32, DcFloorPlanEntry>::new();
|
||||||
@@ -921,18 +931,7 @@ async fn build_family_floor_plan(
|
|||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let key = (*dc, family);
|
let _key = (*dc, family);
|
||||||
let reduce_for_idle = should_reduce_floor_for_idle(
|
|
||||||
pool,
|
|
||||||
key,
|
|
||||||
*dc,
|
|
||||||
endpoints,
|
|
||||||
live_writer_ids_by_addr,
|
|
||||||
bound_clients_by_writer,
|
|
||||||
adaptive_idle_since,
|
|
||||||
adaptive_recover_until,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let base_required = pool.required_writers_for_dc(endpoints.len()).max(1);
|
let base_required = pool.required_writers_for_dc(endpoints.len()).max(1);
|
||||||
let min_required = if is_adaptive {
|
let min_required = if is_adaptive {
|
||||||
adaptive_floor_class_min(pool, endpoints.len(), base_required)
|
adaptive_floor_class_min(pool, endpoints.len(), base_required)
|
||||||
@@ -947,11 +946,11 @@ async fn build_family_floor_plan(
|
|||||||
if max_required < min_required {
|
if max_required < min_required {
|
||||||
max_required = min_required;
|
max_required = min_required;
|
||||||
}
|
}
|
||||||
let desired_raw = if is_adaptive && reduce_for_idle {
|
// We initialize target_required at base_required to prevent 0-writer blackouts
|
||||||
min_required
|
// caused by proactively dropping an idle DC to a single fragile connection.
|
||||||
} else {
|
// The Adaptive Floor constraint loop below will gracefully compress idle DCs
|
||||||
base_required
|
// (prioritized via has_bound_clients = false) to min_required only when global capacity is reached.
|
||||||
};
|
let desired_raw = base_required;
|
||||||
let target_required = desired_raw.clamp(min_required, max_required);
|
let target_required = desired_raw.clamp(min_required, max_required);
|
||||||
let alive = endpoints
|
let alive = endpoints
|
||||||
.iter()
|
.iter()
|
||||||
@@ -1278,43 +1277,6 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn should_reduce_floor_for_idle(
|
|
||||||
pool: &Arc<MePool>,
|
|
||||||
key: (i32, IpFamily),
|
|
||||||
dc: i32,
|
|
||||||
endpoints: &[SocketAddr],
|
|
||||||
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
|
||||||
bound_clients_by_writer: &HashMap<u64, usize>,
|
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
|
||||||
) -> bool {
|
|
||||||
if pool.floor_mode() != MeFloorMode::Adaptive {
|
|
||||||
adaptive_idle_since.remove(&key);
|
|
||||||
adaptive_recover_until.remove(&key);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
let now = Instant::now();
|
|
||||||
let writer_ids = list_writer_ids_for_endpoints(dc, endpoints, live_writer_ids_by_addr);
|
|
||||||
let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer);
|
|
||||||
if has_bound_clients {
|
|
||||||
adaptive_idle_since.remove(&key);
|
|
||||||
adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(recover_until) = adaptive_recover_until.get(&key)
|
|
||||||
&& now < *recover_until
|
|
||||||
{
|
|
||||||
adaptive_idle_since.remove(&key);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
adaptive_recover_until.remove(&key);
|
|
||||||
|
|
||||||
let idle_since = adaptive_idle_since.entry(key).or_insert(now);
|
|
||||||
now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn has_bound_clients_on_endpoint(
|
fn has_bound_clients_on_endpoint(
|
||||||
writer_ids: &[u64],
|
writer_ids: &[u64],
|
||||||
bound_clients_by_writer: &HashMap<u64, usize>,
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
@@ -1364,6 +1326,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
pool.stats.increment_me_reconnect_attempt();
|
||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||||
|
|
||||||
@@ -1439,7 +1402,6 @@ async fn recover_single_endpoint_outage(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
pool.stats.increment_me_reconnect_attempt();
|
|
||||||
let current_ms = *outage_backoff.get(&key).unwrap_or(&min_backoff_ms);
|
let current_ms = *outage_backoff.get(&key).unwrap_or(&min_backoff_ms);
|
||||||
let next_ms = current_ms.saturating_mul(2).min(max_backoff_ms);
|
let next_ms = current_ms.saturating_mul(2).min(max_backoff_ms);
|
||||||
outage_backoff.insert(key, next_ms);
|
outage_backoff.insert(key, next_ms);
|
||||||
|
|||||||
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
mod codec;
|
mod codec;
|
||||||
mod config_updater;
|
mod config_updater;
|
||||||
|
mod fairness;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "tests/fairness_security_tests.rs"]
|
||||||
|
mod fairness_security_tests;
|
||||||
mod handshake;
|
mod handshake;
|
||||||
mod health;
|
mod health;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ pub fn format_sample_line(sample: &MePingSample) -> String {
|
|||||||
fn format_direct_with_config(
|
fn format_direct_with_config(
|
||||||
interface: &Option<String>,
|
interface: &Option<String>,
|
||||||
bind_addresses: &Option<Vec<String>>,
|
bind_addresses: &Option<Vec<String>>,
|
||||||
|
bindtodevice: &Option<String>,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
let mut direct_parts: Vec<String> = Vec::new();
|
let mut direct_parts: Vec<String> = Vec::new();
|
||||||
if let Some(dev) = interface.as_deref().filter(|v| !v.is_empty()) {
|
if let Some(dev) = interface.as_deref().filter(|v| !v.is_empty()) {
|
||||||
@@ -75,6 +76,9 @@ fn format_direct_with_config(
|
|||||||
if let Some(src) = bind_addresses.as_ref().filter(|v| !v.is_empty()) {
|
if let Some(src) = bind_addresses.as_ref().filter(|v| !v.is_empty()) {
|
||||||
direct_parts.push(format!("src={}", src.join(",")));
|
direct_parts.push(format!("src={}", src.join(",")));
|
||||||
}
|
}
|
||||||
|
if let Some(device) = bindtodevice.as_deref().filter(|v| !v.is_empty()) {
|
||||||
|
direct_parts.push(format!("bindtodevice={device}"));
|
||||||
|
}
|
||||||
if direct_parts.is_empty() {
|
if direct_parts.is_empty() {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
@@ -231,8 +235,11 @@ pub async fn format_me_route(
|
|||||||
UpstreamType::Direct {
|
UpstreamType::Direct {
|
||||||
interface,
|
interface,
|
||||||
bind_addresses,
|
bind_addresses,
|
||||||
|
bindtodevice,
|
||||||
} => {
|
} => {
|
||||||
if let Some(route) = format_direct_with_config(interface, bind_addresses) {
|
if let Some(route) =
|
||||||
|
format_direct_with_config(interface, bind_addresses, bindtodevice)
|
||||||
|
{
|
||||||
route
|
route
|
||||||
} else {
|
} else {
|
||||||
detect_direct_route_details(reports, prefer_ipv6, v4_ok, v6_ok)
|
detect_direct_route_details(reports, prefer_ipv6, v4_ok, v6_ok)
|
||||||
|
|||||||
@@ -1422,22 +1422,6 @@ impl MePool {
|
|||||||
MeFloorMode::from_u8(self.floor_runtime.me_floor_mode.load(Ordering::Relaxed))
|
MeFloorMode::from_u8(self.floor_runtime.me_floor_mode.load(Ordering::Relaxed))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_idle_duration(&self) -> Duration {
|
|
||||||
Duration::from_secs(
|
|
||||||
self.floor_runtime
|
|
||||||
.me_adaptive_floor_idle_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_recover_grace_duration(&self) -> Duration {
|
|
||||||
Duration::from_secs(
|
|
||||||
self.floor_runtime
|
|
||||||
.me_adaptive_floor_recover_grace_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_min_writers_multi_endpoint(&self) -> usize {
|
pub(super) fn adaptive_floor_min_writers_multi_endpoint(&self) -> usize {
|
||||||
(self
|
(self
|
||||||
.floor_runtime
|
.floor_runtime
|
||||||
@@ -1659,6 +1643,7 @@ impl MePool {
|
|||||||
&self,
|
&self,
|
||||||
contour: WriterContour,
|
contour: WriterContour,
|
||||||
allow_coverage_override: bool,
|
allow_coverage_override: bool,
|
||||||
|
writer_dc: i32,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await;
|
let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await;
|
||||||
match contour {
|
match contour {
|
||||||
@@ -1670,6 +1655,43 @@ impl MePool {
|
|||||||
if !allow_coverage_override {
|
if !allow_coverage_override {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut endpoints_len = 0;
|
||||||
|
let now_epoch = Self::now_epoch_secs();
|
||||||
|
if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch) {
|
||||||
|
if let Some(addrs) = self.proxy_map_v4.read().await.get(&writer_dc) {
|
||||||
|
endpoints_len += addrs.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch) {
|
||||||
|
if let Some(addrs) = self.proxy_map_v6.read().await.get(&writer_dc) {
|
||||||
|
endpoints_len += addrs.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if endpoints_len > 0 {
|
||||||
|
let base_req =
|
||||||
|
self.required_writers_for_dc_with_floor_mode(endpoints_len, false);
|
||||||
|
let active_for_dc = {
|
||||||
|
let ws = self.writers.read().await;
|
||||||
|
ws.iter()
|
||||||
|
.filter(|w| {
|
||||||
|
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
|
||||||
|
&& w.writer_dc == writer_dc
|
||||||
|
&& matches!(
|
||||||
|
WriterContour::from_u8(
|
||||||
|
w.contour.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
),
|
||||||
|
WriterContour::Active
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.count()
|
||||||
|
};
|
||||||
|
if active_for_dc < base_req {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let coverage_required = self.active_coverage_required_total().await;
|
let coverage_required = self.active_coverage_required_total().await;
|
||||||
active_writers < coverage_required
|
active_writers < coverage_required
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -77,6 +77,12 @@ impl MePool {
|
|||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if endpoints.len() == 1 && self.single_endpoint_outage_disable_quarantine() {
|
||||||
|
let mut guard = self.endpoint_quarantine.lock().await;
|
||||||
|
guard.retain(|_, expiry| *expiry > Instant::now());
|
||||||
|
return endpoints.to_vec();
|
||||||
|
}
|
||||||
|
|
||||||
let mut guard = self.endpoint_quarantine.lock().await;
|
let mut guard = self.endpoint_quarantine.lock().await;
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
guard.retain(|_, expiry| *expiry > now);
|
guard.retain(|_, expiry| *expiry > now);
|
||||||
@@ -236,8 +242,18 @@ impl MePool {
|
|||||||
let fast_retries = self.reconnect_runtime.me_reconnect_fast_retry_count.max(1);
|
let fast_retries = self.reconnect_runtime.me_reconnect_fast_retry_count.max(1);
|
||||||
let mut total_attempts = 0u32;
|
let mut total_attempts = 0u32;
|
||||||
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
||||||
|
let dc_endpoints = self.endpoints_for_dc(writer_dc).await;
|
||||||
|
let single_endpoint_dc = dc_endpoints.len() == 1 && dc_endpoints[0] == addr;
|
||||||
|
let bypass_quarantine_for_single_endpoint =
|
||||||
|
single_endpoint_dc && self.single_endpoint_outage_disable_quarantine();
|
||||||
|
|
||||||
if !same_endpoint_quarantined {
|
if !same_endpoint_quarantined || bypass_quarantine_for_single_endpoint {
|
||||||
|
if same_endpoint_quarantined && bypass_quarantine_for_single_endpoint {
|
||||||
|
debug!(
|
||||||
|
%addr,
|
||||||
|
"Bypassing quarantine for immediate reconnect on single-endpoint DC"
|
||||||
|
);
|
||||||
|
}
|
||||||
for attempt in 0..fast_retries {
|
for attempt in 0..fast_retries {
|
||||||
if total_attempts >= ME_REFILL_TOTAL_ATTEMPT_CAP {
|
if total_attempts >= ME_REFILL_TOTAL_ATTEMPT_CAP {
|
||||||
break;
|
break;
|
||||||
@@ -276,7 +292,6 @@ impl MePool {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let dc_endpoints = self.endpoints_for_dc(writer_dc).await;
|
|
||||||
if dc_endpoints.is_empty() {
|
if dc_endpoints.is_empty() {
|
||||||
self.stats.increment_me_refill_failed_total();
|
self.stats.increment_me_refill_failed_total();
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -342,7 +342,7 @@ impl MePool {
|
|||||||
allow_coverage_override: bool,
|
allow_coverage_override: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if !self
|
if !self
|
||||||
.can_open_writer_for_contour(contour, allow_coverage_override)
|
.can_open_writer_for_contour(contour, allow_coverage_override, writer_dc)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
return Err(ProxyError::Proxy(format!(
|
return Err(ProxyError::Proxy(format!(
|
||||||
|
|||||||
@@ -20,11 +20,15 @@ use crate::protocol::constants::*;
|
|||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
|
|
||||||
use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
|
use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
|
||||||
|
use super::fairness::{
|
||||||
|
AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision, WorkerFairnessConfig,
|
||||||
|
WorkerFairnessSnapshot, WorkerFairnessState,
|
||||||
|
};
|
||||||
use super::registry::RouteResult;
|
use super::registry::RouteResult;
|
||||||
use super::{ConnRegistry, MeResponse};
|
use super::{ConnRegistry, MeResponse};
|
||||||
|
|
||||||
const DATA_ROUTE_MAX_ATTEMPTS: usize = 3;
|
const DATA_ROUTE_MAX_ATTEMPTS: usize = 3;
|
||||||
const DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD: u8 = 3;
|
const DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD: u8 = 3;
|
||||||
|
const FAIRNESS_DRAIN_BUDGET_PER_LOOP: usize = 128;
|
||||||
|
|
||||||
fn should_close_on_route_result_for_data(result: RouteResult) -> bool {
|
fn should_close_on_route_result_for_data(result: RouteResult) -> bool {
|
||||||
matches!(result, RouteResult::NoConn | RouteResult::ChannelClosed)
|
matches!(result, RouteResult::NoConn | RouteResult::ChannelClosed)
|
||||||
@@ -77,6 +81,118 @@ async fn route_data_with_retry(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn route_feedback(result: RouteResult) -> DispatchFeedback {
|
||||||
|
match result {
|
||||||
|
RouteResult::Routed => DispatchFeedback::Routed,
|
||||||
|
RouteResult::NoConn => DispatchFeedback::NoConn,
|
||||||
|
RouteResult::ChannelClosed => DispatchFeedback::ChannelClosed,
|
||||||
|
RouteResult::QueueFullBase | RouteResult::QueueFullHigh => DispatchFeedback::QueueFull,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn report_route_drop(result: RouteResult, stats: &Stats) {
|
||||||
|
match result {
|
||||||
|
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
||||||
|
RouteResult::ChannelClosed => stats.increment_me_route_drop_channel_closed(),
|
||||||
|
RouteResult::QueueFullBase => {
|
||||||
|
stats.increment_me_route_drop_queue_full();
|
||||||
|
stats.increment_me_route_drop_queue_full_base();
|
||||||
|
}
|
||||||
|
RouteResult::QueueFullHigh => {
|
||||||
|
stats.increment_me_route_drop_queue_full();
|
||||||
|
stats.increment_me_route_drop_queue_full_high();
|
||||||
|
}
|
||||||
|
RouteResult::Routed => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_fairness_metrics_delta(
|
||||||
|
stats: &Stats,
|
||||||
|
prev: &mut WorkerFairnessSnapshot,
|
||||||
|
current: WorkerFairnessSnapshot,
|
||||||
|
) {
|
||||||
|
stats.set_me_fair_active_flows_gauge(current.active_flows as u64);
|
||||||
|
stats.set_me_fair_queued_bytes_gauge(current.total_queued_bytes);
|
||||||
|
stats.set_me_fair_standing_flows_gauge(current.standing_flows as u64);
|
||||||
|
stats.set_me_fair_backpressured_flows_gauge(current.backpressured_flows as u64);
|
||||||
|
stats.set_me_fair_pressure_state_gauge(current.pressure_state.as_u8() as u64);
|
||||||
|
stats.add_me_fair_scheduler_rounds_total(
|
||||||
|
current
|
||||||
|
.scheduler_rounds
|
||||||
|
.saturating_sub(prev.scheduler_rounds),
|
||||||
|
);
|
||||||
|
stats.add_me_fair_deficit_grants_total(
|
||||||
|
current.deficit_grants.saturating_sub(prev.deficit_grants),
|
||||||
|
);
|
||||||
|
stats.add_me_fair_deficit_skips_total(current.deficit_skips.saturating_sub(prev.deficit_skips));
|
||||||
|
stats.add_me_fair_enqueue_rejects_total(
|
||||||
|
current.enqueue_rejects.saturating_sub(prev.enqueue_rejects),
|
||||||
|
);
|
||||||
|
stats.add_me_fair_shed_drops_total(current.shed_drops.saturating_sub(prev.shed_drops));
|
||||||
|
stats.add_me_fair_penalties_total(
|
||||||
|
current
|
||||||
|
.fairness_penalties
|
||||||
|
.saturating_sub(prev.fairness_penalties),
|
||||||
|
);
|
||||||
|
stats.add_me_fair_downstream_stalls_total(
|
||||||
|
current
|
||||||
|
.downstream_stalls
|
||||||
|
.saturating_sub(prev.downstream_stalls),
|
||||||
|
);
|
||||||
|
*prev = current;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn drain_fairness_scheduler(
|
||||||
|
fairness: &mut WorkerFairnessState,
|
||||||
|
reg: &ConnRegistry,
|
||||||
|
tx: &mpsc::Sender<WriterCommand>,
|
||||||
|
data_route_queue_full_streak: &mut HashMap<u64, u8>,
|
||||||
|
route_wait_ms: u64,
|
||||||
|
stats: &Stats,
|
||||||
|
) {
|
||||||
|
for _ in 0..FAIRNESS_DRAIN_BUDGET_PER_LOOP {
|
||||||
|
let now = Instant::now();
|
||||||
|
let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let cid = candidate.frame.conn_id;
|
||||||
|
let _pressure_state = candidate.pressure_state;
|
||||||
|
let _flow_class = candidate.flow_class;
|
||||||
|
let routed = route_data_with_retry(
|
||||||
|
reg,
|
||||||
|
cid,
|
||||||
|
candidate.frame.flags,
|
||||||
|
candidate.frame.data.clone(),
|
||||||
|
route_wait_ms,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if matches!(routed, RouteResult::Routed) {
|
||||||
|
data_route_queue_full_streak.remove(&cid);
|
||||||
|
} else {
|
||||||
|
report_route_drop(routed, stats);
|
||||||
|
}
|
||||||
|
let action = fairness.apply_dispatch_feedback(cid, candidate, route_feedback(routed), now);
|
||||||
|
if is_data_route_queue_full(routed) {
|
||||||
|
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
|
||||||
|
*streak = streak.saturating_add(1);
|
||||||
|
if should_close_on_queue_full_streak(*streak) {
|
||||||
|
fairness.remove_flow(cid);
|
||||||
|
data_route_queue_full_streak.remove(&cid);
|
||||||
|
reg.unregister(cid).await;
|
||||||
|
send_close_conn(tx, cid).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if action == DispatchAction::CloseFlow || should_close_on_route_result_for_data(routed) {
|
||||||
|
fairness.remove_flow(cid);
|
||||||
|
data_route_queue_full_streak.remove(&cid);
|
||||||
|
reg.unregister(cid).await;
|
||||||
|
send_close_conn(tx, cid).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn reader_loop(
|
pub(crate) async fn reader_loop(
|
||||||
mut rd: tokio::io::ReadHalf<TcpStream>,
|
mut rd: tokio::io::ReadHalf<TcpStream>,
|
||||||
dk: [u8; 32],
|
dk: [u8; 32],
|
||||||
@@ -98,7 +214,21 @@ pub(crate) async fn reader_loop(
|
|||||||
let mut raw = enc_leftover;
|
let mut raw = enc_leftover;
|
||||||
let mut expected_seq: i32 = 0;
|
let mut expected_seq: i32 = 0;
|
||||||
let mut data_route_queue_full_streak = HashMap::<u64, u8>::new();
|
let mut data_route_queue_full_streak = HashMap::<u64, u8>::new();
|
||||||
|
let mut fairness = WorkerFairnessState::new(
|
||||||
|
WorkerFairnessConfig {
|
||||||
|
worker_id: (writer_id as u16).saturating_add(1),
|
||||||
|
max_active_flows: reg.route_channel_capacity().saturating_mul(4).max(256),
|
||||||
|
max_total_queued_bytes: (reg.route_channel_capacity() as u64)
|
||||||
|
.saturating_mul(16 * 1024)
|
||||||
|
.max(4 * 1024 * 1024),
|
||||||
|
max_flow_queued_bytes: (reg.route_channel_capacity() as u64)
|
||||||
|
.saturating_mul(2 * 1024)
|
||||||
|
.clamp(64 * 1024, 2 * 1024 * 1024),
|
||||||
|
..WorkerFairnessConfig::default()
|
||||||
|
},
|
||||||
|
Instant::now(),
|
||||||
|
);
|
||||||
|
let mut fairness_snapshot = fairness.snapshot();
|
||||||
loop {
|
loop {
|
||||||
let mut tmp = [0u8; 65_536];
|
let mut tmp = [0u8; 65_536];
|
||||||
let n = tokio::select! {
|
let n = tokio::select! {
|
||||||
@@ -181,36 +311,20 @@ pub(crate) async fn reader_loop(
|
|||||||
let data = body.slice(12..);
|
let data = body.slice(12..);
|
||||||
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
|
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
|
||||||
|
|
||||||
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
|
let admission = fairness.enqueue_data(cid, flags, data, Instant::now());
|
||||||
let routed =
|
if !matches!(admission, AdmissionDecision::Admit) {
|
||||||
route_data_with_retry(reg.as_ref(), cid, flags, data, route_wait_ms).await;
|
stats.increment_me_route_drop_queue_full();
|
||||||
if matches!(routed, RouteResult::Routed) {
|
stats.increment_me_route_drop_queue_full_high();
|
||||||
data_route_queue_full_streak.remove(&cid);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
match routed {
|
|
||||||
RouteResult::NoConn => stats.increment_me_route_drop_no_conn(),
|
|
||||||
RouteResult::ChannelClosed => stats.increment_me_route_drop_channel_closed(),
|
|
||||||
RouteResult::QueueFullBase => {
|
|
||||||
stats.increment_me_route_drop_queue_full();
|
|
||||||
stats.increment_me_route_drop_queue_full_base();
|
|
||||||
}
|
|
||||||
RouteResult::QueueFullHigh => {
|
|
||||||
stats.increment_me_route_drop_queue_full();
|
|
||||||
stats.increment_me_route_drop_queue_full_high();
|
|
||||||
}
|
|
||||||
RouteResult::Routed => {}
|
|
||||||
}
|
|
||||||
if should_close_on_route_result_for_data(routed) {
|
|
||||||
data_route_queue_full_streak.remove(&cid);
|
|
||||||
reg.unregister(cid).await;
|
|
||||||
send_close_conn(&tx, cid).await;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if is_data_route_queue_full(routed) {
|
|
||||||
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
|
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
|
||||||
*streak = streak.saturating_add(1);
|
*streak = streak.saturating_add(1);
|
||||||
if should_close_on_queue_full_streak(*streak) {
|
if should_close_on_queue_full_streak(*streak)
|
||||||
|
|| matches!(
|
||||||
|
admission,
|
||||||
|
AdmissionDecision::RejectSaturated
|
||||||
|
| AdmissionDecision::RejectStandingFlow
|
||||||
|
)
|
||||||
|
{
|
||||||
|
fairness.remove_flow(cid);
|
||||||
data_route_queue_full_streak.remove(&cid);
|
data_route_queue_full_streak.remove(&cid);
|
||||||
reg.unregister(cid).await;
|
reg.unregister(cid).await;
|
||||||
send_close_conn(&tx, cid).await;
|
send_close_conn(&tx, cid).await;
|
||||||
@@ -249,12 +363,14 @@ pub(crate) async fn reader_loop(
|
|||||||
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
||||||
reg.unregister(cid).await;
|
reg.unregister(cid).await;
|
||||||
data_route_queue_full_streak.remove(&cid);
|
data_route_queue_full_streak.remove(&cid);
|
||||||
|
fairness.remove_flow(cid);
|
||||||
} else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 {
|
} else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 {
|
||||||
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
debug!(cid, "RPC_CLOSE_CONN from ME");
|
debug!(cid, "RPC_CLOSE_CONN from ME");
|
||||||
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
||||||
reg.unregister(cid).await;
|
reg.unregister(cid).await;
|
||||||
data_route_queue_full_streak.remove(&cid);
|
data_route_queue_full_streak.remove(&cid);
|
||||||
|
fairness.remove_flow(cid);
|
||||||
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
||||||
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
trace!(ping_id, "RPC_PING -> RPC_PONG");
|
trace!(ping_id, "RPC_PING -> RPC_PONG");
|
||||||
@@ -310,6 +426,19 @@ pub(crate) async fn reader_loop(
|
|||||||
"Unknown RPC"
|
"Unknown RPC"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
|
||||||
|
drain_fairness_scheduler(
|
||||||
|
&mut fairness,
|
||||||
|
reg.as_ref(),
|
||||||
|
&tx,
|
||||||
|
&mut data_route_queue_full_streak,
|
||||||
|
route_wait_ms,
|
||||||
|
stats.as_ref(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let current_snapshot = fairness.snapshot();
|
||||||
|
apply_fairness_metrics_delta(stats.as_ref(), &mut fairness_snapshot, current_snapshot);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -140,6 +140,10 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn route_channel_capacity(&self) -> usize {
|
||||||
|
self.route_channel_capacity
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self::with_route_channel_capacity(4096)
|
Self::with_route_channel_capacity(4096)
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
use crate::transport::middle_proxy::fairness::{
|
||||||
|
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
|
||||||
|
WorkerFairnessConfig, WorkerFairnessState,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn enqueue_payload(size: usize) -> Bytes {
|
||||||
|
Bytes::from(vec![0xAB; size])
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fairness_rejects_when_worker_budget_is_exhausted() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut fairness = WorkerFairnessState::new(
|
||||||
|
WorkerFairnessConfig {
|
||||||
|
max_total_queued_bytes: 1024,
|
||||||
|
max_flow_queued_bytes: 1024,
|
||||||
|
..WorkerFairnessConfig::default()
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(1, 0, enqueue_payload(700), now),
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(2, 0, enqueue_payload(400), now),
|
||||||
|
AdmissionDecision::RejectWorkerCap
|
||||||
|
);
|
||||||
|
|
||||||
|
let snapshot = fairness.snapshot();
|
||||||
|
assert!(snapshot.total_queued_bytes <= 1024);
|
||||||
|
assert_eq!(snapshot.enqueue_rejects, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fairness_marks_standing_queue_after_stall_and_age_threshold() {
|
||||||
|
let mut now = Instant::now();
|
||||||
|
let mut fairness = WorkerFairnessState::new(
|
||||||
|
WorkerFairnessConfig {
|
||||||
|
standing_queue_min_age: Duration::from_millis(50),
|
||||||
|
standing_queue_min_backlog_bytes: 256,
|
||||||
|
standing_stall_threshold: 1,
|
||||||
|
max_flow_queued_bytes: 4096,
|
||||||
|
max_total_queued_bytes: 4096,
|
||||||
|
..WorkerFairnessConfig::default()
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(11, 0, enqueue_payload(512), now),
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
);
|
||||||
|
|
||||||
|
now += Duration::from_millis(100);
|
||||||
|
let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) else {
|
||||||
|
panic!("expected dispatch candidate");
|
||||||
|
};
|
||||||
|
|
||||||
|
let action = fairness.apply_dispatch_feedback(11, candidate, DispatchFeedback::QueueFull, now);
|
||||||
|
assert!(matches!(action, DispatchAction::Continue));
|
||||||
|
|
||||||
|
let snapshot = fairness.snapshot();
|
||||||
|
assert_eq!(snapshot.standing_flows, 1);
|
||||||
|
assert!(snapshot.backpressured_flows >= 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fairness_keeps_fast_flow_progress_under_slow_neighbor() {
|
||||||
|
let mut now = Instant::now();
|
||||||
|
let mut fairness = WorkerFairnessState::new(
|
||||||
|
WorkerFairnessConfig {
|
||||||
|
max_total_queued_bytes: 64 * 1024,
|
||||||
|
max_flow_queued_bytes: 32 * 1024,
|
||||||
|
..WorkerFairnessConfig::default()
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
|
||||||
|
for _ in 0..16 {
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(1, 0, enqueue_payload(512), now),
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(2, 0, enqueue_payload(512), now),
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut fast_routed = 0u64;
|
||||||
|
for _ in 0..128 {
|
||||||
|
now += Duration::from_millis(5);
|
||||||
|
let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let cid = candidate.frame.conn_id;
|
||||||
|
let feedback = if cid == 2 {
|
||||||
|
DispatchFeedback::QueueFull
|
||||||
|
} else {
|
||||||
|
fast_routed = fast_routed.saturating_add(1);
|
||||||
|
DispatchFeedback::Routed
|
||||||
|
};
|
||||||
|
let _ = fairness.apply_dispatch_feedback(cid, candidate, feedback, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = fairness.snapshot();
|
||||||
|
assert!(fast_routed > 0, "fast flow must continue making progress");
|
||||||
|
assert!(snapshot.total_queued_bytes <= 64 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fairness_pressure_hysteresis_prevents_instant_flapping() {
|
||||||
|
let mut now = Instant::now();
|
||||||
|
let mut cfg = WorkerFairnessConfig::default();
|
||||||
|
cfg.max_total_queued_bytes = 4096;
|
||||||
|
cfg.max_flow_queued_bytes = 4096;
|
||||||
|
cfg.pressure.evaluate_every_rounds = 1;
|
||||||
|
cfg.pressure.transition_hysteresis_rounds = 3;
|
||||||
|
cfg.pressure.queue_ratio_pressured_pct = 40;
|
||||||
|
cfg.pressure.queue_ratio_shedding_pct = 60;
|
||||||
|
cfg.pressure.queue_ratio_saturated_pct = 80;
|
||||||
|
|
||||||
|
let mut fairness = WorkerFairnessState::new(cfg, now);
|
||||||
|
|
||||||
|
for _ in 0..4 {
|
||||||
|
assert_eq!(
|
||||||
|
fairness.enqueue_data(9, 0, enqueue_payload(900), now),
|
||||||
|
AdmissionDecision::Admit
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for _ in 0..2 {
|
||||||
|
now += Duration::from_millis(1);
|
||||||
|
let _ = fairness.next_decision(now);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fairness.pressure_state(),
|
||||||
|
PressureState::Normal,
|
||||||
|
"state must not flip before hysteresis confirmations"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fairness_randomized_sequence_preserves_memory_bounds() {
|
||||||
|
let mut now = Instant::now();
|
||||||
|
let mut fairness = WorkerFairnessState::new(
|
||||||
|
WorkerFairnessConfig {
|
||||||
|
max_total_queued_bytes: 32 * 1024,
|
||||||
|
max_flow_queued_bytes: 4 * 1024,
|
||||||
|
..WorkerFairnessConfig::default()
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut seed = 0xC0FFEE_u64;
|
||||||
|
for _ in 0..4096 {
|
||||||
|
seed ^= seed << 7;
|
||||||
|
seed ^= seed >> 9;
|
||||||
|
seed ^= seed << 8;
|
||||||
|
let flow = (seed % 32) + 1;
|
||||||
|
let size = ((seed >> 8) % 512 + 64) as usize;
|
||||||
|
let _ = fairness.enqueue_data(flow, 0, enqueue_payload(size), now);
|
||||||
|
|
||||||
|
now += Duration::from_millis(1);
|
||||||
|
if let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) {
|
||||||
|
let feedback = if seed & 0x1 == 0 {
|
||||||
|
DispatchFeedback::Routed
|
||||||
|
} else {
|
||||||
|
DispatchFeedback::QueueFull
|
||||||
|
};
|
||||||
|
let _ =
|
||||||
|
fairness.apply_dispatch_feedback(candidate.frame.conn_id, candidate, feedback, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = fairness.snapshot();
|
||||||
|
assert!(snapshot.total_queued_bytes <= 32 * 1024);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -109,18 +109,16 @@ async fn connectable_endpoints_waits_until_quarantine_expires() {
|
|||||||
|
|
||||||
{
|
{
|
||||||
let mut guard = pool.endpoint_quarantine.lock().await;
|
let mut guard = pool.endpoint_quarantine.lock().await;
|
||||||
guard.insert(addr, Instant::now() + Duration::from_millis(80));
|
guard.insert(addr, Instant::now() + Duration::from_millis(500));
|
||||||
}
|
}
|
||||||
|
|
||||||
let started = Instant::now();
|
let endpoints = tokio::time::timeout(
|
||||||
let endpoints = pool.connectable_endpoints_for_test(&[addr]).await;
|
Duration::from_millis(120),
|
||||||
let elapsed = started.elapsed();
|
pool.connectable_endpoints_for_test(&[addr]),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("single-endpoint outage mode should bypass quarantine delay");
|
||||||
assert_eq!(endpoints, vec![addr]);
|
assert_eq!(endpoints, vec![addr]);
|
||||||
assert!(
|
|
||||||
elapsed >= Duration::from_millis(50),
|
|
||||||
"single-endpoint DC should honor quarantine before retry"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user