Compare commits

...

52 Commits

Author SHA1 Message Date
Alexey 326c0ecdb9 Rustfmt 2026-09-25 19:49:36 +03:00
Alexey 2d63fcf376 Update Cargo.toml 2026-09-25 19:44:47 +03:00
Alexey c12c5c73c1 Conntrack: Firewall reconciliation process-owned&transactional 2026-09-25 19:35:31 +03:00
Alexey 26a574780e CAS Contention in Traffic Bucket bounded 2026-09-25 06:32:01 +03:00
Alexey 08109d53e8 WEB Data Budget: Session effects until after state unlock deferred 2026-09-24 01:02:07 +03:00
Alexey f1107c21d9 Process-wide concurrency + Cancellation ownership fixes 2026-09-23 22:30:51 +03:00
Alexey baa9bfbb01 ME Authority + Quota Resets + WEB Replacement Rollback fixes 2026-09-22 03:01:17 +03:00
Alexey 51e706770c WEB Manager Session + Generation Fence tests 2026-09-21 21:08:34 +03:00
Alexey d706b3f3ba Hardswap Invariants in tests + Quota fixes 2026-09-20 00:28:52 +03:00
Alexey 89dacbd17e TOCTOU and lifecycle races across runtime boundaries fixes 2026-09-19 22:37:22 +03:00
Alexey acad414cc7 Runtime paths symlink preserved + User Admission Registration 2026-09-18 23:11:09 +03:00
Alexey 02f66c542e Descriptor-anchored Secured Filesystem Operations 2026-09-17 22:45:48 +03:00
Alexey 9a683d8b3d Slot Budget + Config Store Atomic Writer fixes + Trusted Command 2026-09-16 22:06:59 +03:00
Alexey 55f3d19ee0 Direct-since-epoch 2026-09-15 20:37:41 +03:00
Alexey 5ec9f85530 User Admission Tests 2026-09-14 20:21:41 +03:00
Alexey 0ac236955a Proxy Shared User Drafts 2026-09-14 20:19:48 +03:00
Alexey b37f1ebdeb TLS Replay Digest In-flight Ownership 2026-09-13 21:56:20 +03:00
Alexey 1ea5f7a1d6 Atomic Lifecycle for Writer Refresh + Hardswap 2026-09-12 20:14:41 +03:00
Alexey 54197e848a Update tests.rs 2026-09-11 20:33:01 +03:00
Alexey 479899240a WEB: websocket-lanes upgrade-fail recovery #925 2026-09-10 23:11:36 +03:00
Alexey 844e41ea34 Races in admission + accounting + publication,+ PID fixed 2026-09-09 22:45:02 +03:00
Alexey 021ad1fe68 WEB Bridge Sideband 2026-09-08 21:29:32 +03:00
Alexey 4ca7418442 Merge pull request #923 from telemt/staled-ws
Stale Websocket-lanes recovers after restart + Native macOS status-schema preserved
2026-09-08 10:02:48 +03:00
Alexey 464bc180e1 Rustfmt 2026-09-07 21:01:56 +03:00
Alexey 8b0ee9bf3d Bump -> 3.5.7 2026-09-07 20:59:31 +03:00
Alexey 24c4cefc74 Native macOS status-schema preserved
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-07 18:33:12 +03:00
Alexey 97c046757a Stale Websocket-lanes recovers after restart
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-07 18:08:24 +03:00
Alexey 4e3d560a88 Merge pull request #919 from nikoano/fix/install-port-check-tcp-only
fix(install): probe only TCP when validating the target port
2026-09-07 00:03:33 +03:00
Alexey 3693d1e2a8 GHA Workflows updated 2026-09-06 20:17:30 +03:00
Alexey f857aefd06 Dependencies Update + Rust 1.88 2026-09-06 19:52:12 +03:00
Alexey 3cb123fbf0 Create web_decoy_fasttrack.rs 2026-09-06 18:54:17 +03:00
Alexey 597b6b0226 Bump -> 3.5.6 2026-09-06 18:52:28 +03:00
Alexey 63b9cce25f Docs for Decoy Fast-Track
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-06 18:51:39 +03:00
Alexey 20a4d50524 Restart-gated Decoy Fast-Track
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-06 18:51:13 +03:00
Alexey 106b26a5b7 Decoy Fasttrack Drafts
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-05 21:18:48 +03:00
nikoano 9908e04e04 fix(install): probe only TCP when validating the target port 2026-09-05 17:03:34 +03:00
Alexey 9f023ff9c7 WEB Bounded Bridge Recovery added
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-04 19:01:56 +03:00
Alexey 0d044c7372 WEB WS Downlink correctness + peer-lease fixed
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-03 20:24:06 +03:00
Alexey e796cb112c Update activity.rs
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-03 20:13:37 +03:00
Alexey 7de6edda98 Security-regression fixed + Uplink moduled
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-03 20:11:59 +03:00
Alexey 7f4e637b99 Decoy Contract sanitized
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-03 19:01:52 +03:00
Alexey 50faaba8c5 Generation Fence against Cancellation race after HTTP-Runner Fresh Reincarnation
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-03 19:00:06 +03:00
Alexey 9170e347f5 Concurrent Uplinks w/ wrong single-flight recovery fixes
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-02 18:55:32 +03:00
Alexey 0d8d331c7c Bridge Request/Buffers/Recovery
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-02 18:49:10 +03:00
Alexey 01ffca5d34 WEB Session Lifecycle Observability + Bridge Recovery Drafts
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-09-01 19:19:52 +03:00
Alexey aec7d1619a Docs for WEB Carrier Status
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-31 21:50:17 +03:00
Alexey 718ce0847e WEB Carrier Counters + Status
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-31 21:39:09 +03:00
Alexey 66f2b8889f Split oversized runtime modules + Async tests hardened
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-30 09:39:33 +03:00
Alexey 281f63f940 Runtime Ownership hardened
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-30 08:38:03 +03:00
Alexey 1bb6b0bdda Docs for Web Lifecycle API
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-29 16:19:43 +03:00
Alexey 084834f5ec API for WEB: bounded lifecycle and overload observability added
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-29 16:19:21 +03:00
Alexey 012dc07a98 WEB Ephemeral Lifecycle
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-08-28 22:39:18 +03:00
436 changed files with 61140 additions and 32067 deletions
+4 -4
View File
@@ -19,13 +19,13 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v7.0.1
- name: Install latest stable Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry & build artifacts
uses: actions/cache@v4
uses: actions/cache@v6.1.0
with:
path: |
~/.cargo/registry
@@ -39,7 +39,7 @@ jobs:
run: cargo build --release --verbose
- name: Upload binary artifact
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7.0.1
with:
name: telemt
path: target/release/telemt
path: target/release/telemt
+40 -24
View File
@@ -14,9 +14,7 @@ concurrency:
cancel-in-progress: true
jobs:
# ==========================
# Formatting
# ==========================
# Rust formatting validation
fmt:
name: Fmt
runs-on: ubuntu-latest
@@ -25,7 +23,7 @@ jobs:
contents: read
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@stable
with:
@@ -33,9 +31,23 @@ jobs:
- run: cargo fmt -- --check
# ==========================
# Tests
# ==========================
# Minimum supported Rust version validation
msrv:
name: MSRV
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@1.88.0
- name: Check all targets on MSRV
run: cargo check --all-targets --locked
# Rust test suite
test:
name: Test
runs-on: ubuntu-latest
@@ -46,32 +58,34 @@ jobs:
checks: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@stable
- name: Cache cargo
uses: actions/cache@v4
uses: actions/cache@v6.1.0
with:
path: |
~/.cargo/bin
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}
key: ${{ runner.os }}-cargo-nextest-0.9.143-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-nextest-0.9.143-
${{ runner.os }}-cargo-nextest-
${{ runner.os }}-cargo-
- name: Install cargo-nextest
run: cargo install --locked cargo-nextest || true
run: |
if ! cargo-nextest --version 2>/dev/null | grep -Fq "cargo-nextest 0.9.143"; then
cargo install --locked --version 0.9.143 --force cargo-nextest
fi
- name: Run tests with nextest
run: cargo nextest run -j "$(nproc)"
# ==========================
# Clippy
# ==========================
# Rust lint validation
clippy:
name: Clippy
runs-on: ubuntu-latest
@@ -81,14 +95,14 @@ jobs:
checks: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- name: Cache cargo
uses: actions/cache@v4
uses: actions/cache@v6.1.0
with:
path: |
~/.cargo/registry
@@ -102,9 +116,7 @@ jobs:
- name: Run clippy
run: cargo clippy -j "$(nproc)" -- --cap-lints warn
# ==========================
# Udeps
# ==========================
# Unused dependency validation
udeps:
name: Udeps
runs-on: ubuntu-latest
@@ -113,27 +125,31 @@ jobs:
contents: read
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- name: Cache cargo
uses: actions/cache@v4
uses: actions/cache@v6.1.0
with:
path: |
~/.cargo/bin
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-udeps-${{ hashFiles('**/Cargo.lock') }}
key: ${{ runner.os }}-cargo-udeps-0.1.61-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-udeps-0.1.61-
${{ runner.os }}-cargo-udeps-
${{ runner.os }}-cargo-
- name: Install cargo-udeps
run: cargo install --locked cargo-udeps || true
run: |
if ! cargo-udeps --version 2>/dev/null | grep -Fq "cargo-udeps 0.1.61"; then
cargo install --locked --version 0.1.61 --force cargo-udeps
fi
- name: Run udeps
run: cargo udeps -j "$(nproc)" || true
+3 -3
View File
@@ -30,16 +30,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v7.0.1
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v4.37.9
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: .github/codeql/codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v4.37.9
with:
category: "/language:${{ matrix.language }}"
+9 -5
View File
@@ -17,33 +17,37 @@ jobs:
contents: read
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
- name: Cache cargo
uses: actions/cache@v4
uses: actions/cache@v6.1.0
with:
path: |
~/.cargo/bin
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}
key: ${{ runner.os }}-cargo-llvm-cov-0.9.0-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-llvm-cov-0.9.0-
${{ runner.os }}-cargo-llvm-cov-
${{ runner.os }}-cargo-
- name: Install cargo-llvm-cov
run: cargo install --locked cargo-llvm-cov || true
run: |
if ! cargo-llvm-cov --version 2>/dev/null | grep -Fq "cargo-llvm-cov 0.9.0"; then
cargo install --locked --version 0.9.0 --force cargo-llvm-cov
fi
- name: Generate LCOV report
run: cargo llvm-cov --locked --lcov --output-path lcov.info
- name: Upload LCOV report
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7.0.1
with:
name: telemt-lcov
path: lcov.info
+15 -15
View File
@@ -88,9 +88,9 @@ jobs:
cpu: generic
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: dtolnay/rust-toolchain@v1
- uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
targets: |
@@ -108,7 +108,7 @@ jobs:
gcc-aarch64-linux-gnu \
g++-aarch64-linux-gnu
- uses: actions/cache@v4
- uses: actions/cache@v6.1.0
with:
path: |
/usr/local/cargo/registry
@@ -166,7 +166,7 @@ jobs:
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
- uses: actions/upload-artifact@v4
- uses: actions/upload-artifact@v7.0.1
with:
name: ${{ matrix.asset }}
path: dist/*
@@ -199,7 +199,7 @@ jobs:
cpu: generic
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- name: Install deps
run: |
@@ -209,7 +209,7 @@ jobs:
pkg-config \
curl
- uses: actions/cache@v4
- uses: actions/cache@v6.1.0
if: matrix.target == 'aarch64-unknown-linux-musl'
with:
path: ~/.musl-aarch64
@@ -244,7 +244,7 @@ jobs:
- name: Add rust target
run: rustup target add ${{ matrix.target }}
- uses: actions/cache@v4
- uses: actions/cache@v6.1.0
with:
path: |
/usr/local/cargo/registry
@@ -302,7 +302,7 @@ jobs:
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
- uses: actions/upload-artifact@v4
- uses: actions/upload-artifact@v7.0.1
with:
name: ${{ matrix.asset }}
path: dist/*
@@ -319,7 +319,7 @@ jobs:
contents: write
steps:
- uses: actions/download-artifact@v4
- uses: actions/download-artifact@v8.0.1
with:
path: artifacts
@@ -331,7 +331,7 @@ jobs:
find artifacts -type f -exec cp {} dist/ \;
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@v3.0.3
with:
tag_name: ${{ needs.prepare.outputs.version }}
target_commitish: ${{ github.sha }}
@@ -353,13 +353,13 @@ jobs:
packages: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7.0.1
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-qemu-action@v4.3.0
- uses: docker/setup-buildx-action@v3
- uses: docker/setup-buildx-action@v4.3.0
- uses: docker/login-action@v3
- uses: docker/login-action@v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -407,7 +407,7 @@ jobs:
} >> "${GITHUB_OUTPUT}"
- name: Build & Push
uses: docker/build-push-action@v6
uses: docker/build-push-action@v7.3.0
with:
context: .
push: true
Generated
+321 -468
View File
File diff suppressed because it is too large Load Diff
+36 -27
View File
@@ -1,28 +1,29 @@
[package]
name = "telemt"
version = "3.5.5"
version = "3.5.7"
edition = "2024"
rust-version = "1.88"
[features]
redteam_offline_expected_fail = []
[dependencies]
# C
libc = "0.2.186"
libc = "0.2.189"
# Async runtime
tokio = { version = "1.52.3", features = ["full", "tracing"] }
tokio-util = { version = "0.7.18", features = ["full"] }
tokio = { version = "1.53.1", features = ["full", "tracing"] }
tokio-util = { version = "0.7.19", features = ["full"] }
# Crypto
aes = { version = "0.8.4", features = ["zeroize"] }
ctr = { version = "0.9.2", features = ["zeroize"] }
cbc = "0.1.2"
sha2 = "0.10.9"
sha1 = "0.10.6"
sha1 = "0.10.7"
md-5 = "0.10.6"
hmac = "0.12.1"
crc32fast = "1.5.0"
crc32fast = "1.5.1"
crc32c = "0.6.8"
zeroize = { version = "1.9.0", features = ["derive"] }
subtle = "2.6.1"
@@ -30,75 +31,83 @@ static_assertions = "1.1.0"
ml-kem = { version = "0.3.2", default-features = false, features = ["alloc", "zeroize"] }
# Network
socket2 = { version = "0.6.4", features = ["all"] }
socket2 = { version = "0.6.5", features = ["all"] }
nix = { version = "0.31.3", default-features = false, features = [
"net",
"user",
"process",
"dir",
"fs",
"signal",
] }
shadowsocks = { version = "1.24.0", features = ["aead-cipher-2022"] }
shadowsocks = { version = "1.24.0", default-features = false, features = [
"aead-cipher",
"aead-cipher-2022",
] }
# Serialization
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.150"
toml = "1.1"
serde = { version = "1.0.229", features = ["derive"] }
serde_json = "1.0.151"
toml = "1.1.5"
x509-parser = "0.18.1"
# Utils
bytes = "1.12.0"
thiserror = "2.0.18"
bytes = "1.12.1"
thiserror = "2.0.20"
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
tracing-appender = "0.2.5"
parking_lot = "0.12.5"
dashmap = "6.2.1"
arc-swap = "1.9.1"
lru = "0.16.4"
rand = "0.10.1"
arc-swap = "1.9.2"
lru = "0.18.4"
rand = "0.10.2"
chrono = { version = "0.4.45", features = ["serde"] }
hex = "0.4.3"
base64 = "0.22.1"
url = "2.5.8"
regex = "1.12.4"
crossbeam-queue = "0.3.12"
num-bigint = "0.4.6"
regex = "1.13.1"
crossbeam-queue = "0.3.14"
num-bigint = "0.4.8"
num-traits = "0.2.19"
x25519-dalek = "2.0.1"
anyhow = "1.0.102"
anyhow = "1.0.104"
# HTTP
reqwest = { version = "0.13.4", features = ["rustls"], default-features = false }
notify = "8.2.0"
ipnetwork = { version = "0.21.1", features = ["serde"] }
hyper = { version = "1.10.1", features = ["client", "server", "http1"] }
hyper = { version = "1.11.1", features = ["client", "server", "http1"] }
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
http-body-util = "0.1.3"
http-body-util = "0.1.5"
httpdate = "1.0.3"
tokio-tungstenite = { version = "0.30.0", default-features = false }
futures-util = { version = "0.3.32", default-features = false, features = ["sink", "std"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = [
futures-util = { version = "0.3.34", default-features = false, features = ["sink", "std"] }
tokio-rustls = { version = "0.26.5", default-features = false, features = [
"tls12",
] }
rustls = { version = "0.23.41", default-features = false, features = [
rustls = { version = "0.23.43", default-features = false, features = [
"std",
"tls12",
"ring",
] }
webpki-roots = "1.0.8"
webpki-roots = "1.0.9"
[dev-dependencies]
tokio-test = "0.4.5"
criterion = "0.8.2"
proptest = "1.11.0"
futures = "0.3.32"
futures = "0.3.34"
tempfile = "3.27.0"
[[bench]]
name = "crypto_bench"
harness = false
[[bench]]
name = "web_decoy_fasttrack"
harness = false
[profile.release]
lto = "fat"
codegen-units = 1
+73
View File
@@ -0,0 +1,73 @@
use std::hint::black_box;
use std::sync::atomic::{AtomicU64, Ordering};
use base64::Engine as _;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
#[allow(dead_code)]
#[path = "../src/web/http/capability.rs"]
mod capability;
fn capability_at(index: usize) -> [u8; 32] {
let mut capability = [0xa5u8; 32];
capability[..8].copy_from_slice(&(index as u64).to_le_bytes());
capability
}
fn consume_scan(scan: capability::CapabilityScan) {
black_box(scan.matched.unwrap_u8());
black_box(scan.matched_index);
}
fn bench_decoy_fasttrack(c: &mut Criterion) {
for profile_count in [1usize, 32, 256, 1024] {
let capabilities = (0..profile_count).map(capability_at).collect::<Vec<_>>();
let miss = [0x5au8; 32];
let first = capabilities[0];
let middle = capabilities[profile_count / 2];
let last = capabilities[profile_count - 1];
let telemetry = AtomicU64::new(0);
let mut group = c.benchmark_group(format!("web_decoy_fasttrack/{profile_count}"));
group.bench_function(BenchmarkId::new("ordinary_enforce", profile_count), |b| {
b.iter(|| {
let candidate = capability::bridge_candidate(black_box(None));
telemetry.fetch_add(1, Ordering::Relaxed);
black_box(candidate.is_canonical());
});
});
group.bench_function(BenchmarkId::new("ordinary_shadow", profile_count), |b| {
b.iter(|| {
let candidate = capability::bridge_candidate(black_box(None));
telemetry.fetch_add(1, Ordering::Relaxed);
consume_scan(capability::scan_capabilities(
black_box(&capabilities),
candidate.scan_bytes(),
));
});
});
for (name, candidate) in [
("canonical_miss", miss),
("canonical_hit_first", first),
("canonical_hit_middle", middle),
("canonical_hit_last", last),
] {
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(candidate);
let query = format!("bridge={token}");
group.bench_function(BenchmarkId::new(name, profile_count), |b| {
b.iter(|| {
let candidate = capability::bridge_candidate(black_box(Some(&query)));
telemetry.fetch_add(1, Ordering::Relaxed);
consume_scan(capability::scan_capabilities(
black_box(&capabilities),
candidate.scan_bytes(),
));
});
});
}
group.finish();
}
}
criterion_group!(benches, bench_decoy_fasttrack);
criterion_main!(benches);
+66 -8
View File
@@ -111,6 +111,9 @@ Notes:
| `GET` | `/v1/runtime/web/operations/{operation_id}` | none | `200` | `ControlOperationStatus` |
| `POST` | `/v1/runtime/web/debug/clear` | `RuntimeInstanceRequest` | `200` | `DebugClearData` |
| `POST` | `/v1/runtime/web/carrier-learning/reset` | `RuntimeInstanceRequest` | `200` | `LearningResetData` |
| `POST` | `/v1/runtime/web/lifecycle/pause` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
| `POST` | `/v1/runtime/web/lifecycle/drain` | `DrainRequest` | `202` | `OperatorLifecycleStatus` |
| `POST` | `/v1/runtime/web/lifecycle/resume` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
@@ -159,6 +162,9 @@ Notes:
| `GET /v1/runtime/web/operations/{operation_id}` | Returns one of the 32 most recently retained WEB close-operation states. |
| `POST /v1/runtime/web/debug/clear` | Clears the bounded WEB debug ring under an epoch fence. |
| `POST /v1/runtime/web/carrier-learning/reset` | Clears process-local carrier-learning evidence without changing live attempt chains. |
| `POST /v1/runtime/web/lifecycle/pause` | Ephemerally closes new WEB work admission without closing existing sessions or streams. |
| `POST /v1/runtime/web/lifecycle/drain` | Starts one asynchronous graceful WEB drain under a bounded monotonic deadline. |
| `POST /v1/runtime/web/lifecycle/resume` | Cancels an active drain, if any, and reopens only the operator-owned admission fence. |
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
@@ -193,6 +199,7 @@ Notes:
| `409` | `web_runtime_mismatch` | A runtime instance, session reference, or operation reference belongs to another WEB process instance. |
| `409` | `web_issuance_enabled` | A WEB close-all operation was requested while effective issuance remained enabled. |
| `409` | `web_operation_in_progress` | Another bounded WEB close operation is active. |
| `409` | `web_lifecycle_in_progress` | Another WEB drain operation is active. |
| `409` | `user_exists` | User already exists on create. |
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
@@ -328,7 +335,7 @@ Sections absent from the config file are absent from the response (not `null`).
### WEB runtime identity and lifecycle
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>` and close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all three as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>`, close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`, and drain references use `wd1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all references as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
`GET /v1/config` is the desired on-disk configuration view. `GET /v1/runtime/web/status` is the effective process view. Its envelope `revision` still identifies the current source graph and can therefore be newer than the active runtime generation while a reload is pending.
@@ -339,14 +346,37 @@ The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit
| `lifecycle` | `string` | `starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`. |
| `lifecycle_epoch` | `u64` | Monotonic publication epoch. |
| `lifecycle_age_ms` | `u64` | Monotonic age of the current lifecycle publication. |
| `available` | `bool` | Whether a readable process runtime is currently published. |
| `available` | `bool` | Backward-compatible readable-runtime flag; it is not public TLS or private acceptor readiness. |
| `reason` | `string?` | Stable unavailability reason when `available=false`. |
| `listeners` | `string[]` | Effective bound WEB listener addresses. |
| `effective_config_enabled` | `bool` | `web.enabled` in the API request's active runtime generation. |
| `ingress` | `WebIngressStatus` | Process-owned listener/acceptor liveness and TCP accept counters. |
| `capacity` | `WebCapacityStatus` | Effective accepted-socket policy, fixed global resources, and typed rejection counters. |
| `decoy_upstream` | `WebDecoyUpstreamStatus` | Passive outcomes for Telemt's internal plain-HTTP decoy origin hop. |
| `decoy_fasttrack` | `WebDecoyFastTrackStatus` | Effective restart-frozen capability policy and fixed process-lifetime routing counters. |
| `carrier_negotiation` | `WebCarrierNegotiationStatus` | Fixed process-lifetime selection, reported-failure, and health/learning outcome counters. |
| `lifecycle_counters` | `WebLifecycleCountersStatus` | Fixed close-reason, post-gap observation, and bridge-recovery counters plus the effective recovery deadline. |
| `operator_lifecycle` | `OperatorLifecycleStatus?` | Process-local reversible admission and active/latest drain status while a runtime is published. |
| `runtime` | `WebRuntimeStatus?` | Present while the weak process-runtime publication can be upgraded. |
`WebIngressStatus` contains `configured_listeners`, `live_acceptors`, `accepting_connections`, optional `reason`, `tcp_accept_total`, and `tcp_accept_error_total`. Accepting requires lifecycle `running`, a readable runtime, at least one effective WEB listener, and one live accept loop per listener. Stable non-accepting reasons are `starting`, `no_web_listener`, `ingress_draining`, `ingress_drained`, `deadline_exceeded`, `runtime_released`, and `acceptor_unavailable`. Accept errors are `accept(2)` failures observed by Telemt; they are not kernel backlog drops or failed connection attempts that never reached the process.
`WebCapacityStatus` contains `http_connection_capacity_action`, `max_http_overload_connections`, `http_overload_timeout_ms`, fixed `resources`, `saturated_resources`, `partial`, `rejections`, and `http_connection_overload_outcomes`. Each resource has a closed-set `resource`, `unit`, `used`, `available`, `limit`, and terminal `closed` flag. Saturation is an instantaneous plane-local observation and never changes `available` or ingress readiness. Rejections are monotonic admission decisions indexed only by a closed reason enum; an internally retried queue or byte-budget decision may later make progress. Accepted-socket outcomes are `dropped`, `wait_admitted`, `wait_timeout_503`, `responded_503`, `overflow_capacity_drop`, `response_error_drop`, and `shutdown_drop`; `wait_admitted` is not a rejection.
`WebDecoyUpstreamStatus` contains the complete fixed outcome set plus optional `last_outcome` and `last_outcome_age_ms`. Outcomes distinguish `success`, `deadline_exhausted`, `connect_refused`, `connect_timeout`, `connect_error`, `http_handshake_timeout`, `http_handshake_error`, `response_head_timeout`, and `request_error`. This describes only Telemt to the configured decoy origin. A public client to NGINX refusal, or an NGINX to Telemt refusal before `accept(2)`, is outside this counter plane.
`WebDecoyFastTrackStatus` contains effective `mode` and the complete fixed `requests` disposition array. Dispositions are `shadow_would_fasttrack`, `shadow_candidate_full_scan`, `enforce_fasttrack`, and `enforce_candidate_full_scan`. `off` performs no fast-track counter writes. The complete set remains visible and retains its process-lifetime totals after runtime release because telemetry is process-owned.
`WebCarrierNegotiationStatus` remains present when the process runtime is unavailable because its counters belong to the WEB publication. `selections` is the complete carrier x disposition matrix (`profile_disabled`, `policy_disabled`, `policy_pending`, `epoch_exhausted`, `cold`, `applied`). `reported_failures` is the complete carrier x phase x canonical reason matrix, where phase is `provisional` or `committed` and reason is `timeout`, `network`, `upgrade`, `http`, or `protocol`. `learning_outcomes` distinguishes `recorded`, `not_eligible`, `policy_disabled`, `stale_epoch`, `capacity_rejected`, `sequence_exhausted`, `missing_chain`, `phase_mismatch`, `session_mismatch`, `owner_not_live`, and `closed_before_health`. Reported failures and rejection outcomes are diagnostic only and never create negative ranking evidence.
`WebLifecycleCountersStatus` always contains `bridge_recovery_secs`, the complete carrier x close-reason matrix, the complete carrier x lifecycle-observation matrix, and all recovery milestones. Close reasons are `client_delete`, `bridge_recovery`, `peer_idle`, `negotiation_timeout`, `carrier_superseded`, `protocol`, `backpressure`, `websocket_ended`, `api_close`, `operator_force`, and `runtime_shutdown`. Observations are `http_activity_after_gap`, `websocket_activity_after_gap`, and `request_after_close`. Recovery events are `bootstrap_issued`, `session_created`, `committed`, `expired_unused`, and `closed_before_commit`. All counters are process-owned, monotonic, fixed-cardinality, and remain present at zero while the runtime is unavailable.
`WebRuntimeStatus` includes `runtime_instance`, `generation_id`, immutable effective `limits`, manager/stream/budget/WebSocket/learning/debug planes, permit usage, task/counter totals, and `partial`. Plane locks are read with `try_lock`; a contended plane is omitted and named in `partial`. Status collection performs no cleanup, waits, or data-plane mutation, so fields are plane-local observations rather than one globally atomic snapshot. `runtime.manager.issuance_enabled` is the authority to check before close-all.
`OperatorLifecycleStatus` is a lock-free process snapshot with `state`, monotonic `epoch`, `age_ms`, `admission_open`, `effective_new_work_admission`, and the active or latest `drain`. States are `running`, `paused`, `draining`, `force_closing`, and `drained`. Drain status contains its opaque id, phase/outcome, frozen timeout, wall-clock correlation timestamps, latest session/stream/WebSocket remainder, and `force_close_signalled`. The response envelope `revision` remains a config source-graph revision and is not a lifecycle version.
The Prometheus endpoint exports the same process-owned observations through fixed-cardinality `telemt_web_*` families: ingress/operator lifecycle states, independent ingress flags, listener and TCP accept counts, resource usage/closure/saturation, typed rejection totals, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier aggregate totals. Decoy routing adds `telemt_web_decoy_fasttrack_mode{mode}` and `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation adds `telemt_web_carrier_selections_total{carrier,disposition}`, `telemt_web_carrier_reported_failures_total{carrier,phase,reason}`, `telemt_web_carrier_learning_outcomes_total{carrier,outcome}`, one-hot `telemt_web_carrier_learning_state{state}`, `telemt_web_carrier_learning_entries{kind}`, and one-hot `telemt_web_carrier_learning_policy{aggressiveness}`. Lifecycle recovery adds `telemt_web_session_closures_total{carrier,reason}`, `telemt_web_session_lifecycle_observations_total{carrier,observation}`, `telemt_web_bridge_recovery_events_total{event}`, and `telemt_web_bridge_recovery_seconds`. The learning states are `unavailable`, `partial`, `pending`, `exhausted`, `disabled`, and `enabled`; `pending` explicitly exposes a generation/policy publication mismatch instead of silently treating it as cold evidence. WEB labels never contain a host, user, client IP, listener address, token, session reference, profile key, runtime instance, or generation ID. Telemt does not claim health for the externally owned NGINX or HAProxy TLS endpoint; that boundary requires terminator telemetry and an external TCP/TLS probe.
### WEB session enumeration
`GET /v1/runtime/web/sessions` defaults to `limit=50`, permits `1..=200`, and scans at most 1000 ordered candidates. `next_cursor` continues after the last scanned opaque session reference. `scan_truncated` reports the scan bound, `partial_sessions` counts contended per-session snapshots, and `partial` names an unavailable manager plane. The complete serialized page remains below the API response envelope because every string and row count is bounded.
@@ -364,12 +394,32 @@ Filters are exact unless stated otherwise:
| `carrier` | `https`, `https-lanes`, `websocket`, or `websocket-lanes`. |
| `state` | `provisional`, `replacing`, `committed`, `healthy`, `closing`, `superseded`, or transient live-index `closed`. |
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, stream/task/lane/WebSocket counts, pending/control usage, age/idle timing, and optional negotiation time remaining. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `{state:"closed", attempt}` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, health publication, stream/task/lane/WebSocket counts, pending/control usage, progress idle time, authenticated peer idle time, frozen reconnect grace, remaining peer deadline, and optional negotiation time remaining. Server-only progress and empty long polls do not extend the authenticated peer deadline. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `state`, `attempt`, `carrier`, `reason`, and `closed_age_ms` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
### WEB runtime mutations
Every WEB runtime POST requires the currently published `runtime_instance`, exactly one `Content-Type: application/json` header, no query parameters, and a JSON object with no unknown fields. All mutations inherit API authentication, direct-peer whitelist, body limit, audit recording, and `read_only` enforcement.
Operator lifecycle requests are:
```json
{"runtime_instance":"0123456789abcdef0123456789abcdef"}
```
for `POST /v1/runtime/web/lifecycle/pause` and `/resume`, and:
```json
{"runtime_instance":"0123456789abcdef0123456789abcdef","timeout_secs":30}
```
for `POST /v1/runtime/web/lifecycle/drain`, where `timeout_secs` is bounded to `1..=3600`. Pause and resume return `200`; drain freezes one monotonic absolute deadline and returns `202` without waiting for completion. A second drain while one is `draining` or `force_closing` returns `409 web_lifecycle_in_progress` and cannot alter the first deadline. Repeated pause/resume requests already satisfied by the current state are idempotent and do not advance the lifecycle epoch. Pause during an active drain leaves that drain running. Resume cancels an active drain and opens admission; if the deadline already committed its forced-close snapshot, those old session close signals remain effective.
Pause and drain block bootstrap issuance, initial/replacement session creation, and logical-stream admission. Exact session-creation replay, existing DATA/WINDOW/CLOSE, carrier polling/WebSocket exchanges, and explicit session DELETE remain available. Rejection does not consume bootstrap/session/stream rate or quota state: authenticated session creation returns retryable `503` with `Retry-After: 1`, while bridge issuance preserves the decoy route and a rejected logical `OPEN` receives a stream-local close.
Drain remains graceful until either all live sessions, logical-stream ownership, and session-owned WebSockets reach zero or its deadline fires. The deadline is the latest time to commit close signals, not a claim that cooperative task teardown is already complete. At the deadline every remaining live session receives an idempotent close signal outside manager locks, status becomes `force_closing`, and only confirmed zero publishes `drained` with outcome `forced`. Natural zero publishes outcome `graceful`. Both outcomes keep operator admission closed until explicit resume.
This lifecycle is ephemeral: it survives in-process generation reload because its authority is process-owned, is not written to configuration, and starts as `running` after process restart. Resume never overrides `web.enabled=false`, disabled-user policy, generation health admission, or terminal process shutdown. The global health/readiness and native TCP/Unix admission contracts are unchanged.
`POST /v1/runtime/web/sessions/close` accepts:
```json
@@ -416,8 +466,8 @@ Returned by `PATCH /v1/config` on success (`200`, or `202` when a reload was acc
| `revision` | `string` | SHA-256 hex of the config file after the patch was written. |
| `restart_required` | `bool` | Legacy classifier result: `true` when the old file watcher alone cannot apply every changed field. Use `runtime_reload_required` and `process_restart_required` for new integrations. |
| `runtime_reload_required` | `bool` | `true` when full effect requires a Maestro runtime-generation reload rather than the legacy hot-field overlay. |
| `process_restart_required` | `bool` | `true` when process-owned sockets or paths changed and remain deferred after an in-process reload. |
| `deferred_process_fields` | `string[]` | Process-owned fields that the active process cannot rebind during generation activation. |
| `process_restart_required` | `bool` | `true` when a process-owned field changed and remains deferred after an in-process reload. |
| `deferred_process_fields` | `string[]` | Process-owned sockets, paths, capacities, or policies retained by the active process. |
| `changed` | `string[]` | Top-level section names that differed between the old and new config (e.g. `["censorship"]`). |
| `reload` | `ReloadAccepted?` | Present only when the patch included a valid reload query and Maestro accepted the operation. |
@@ -668,8 +718,11 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
| --- | --- | --- |
| `active_generation` | `u64` | Active pool generation id. |
| `warm_generation` | `u64` | Warm pool generation id. |
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
| `pending_hardswap_generation` | `u64` | Pending hardswap generation id (`0` when none). |
| `pending_hardswap_age_secs` | `u64?` | Age of pending hardswap generation in seconds. |
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
| `draining_generations` | `u64[]` | Distinct generation ids currently draining. |
#### `RuntimeMePoolStateHardswapData`
@@ -707,6 +760,8 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
| --- | --- | --- |
| `inflight_endpoints_total` | `usize` | Total in-flight endpoint refill operations. |
| `inflight_dc_total` | `usize` | Number of distinct DC+family keys with refill in flight. |
| `running_dc_total` | `usize` | DC+family refill workers currently running. |
| `pending_dc_total` | `usize` | Running DC+family workers with one coalesced pending endpoint. |
| `by_dc` | `RuntimeMePoolStateRefillDcData[]` | Per-DC refill rows. |
#### `RuntimeMePoolStateRefillDcData`
@@ -1264,8 +1319,11 @@ JA3 follows the Salesforce ClientHello field order. JA4 follows the FoxIO TLS-cl
| --- | --- | --- |
| `active_generation` | `u64` | Active pool generation. |
| `warm_generation` | `u64` | Warm pool generation. |
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
| `pending_hardswap_generation` | `u64` | Pending hardswap generation. |
| `pending_hardswap_age_secs` | `u64?` | Pending hardswap age in seconds. |
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
| `hardswap_enabled` | `bool` | Hardswap mode toggle. |
| `floor_mode` | `string` | Writer floor mode. |
| `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. |
@@ -1566,7 +1624,7 @@ Without a `reload` query parameter, the endpoint writes the patch and the file w
- `revision` — SHA-256 hex of the canonical source manifest after the write, including every recursive include path and its raw bytes.
- `restart_required` — legacy file-watcher classification retained for compatibility.
- `runtime_reload_required` — reports whether a full Maestro generation reload is needed for runtime effect.
- `process_restart_required` and `deferred_process_fields` — report socket policies or process-owned paths that remain unchanged by an in-process reload. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
- `process_restart_required` and `deferred_process_fields` — report process-owned sockets, paths, capacities, or policies that remain unchanged by an in-process reload, including `web.decoy_fasttrack_mode`. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
- `changed` — list of top-level section names that differed.
- `reload` — accepted operation metadata; omitted without a reload query and for process-only patches that cannot change the active generation.
@@ -1632,10 +1690,10 @@ The API exposes WEB desired configuration through the common config resource, pr
| Operation | Current contract |
| --- | --- |
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale, and `web.limits` remains process-deferred. |
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale; `web.limits` and `web.decoy_fasttrack_mode` remain process-deferred. |
| Persist `server.listeners` | Supported through `PATCH /v1/config`. Arrays replace wholesale. A changed WEB listener is process-owned and remains deferred until process restart. |
| Apply an externally edited WEB config | Update the owning TOML source, call `POST /v1/system/reload`, then poll `GET /v1/system/reload/{id}`. |
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners` and `web.limits` require process restart. |
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners`, `web.limits`, and `web.decoy_fasttrack_mode` require process restart. |
| Inspect WEB lifecycle, capacity, sessions, operations, learning, and debug state | Use the authenticated `GET /v1/runtime/web/*` routes documented above. |
| Close selected or all point-in-time sessions | Use `POST /v1/runtime/web/sessions/close`; close-all first requires effective issuance to be disabled. |
| Clear debug records or reset carrier learning | Use `POST /v1/runtime/web/debug/clear` or `/carrier-learning/reset` with the current `runtime_instance`. |
+24 -5
View File
@@ -308,6 +308,7 @@ This document lists all configuration keys accepted by `config.toml`.
| [`proxy_secret_auto_reload_secs`](#proxy_secret_auto_reload_secs) | `u64` | `3600` | `✔` |
| [`proxy_config_auto_reload_secs`](#proxy_config_auto_reload_secs) | `u64` | `3600` | `✔` |
| [`me_reinit_singleflight`](#me_reinit_singleflight) | `bool` | `true` | `✔` |
| [`me_reinit_max_concurrency`](#me_reinit_max_concurrency) | `usize` | `2` | `✔` |
| [`me_reinit_trigger_channel`](#me_reinit_trigger_channel) | `usize` | `64` | `✘` |
| [`me_reinit_coalesce_window_ms`](#me_reinit_coalesce_window_ms) | `u64` | `200` | `✔` |
| [`me_deterministic_writer_sort`](#me_deterministic_writer_sort) | `bool` | `true` | `✔` |
@@ -1547,8 +1548,17 @@ This document lists all configuration keys accepted by `config.toml`.
[general]
me_reinit_singleflight = true
```
## me_reinit_max_concurrency
- **Constraints / validation**: Must be within `[1, 8]`. The effective value is `1` while `me_reinit_singleflight = true`.
- **Description**: Bounds concurrent ME generation warmups. Excess triggers are coalesced into one pending rerun.
- **Example**:
```toml
[general]
me_reinit_max_concurrency = 2
```
## me_reinit_trigger_channel
- **Constraints / validation**: Must be `> 0`.
- **Constraints / validation**: Must be within `[1, 4096]`.
- **Description**: Trigger queue capacity for reinit scheduler.
- **Example**:
@@ -2561,6 +2571,8 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
| `carriers` | `false` or a non-empty array of unique carriers | `false` | `✔` |
| `carrier_learning` | `bool` | `true` | `✔` |
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"`, or `"aggressive"` | `"conservative"` | `✔` |
| `decoy_fasttrack_mode` | `"off"`, `"shadow"`, or `"enforce"` | `"off"` | `✘` |
| `http_connection_capacity_action` | `"drop"`, `"wait"`, or `"respond"` | `"drop"` | `✔` |
| `debug` | table | disabled, bounded defaults | `✔` |
| `limits` | table | bounded defaults | `✘` |
| `timeouts` | table | bounded defaults | `✔` |
@@ -2570,7 +2582,11 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
When `carriers` is missing or `false`, auto-negotiation and learning are disabled and `carrier` is the only mode. A non-empty `carriers` array enables startup-only negotiation in its configured order; `carrier` is appended exactly once as the final fallback. Empty arrays, duplicates, and `true` are rejected. The client advances candidates only before carrier commit and must create a new session to change carrier after commit. A metadata-free native client, including Telegram iOS, always uses the configured fixed `carrier`, even when negotiation is enabled. Current iOS supports only `https`, so such deployments must configure `carrier = "https"`. CFNetwork and Darwin User-Agent classification does not infer carrier support. Explicit native iOS capabilities are intersected with `{https}`; other explicit client capabilities participate as reported.
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload applies the policy to new negotiation chains and invalidates incompatible retained evidence. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
`http_connection_capacity_action` applies only after Telemt has accepted a private WEB TCP connection and `max_http_connections` is exhausted. `drop` preserves the legacy immediate close. `respond` emits an empty `503 Service Unavailable` with `Retry-After: 1`, `Cache-Control: no-store`, and `Connection: close`. `wait` waits for ordinary connection capacity for at most `http_overload_timeout_ms`, then enters normal HTTP handling; timeout emits the same bounded `503`. At most `max_http_overload_connections` accepted sockets may wait or respond outside ordinary connection capacity. This policy cannot observe or cause a TCP connect refusal before Telemt accepts the socket.
`decoy_fasttrack_mode` is restart-only. `off` preserves legacy root-request scanning and collects no fast-track decisions. `shadow` records eligible requests while preserving the full scan. `enforce` skips scans only for `HEAD` or absent/noncanonical bridge queries. A canonical bridge-shaped `GET`, including an unknown capability, always scans every profile in the selected vhost. The optimization does not bound hostile canonical probes and enforce mode must be validated for request-shape timing distinguishability behind the production TLS terminator.
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload preserves evidence across a generation change only when enabled state, aggressiveness, evidence lifetime, and health window are identical; any semantic change advances the evidence epoch and fences stale outcomes. Because `[web.limits]` is process-owned, a reload that enables learning or negotiation using only a desired larger `max_carrier_learning_entries` atomically defers the dependent learning/carrier field rather than publishing an invalid effective combination. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
# [web.debug]
@@ -2595,7 +2611,7 @@ Authenticated JSON control may clear the ring explicitly with `POST /v1/runtime/
# [web.limits]
These process-wide ceilings make every WEB registry, queue, request body, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
These process-wide ceilings make every WEB registry, queue, request body, capability index, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
| Key | Type | Default | Description |
| --- | --- | --- | --- |
@@ -2605,6 +2621,7 @@ These process-wide ceilings make every WEB registry, queue, request body, static
| `carrier_batch_bytes` | `usize` | `2097152` | Maximum encoded downlink batch. |
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
| `max_http_overload_connections` | `usize` | `64` | Accepted saturated sockets allowed to wait or emit the bounded retryable response outside ordinary HTTP capacity. |
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
| `max_lane_open_waits_per_session` | `usize` | `16` | Canonical cursor-zero downlink polls allowed to wait for a racing lane `OPEN` in one session. |
| `pending_bytes_per_lane` | `usize` | `8388608` | Queued and resident `DATA` bytes allowed for one independent HTTPS or WebSocket lane. |
@@ -2638,7 +2655,7 @@ These process-wide ceilings make every WEB registry, queue, request body, static
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
| `debug_records_capacity` | `usize` | `65536` | Maximum retained WEB debug record count. |
| `debug_bytes_global` | `usize` | `67108864` | Retained plus in-flight WEB debug byte ceiling; minimum 4096. |
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, capability indexes, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
@@ -2659,6 +2676,7 @@ Unless a row states otherwise, timeouts are measured in seconds and must be with
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-side deadline for one HTTP attempt through complete response-body consumption; `/down` additionally allows `long_poll_secs`. Validated within `1..=60`. |
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolute bridge retry window including attempts and backoff; validated within `1..=300` and no lower than `bridge_request_secs`. |
| `bridge_recovery_secs` | `u64` | `15` | `✔` | Absolute post-commit recovery window for a surviving bridge document; validated within `1..=60` and frozen when recovery starts. |
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optional bridge wait after `OPEN` for matching `DATA`; milliseconds within `0..=10`, where `0` preserves immediate probing. |
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wait for a canonical cursor-zero downlink that races its lane `OPEN`; no greater than `long_poll_secs`. |
| `carrier_health_secs` | `u64` | `30` | `✔` | Post-commit observation interval required before a carrier can contribute learning evidence. |
@@ -2670,8 +2688,9 @@ Unless a row states otherwise, timeouts are measured in seconds and must be with
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Strictly increasing cumulative offsets used by the bridge before its first `/session` request and by the server when accepting the first automatic attempt. Checkpoints for one through four candidates are `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always uses `d3`. |
| `carrier_learning_secs` | `u64` | `600` | `✔` | Fixed two-window process-local evidence lifetime; validated within `2..=86400`. |
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum carrier inactivity before session closure. |
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum validated peer inactivity before session closure; empty polls and backend-only progress do not renew this lease. |
| `http_idle_secs` | `u64` | `75` | `✔` | Idle limit between HTTP exchanges and while an emitted response body makes no progress. Explicitly bounded request-body, long-poll, decoy, and pending-Upgrade phases keep their own deadlines instead of being truncated by this timer. The value is frozen when the connection is accepted. |
| `http_overload_timeout_ms` | `u64` | `250` | `✔` | Per-phase deadline in milliseconds for an accepted saturated socket to wait for capacity or write its retryable response; validated within `1..=60000`. A timed-out wait and its response write each receive at most one phase budget. |
| `shutdown_secs` | `u64` | `15` | `✔` | One absolute process-shutdown budget shared by all listener acceptors and connections plus WEB session and auxiliary-task drains. The active value is captured once when shutdown starts. |
| `decoy_header_secs` | `u64` | `30` | `✔` | Connect and response-head deadline for an HTTP decoy. |
+42 -10
View File
@@ -76,6 +76,8 @@ web_trusted_proxy_cidrs = ["127.0.0.1/32"]
[web]
enabled = true
carrier = "https-lanes"
decoy_fasttrack_mode = "off"
http_connection_capacity_action = "drop"
[[web.vhosts]]
host = "proxy.example.com"
@@ -93,6 +95,10 @@ max_streams = 512
max_streams_per_session = 64
```
Accepted-socket overload handling is independently configurable. `drop` preserves the legacy close after `accept(2)`. `respond` writes an empty retryable `503` without parsing a request. `wait` waits outside the accept loop for ordinary connection capacity and then enters normal HTTP handling; timeout writes the same `503`. Both waiting and response writing use `web.timeouts.http_overload_timeout_ms` per phase. `web.limits.max_http_overload_connections` bounds sockets outside ordinary capacity and requires a process restart when changed; the action and timeout are hot-reloadable.
`decoy_fasttrack_mode` controls only capability work for `GET/HEAD /`. `off` is the default and preserves the legacy full scan without fast-track counters. `shadow` records which structurally impossible requests could bypass the scan but still performs the complete legacy scan. `enforce` bypasses capability work only for `HEAD` or an absent/noncanonical `bridge` query. Every exact canonical `GET /?bridge=<43-character-base64url>` performs a complete scan across all profiles of the selected vhost, for both matches and misses. The setting requires a process restart; reload persists the desired value but reports `web.decoy_fasttrack_mode` as deferred. Fast-track does not protect against adversarial CPU load because a scanner can always submit canonical candidates, and enforce mode may expose a public request-shape timing class, especially with a static decoy. Do not enable enforce without external timing measurements through the production TLS terminator.
## Server-side carrier negotiation
Auto-negotiation is optional and disabled unless `carriers` is an explicit non-empty array. The configured `carrier` remains the final fallback and is appended exactly once, even when it also appears in the array:
@@ -111,20 +117,29 @@ carrier_health_secs = 30
carrier_learning_secs = 600
bridge_request_secs = 10
bridge_retry_secs = 90
bridge_recovery_secs = 15
carrier_probe_coalesce_ms = 0
```
The generated bridge sends canonical `X-Carrier-Capabilities`, `X-Carrier-Attempt`, and, after the first attempt, `X-Carrier-Failure` headers on `/session`. Every successful automatic response returns `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline`, and `X-Carrier-State`. The bridge starts its local cumulative clock immediately before the first `/session` request; the server freezes its separate absolute chain deadline when it accepts the first automatic attempt. Both use the configured offsets, and neither resets across replacement attempts. For one through four effective candidates, the attempt checkpoints are respectively `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always owns `d3`. A successor remains admissible until its own checkpoint. The states are `provisional`, `committed`, and `healthy`.
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit dynamic switching is deliberately unsupported: reconnect with a new session instead.
The bridge emits additive v1 status objects with `state`, `phase`, `reason`, and `deadline_ms`. `phase=provisional` follows the authenticated `WELCOME`; `state=connected,phase=committed` is emitted only after the selected transport acknowledges real `OPEN` or `DATA` progress. The initialization port has its own `bridge_request_secs` pre-`HELLO` deadline, and page navigation is terminal for that document instance. A later initialization message cannot resurrect a closed or BFCache-retained bridge.
Each bridge HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the pre-commit replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit in-place carrier switching remains unsupported; a surviving bridge recovers by creating a fresh server session.
After commit, an HTTP failure first replays the exact frozen request against the current bearer. A successful replay keeps the current session. WebSocket loss, or a foreground/online/native event after at least `reconnect_grace_secs` of scheduler gap, starts one recovery epoch. The bridge performs exactly one `GET /?bridge=<capability>` with `Accept: application/vnd.telemt.web-recovery+json` and optional current bearer authorization. A positive response is an uncacheable JSON document of at most 1024 bytes containing a fresh bootstrap plus current limits, timeouts, and negotiation policy. Telemt issues that bootstrap before synchronously retiring a matching current session, so recreation remains possible with a one-session capacity. Unknown or already retired bearer authorization receives the same positive representation; malformed recovery headers, disabled admission, pause, drain, and capacity rejection follow the sanitized decoy path.
The recovery epoch has one dual wall/monotonic absolute `bridge_recovery_secs` deadline, a single recovery-document request, and bounded carrier retries with 250 ms through 2 s backoff. Recovery status is repeated at most every 2.5 seconds while active. A fresh incarnation aborts and releases old requests, sockets, lanes, and queues, sends one synthetic `CLOSE` for each still-active native stream, suppresses a second `WELCOME`, and commits only after real carrier progress. Retired stream IDs are retained in a bounded set so valid late frames cannot enter a new stream; the native side must allocate a new stream ID. Frequent native reconnect attempts are valid, but they neither extend the recovery epoch nor retain old incarnation state. Destroying the WebView destroys this recovery owner; a native supervisor must then create a new bridge document.
Each ordinary bridge carrier HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
Response bodies are streamed into explicit endpoint bounds: `/session` is exactly eight bytes, a successful `/down` is at most `carrier_batch_bytes`, and bodyless responses accept zero bytes. Declared overflow is rejected before reading, streamed overflow or excessive chunk count cancels the reader, and retryable response bodies are canceled before backoff. Terminal bridge cleanup sends at most one authenticated `DELETE`; canonical transport failures are copied to `X-Carrier-Failure` for diagnostics, while navigation and explicit close remain non-learning reasons.
Automatic WebSockets use `tproxy-auto-v1.<session-token>` or `tproxy-auto-lane-v1.<session-token>.<stream-id>`. The first accepted binary message containing real `OPEN` or `DATA` progress commits the carrier; the server then writes an empty binary commit acknowledgement to that exact connection. Ping/Pong does not commit a carrier and does not count as learning evidence.
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Closing earlier is neutral and records no learning result.
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Health publication, owner eviction, and close have one terminal winner. Closing earlier remains neutral for ranking evidence but is visible as the diagnostic `closed_before_health` outcome.
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. Disabling learning or changing its policy on reload clears incompatible evidence without changing in-flight sessions.
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. A generation change with identical learning semantics preserves evidence and atomically republishes its generation fence. Disabling learning or changing aggressiveness, evidence lifetime, or health window advances the evidence epoch and detaches incompatible state; stale outcomes cannot repopulate it.
`https` remains the default and preserves the original serialized behavior. `https-lanes` assigns lane zero to session control and one lane to every non-zero logical stream. Each lane has its own uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, queue, and newest-poll-wins lifecycle. A slow stream therefore does not block another stream at the WEB protocol layer.
@@ -144,7 +159,7 @@ Every pre-Upgrade authentication, shape, lane-reservation, or capacity failure f
The WEB listener must use `proxy_protocol = false` and `reuse_allow = false`. It cannot use `client_mss`, `synlimit`, `announce`, or `announce_ip`. `web_trusted_proxy_cidrs` must be non-empty and must contain only the immediate NGINX or HAProxy peers; `/0` networks are rejected.
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy.
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy. A literal decoy endpoint that exactly matches an effective WEB listener, or is covered by its same-family wildcard address on the same port, is rejected. Indirect loops through DNS, NGINX, HAProxy, or another forwarding layer cannot be proven from Telemt configuration and must be excluded operationally.
An immutable static-site snapshot can be used instead:
@@ -205,6 +220,14 @@ Place the `map` in NGINX's `http` context. `client_max_body_size` must be at lea
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. WebSocket Upgrade requires HTTP/1.1, so the public endpoint must also permit HTTP/1.1 and the private NGINX-to-Telemt hop remains HTTP/1.1. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*` exactly as shown. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls or WebSocket lanes; `keepalive` controls the idle pool and is not a concurrency limit.
### Distinguishing refusal from WEB capacity
`connect() failed (111: Connection refused) while connecting to upstream` is a TCP-connect failure before Telemt accepts a socket. Check that the Telemt process is running, the effective WEB listener address and port match the NGINX upstream, both processes share the expected network namespace and address family, and no local firewall actively rejects the connection. Startup bind failure, terminal listener removal, or switching NGINX to a desired port before a restart-only listener change becomes effective can produce this symptom. Kernel listen-backlog pressure is separate and normally requires host `ListenOverflows`/`ListenDrops` telemetry.
WEB capacity is enforced after successful `accept(2)`. Exhausting `max_http_connections` therefore produces the configured `drop`, `wait`, or `respond` outcome; it does not produce an upstream connect refusal. Handler, body, lane, stream, queue, and WebSocket limits have their own HTTP, decoy, or stream-local failure boundaries. Operator pause and drain also leave the WEB listener bound, so they cannot by themselves cause a refusal.
Use `GET /v1/runtime/web/status` to correlate only Telemt-owned state. `ingress.accepting_connections` requires a running publication, a readable runtime, and one live acceptor for every effective WEB listener. `capacity.saturated_resources`, typed rejection totals, and overload outcomes identify failures after acceptance. `decoy_upstream` describes only Telemt's outgoing plain-HTTP decoy hop. None of these fields claims that the public NGINX TLS endpoint is reachable; use an external TCP/TLS probe and NGINX or HAProxy telemetry for that boundary.
## HAProxy TLS termination
```haproxy
@@ -236,7 +259,8 @@ The frontend or `defaults` section must also set `timeout client 65s` or longer
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
| `web.enabled`, carrier/negotiation policy, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, and probe-coalescing values. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
| Operator pause/drain state | Process-owned and ephemeral; survives generation reload, never writes config, and resets to `running` after process restart. |
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, recovery, and probe-coalescing values. A recovery epoch freezes its current bridge budget, while a successful recovery representation refreshes the policy used by later epochs and the fresh session. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
| Process shutdown | Captures the latest reloaded `web.timeouts.shutdown_secs` once and shares that single absolute deadline across listener acceptors and connections plus WEB sessions and auxiliary tasks. The waits do not receive sequential per-component budgets. |
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
@@ -257,6 +281,7 @@ WEB configuration, runtime status, and bounded runtime controls share the authen
| Inspect bounded server-side WEB request and lifecycle details | Yes, through authenticated `GET /web-status`. |
| Inspect lifecycle, capacity planes, learning/debug state, and live sessions | Yes, through `GET /v1/runtime/web/status` and `/v1/runtime/web/sessions`. |
| Close selected live WEB sessions | Yes, through the asynchronous `POST /v1/runtime/web/sessions/close` operation. |
| Pause, deadline-drain, or resume new WEB work | Yes, through `/v1/runtime/web/lifecycle/{pause,drain,resume}`. |
| Clear debug records or reset carrier learning | Yes, through the corresponding runtime POST endpoints. |
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
@@ -276,18 +301,25 @@ The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For
### Runtime status and control
`GET /v1/runtime/web/status` always returns the published lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and availability. When the process-owned WEB runtime is alive, `runtime` adds its random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Status collection uses non-blocking plane reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
`GET /v1/runtime/web/status` always returns the published ingress lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and backward-compatible runtime availability. `ingress` independently reports configured listeners, live acceptors, accepting state, accept totals, and a stable reason. `capacity` reports effective accepted-socket overload policy, fixed resource usage, instantaneous saturation, partial planes, typed rejection decisions, and overload outcomes. `decoy_upstream` reports fixed outcomes and the age of the latest internal origin result. `decoy_fasttrack` reports the effective restart-frozen mode and the complete fixed disposition set even while the runtime manager is unavailable. `carrier_negotiation` always reports fixed selection, client-failure, and terminal health/learning outcome matrices from publication ownership. When the process-owned WEB runtime is alive, `operator_lifecycle` independently exposes `running`, `paused`, `draining`, `force_closing`, or `drained`, its own epoch/admission flags, and the active or latest drain. `runtime` adds the random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Runtime plane collection uses non-blocking reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
Prometheus exports the same process-owned planes as fixed-cardinality `telemt_web_*` families: ingress and operator one-hot states, listener/accept counters, capacity usage and saturation, typed terminal rejections, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier totals. Decoy routing adds one-hot `telemt_web_decoy_fasttrack_mode` and fixed `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation uses `telemt_web_carrier_selections_total`, `telemt_web_carrier_reported_failures_total`, `telemt_web_carrier_learning_outcomes_total`, one-hot learning state/policy gauges, and used/limit entry gauges. Labels are closed enums or fixed resource names; user, host, client IP, token, profile key, runtime instance, listener address, and generation ID are never labels. A successful `wait` outcome does not increment a rejection counter.
`GET /v1/runtime/web/sessions` returns at most 50 sessions by default and at most 200 when `limit` is supplied. Its ordered scan is capped at 1000 candidates. `cursor` and `session_ref` use the opaque canonical form `ws1.<runtime-instance>.<lowercase-hex-id>`; exact `session_ref` is mutually exclusive with `cursor` and `limit`. Filters are `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, and `state`; duplicate or unknown query fields are rejected. The detail route is `GET /v1/runtime/web/sessions/{session_ref}`. A retained closed-session tombstone returns `410`; a contended exact snapshot returns `503 web_snapshot_busy`. Responses expose bounded non-secret metadata and never expose bootstrap/session bearers, capabilities, secret hashes, or synthetic/KDF ports.
Every runtime POST requires `Content-Type: application/json` exactly, rejects unknown JSON fields, obeys API authentication, whitelist, and `read_only`, and carries the current `runtime_instance` as an ABA fence. Available controls are:
- `POST /v1/runtime/web/lifecycle/pause` with `{"runtime_instance":"..."}`. It blocks new bootstrap, session incarnation, replacement, and logical-stream admission after a linearizable fence. Existing carrier exchanges and streams continue, exact session replay remains available, and bridge rejection stays on the decoy route.
- `POST /v1/runtime/web/lifecycle/drain` with `{"runtime_instance":"...","timeout_secs":30}`. It returns `202`, keeps the same admission fence closed, and waits asynchronously for sessions, streams, and session-owned WebSockets. At the monotonic deadline it signals close to every remaining live session and reports `force_closing` until zero is confirmed. Natural and forced completion both remain closed until resume. A concurrent second drain returns `409 web_lifecycle_in_progress`.
- `POST /v1/runtime/web/lifecycle/resume` with `{"runtime_instance":"..."}`. It cancels an active drain and reopens only operator admission. If forced close already committed, old session cancellation cannot be undone. Config, user, generation, and terminal shutdown gates still dominate.
- `POST /v1/runtime/web/sessions/close` with one selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}`, or `{"kind":"all"}`. Exact refs are limited to 200, a filter must be non-empty, only one close operation may run, and `all` is rejected while effective issuance remains enabled. The `202` response returns `operation_id`; poll `GET /v1/runtime/web/operations/{operation_id}`. The operation scans only sessions at or below its submission high-water mark in chunks of 128.
- `POST /v1/runtime/web/debug/clear` with `{"runtime_instance":"..."}`. The response reports cleared records, bytes still leased by already rendered snapshots, and the new epoch. In-flight writers from the old epoch cannot repopulate the ring.
- `POST /v1/runtime/web/carrier-learning/reset` with the same body shape. It clears retained process-local evidence and advances the learning epoch; already frozen attempt chains and live sessions are unchanged.
For a deterministic close-all, patch `{"web":{"enabled":false}}` with runtime reload enabled, wait until `runtime.manager.issuance_enabled` is `false`, submit the `all` selector using that same `runtime_instance`, and poll the operation to a terminal state. Disabling WEB stops new bootstrap/session issuance but never implicitly closes existing sessions.
Operator lifecycle is WEB-only and does not change global readiness, liveness, native TCP/Unix listeners, TLS-fronting, or fallback behavior. A pre-pause WebSocket lane reservation is already admitted logical work: it may finish opening and remains included in drain accounting. Lifecycle rejection consumes no rate/quota tokens and adds no hot-path relay lock.
### Server-side WEB debug view
Enable bounded collection in the owned configuration file:
@@ -306,7 +338,7 @@ default_window_secs = 180
max_window_secs = 3600
```
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, and reported-failure transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, reported-failure, exact close reason, peer gap, and recovered-session predecessor transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
The process-owned ring survives runtime generation replacement. Capture-policy changes clear incompatible retained records; window-only changes do not. The ring defaults to 65536 records and 64 MiB retained plus in-flight bytes, the HTML response is capped at 8 MiB, grouping is capped at 1024 groups, and no more than two response bodies retain page permits concurrently. Change `web.limits.debug_records_capacity` or `web.limits.debug_bytes_global` only with a process restart. A hot prefix that fits only a simultaneously increased restart-only capacity is deferred until that restart.
@@ -348,7 +380,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
- Never expose the plain HTTP WEB listener to an untrusted network. Enforce the restriction with host firewall rules even when it binds to loopback.
- Disable request-target and authorization logging at the TLS terminator, or use a verified redacted format. Raw queries contain bridge capabilities and `Authorization` contains bootstrap or session bearer credentials.
- Keep one stable public address per vhost. If DNS returns several ingress addresses, each deployment must use the address matching its external path.
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: bridge GET, session creation, uplink, downlink, and DELETE. A single Telemt process needs no extra affinity.
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: initial and recovery root GET, session creation, uplink, downlink, WebSocket Upgrade, and DELETE. A single Telemt process needs no extra affinity.
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
- The decoy is part of the anti-probing contract. Verify its ordinary 404 behavior and response timing through the public TLS endpoint before distributing links.
@@ -360,7 +392,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
4. Import the printed `tg://webproxy` link in the intended Telegram Desktop build and establish a proxy connection.
5. For `https-lanes`, confirm that the public connection negotiated HTTP/2 and exercise at least two simultaneous logical streams; the private Telemt hop remains HTTP/1.1.
6. For `websocket`, confirm one `101` response, binary relay traffic, and RFC 6455 Ping/Pong beyond 25 seconds. For `websocket-lanes`, exercise at least two simultaneous stream sockets and verify that closing or corrupting one lane does not close its sibling or parent session.
7. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
7. Exercise one HTTP replay and one fresh-session recovery after a scheduler gap, then keep a long poll open beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
8. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
9. When auto-negotiation is enabled, verify the configured sequence, exact-attempt replay after an intentionally lost response, terminal behavior after commit, and `carrier_committed`/`carrier_healthy` lifecycle rows in `/web-status`. Verify that a metadata-free native client uses the fixed `carrier` without automatic response headers and that explicit capabilities remain unchanged.
+2 -2
View File
@@ -447,9 +447,9 @@ check_port_availability() {
port_info=""
if command -v ss >/dev/null 2>&1; then
port_info=$($SUDO ss -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
port_info=$($SUDO ss -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
elif command -v netstat >/dev/null 2>&1; then
port_info=$($SUDO netstat -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
port_info=$($SUDO netstat -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
elif command -v lsof >/dev/null 2>&1; then
port_info=$($SUDO lsof -i :${SERVER_PORT} 2>/dev/null | grep LISTEN || true)
else
+76 -11
View File
@@ -6,34 +6,50 @@ use serde_json::Value as Json;
use toml::Value as Toml;
use super::ApiShared;
#[cfg(test)]
use super::config_store::write_atomic;
use super::config_store::{
EDITABLE_SECTIONS, EDITABLE_SERVER_FIELDS, compute_snapshot_revision, is_editable_section,
load_candidate_snapshot, load_config_snapshot, render_server_listeners,
render_top_level_section, resolve_single_source_owner, upsert_toml_table, write_atomic,
render_top_level_section, resolve_single_source_owner, upsert_toml_table,
write_atomic_if_unchanged,
};
use super::model::ApiFailure;
use crate::config::ProxyConfig;
use crate::config::hot_reload::classify_config_changes;
use crate::maestro::reload::{ReloadAccepted, ReloadRequest, ReloadSubmitError};
use crate::maestro::runtime_build::{deferred_process_fields, resolve_reload_config};
use crate::maestro::runtime_build::{
ResolvedReloadConfig, deferred_process_fields, resolve_reload_config,
};
use serde::Serialize;
use std::path::{Path, PathBuf};
use std::sync::Arc;
/// Result of one validated managed-config mutation.
#[derive(Debug, Serialize)]
pub(super) struct PatchConfigResponse {
/// Revision of the persisted desired configuration.
pub revision: String,
/// Whether any changed field is not hot-reloadable.
pub restart_required: bool,
/// Whether the effective runtime snapshot must be reloaded.
pub runtime_reload_required: bool,
/// Whether any desired field remains deferred until process restart.
pub process_restart_required: bool,
/// Stable paths of desired fields retained from the active process.
pub deferred_process_fields: Vec<String>,
/// Top-level managed sections changed by the mutation.
pub changed: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
/// Accepted runtime reload when one was requested and required.
pub reload: Option<ReloadAccepted>,
}
struct PreparedConfigPatch {
config_path: PathBuf,
expected_revision: String,
owner_path: PathBuf,
expected_owner_contents: String,
owner_contents: String,
desired_config: Arc<ProxyConfig>,
response: PatchConfigResponse,
@@ -46,15 +62,31 @@ pub(super) async fn patch_config(
expected_revision: Option<String>,
reload_request: Option<ReloadRequest>,
shared: &ApiShared,
) -> Result<PatchConfigResponse, ApiFailure> {
let shared = shared.clone();
shared
.clone()
.run_mutation_completion(async move {
patch_config_to_completion(patch_json, expected_revision, reload_request, &shared).await
})
.await
}
async fn patch_config_to_completion(
patch_json: Json,
expected_revision: Option<String>,
reload_request: Option<ReloadRequest>,
shared: &ApiShared,
) -> Result<PatchConfigResponse, ApiFailure> {
let _guard = shared.mutation_lock.lock().await;
let active_config = shared.active_runtime.load_full().config();
let mut prepared =
prepare_patch_to_path(&shared.config_path, &patch_json, expected_revision).await?;
let resolved = resolve_reload_config(&active_config, &prepared.desired_config);
prepared.response.runtime_reload_required = resolved.runtime_changed;
prepared.response.process_restart_required = !resolved.deferred_process_fields.is_empty();
prepared.response.deferred_process_fields = resolved.deferred_process_fields;
let resolved = reconcile_runtime_effect(
&mut prepared.response,
&active_config,
&prepared.desired_config,
)?;
let reservation = if let Some(request) = reload_request.filter(|_| resolved.runtime_changed) {
Some(
shared
@@ -66,7 +98,14 @@ pub(super) async fn patch_config(
} else {
None
};
write_atomic(prepared.owner_path, prepared.owner_contents).await?;
prepared.response.revision = write_atomic_if_unchanged(
prepared.config_path,
prepared.expected_revision,
prepared.owner_path,
prepared.expected_owner_contents,
prepared.owner_contents,
)
.await?;
if let Some(reservation) = reservation {
prepared.response.reload = Some(reservation.enqueue(prepared.desired_config));
}
@@ -78,6 +117,19 @@ pub(super) async fn patch_config(
Ok(resp)
}
fn reconcile_runtime_effect(
response: &mut PatchConfigResponse,
active_config: &ProxyConfig,
desired_config: &ProxyConfig,
) -> Result<ResolvedReloadConfig, ApiFailure> {
let resolved =
resolve_reload_config(active_config, desired_config).map_err(ApiFailure::bad_request)?;
response.runtime_reload_required = resolved.runtime_changed;
response.process_restart_required = !resolved.deferred_process_fields.is_empty();
response.deferred_process_fields = resolved.deferred_process_fields.clone();
Ok(resolved)
}
/// Core patch logic, decoupled from hyper/shared-state so it is unit-testable
/// against a temp file. The route handler holds `mutation_lock` while calling this.
#[cfg(test)]
@@ -86,8 +138,16 @@ pub(super) async fn apply_patch_to_path(
patch_json: &Json,
expected_revision: Option<String>,
) -> Result<PatchConfigResponse, ApiFailure> {
let prepared = prepare_patch_to_path(config_path, patch_json, expected_revision).await?;
write_atomic(prepared.owner_path, prepared.owner_contents).await?;
let mut prepared = prepare_patch_to_path(config_path, patch_json, expected_revision).await?;
let revision = write_atomic_if_unchanged(
prepared.config_path,
prepared.expected_revision,
prepared.owner_path,
prepared.expected_owner_contents,
prepared.owner_contents,
)
.await?;
prepared.response.revision = revision;
Ok(prepared.response)
}
@@ -173,6 +233,7 @@ async fn prepare_patch_to_path(
.get(&owner_path)
.cloned()
.ok_or_else(|| ApiFailure::internal("config source owner is missing from snapshot"))?;
let expected_owner_contents = owner_contents.clone();
for section in &touched {
if *section == "server" {
let rendered = render_server_listeners(&requested_cfg)?;
@@ -205,10 +266,14 @@ async fn prepare_patch_to_path(
let revision = compute_snapshot_revision(&candidate);
let new_cfg = candidate.config;
let class = classify_config_changes(&old_cfg, &new_cfg);
let deferred_process_fields = deferred_process_fields(&old_cfg, &new_cfg);
let deferred_process_fields =
deferred_process_fields(&old_cfg, &new_cfg).map_err(ApiFailure::bad_request)?;
Ok(PreparedConfigPatch {
config_path: config_path.to_path_buf(),
expected_revision: current,
owner_path,
expected_owner_contents,
owner_contents,
desired_config: Arc::new(new_cfg),
response: PatchConfigResponse {
@@ -238,7 +303,7 @@ fn reload_submit_failure(error: ReloadSubmitError) -> ApiFailure {
}
}
/// Return only the editable config sections + current revision.
/// Returns only the editable config sections and current revision.
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
let loaded = load_config_snapshot(config_path, false).await?;
let revision = compute_snapshot_revision(&loaded);
+57 -2
View File
@@ -107,14 +107,24 @@ async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() {
#[tokio::test]
async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
let (path, _directory) = temp_config("[web]\nenabled = false\n");
let active = ProxyConfig::load(&path).unwrap();
let debug_patch: Json = serde_json::json!({
"web": {"debug": {"enabled": true, "capture_headers": false}}
"web": {"debug": {
"enabled": true,
"sideband": true,
"capture_headers": false
}}
});
let debug = apply_patch_to_path(&path, &debug_patch, None)
let mut debug = apply_patch_to_path(&path, &debug_patch, None)
.await
.unwrap();
let desired = ProxyConfig::load(&path).unwrap();
reconcile_runtime_effect(&mut debug, &active, &desired).unwrap();
assert!(!debug.restart_required);
assert!(debug.runtime_reload_required);
assert!(!debug.process_restart_required);
assert!(debug.changed.iter().any(|section| section == "web"));
assert!(desired.web.debug.sideband);
let limits_patch: Json = serde_json::json!({
"web": {"limits": {"max_http_connections": 2049}}
@@ -131,6 +141,27 @@ async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
);
}
#[tokio::test]
async fn patch_web_decoy_fasttrack_requires_only_process_restart() {
let (path, _directory) = temp_config("[web]\nenabled = false\n");
let active = ProxyConfig::load(&path).unwrap();
let patch: Json = serde_json::json!({
"web": {"decoy_fasttrack_mode": "shadow"}
});
let mut prepared = prepare_patch_to_path(&path, &patch, None).await.unwrap();
reconcile_runtime_effect(&mut prepared.response, &active, &prepared.desired_config).unwrap();
let response = prepared.response;
assert!(response.restart_required);
assert!(!response.runtime_reload_required);
assert!(response.process_restart_required);
assert_eq!(
response.deferred_process_fields,
vec!["web.decoy_fasttrack_mode".to_string()]
);
}
#[tokio::test]
async fn invalid_web_patch_does_not_modify_the_source() {
let (path, _directory) = temp_config("[web]\nenabled = false\n");
@@ -292,6 +323,30 @@ async fn patch_writes_the_included_section_owner_only() {
);
}
#[tokio::test]
async fn prepared_patch_rejects_external_edit_before_commit() {
let (path, _directory) = temp_config("[censorship]\ntls_domain = \"old.example\"\n");
let patch: Json = serde_json::json!({
"censorship": {"tls_domain": "api.example"}
});
let prepared = prepare_patch_to_path(&path, &patch, None).await.unwrap();
let external = "[censorship]\ntls_domain = \"external.example\"\n";
tokio::fs::write(&path, external).await.unwrap();
let error = write_atomic_if_unchanged(
prepared.config_path,
prepared.expected_revision,
prepared.owner_path,
prepared.expected_owner_contents,
prepared.owner_contents,
)
.await
.unwrap_err();
assert_eq!(error.code, "revision_conflict");
assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), external);
}
#[tokio::test]
async fn patch_rejects_multiple_source_owners_without_writing() {
let dir = tempfile::tempdir().unwrap();
+28 -17
View File
@@ -10,12 +10,15 @@ use super::model::ApiFailure;
// Source-preserving TOML rendering and atomic persistence helpers.
mod persistence;
// Compare-and-replace file persistence and metadata preservation.
mod atomic;
pub(in crate::api) use atomic::{write_atomic, write_atomic_if_unchanged};
#[cfg(test)]
use persistence::{find_toml_table_bounds, render_access_section, save_sections_to_disk};
pub(in crate::api) use persistence::{
render_server_listeners, render_top_level_section, save_access_sections_to_disk,
upsert_toml_table, write_atomic,
save_access_sections_to_disk_if_revision, upsert_toml_table,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -54,22 +57,21 @@ pub(super) fn parse_if_match(headers: &hyper::HeaderMap) -> Option<String> {
.map(|value| value.trim_matches('"').to_string())
}
pub(super) async fn ensure_expected_revision(
/// Loads one mutation base and validates its revision from the same source snapshot.
pub(super) async fn load_config_for_mutation(
config_path: &Path,
expected_revision: Option<&str>,
) -> Result<(), ApiFailure> {
let Some(expected) = expected_revision else {
return Ok(());
};
let current = current_revision(config_path).await?;
if current != expected {
) -> Result<(ProxyConfig, String), ApiFailure> {
let loaded = load_config_snapshot(config_path, false).await?;
let revision = compute_snapshot_revision(&loaded);
if expected_revision.is_some_and(|expected| expected != revision) {
return Err(ApiFailure::new(
hyper::StatusCode::CONFLICT,
"revision_conflict",
"Config revision mismatch",
));
}
Ok(())
Ok((loaded.config, revision))
}
pub(super) async fn current_revision(config_path: &Path) -> Result<String, ApiFailure> {
@@ -243,15 +245,24 @@ pub(super) async fn load_candidate_snapshot(
}
fn normalize_source_path(path: &Path) -> PathBuf {
path.canonicalize().unwrap_or_else(|_| {
if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
};
let mut normalized = PathBuf::new();
for component in absolute.components() {
match component {
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
normalized.pop();
}
component => normalized.push(component.as_os_str()),
}
})
}
normalized
}
pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
+432
View File
@@ -0,0 +1,432 @@
use std::fs::File;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
#[cfg(unix)]
use std::os::unix::fs::{MetadataExt, PermissionsExt};
#[cfg(unix)]
use nix::fcntl::{Flock, FlockArg, OFlag, openat, renameat};
#[cfg(unix)]
use nix::sys::stat::Mode;
#[cfg(unix)]
use nix::unistd::{UnlinkatFlags, fsync, unlinkat};
#[cfg(unix)]
use tracing::warn;
use super::compute_source_revision;
use crate::api::model::ApiFailure;
use crate::config::ProxyConfig;
#[cfg(unix)]
use crate::util::secure_fs::AnchoredPath;
const MAX_CONFIG_SOURCE_BYTES: u64 = 8 * 1024 * 1024;
enum AtomicWriteError {
Conflict,
ReadGraph(String),
Io(std::io::Error),
}
struct ExistingTarget {
contents: String,
metadata: std::fs::Metadata,
}
struct GraphFence<'a> {
config_path: &'a Path,
expected_revision: &'a str,
}
struct ConfigWriteLock {
#[cfg(unix)]
_file: Flock<File>,
}
impl ConfigWriteLock {
fn acquire(path: &Path) -> std::io::Result<Self> {
let path = normalize_path(path);
#[cfg(unix)]
{
let lock_path = sibling_lock_path(&path);
let anchored = AnchoredPath::open_creating_parents(&lock_path, 0o750)?;
let descriptor = openat(
anchored.parent(),
anchored.name(),
OFlag::O_RDWR | OFlag::O_CREAT | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC,
Mode::from_bits_truncate(0o600),
)
.map_err(errno_to_io)?;
let file = File::from(descriptor);
let metadata = file.metadata()?;
if !metadata.is_file() || metadata.nlink() != 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"config lock must be a regular file with one directory entry",
));
}
let file = Flock::lock(file, FlockArg::LockExclusive)
.map_err(|(_, error)| errno_to_io(error))?;
Ok(Self { _file: file })
}
#[cfg(not(unix))]
{
let _ = path;
Ok(Self {})
}
}
}
/// Replaces one config source through a same-directory rename after syncing file data.
pub(in crate::api) async fn write_atomic(
path: PathBuf,
contents: String,
) -> Result<(), ApiFailure> {
tokio::task::spawn_blocking(move || {
let _lock = ConfigWriteLock::acquire(&path)?;
write_atomic_sync(&path, None, &contents, None).map(|_| ())
})
.await
.map_err(|error| ApiFailure::internal(format!("failed to join writer: {error}")))?
.map_err(|error| ApiFailure::internal(format!("failed to write config: {error}")))
}
/// Replaces one source only if both its graph revision and owner contents are unchanged.
pub(in crate::api) async fn write_atomic_if_unchanged(
config_path: PathBuf,
expected_revision: String,
path: PathBuf,
expected_contents: String,
contents: String,
) -> Result<String, ApiFailure> {
tokio::task::spawn_blocking(move || {
let config_path = normalize_path(&config_path);
let path = normalize_path(&path);
// Every API mutation locks the root source so writes to different includes serialize.
let _lock = ConfigWriteLock::acquire(&config_path).map_err(AtomicWriteError::Io)?;
let graph = ProxyConfig::read_source_graph(&config_path)
.map_err(|error| AtomicWriteError::ReadGraph(error.to_string()))?;
if compute_source_revision(&graph) != expected_revision {
return Err(AtomicWriteError::Conflict);
}
write_atomic_sync(
&path,
Some(&expected_contents),
&contents,
Some(GraphFence {
config_path: &config_path,
expected_revision: &expected_revision,
}),
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::AlreadyExists {
AtomicWriteError::Conflict
} else {
AtomicWriteError::Io(error)
}
})?
.ok_or_else(|| {
AtomicWriteError::Io(std::io::Error::other(
"config graph fence did not produce a committed revision",
))
})
})
.await
.map_err(|error| ApiFailure::internal(format!("failed to join writer: {error}")))?
.map_err(|error| match error {
AtomicWriteError::Conflict => revision_conflict(),
AtomicWriteError::ReadGraph(error) => {
ApiFailure::internal(format!("failed to verify config graph: {error}"))
}
AtomicWriteError::Io(error) => {
ApiFailure::internal(format!("failed to write config: {error}"))
}
})
}
fn revision_conflict() -> ApiFailure {
ApiFailure::new(
hyper::StatusCode::CONFLICT,
"revision_conflict",
"Config revision changed before persistence",
)
}
fn sibling_lock_path(path: &Path) -> PathBuf {
let mut name = path
.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("config.toml"))
.to_os_string();
name.push(".lock");
path.parent().unwrap_or_else(|| Path::new(".")).join(name)
}
fn normalize_path(path: &Path) -> PathBuf {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|current| current.join(path))
.unwrap_or_else(|_| path.to_path_buf())
};
let mut normalized = PathBuf::new();
for component in absolute.components() {
match component {
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
normalized.pop();
}
component => normalized.push(component.as_os_str()),
}
}
normalized
}
fn fenced_post_commit_revision(
fence: GraphFence<'_>,
path: &Path,
contents: &str,
) -> std::io::Result<String> {
let mut graph = ProxyConfig::read_source_graph(fence.config_path)
.map_err(|error| std::io::Error::other(error.to_string()))?;
if compute_source_revision(&graph) != fence.expected_revision {
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"config graph changed during persistence",
));
}
let path = normalize_path(path);
let Some(owner) = graph.source_contents.get_mut(&path) else {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"config source owner left the source graph during persistence",
));
};
*owner = contents.to_string();
Ok(compute_source_revision(&graph))
}
#[cfg(unix)]
fn open_existing_target(anchored: &AnchoredPath) -> std::io::Result<Option<ExistingTarget>> {
let descriptor = match openat(
anchored.parent(),
anchored.name(),
OFlag::O_RDONLY | OFlag::O_NONBLOCK | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC,
Mode::empty(),
) {
Ok(descriptor) => descriptor,
Err(nix::errno::Errno::ENOENT) => return Ok(None),
Err(error) => return Err(errno_to_io(error)),
};
let mut file = File::from(descriptor);
let metadata = file.metadata()?;
if !metadata.is_file() || metadata.nlink() != 1 || metadata.len() > MAX_CONFIG_SOURCE_BYTES {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"config target must be a bounded regular file with one directory entry",
));
}
let mut contents = String::with_capacity(metadata.len() as usize);
Read::take(&mut file, MAX_CONFIG_SOURCE_BYTES + 1).read_to_string(&mut contents)?;
if contents.len() as u64 > MAX_CONFIG_SOURCE_BYTES {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"config target exceeds the source size limit",
));
}
let completed = file.metadata()?;
if !same_target(&metadata, &completed) || metadata.len() != completed.len() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"config target changed while it was read",
));
}
Ok(Some(ExistingTarget { contents, metadata }))
}
#[cfg(not(unix))]
fn open_existing_target(path: &Path) -> std::io::Result<Option<ExistingTarget>> {
let mut file = match File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error),
};
let metadata = file.metadata()?;
if !metadata.is_file() || metadata.len() > MAX_CONFIG_SOURCE_BYTES {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"config target must be a bounded regular file",
));
}
let mut contents = String::new();
file.read_to_string(&mut contents)?;
Ok(Some(ExistingTarget { contents, metadata }))
}
fn same_target(left: &std::fs::Metadata, right: &std::fs::Metadata) -> bool {
#[cfg(unix)]
{
left.dev() == right.dev() && left.ino() == right.ino()
}
#[cfg(not(unix))]
{
left.len() == right.len() && left.modified().ok() == right.modified().ok()
}
}
#[cfg(unix)]
fn write_atomic_sync(
path: &Path,
expected_contents: Option<&str>,
contents: &str,
graph_fence: Option<GraphFence<'_>>,
) -> std::io::Result<Option<String>> {
let anchored = AnchoredPath::open_creating_parents(path, 0o750)?;
let existing = open_existing_target(&anchored)?;
validate_expected_contents(existing.as_ref(), expected_contents)?;
let temp_name = format!(
".{}.tmp-{}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("config.toml"),
rand::random::<u64>()
);
let descriptor = openat(
anchored.parent(),
temp_name.as_str(),
OFlag::O_WRONLY | OFlag::O_CREAT | OFlag::O_EXCL | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC,
Mode::from_bits_truncate(0o600),
)
.map_err(errno_to_io)?;
let write_result = write_and_publish(
descriptor,
path,
&anchored,
&temp_name,
existing.as_ref(),
contents,
graph_fence,
);
if write_result.is_err() {
let _ = unlinkat(
anchored.parent(),
temp_name.as_str(),
UnlinkatFlags::NoRemoveDir,
);
}
write_result
}
#[cfg(unix)]
fn write_and_publish(
descriptor: std::os::fd::OwnedFd,
path: &Path,
anchored: &AnchoredPath,
temp_name: &str,
existing: Option<&ExistingTarget>,
contents: &str,
graph_fence: Option<GraphFence<'_>>,
) -> std::io::Result<Option<String>> {
let mut file = File::from(descriptor);
if let Some(existing) = existing {
use nix::unistd::{Gid, Uid, fchown};
fchown(
&file,
Some(Uid::from_raw(existing.metadata.uid())),
Some(Gid::from_raw(existing.metadata.gid())),
)
.map_err(errno_to_io)?;
file.set_permissions(std::fs::Permissions::from_mode(
existing.metadata.mode() & 0o7777,
))?;
}
file.write_all(contents.as_bytes())?;
file.sync_all()?;
let current = open_existing_target(anchored)?;
if !target_unchanged(existing, current.as_ref()) {
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"config target changed during persistence",
));
}
let committed_revision = graph_fence
.map(|fence| fenced_post_commit_revision(fence, path, contents))
.transpose()?;
renameat(
anchored.parent(),
temp_name,
anchored.parent(),
anchored.name(),
)
.map_err(errno_to_io)?;
// Rename is the commit boundary. A later directory-sync error cannot be reported as an
// uncommitted mutation because mandatory in-process publication must still run.
if let Err(error) = fsync(anchored.parent()).map_err(errno_to_io) {
warn!(
path = %path.display(),
%error,
"Config rename committed but directory durability sync failed"
);
}
Ok(committed_revision)
}
#[cfg(not(unix))]
fn write_atomic_sync(
path: &Path,
expected_contents: Option<&str>,
contents: &str,
graph_fence: Option<GraphFence<'_>>,
) -> std::io::Result<Option<String>> {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
std::fs::create_dir_all(parent)?;
let existing = open_existing_target(path)?;
validate_expected_contents(existing.as_ref(), expected_contents)?;
let temp = parent.join(format!(".telemt.tmp-{}", rand::random::<u64>()));
std::fs::write(&temp, contents)?;
let current = open_existing_target(path)?;
if !target_unchanged(existing.as_ref(), current.as_ref()) {
let _ = std::fs::remove_file(&temp);
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"config target changed during persistence",
));
}
let committed_revision = graph_fence
.map(|fence| fenced_post_commit_revision(fence, path, contents))
.transpose()?;
std::fs::rename(temp, path)?;
Ok(committed_revision)
}
fn validate_expected_contents(
existing: Option<&ExistingTarget>,
expected_contents: Option<&str>,
) -> std::io::Result<()> {
if expected_contents
.is_some_and(|expected| existing.is_none_or(|target| target.contents != expected))
{
return Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"config source changed before persistence",
));
}
Ok(())
}
fn target_unchanged(existing: Option<&ExistingTarget>, current: Option<&ExistingTarget>) -> bool {
match (existing, current) {
(Some(expected), Some(current)) => {
same_target(&expected.metadata, &current.metadata)
&& expected.contents == current.contents
}
(None, None) => true,
_ => false,
}
}
#[cfg(unix)]
fn errno_to_io(error: nix::errno::Errno) -> std::io::Error {
std::io::Error::from_raw_os_error(error as i32)
}
+35 -47
View File
@@ -1,12 +1,14 @@
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::path::Path;
use chrono::{DateTime, Utc};
use serde::Serialize;
use crate::config::{ProxyConfig, RateLimitBps};
#[cfg(test)]
use super::atomic::write_atomic;
use super::atomic::write_atomic_if_unchanged;
#[cfg(test)]
use super::compute_revision;
use super::{
@@ -99,8 +101,22 @@ pub(in crate::api) async fn save_access_sections_to_disk(
config_path: &Path,
cfg: &ProxyConfig,
sections: &[AccessSection],
) -> Result<String, ApiFailure> {
save_access_sections_to_disk_if_revision(config_path, cfg, sections, None).await
}
/// Persists access tables only while the complete source graph remains unchanged.
pub(in crate::api) async fn save_access_sections_to_disk_if_revision(
config_path: &Path,
cfg: &ProxyConfig,
sections: &[AccessSection],
expected_revision: Option<&str>,
) -> Result<String, ApiFailure> {
let loaded = load_config_snapshot(config_path, false).await?;
let loaded_revision = compute_snapshot_revision(&loaded);
if expected_revision.is_some_and(|expected| expected != loaded_revision) {
return Err(revision_conflict());
}
let mut applied = Vec::new();
for section in sections {
if applied.contains(section) {
@@ -117,7 +133,7 @@ pub(in crate::api) async fn save_access_sections_to_disk(
})
});
if applied.is_empty() {
return Ok(compute_snapshot_revision(&loaded));
return Ok(loaded_revision);
}
let targets = applied
@@ -130,6 +146,7 @@ pub(in crate::api) async fn save_access_sections_to_disk(
.get(&owner_path)
.cloned()
.ok_or_else(|| ApiFailure::internal("config source owner is missing from snapshot"))?;
let expected_owner_contents = owner_contents.clone();
for section in applied {
let rendered = render_access_section(cfg, section)?;
owner_contents = upsert_toml_table(&owner_contents, section.table_name(), &rendered);
@@ -142,8 +159,15 @@ pub(in crate::api) async fn save_access_sections_to_disk(
owner_contents.clone(),
)
.await?;
let revision = compute_snapshot_revision(&candidate);
write_atomic(owner_path, owner_contents).await?;
let _candidate_revision = compute_snapshot_revision(&candidate);
let revision = write_atomic_if_unchanged(
config_path.to_path_buf(),
loaded_revision,
owner_path,
expected_owner_contents,
owner_contents,
)
.await?;
Ok(revision)
}
@@ -373,46 +397,10 @@ fn find_all_table_blocks(source: &str, table_name: &str) -> Vec<(usize, usize)>
blocks
}
/// Replaces one config source through a durable same-directory rename.
pub(in crate::api) async fn write_atomic(
path: PathBuf,
contents: String,
) -> Result<(), ApiFailure> {
tokio::task::spawn_blocking(move || write_atomic_sync(&path, &contents))
.await
.map_err(|e| ApiFailure::internal(format!("failed to join writer: {}", e)))?
.map_err(|e| ApiFailure::internal(format!("failed to write config: {}", e)))
}
fn write_atomic_sync(path: &Path, contents: &str) -> std::io::Result<()> {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
std::fs::create_dir_all(parent)?;
let tmp_name = format!(
".{}.tmp-{}",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("config.toml"),
rand::random::<u64>()
);
let tmp_path = parent.join(tmp_name);
let write_result = (|| {
let mut file = std::fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(&tmp_path)?;
file.write_all(contents.as_bytes())?;
file.sync_all()?;
std::fs::rename(&tmp_path, path)?;
if let Ok(dir) = std::fs::File::open(parent) {
let _ = dir.sync_all();
}
Ok(())
})();
if write_result.is_err() {
let _ = std::fs::remove_file(&tmp_path);
}
write_result
fn revision_conflict() -> ApiFailure {
ApiFailure::new(
hyper::StatusCode::CONFLICT,
"revision_conflict",
"Config revision changed before persistence",
)
}
+112
View File
@@ -260,6 +260,104 @@ async fn access_mutation_writes_only_the_single_included_owner() {
assert_eq!(revision, current_revision(&root).await.unwrap());
}
#[tokio::test]
async fn access_mutation_rejects_source_graph_change_after_snapshot() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("config.toml");
let included = dir.path().join("users.toml");
let root_body = "include = \"users.toml\"\n[censorship]\ntls_domain = \"one.example\"\n";
let external_root = "include = \"users.toml\"\n[censorship]\ntls_domain = \"two.example\"\n";
let included_body = "[access.users]\nalice = \"00000000000000000000000000000000\"\n";
tokio::fs::write(&root, root_body).await.unwrap();
tokio::fs::write(&included, included_body).await.unwrap();
let (mut cfg, revision) = load_config_for_mutation(&root, None).await.unwrap();
cfg.access.users.insert(
"bob".to_string(),
"11111111111111111111111111111111".to_string(),
);
tokio::fs::write(&root, external_root).await.unwrap();
let error = save_access_sections_to_disk_if_revision(
&root,
&cfg,
&[AccessSection::Users],
Some(&revision),
)
.await
.unwrap_err();
assert_eq!(error.code, "revision_conflict");
assert_eq!(
tokio::fs::read_to_string(&root).await.unwrap(),
external_root
);
assert_eq!(
tokio::fs::read_to_string(&included).await.unwrap(),
included_body
);
}
#[cfg(unix)]
#[tokio::test]
async fn atomic_write_preserves_existing_file_mode() {
use std::os::unix::fs::{MetadataExt, PermissionsExt};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("config.toml");
tokio::fs::write(&path, "old").await.unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o640)).unwrap();
let before = std::fs::metadata(&path).unwrap();
write_atomic(path.clone(), "new".to_string()).await.unwrap();
let after = std::fs::metadata(&path).unwrap();
assert_eq!(after.mode() & 0o7777, 0o640);
assert_eq!(after.uid(), before.uid());
assert_eq!(after.gid(), before.gid());
}
#[tokio::test]
async fn config_sidecar_lock_serializes_competing_revision_writers() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("config.toml");
let original = concat!(
"[censorship]\n",
"tls_domain = \"original.example\"\n",
"[access.users]\n",
"alice = \"00000000000000000000000000000000\"\n"
);
tokio::fs::write(&path, original).await.unwrap();
let graph = ProxyConfig::read_source_graph(&path).unwrap();
let revision = compute_source_revision(&graph);
let first = tokio::spawn(write_atomic_if_unchanged(
path.clone(),
revision.clone(),
path.clone(),
original.to_string(),
original.replace("original.example", "first.example"),
));
let second = tokio::spawn(write_atomic_if_unchanged(
path.clone(),
revision,
path.clone(),
original.to_string(),
original.replace("original.example", "second.example"),
));
let first = first.await.unwrap();
let second = second.await.unwrap();
assert_ne!(first.is_ok(), second.is_ok());
let (winner_revision, conflict) = match (first, second) {
(Ok(revision), Err(error)) | (Err(error), Ok(revision)) => (revision, error),
_ => unreachable!("exactly one cooperative writer must commit"),
};
assert_eq!(conflict.code, "revision_conflict");
assert_eq!(winner_revision, current_revision(&path).await.unwrap());
let persisted = tokio::fs::read_to_string(&path).await.unwrap();
assert!(persisted.contains("first.example") || persisted.contains("second.example"));
}
#[tokio::test]
async fn access_mutation_rejects_sections_with_different_source_owners() {
let dir = tempfile::tempdir().unwrap();
@@ -307,3 +405,17 @@ fn render_user_rate_limits_section() {
assert!(rendered.starts_with("[access.user_rate_limits]\n"));
assert!(rendered.contains("alice = { up_bps = 1024, down_bps = 2048 }"));
}
#[cfg(unix)]
#[test]
fn source_owner_normalization_preserves_symlinks() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let real = dir.path().join("real.toml");
let linked = dir.path().join("linked.toml");
std::fs::write(&real, "").unwrap();
symlink(&real, &linked).unwrap();
assert_eq!(normalize_source_path(&linked), linked);
}
+187
View File
@@ -0,0 +1,187 @@
use super::*;
// Read-only fixed API endpoints.
mod read_routes;
// Fixed configuration and lifecycle mutations.
mod fixed_routes;
// Dynamic reload and user-resource routes.
mod user_routes;
pub(super) async fn handle(
req: Request<Incoming>,
peer: SocketAddr,
shared: Arc<ApiShared>,
) -> Result<Response<Full<Bytes>>, IoError> {
let runtime = shared.active_runtime.load_full();
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
if previous_cache_generation != runtime.id {
*shared.minimal_cache.lock().await = None;
*shared.runtime_edge_connections_cache.lock().await = None;
}
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
let config_rx = runtime.config_rx.clone();
shared
.runtime_state
.admission_open
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
let request_id = shared.next_request_id();
let cfg = config_rx.borrow().clone();
let api_cfg = &cfg.server.api;
if !api_cfg.enabled {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::SERVICE_UNAVAILABLE,
"api_disabled",
"API is disabled",
),
));
}
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
{
return match api_cfg.gray_action {
ApiGrayAction::Api => Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"forbidden",
"Source IP is not allowed",
),
)),
ApiGrayAction::Ok200 => Ok(Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/html; charset=utf-8")
.body(Full::new(Bytes::new()))
.unwrap()),
ApiGrayAction::Drop => Err(IoError::new(
ErrorKind::ConnectionAborted,
"api request dropped by gray_action=drop",
)),
};
}
if !api_cfg.auth_header.is_empty() {
let auth_ok = req
.headers()
.get(AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
.unwrap_or(false);
if !auth_ok {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::UNAUTHORIZED,
"unauthorized",
"Missing or invalid Authorization header",
),
));
}
}
let method = req.method().clone();
let path = req.uri().path().to_string();
let normalized_path = if path.len() > 1 {
path.trim_end_matches('/')
} else {
path.as_str()
};
let query = req.uri().query().map(str::to_string);
let body_limit = api_cfg.request_body_limit_bytes;
let result = dispatch(
req,
method,
&path,
normalized_path,
query.as_deref(),
body_limit,
&shared,
cfg.as_ref(),
&config_rx,
request_id,
)
.await;
match result {
Ok(resp) => Ok(resp),
Err(error) => Ok(error_response(request_id, error)),
}
}
async fn dispatch(
req: Request<Incoming>,
method: Method,
path: &str,
normalized_path: &str,
query: Option<&str>,
body_limit: usize,
shared: &Arc<ApiShared>,
cfg: &ProxyConfig,
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
request_id: u64,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
if web_runtime::is_route(normalized_path) {
let web_mutation = method == Method::POST;
let result = web_runtime::handle(
method,
normalized_path,
query,
req,
shared.as_ref(),
cfg,
request_id,
body_limit,
)
.await;
if web_mutation && let Err(error) = &result {
shared.runtime_events.record(
"api.web.control.failed",
format!("path={} code={}", normalized_path, error.code),
);
}
return result;
}
if let Some(response) = read_routes::handle(
&method,
normalized_path,
query,
shared.as_ref(),
cfg,
config_rx,
)
.await?
{
return Ok(response);
}
match (method.as_str(), normalized_path) {
("POST", "/v1/users") => {
fixed_routes::create_user_route(req, shared, cfg, config_rx, request_id, body_limit)
.await
}
("GET", "/v1/config") => fixed_routes::get_config_route(shared).await,
("POST", "/v1/system/reload") => {
fixed_routes::reload_route(req, shared, cfg, request_id, body_limit).await
}
("PATCH", "/v1/config") => {
fixed_routes::patch_config_route(req, shared, cfg, query, request_id, body_limit).await
}
_ => {
user_routes::handle(
req,
&method,
path,
normalized_path,
shared,
cfg,
config_rx,
request_id,
body_limit,
)
.await
}
}
}
+129
View File
@@ -0,0 +1,129 @@
use super::*;
pub(super) async fn create_user_route(
req: Request<Incoming>,
shared: &Arc<ApiShared>,
cfg: &ProxyConfig,
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
request_id: u64,
body_limit: usize,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let api_cfg = &cfg.server.api;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
let result = create_user(body, expected_revision, shared).await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared
.runtime_events
.record("api.user.create.failed", error.code);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.create.ok",
format!("username={}", data.user.username),
);
let status = if data.user.in_runtime {
StatusCode::CREATED
} else {
StatusCode::ACCEPTED
};
Ok(success_response(status, data, revision))
}
pub(super) async fn get_config_route(
shared: &Arc<ApiShared>,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let (value, revision) = config_edit::read_managed_config(&shared.config_path).await?;
Ok(success_response(StatusCode::OK, value, revision))
}
pub(super) async fn reload_route(
req: Request<Incoming>,
shared: &Arc<ApiShared>,
cfg: &ProxyConfig,
request_id: u64,
body_limit: usize,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let api_cfg = &cfg.server.api;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
.await?
.unwrap_or_default();
request.validate().map_err(ApiFailure::bad_request)?;
let (accepted, revision) = submit_reload_from_disk(
&shared.config_path,
shared.mutation_lock.as_ref(),
&shared.reload_control,
expected_revision.as_deref(),
request,
)
.await?;
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
}
pub(super) async fn patch_config_route(
req: Request<Incoming>,
shared: &Arc<ApiShared>,
cfg: &ProxyConfig,
query: Option<&str>,
request_id: u64,
body_limit: usize,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let api_cfg = &cfg.server.api;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let reload_request = ReloadRequest::from_query(query).map_err(ApiFailure::bad_request)?;
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
match config_edit::patch_config(body, expected_revision, reload_request, shared).await {
Ok(resp) => {
let revision = resp.revision.clone();
let status = if resp.reload.is_some() {
StatusCode::ACCEPTED
} else {
StatusCode::OK
};
Ok(success_response(status, resp, revision))
}
Err(error) => {
shared
.runtime_events
.record("api.config.patch.failed", error.code);
Err(error)
}
}
}
+210
View File
@@ -0,0 +1,210 @@
use super::*;
pub(super) async fn handle(
method: &Method,
normalized_path: &str,
query: Option<&str>,
shared: &ApiShared,
cfg: &ProxyConfig,
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
let api_cfg = &cfg.server.api;
match (method.as_str(), normalized_path) {
("GET", "/web-status") => Ok(web_status::render(query, &shared.web_trace).await),
("GET", "/v1/health") => {
let revision = current_revision(&shared.config_path).await?;
let data = HealthData {
status: "ok",
read_only: api_cfg.read_only,
};
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/health/ready") => {
let revision = current_revision(&shared.config_path).await?;
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
let upstream_health = shared.upstream_manager.api_health_summary().await;
let ready = admission_open && upstream_health.healthy_total > 0;
let reason = if ready {
None
} else if !admission_open {
Some("admission_closed")
} else {
Some("no_healthy_upstreams")
};
let data = HealthReadyData {
ready,
status: if ready { "ready" } else { "not_ready" },
reason,
admission_open,
healthy_upstreams: upstream_health.healthy_total,
total_upstreams: upstream_health.configured_total,
};
let status_code = if ready {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
};
Ok(success_response(status_code, data, revision))
}
("GET", "/v1/system/info") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_system_info_data(shared, cfg, &revision);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/gates") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_gates_data(shared, cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/initialization") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_initialization_data(shared).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/limits/effective") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_limits_effective_data(cfg);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/security/posture") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_security_posture_data(cfg);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/security/whitelist") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_security_whitelist_data(cfg);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/summary") => {
let revision = current_revision(&shared.config_path).await?;
let connections_bad_by_class = shared
.stats
.get_connects_bad_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let handshake_failures_by_class = shared
.stats
.get_handshake_failure_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let data = SummaryData {
uptime_seconds: shared.stats.uptime_secs(),
connections_total: shared.stats.get_connects_all(),
connections_bad_total: shared.stats.get_connects_bad(),
connections_bad_by_class,
handshake_failures_by_class,
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
configured_users: cfg.access.users.len(),
};
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/zero/all") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/upstreams") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_upstreams_data(shared, api_cfg);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/minimal/all") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_minimal_all_data(shared, api_cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/me-writers") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_me_writers_data(shared, api_cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/dcs") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_dcs_data(shared, api_cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_me_pool_state_data(shared).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_me_quality_data(shared).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_upstream_quality_data(shared).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_nat_stun_data(shared).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/me-selftest") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_me_selftest_data(shared, cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/connections/summary") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_connections_summary_data(shared, cfg).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/events/recent") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_events_recent_data(shared, cfg, query);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/tls-fingerprints") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_tls_fingerprints_data(shared, cfg, query);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/users/active-ips") => {
let revision = current_revision(&shared.config_path).await?;
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
let mut data: Vec<UserActiveIps> = active_ips_map
.into_iter()
.filter(|(_, ips)| !ips.is_empty())
.map(|(username, active_ips)| UserActiveIps {
username,
active_ips,
})
.collect();
data.sort_by(|a, b| a.username.cmp(&b.username));
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
let runtime_cfg = config_rx.borrow().clone();
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
let users = users_from_config(
&disk_cfg,
&shared.stats,
&shared.ip_tracker,
detected_ip_v4,
detected_ip_v6,
Some(runtime_cfg.as_ref()),
)
.await;
Ok(success_response(StatusCode::OK, users, revision))
}
("GET", "/v1/stats/users/quota") => {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
Ok(success_response(StatusCode::OK, data, revision))
}
_ => return Ok(None),
}
.map(Some)
}
+373
View File
@@ -0,0 +1,373 @@
use super::*;
pub(super) async fn handle(
req: Request<Incoming>,
method: &Method,
path: &str,
normalized_path: &str,
shared: &Arc<ApiShared>,
cfg: &ProxyConfig,
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
request_id: u64,
body_limit: usize,
) -> Result<Response<Full<Bytes>>, ApiFailure> {
let api_cfg = &cfg.server.api;
if method == Method::GET
&& let Some(reload_id) = reload_status_route_id(normalized_path)
{
let revision = current_revision(&shared.config_path).await?;
let status = shared
.reload_control
.status(reload_id)
.await
.ok_or_else(|| {
ApiFailure::new(
StatusCode::NOT_FOUND,
"reload_not_found",
format!("Reload {} was not found", reload_id),
)
})?;
return Ok(success_response(StatusCode::OK, status, revision));
}
if method == Method::POST
&& let Some(base_user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/enable"))
&& !base_user.is_empty()
&& !base_user.contains('/')
{
let base_user = parse_route_username(base_user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let result = set_user_enabled(base_user, true, expected_revision, shared).await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.enable.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
shared
.runtime_events
.record("api.user.enable.ok", format!("username={}", base_user));
let status = if data.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if method == Method::POST
&& let Some(base_user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/disable"))
&& !base_user.is_empty()
&& !base_user.contains('/')
{
let base_user = parse_route_username(base_user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let result = set_user_enabled(base_user, false, expected_revision, shared).await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.disable.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
shared
.runtime_events
.record("api.user.disable.ok", format!("username={}", base_user));
let status = if data.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if method == Method::POST
&& let Some(user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/reset-quota"))
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let completion_shared = shared.as_ref().clone();
let user_owned = user.to_string();
let completion = shared
.run_mutation_completion(async move {
let _mutation_guard = completion_shared.mutation_lock.lock().await;
let (disk_cfg, _) = load_config_for_mutation(
&completion_shared.config_path,
expected_revision.as_deref(),
)
.await?;
if !disk_cfg.access.users.contains_key(&user_owned) {
return Err(ApiFailure::new(
StatusCode::NOT_FOUND,
"not_found",
"User not found",
));
}
let configured_users = disk_cfg
.access
.users
.keys()
.cloned()
.collect::<BTreeSet<_>>();
let snapshot = completion_shared
.quota_state
.reset_user(&configured_users, &user_owned)
.await
.map_err(|error| {
completion_shared.runtime_events.record(
"api.user.reset_quota.failed",
format!("username={} error={}", user_owned, error),
);
ApiFailure::internal(format!("Failed to reset user quota: {}", error))
})?;
completion_shared.runtime_events.record(
"api.user.reset_quota.ok",
format!("username={}", user_owned),
);
let revision = current_revision(&completion_shared.config_path).await?;
Ok((snapshot, revision))
})
.await;
let (snapshot, revision) = match completion {
Ok(result) => result,
Err(error) if error.code == "not_found" => {
return Ok(error_response(request_id, error));
}
Err(error) => return Err(error),
};
return Ok(success_response(
StatusCode::OK,
ResetUserQuotaResponse {
username: user.to_string(),
used_bytes: snapshot.used_bytes,
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
},
revision,
));
}
if method == Method::POST
&& let Some(base_user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/rotate-secret"))
&& !base_user.is_empty()
&& !base_user.contains('/')
{
let base_user = parse_route_username(base_user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body = read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit).await?;
let result = rotate_secret(
base_user,
body.unwrap_or_default(),
expected_revision,
shared,
)
.await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.rotate_secret.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.rotate_secret.ok",
format!("username={}", base_user),
);
let status = if data.user.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if method == Method::GET {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
let runtime_cfg = config_rx.borrow().clone();
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
let users = users_from_config(
&disk_cfg,
&shared.stats,
&shared.ip_tracker,
detected_ip_v4,
detected_ip_v6,
Some(runtime_cfg.as_ref()),
)
.await;
if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) {
return Ok(success_response(StatusCode::OK, user_info, revision));
}
return Ok(error_response(
request_id,
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
));
}
if method == Method::PATCH {
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
let result = patch_user(user, body, expected_revision, shared).await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.patch.failed",
format!("username={} code={}", user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
shared
.runtime_events
.record("api.user.patch.ok", format!("username={}", data.username));
let status = if data.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if method == Method::DELETE {
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let result = delete_user(user, expected_revision, shared).await;
let (deleted_user, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.delete.failed",
format!("username={} code={}", user, error.code),
);
return Err(error);
}
};
shared
.runtime_events
.record("api.user.delete.ok", format!("username={}", deleted_user));
let runtime_cfg = config_rx.borrow().clone();
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
let response = DeleteUserResponse {
username: deleted_user,
in_runtime,
};
let status = if response.in_runtime {
StatusCode::ACCEPTED
} else {
StatusCode::OK
};
return Ok(success_response(status, response, revision));
}
if method == Method::POST {
return Ok(error_response(
request_id,
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
));
}
return Ok(error_response(
request_id,
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
));
}
if let Some(allow) = allowed_methods_for_path(normalized_path) {
return Ok(error_response(
request_id,
ApiFailure::method_not_allowed(allow),
));
}
debug!(
method = method.as_str(),
path = %path,
normalized_path = %normalized_path,
"API route not found"
);
Ok(error_response(
request_id,
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
))
}
+47 -850
View File
File diff suppressed because it is too large Load Diff
+11 -161
View File
@@ -124,6 +124,10 @@ pub(super) struct ZeroCoreData {
pub(super) conntrack_pressure_active: bool,
pub(super) conntrack_event_queue_depth: u64,
pub(super) conntrack_rule_apply_ok: bool,
pub(super) conntrack_rule_reconcile_success_total: u64,
pub(super) conntrack_rule_reconcile_error_total: u64,
pub(super) conntrack_rule_rollback_success_total: u64,
pub(super) conntrack_rule_rollback_error_total: u64,
pub(super) conntrack_delete_attempt_total: u64,
pub(super) conntrack_delete_success_total: u64,
pub(super) conntrack_delete_not_found_total: u64,
@@ -211,6 +215,7 @@ pub(super) struct ZeroMiddleProxyData {
pub(super) reconnect_success_total: u64,
pub(super) handshake_reject_total: u64,
pub(super) handshake_error_codes: Vec<ZeroCodeCount>,
pub(super) handshake_error_code_overflow_total: u64,
pub(super) reader_eof_total: u64,
pub(super) idle_close_by_peer_total: u64,
pub(super) route_drop_no_conn_total: u64,
@@ -388,8 +393,11 @@ pub(super) struct MinimalDcPathData {
pub(super) struct MinimalMeRuntimeData {
pub(super) active_generation: u64,
pub(super) warm_generation: u64,
pub(super) warm_generations: Vec<u64>,
pub(super) pending_hardswap_generation: u64,
pub(super) pending_hardswap_age_secs: Option<u64>,
pub(super) reinit_inflight: usize,
pub(super) reinit_max_concurrency_effective: usize,
pub(super) hardswap_enabled: bool,
pub(super) floor_mode: &'static str,
pub(super) adaptive_floor_idle_secs: u64,
@@ -462,164 +470,6 @@ pub(super) struct MinimalAllData {
pub(super) data: Option<MinimalAllPayload>,
}
#[derive(Serialize)]
pub(super) struct UserLinks {
pub(super) classic: Vec<String>,
pub(super) secure: Vec<String>,
pub(super) tls: Vec<String>,
pub(super) tls_domains: Vec<TlsDomainLink>,
}
#[derive(Serialize)]
pub(super) struct TlsDomainLink {
pub(super) domain: String,
pub(super) link: String,
}
#[derive(Serialize)]
pub(super) struct UserInfo {
pub(super) username: String,
pub(super) enabled: bool,
pub(super) in_runtime: bool,
pub(super) user_ad_tag: Option<String>,
pub(super) max_tcp_conns: Option<usize>,
pub(super) expiration_rfc3339: Option<String>,
pub(super) data_quota_bytes: Option<u64>,
pub(super) rate_limit_up_bps: Option<u64>,
pub(super) rate_limit_down_bps: Option<u64>,
pub(super) max_unique_ips: Option<usize>,
pub(super) current_connections: u64,
pub(super) active_unique_ips: usize,
pub(super) active_unique_ips_list: Vec<IpAddr>,
pub(super) recent_unique_ips: usize,
pub(super) recent_unique_ips_list: Vec<IpAddr>,
pub(super) total_octets: u64,
pub(super) links: UserLinks,
}
#[derive(Serialize)]
pub(super) struct UserActiveIps {
pub(super) username: String,
pub(super) active_ips: Vec<IpAddr>,
}
#[derive(Serialize)]
pub(super) struct CreateUserResponse {
pub(super) user: UserInfo,
pub(super) secret: String,
}
#[derive(Serialize)]
pub(super) struct DeleteUserResponse {
pub(super) username: String,
pub(super) in_runtime: bool,
}
#[derive(Serialize)]
pub(super) struct ResetUserQuotaResponse {
pub(super) username: String,
pub(super) used_bytes: u64,
pub(super) last_reset_epoch_secs: u64,
}
#[derive(Serialize)]
pub(super) struct UserQuotaListData {
pub(super) users: Vec<UserQuotaEntry>,
}
#[derive(Serialize)]
pub(super) struct UserQuotaEntry {
pub(super) username: String,
pub(super) data_quota_bytes: u64,
pub(super) used_bytes: u64,
pub(super) last_reset_epoch_secs: u64,
}
#[derive(Deserialize)]
pub(super) struct CreateUserRequest {
pub(super) username: String,
pub(super) secret: Option<String>,
pub(super) user_ad_tag: Option<String>,
pub(super) max_tcp_conns: Option<usize>,
pub(super) expiration_rfc3339: Option<String>,
pub(super) data_quota_bytes: Option<u64>,
pub(super) rate_limit_up_bps: Option<u64>,
pub(super) rate_limit_down_bps: Option<u64>,
pub(super) max_unique_ips: Option<usize>,
pub(super) enabled: Option<bool>,
}
#[derive(Deserialize)]
pub(super) struct PatchUserRequest {
pub(super) secret: Option<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) user_ad_tag: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) max_tcp_conns: Patch<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) expiration_rfc3339: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) data_quota_bytes: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) rate_limit_up_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) rate_limit_down_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) max_unique_ips: Patch<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) enabled: Patch<bool>,
}
#[derive(Default, Deserialize)]
pub(super) struct RotateSecretRequest {
pub(super) secret: Option<String>,
}
pub(super) fn parse_optional_expiration(
value: Option<&str>,
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
let Some(raw) = value else {
return Ok(None);
};
let parsed = DateTime::parse_from_rfc3339(raw)
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
Ok(Some(parsed.with_timezone(&Utc)))
}
pub(super) fn parse_patch_expiration(
value: &Patch<String>,
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
match value {
Patch::Unchanged => Ok(Patch::Unchanged),
Patch::Remove => Ok(Patch::Remove),
Patch::Set(raw) => {
let parsed = DateTime::parse_from_rfc3339(raw)
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
Ok(Patch::Set(parsed.with_timezone(&Utc)))
}
}
}
pub(super) fn is_valid_user_secret(secret: &str) -> bool {
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
}
pub(super) fn is_valid_ad_tag(tag: &str) -> bool {
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
}
pub(super) fn is_valid_username(user: &str) -> bool {
!user.is_empty()
&& user.len() <= MAX_USERNAME_LEN
&& user
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
}
pub(super) fn random_user_secret() -> String {
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
let mut bytes = [0u8; 16];
rng.fill(&mut bytes);
hex::encode(bytes)
}
// User-management request, response, and validation models.
mod users;
pub(super) use users::*;
+163
View File
@@ -0,0 +1,163 @@
use super::*;
#[derive(Serialize)]
pub(in crate::api) struct UserLinks {
pub(in crate::api) classic: Vec<String>,
pub(in crate::api) secure: Vec<String>,
pub(in crate::api) tls: Vec<String>,
pub(in crate::api) tls_domains: Vec<TlsDomainLink>,
}
#[derive(Serialize)]
pub(in crate::api) struct TlsDomainLink {
pub(in crate::api) domain: String,
pub(in crate::api) link: String,
}
#[derive(Serialize)]
pub(in crate::api) struct UserInfo {
pub(in crate::api) username: String,
pub(in crate::api) enabled: bool,
pub(in crate::api) in_runtime: bool,
pub(in crate::api) user_ad_tag: Option<String>,
pub(in crate::api) max_tcp_conns: Option<usize>,
pub(in crate::api) expiration_rfc3339: Option<String>,
pub(in crate::api) data_quota_bytes: Option<u64>,
pub(in crate::api) rate_limit_up_bps: Option<u64>,
pub(in crate::api) rate_limit_down_bps: Option<u64>,
pub(in crate::api) max_unique_ips: Option<usize>,
pub(in crate::api) current_connections: u64,
pub(in crate::api) active_unique_ips: usize,
pub(in crate::api) active_unique_ips_list: Vec<IpAddr>,
pub(in crate::api) recent_unique_ips: usize,
pub(in crate::api) recent_unique_ips_list: Vec<IpAddr>,
pub(in crate::api) total_octets: u64,
pub(in crate::api) links: UserLinks,
}
#[derive(Serialize)]
pub(in crate::api) struct UserActiveIps {
pub(in crate::api) username: String,
pub(in crate::api) active_ips: Vec<IpAddr>,
}
#[derive(Serialize)]
pub(in crate::api) struct CreateUserResponse {
pub(in crate::api) user: UserInfo,
pub(in crate::api) secret: String,
}
#[derive(Serialize)]
pub(in crate::api) struct DeleteUserResponse {
pub(in crate::api) username: String,
pub(in crate::api) in_runtime: bool,
}
#[derive(Serialize)]
pub(in crate::api) struct ResetUserQuotaResponse {
pub(in crate::api) username: String,
pub(in crate::api) used_bytes: u64,
pub(in crate::api) last_reset_epoch_secs: u64,
}
#[derive(Serialize)]
pub(in crate::api) struct UserQuotaListData {
pub(in crate::api) users: Vec<UserQuotaEntry>,
}
#[derive(Serialize)]
pub(in crate::api) struct UserQuotaEntry {
pub(in crate::api) username: String,
pub(in crate::api) data_quota_bytes: u64,
pub(in crate::api) used_bytes: u64,
pub(in crate::api) last_reset_epoch_secs: u64,
}
#[derive(Deserialize)]
pub(in crate::api) struct CreateUserRequest {
pub(in crate::api) username: String,
pub(in crate::api) secret: Option<String>,
pub(in crate::api) user_ad_tag: Option<String>,
pub(in crate::api) max_tcp_conns: Option<usize>,
pub(in crate::api) expiration_rfc3339: Option<String>,
pub(in crate::api) data_quota_bytes: Option<u64>,
pub(in crate::api) rate_limit_up_bps: Option<u64>,
pub(in crate::api) rate_limit_down_bps: Option<u64>,
pub(in crate::api) max_unique_ips: Option<usize>,
pub(in crate::api) enabled: Option<bool>,
}
#[derive(Deserialize)]
pub(in crate::api) struct PatchUserRequest {
pub(in crate::api) secret: Option<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) user_ad_tag: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) max_tcp_conns: Patch<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) expiration_rfc3339: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) data_quota_bytes: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) rate_limit_up_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) rate_limit_down_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) max_unique_ips: Patch<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(in crate::api) enabled: Patch<bool>,
}
#[derive(Default, Deserialize)]
pub(in crate::api) struct RotateSecretRequest {
pub(in crate::api) secret: Option<String>,
}
pub(in crate::api) fn parse_optional_expiration(
value: Option<&str>,
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
let Some(raw) = value else {
return Ok(None);
};
let parsed = DateTime::parse_from_rfc3339(raw)
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
Ok(Some(parsed.with_timezone(&Utc)))
}
pub(in crate::api) fn parse_patch_expiration(
value: &Patch<String>,
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
match value {
Patch::Unchanged => Ok(Patch::Unchanged),
Patch::Remove => Ok(Patch::Remove),
Patch::Set(raw) => {
let parsed = DateTime::parse_from_rfc3339(raw)
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
Ok(Patch::Set(parsed.with_timezone(&Utc)))
}
}
}
pub(in crate::api) fn is_valid_user_secret(secret: &str) -> bool {
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
}
pub(in crate::api) fn is_valid_ad_tag(tag: &str) -> bool {
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
}
pub(in crate::api) fn is_valid_username(user: &str) -> bool {
!user.is_empty()
&& user.len() <= MAX_USERNAME_LEN
&& user
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
}
pub(in crate::api) fn random_user_secret() -> String {
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
let mut bytes = [0u8; 16];
rng.fill(&mut bytes);
hex::encode(bytes)
}
+1 -3
View File
@@ -314,9 +314,7 @@ async fn recompute_connections_payload(
let mut active_users = 0usize;
for entry in shared.stats.iter_user_stats() {
let user_stats = entry.value();
let current_connections = user_stats
.curr_connects
.load(std::sync::atomic::Ordering::Relaxed);
let current_connections = shared.stats.get_process_user_curr_connects(entry.key());
let total_octets = user_stats
.octets_from_client
.load(std::sync::atomic::Ordering::Relaxed)
+7 -214
View File
@@ -1,4 +1,3 @@
use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::Serialize;
@@ -17,77 +16,6 @@ pub(super) struct SecurityWhitelistData {
pub(super) entries: Vec<String>,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateGenerationData {
pub(super) active_generation: u64,
pub(super) warm_generation: u64,
pub(super) pending_hardswap_generation: u64,
pub(super) pending_hardswap_age_secs: Option<u64>,
pub(super) draining_generations: Vec<u64>,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateHardswapData {
pub(super) enabled: bool,
pub(super) pending: bool,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateWriterContourData {
pub(super) warm: usize,
pub(super) active: usize,
pub(super) draining: usize,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateWriterHealthData {
pub(super) healthy: usize,
pub(super) degraded: usize,
pub(super) draining: usize,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateWriterData {
pub(super) total: usize,
pub(super) alive_non_draining: usize,
pub(super) draining: usize,
pub(super) degraded: usize,
pub(super) contour: RuntimeMePoolStateWriterContourData,
pub(super) health: RuntimeMePoolStateWriterHealthData,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateRefillDcData {
pub(super) dc: i16,
pub(super) family: &'static str,
pub(super) inflight: usize,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateRefillData {
pub(super) inflight_endpoints_total: usize,
pub(super) inflight_dc_total: usize,
pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStatePayload {
pub(super) generations: RuntimeMePoolStateGenerationData,
pub(super) hardswap: RuntimeMePoolStateHardswapData,
pub(super) writers: RuntimeMePoolStateWriterData,
pub(super) refill: RuntimeMePoolStateRefillData,
}
#[derive(Serialize)]
pub(super) struct RuntimeMePoolStateData {
pub(super) enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) reason: Option<&'static str>,
pub(super) generated_at_epoch_secs: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) data: Option<RuntimeMePoolStatePayload>,
}
#[derive(Serialize)]
pub(super) struct RuntimeMeQualityCountersData {
pub(super) idle_close_by_peer_total: u64,
@@ -280,96 +208,6 @@ pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitel
}
}
pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData {
let now_epoch_secs = now_epoch_secs();
let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeMePoolStateData {
enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON),
generated_at_epoch_secs: now_epoch_secs,
data: None,
};
};
let status = pool.api_status_snapshot().await;
let runtime = pool.api_runtime_snapshot().await;
let refill = pool.api_refill_snapshot().await;
let mut draining_generations = BTreeSet::<u64>::new();
let mut contour_warm = 0usize;
let mut contour_active = 0usize;
let mut contour_draining = 0usize;
let mut draining = 0usize;
let mut degraded = 0usize;
let mut healthy = 0usize;
for writer in &status.writers {
if writer.draining {
draining_generations.insert(writer.generation);
draining += 1;
}
if writer.degraded && !writer.draining {
degraded += 1;
}
if !writer.degraded && !writer.draining {
healthy += 1;
}
match writer.state {
"warm" => contour_warm += 1,
"active" => contour_active += 1,
_ => contour_draining += 1,
}
}
RuntimeMePoolStateData {
enabled: true,
reason: None,
generated_at_epoch_secs: status.generated_at_epoch_secs,
data: Some(RuntimeMePoolStatePayload {
generations: RuntimeMePoolStateGenerationData {
active_generation: runtime.active_generation,
warm_generation: runtime.warm_generation,
pending_hardswap_generation: runtime.pending_hardswap_generation,
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
draining_generations: draining_generations.into_iter().collect(),
},
hardswap: RuntimeMePoolStateHardswapData {
enabled: runtime.hardswap_enabled,
pending: runtime.pending_hardswap_generation != 0,
},
writers: RuntimeMePoolStateWriterData {
total: status.writers.len(),
alive_non_draining: status.writers.len().saturating_sub(draining),
draining,
degraded,
contour: RuntimeMePoolStateWriterContourData {
warm: contour_warm,
active: contour_active,
draining: contour_draining,
},
health: RuntimeMePoolStateWriterHealthData {
healthy,
degraded,
draining,
},
},
refill: RuntimeMePoolStateRefillData {
inflight_endpoints_total: refill.inflight_endpoints_total,
inflight_dc_total: refill.inflight_dc_total,
by_dc: refill
.by_dc
.into_iter()
.map(|entry| RuntimeMePoolStateRefillDcData {
dc: entry.dc,
family: entry.family,
inflight: entry.inflight,
})
.collect(),
},
}),
}
}
pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData {
let now_epoch_secs = now_epoch_secs();
let Some(pool) = shared.me_pool.read().await.clone() else {
@@ -532,55 +370,10 @@ pub(super) async fn build_runtime_upstream_quality_data(
}
}
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
let now_epoch_secs = now_epoch_secs();
let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeNatStunData {
enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON),
generated_at_epoch_secs: now_epoch_secs,
data: None,
};
};
let snapshot = pool.api_nat_stun_snapshot().await;
RuntimeNatStunData {
enabled: true,
reason: None,
generated_at_epoch_secs: now_epoch_secs,
data: Some(RuntimeNatStunPayload {
flags: RuntimeNatStunFlagsData {
nat_probe_enabled: snapshot.nat_probe_enabled,
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
nat_probe_attempts: snapshot.nat_probe_attempts,
},
servers: RuntimeNatStunServersData {
configured: snapshot.configured_servers,
live: snapshot.live_servers.clone(),
live_total: snapshot.live_servers.len(),
},
reflection: RuntimeNatStunReflectionBlockData {
v4: snapshot
.reflection_v4
.map(|entry| RuntimeNatStunReflectionData {
addr: entry.addr.to_string(),
age_secs: entry.age_secs,
}),
v6: snapshot
.reflection_v6
.map(|entry| RuntimeNatStunReflectionData {
addr: entry.addr.to_string(),
age_secs: entry.age_secs,
}),
},
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
}),
}
}
fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
// ME pool runtime-state projection.
mod me_pool;
// NAT/STUN runtime projection and timestamping.
mod nat;
pub(super) use me_pool::build_runtime_me_pool_state_data;
pub(super) use nat::build_runtime_nat_stun_data;
use nat::now_epoch_secs;
+192
View File
@@ -0,0 +1,192 @@
//! ME pool runtime-state projection.
use std::collections::BTreeSet;
use serde::Serialize;
use super::{ApiShared, SOURCE_UNAVAILABLE_REASON, now_epoch_secs};
#[derive(Serialize)]
struct RuntimeMePoolStateGenerationData {
active_generation: u64,
warm_generation: u64,
warm_generations: Vec<u64>,
pending_hardswap_generation: u64,
pending_hardswap_age_secs: Option<u64>,
reinit_inflight: usize,
reinit_max_concurrency_effective: usize,
draining_generations: Vec<u64>,
}
#[derive(Serialize)]
struct RuntimeMePoolStateHardswapData {
enabled: bool,
pending: bool,
pending_writers_current: usize,
pending_writer_deficit: usize,
pending_missing_dc_groups: usize,
pending_map_current: Option<bool>,
orphan_warm_writers_current: usize,
replacement_preparing_current: usize,
replacement_retiring_current: usize,
}
#[derive(Serialize)]
struct RuntimeMePoolStateWriterContourData {
warm: usize,
active: usize,
draining: usize,
}
#[derive(Serialize)]
struct RuntimeMePoolStateWriterHealthData {
healthy: usize,
degraded: usize,
draining: usize,
}
#[derive(Serialize)]
struct RuntimeMePoolStateWriterData {
total: usize,
alive_non_draining: usize,
draining: usize,
degraded: usize,
contour: RuntimeMePoolStateWriterContourData,
health: RuntimeMePoolStateWriterHealthData,
}
#[derive(Serialize)]
struct RuntimeMePoolStateRefillDcData {
dc: i16,
family: &'static str,
inflight: usize,
}
#[derive(Serialize)]
struct RuntimeMePoolStateRefillData {
inflight_endpoints_total: usize,
inflight_dc_total: usize,
running_dc_total: usize,
pending_dc_total: usize,
by_dc: Vec<RuntimeMePoolStateRefillDcData>,
}
#[derive(Serialize)]
struct RuntimeMePoolStatePayload {
generations: RuntimeMePoolStateGenerationData,
hardswap: RuntimeMePoolStateHardswapData,
writers: RuntimeMePoolStateWriterData,
refill: RuntimeMePoolStateRefillData,
}
#[derive(Serialize)]
struct RuntimeMePoolStateData {
enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
generated_at_epoch_secs: u64,
#[serde(skip_serializing_if = "Option::is_none")]
data: Option<RuntimeMePoolStatePayload>,
}
/// Builds the bounded runtime ME pool response projection.
pub(in crate::api) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> impl Serialize {
let now_epoch_secs = now_epoch_secs();
let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeMePoolStateData {
enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON),
generated_at_epoch_secs: now_epoch_secs,
data: None,
};
};
let (status, runtime) = pool.api_coherent_snapshots().await;
let refill = pool.api_refill_snapshot().await;
let mut draining_generations = BTreeSet::<u64>::new();
let mut contour_warm = 0usize;
let mut contour_active = 0usize;
let mut contour_draining = 0usize;
let mut draining = 0usize;
let mut degraded = 0usize;
let mut healthy = 0usize;
for writer in &status.writers {
if writer.draining {
draining_generations.insert(writer.generation);
draining += 1;
}
if writer.degraded && !writer.draining {
degraded += 1;
}
if !writer.degraded && !writer.draining {
healthy += 1;
}
match writer.state {
"warm" => contour_warm += 1,
"active" => contour_active += 1,
_ => contour_draining += 1,
}
}
RuntimeMePoolStateData {
enabled: true,
reason: None,
generated_at_epoch_secs: status.generated_at_epoch_secs,
data: Some(RuntimeMePoolStatePayload {
generations: RuntimeMePoolStateGenerationData {
active_generation: runtime.active_generation,
warm_generation: runtime.warm_generation,
warm_generations: runtime.warm_generations,
pending_hardswap_generation: runtime.pending_hardswap_generation,
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
reinit_inflight: runtime.reinit_inflight,
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
draining_generations: draining_generations.into_iter().collect(),
},
hardswap: RuntimeMePoolStateHardswapData {
enabled: runtime.hardswap_enabled,
pending: runtime.pending_hardswap_generation != 0,
pending_writers_current: runtime.pending_writers_current,
pending_writer_deficit: runtime.pending_writer_deficit,
pending_missing_dc_groups: runtime.pending_missing_dc_groups,
pending_map_current: runtime.pending_map_current,
orphan_warm_writers_current: runtime.orphan_warm_writers_current,
replacement_preparing_current: runtime.replacement_preparing_current,
replacement_retiring_current: runtime.replacement_retiring_current,
},
writers: RuntimeMePoolStateWriterData {
total: status.writers.len(),
alive_non_draining: status.writers.len().saturating_sub(draining),
draining,
degraded,
contour: RuntimeMePoolStateWriterContourData {
warm: contour_warm,
active: contour_active,
draining: contour_draining,
},
health: RuntimeMePoolStateWriterHealthData {
healthy,
degraded,
draining,
},
},
refill: RuntimeMePoolStateRefillData {
inflight_endpoints_total: refill.inflight_endpoints_total,
inflight_dc_total: refill.inflight_dc_total,
running_dc_total: refill.running_dc_total,
pending_dc_total: refill.pending_dc_total,
by_dc: refill
.by_dc
.into_iter()
.map(|entry| RuntimeMePoolStateRefillDcData {
dc: entry.dc,
family: entry.family,
inflight: entry.inflight,
})
.collect(),
},
}),
}
}
+54
View File
@@ -0,0 +1,54 @@
use super::*;
pub(in crate::api) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
let now_epoch_secs = now_epoch_secs();
let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeNatStunData {
enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON),
generated_at_epoch_secs: now_epoch_secs,
data: None,
};
};
let snapshot = pool.api_nat_stun_snapshot().await;
RuntimeNatStunData {
enabled: true,
reason: None,
generated_at_epoch_secs: now_epoch_secs,
data: Some(RuntimeNatStunPayload {
flags: RuntimeNatStunFlagsData {
nat_probe_enabled: snapshot.nat_probe_enabled,
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
nat_probe_attempts: snapshot.nat_probe_attempts,
},
servers: RuntimeNatStunServersData {
configured: snapshot.configured_servers,
live: snapshot.live_servers.clone(),
live_total: snapshot.live_servers.len(),
},
reflection: RuntimeNatStunReflectionBlockData {
v4: snapshot
.reflection_v4
.map(|entry| RuntimeNatStunReflectionData {
addr: entry.addr.to_string(),
age_secs: entry.age_secs,
}),
v6: snapshot
.reflection_v6
.map(|entry| RuntimeNatStunReflectionData {
addr: entry.addr.to_string(),
age_secs: entry.age_secs,
}),
},
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
}),
}
}
pub(super) fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
+14 -56
View File
@@ -61,6 +61,12 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
conntrack_pressure_active: stats.get_conntrack_pressure_active(),
conntrack_event_queue_depth: stats.get_conntrack_event_queue_depth(),
conntrack_rule_apply_ok: stats.get_conntrack_rule_apply_ok(),
conntrack_rule_reconcile_success_total: stats
.get_conntrack_rule_reconcile_success_total(),
conntrack_rule_reconcile_error_total: stats.get_conntrack_rule_reconcile_error_total(),
conntrack_rule_rollback_success_total: stats
.get_conntrack_rule_rollback_success_total(),
conntrack_rule_rollback_error_total: stats.get_conntrack_rule_rollback_error_total(),
conntrack_delete_attempt_total: stats.get_conntrack_delete_attempt_total(),
conntrack_delete_success_total: stats.get_conntrack_delete_success_total(),
conntrack_delete_not_found_total: stats.get_conntrack_delete_not_found_total(),
@@ -84,6 +90,7 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
reconnect_success_total: stats.get_me_reconnect_success(),
handshake_reject_total: stats.get_me_handshake_reject_total(),
handshake_error_codes,
handshake_error_code_overflow_total: stats.get_me_handshake_error_code_overflow_total(),
reader_eof_total: stats.get_me_reader_eof_total(),
idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
@@ -342,8 +349,7 @@ async fn get_minimal_payload_cached(
}
let pool = shared.me_pool.read().await.clone()?;
let status = pool.api_status_snapshot().await;
let runtime = pool.api_runtime_snapshot().await;
let (status, runtime) = pool.api_coherent_snapshots().await;
let generated_at_epoch_secs = status.generated_at_epoch_secs;
let me_writers = MeWritersData {
@@ -425,8 +431,11 @@ async fn get_minimal_payload_cached(
let me_runtime = MinimalMeRuntimeData {
active_generation: runtime.active_generation,
warm_generation: runtime.warm_generation,
warm_generations: runtime.warm_generations,
pending_hardswap_generation: runtime.pending_hardswap_generation,
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
reinit_inflight: runtime.reinit_inflight,
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
hardswap_enabled: runtime.hardswap_enabled,
floor_mode: runtime.floor_mode,
adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs,
@@ -523,57 +532,6 @@ async fn get_minimal_payload_cached(
Some((generated_at_epoch_secs, payload))
}
fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
MeWritersData {
middle_proxy_enabled: false,
reason: Some(reason),
generated_at_epoch_secs: now_epoch_secs,
summary: MeWritersSummary {
configured_dc_groups: 0,
configured_endpoints: 0,
available_endpoints: 0,
available_pct: 0.0,
required_writers: 0,
alive_writers: 0,
coverage_pct: 0.0,
fresh_alive_writers: 0,
fresh_coverage_pct: 0.0,
},
writers: Vec::new(),
}
}
fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
DcStatusData {
middle_proxy_enabled: false,
reason: Some(reason),
generated_at_epoch_secs: now_epoch_secs,
dcs: Vec::new(),
}
}
fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
match value {
UpstreamRouteKind::Direct => "direct",
UpstreamRouteKind::Socks4 => "socks4",
UpstreamRouteKind::Socks5 => "socks5",
UpstreamRouteKind::Shadowsocks => "shadowsocks",
}
}
fn map_ip_preference(value: IpPreference) -> &'static str {
match value {
IpPreference::Unknown => "unknown",
IpPreference::PreferV6 => "prefer_v6",
IpPreference::PreferV4 => "prefer_v4",
IpPreference::BothWork => "both_work",
IpPreference::Unavailable => "unavailable",
}
}
fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
// Disabled-state builders and stable upstream enum mappings.
mod helpers;
use helpers::*;
+56
View File
@@ -0,0 +1,56 @@
use super::*;
pub(super) fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
MeWritersData {
middle_proxy_enabled: false,
reason: Some(reason),
generated_at_epoch_secs: now_epoch_secs,
summary: MeWritersSummary {
configured_dc_groups: 0,
configured_endpoints: 0,
available_endpoints: 0,
available_pct: 0.0,
required_writers: 0,
alive_writers: 0,
coverage_pct: 0.0,
fresh_alive_writers: 0,
fresh_coverage_pct: 0.0,
},
writers: Vec::new(),
}
}
pub(super) fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
DcStatusData {
middle_proxy_enabled: false,
reason: Some(reason),
generated_at_epoch_secs: now_epoch_secs,
dcs: Vec::new(),
}
}
pub(super) fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
match value {
UpstreamRouteKind::Direct => "direct",
UpstreamRouteKind::Socks4 => "socks4",
UpstreamRouteKind::Socks5 => "socks5",
UpstreamRouteKind::Shadowsocks => "shadowsocks",
}
}
pub(super) fn map_ip_preference(value: IpPreference) -> &'static str {
match value {
IpPreference::Unknown => "unknown",
IpPreference::PreferV6 => "prefer_v6",
IpPreference::PreferV4 => "prefer_v4",
IpPreference::BothWork => "both_work",
IpPreference::Unavailable => "unavailable",
}
}
pub(super) fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
+34 -15
View File
@@ -4,6 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::watch;
use crate::maestro::control_plane::ProcessControlPlane;
use crate::maestro::generation::RuntimeWatchState;
use super::ApiRuntimeState;
@@ -13,22 +14,29 @@ pub(super) fn spawn_runtime_watchers(
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
runtime_state: Arc<ApiRuntimeState>,
runtime_events: Arc<ApiEventStore>,
control_plane: &ProcessControlPlane,
) {
let _config_watcher = spawn_config_watcher(
spawn_config_watcher(
runtime_watch_rx.clone(),
runtime_state.clone(),
runtime_events.clone(),
control_plane,
);
spawn_admission_watcher(
runtime_watch_rx,
runtime_state,
runtime_events,
control_plane,
);
let _admission_watcher =
spawn_admission_watcher(runtime_watch_rx, runtime_state, runtime_events);
}
fn spawn_config_watcher(
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
runtime_state: Arc<ApiRuntimeState>,
runtime_events: Arc<ApiEventStore>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
control_plane: &ProcessControlPlane,
) {
let _ = control_plane.spawn(async move {
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
return;
};
@@ -78,15 +86,16 @@ fn spawn_config_watcher(
}
}
}
})
});
}
fn spawn_admission_watcher(
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
runtime_state: Arc<ApiRuntimeState>,
runtime_events: Arc<ApiEventStore>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
control_plane: &ProcessControlPlane,
) {
let _ = control_plane.spawn(async move {
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
return;
};
@@ -124,7 +133,7 @@ fn spawn_admission_watcher(
}
}
}
})
});
}
fn active_generation_id(
@@ -246,7 +255,13 @@ mod tests {
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
let runtime_state = runtime_state();
let events = Arc::new(ApiEventStore::new(16));
spawn_runtime_watchers(runtime_watch_rx, runtime_state.clone(), events.clone());
let control_plane = ProcessControlPlane::new();
spawn_runtime_watchers(
runtime_watch_rx,
runtime_state.clone(),
events.clone(),
&control_plane,
);
tokio::task::yield_now().await;
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
@@ -283,6 +298,7 @@ mod tests {
.count(),
3
);
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
}
#[tokio::test]
@@ -291,7 +307,13 @@ mod tests {
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
let runtime_state = runtime_state();
let events = Arc::new(ApiEventStore::new(16));
let watcher = spawn_config_watcher(runtime_watch_rx, runtime_state.clone(), events.clone());
let control_plane = ProcessControlPlane::new();
spawn_config_watcher(
runtime_watch_rx,
runtime_state.clone(),
events.clone(),
&control_plane,
);
drop(initial_config_tx);
tokio::task::yield_now().await;
@@ -302,10 +324,7 @@ mod tests {
wait_for_count(&runtime_state, 2).await;
drop(runtime_watch_tx);
tokio::time::timeout(Duration::from_secs(1), watcher)
.await
.unwrap()
.unwrap();
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
assert_eq!(
events
.snapshot(16)
+1 -1
View File
@@ -200,7 +200,7 @@ pub(super) async fn build_runtime_gates_data(
&& cfg.general.me2dc_fallback
&& matches!(route_state.mode, RelayRouteMode::Direct);
let reroute_to_direct_at_epoch_secs = if reroute_active {
shared.route_runtime.direct_since_epoch_secs()
route_state.direct_since_epoch_secs
} else {
None
};
+3 -2
View File
@@ -5,12 +5,13 @@ use hyper::StatusCode;
use crate::config::ProxyConfig;
use crate::config::RateLimitBps;
use crate::ip_tracker::UserIpTracker;
use crate::proxy::user_admission::credential_id_from_hex;
use crate::stats::Stats;
use super::ApiShared;
use super::config_store::{
AccessSection, current_revision, ensure_expected_revision, load_config_from_disk,
save_access_sections_to_disk,
AccessSection, current_revision, load_config_for_mutation,
save_access_sections_to_disk_if_revision,
};
use super::model::{
ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest,
+31 -5
View File
@@ -4,6 +4,20 @@ pub(in crate::api) async fn create_user(
body: CreateUserRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(CreateUserResponse, String), ApiFailure> {
let shared = shared.clone();
shared
.clone()
.run_mutation_completion(async move {
create_user_to_completion(body, expected_revision, &shared).await
})
.await
}
async fn create_user_to_completion(
body: CreateUserRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(CreateUserResponse, String), ApiFailure> {
let touches_user_ad_tags = body.user_ad_tag.is_some();
let touches_user_max_tcp_conns = body.max_tcp_conns.is_some();
@@ -41,9 +55,11 @@ pub(in crate::api) async fn create_user(
}
let expiration = parse_optional_expiration(body.expiration_rfc3339.as_deref())?;
let credential_id = credential_id_from_hex(&secret)
.ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
let (mut cfg, base_revision) =
load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
if cfg.access.users.contains_key(&body.username) {
return Err(ApiFailure::new(
@@ -122,9 +138,18 @@ pub(in crate::api) async fn create_user(
touched_sections.push(AccessSection::UserEnabled);
}
let revision =
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
drop(_guard);
let revision = save_access_sections_to_disk_if_revision(
&shared.config_path,
&cfg,
&touched_sections,
Some(&base_revision),
)
.await?;
shared.proxy_shared.stage_user_credential(
&body.username,
credential_id,
cfg.access.is_user_enabled(&body.username),
);
if let Some(limit) = updated_limit {
shared
@@ -132,6 +157,7 @@ pub(in crate::api) async fn create_user(
.set_user_limit(&body.username, limit)
.await;
}
drop(_guard);
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
let users = users_from_config(
+75 -10
View File
@@ -1,10 +1,27 @@
use super::*;
use tracing::warn;
pub(in crate::api) async fn rotate_secret(
user: &str,
body: RotateSecretRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(CreateUserResponse, String), ApiFailure> {
let shared = shared.clone();
let user = user.to_string();
shared
.clone()
.run_mutation_completion(async move {
rotate_secret_to_completion(&user, body, expected_revision, &shared).await
})
.await
}
async fn rotate_secret_to_completion(
user: &str,
body: RotateSecretRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(CreateUserResponse, String), ApiFailure> {
let secret = body.secret.unwrap_or_else(random_user_secret);
if !is_valid_user_secret(&secret) {
@@ -12,10 +29,12 @@ pub(in crate::api) async fn rotate_secret(
"secret must be exactly 32 hex characters",
));
}
let credential_id = credential_id_from_hex(&secret)
.ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
let (mut cfg, base_revision) =
load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
if !cfg.access.users.contains_key(user) {
return Err(ApiFailure::new(
@@ -28,8 +47,18 @@ pub(in crate::api) async fn rotate_secret(
cfg.access.users.insert(user.to_string(), secret.clone());
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let revision =
save_access_sections_to_disk(&shared.config_path, &cfg, &[AccessSection::Users]).await?;
let revision = save_access_sections_to_disk_if_revision(
&shared.config_path,
&cfg,
&[AccessSection::Users],
Some(&base_revision),
)
.await?;
shared.proxy_shared.stage_user_credential(
user,
credential_id,
cfg.access.is_user_enabled(user),
);
drop(_guard);
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
@@ -60,10 +89,25 @@ pub(in crate::api) async fn delete_user(
user: &str,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(String, String), ApiFailure> {
let shared = shared.clone();
let user = user.to_string();
shared
.clone()
.run_mutation_completion(async move {
delete_user_to_completion(&user, expected_revision, &shared).await
})
.await
}
async fn delete_user_to_completion(
user: &str,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(String, String), ApiFailure> {
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
let (mut cfg, base_revision) =
load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
if !cfg.access.users.contains_key(user) {
return Err(ApiFailure::new(
@@ -106,11 +150,32 @@ pub(in crate::api) async fn delete_user(
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let revision =
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
drop(_guard);
let revision = save_access_sections_to_disk_if_revision(
&shared.config_path,
&cfg,
&touched_sections,
Some(&base_revision),
)
.await?;
let deleted_incarnation = shared.proxy_shared.delete_user(user).incarnation;
let configured_users = cfg.access.users.keys().cloned().collect();
if let Err(error) = shared
.quota_state
.remove_user(&configured_users, user)
.await
{
warn!(
user,
error = %error,
"Deleted user quota checkpoint cleanup will be reconciled on restart"
);
}
shared.ip_tracker.remove_user_limit(user).await;
shared.ip_tracker.clear_user_ips(user).await;
shared
.ip_tracker
.clear_user_ips_if_not_newer(user, deleted_incarnation)
.await;
drop(_guard);
Ok((user.to_string(), revision))
}
+80 -9
View File
@@ -5,6 +5,22 @@ pub(in crate::api) async fn patch_user(
body: PatchUserRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let shared = shared.clone();
let user = user.to_string();
shared
.clone()
.run_mutation_completion(async move {
patch_user_to_completion(&user, body, expected_revision, &shared).await
})
.await
}
async fn patch_user_to_completion(
user: &str,
body: PatchUserRequest,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let touches_users = body.secret.is_some();
let touches_user_ad_tags = !matches!(&body.user_ad_tag, Patch::Unchanged);
@@ -32,8 +48,8 @@ pub(in crate::api) async fn patch_user(
}
let expiration = parse_patch_expiration(&body.expiration_rfc3339)?;
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
let (mut cfg, base_revision) =
load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
if !cfg.access.users.contains_key(user) {
return Err(ApiFailure::new(
@@ -138,6 +154,19 @@ pub(in crate::api) async fn patch_user(
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let staged_credential =
if touches_users || touches_user_enabled {
let secret = cfg
.access
.users
.get(user)
.ok_or_else(|| ApiFailure::internal("updated user secret is missing"))?;
Some(credential_id_from_hex(secret).ok_or_else(|| {
ApiFailure::internal("validated user secret could not be decoded")
})?)
} else {
None
};
let mut touched_sections = Vec::new();
if touches_users {
@@ -168,14 +197,27 @@ pub(in crate::api) async fn patch_user(
let revision = if touched_sections.is_empty() {
current_revision(&shared.config_path).await?
} else {
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?
save_access_sections_to_disk_if_revision(
&shared.config_path,
&cfg,
&touched_sections,
Some(&base_revision),
)
.await?
};
drop(_guard);
if let Some(credential_id) = staged_credential {
shared.proxy_shared.stage_user_credential(
user,
credential_id,
cfg.access.is_user_enabled(user),
);
}
match max_unique_ips_change {
Some(Some(limit)) => shared.ip_tracker.set_user_limit(user, limit).await,
Some(None) => shared.ip_tracker.remove_user_limit(user).await,
None => {}
}
drop(_guard);
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
let users = users_from_config(
&cfg,
@@ -199,10 +241,26 @@ pub(in crate::api) async fn set_user_enabled(
enabled: bool,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let shared = shared.clone();
let user = user.to_string();
shared
.clone()
.run_mutation_completion(async move {
set_user_enabled_to_completion(&user, enabled, expected_revision, &shared).await
})
.await
}
async fn set_user_enabled_to_completion(
user: &str,
enabled: bool,
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
let (mut cfg, base_revision) =
load_config_for_mutation(&shared.config_path, expected_revision.as_deref()).await?;
if !cfg.access.users.contains_key(user) {
return Err(ApiFailure::new(
@@ -220,9 +278,22 @@ pub(in crate::api) async fn set_user_enabled(
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let revision =
save_access_sections_to_disk(&shared.config_path, &cfg, &[AccessSection::UserEnabled])
.await?;
let credential_id = cfg
.access
.users
.get(user)
.and_then(|secret| credential_id_from_hex(secret))
.ok_or_else(|| ApiFailure::internal("validated user secret could not be decoded"))?;
let revision = save_access_sections_to_disk_if_revision(
&shared.config_path,
&cfg,
&[AccessSection::UserEnabled],
Some(&base_revision),
)
.await?;
shared
.proxy_shared
.stage_user_credential(user, credential_id, enabled);
drop(_guard);
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
+1 -1
View File
@@ -71,7 +71,7 @@ pub(in crate::api) async fn users_from_config(
.filter(|limit| *limit > 0)
.or((cfg.access.user_max_unique_ips_global_each > 0)
.then_some(cfg.access.user_max_unique_ips_global_each)),
current_connections: stats.get_user_curr_connects(&username),
current_connections: stats.get_process_user_curr_connects(&username),
active_unique_ips: active_ip_list.len(),
active_unique_ips_list: active_ip_list,
recent_unique_ips: recent_ip_list.len(),
+138 -49
View File
@@ -1,5 +1,5 @@
use std::sync::Arc;
use std::time::Instant;
use std::time::{Duration, Instant};
use http_body_util::Full;
use hyper::body::{Bytes, Incoming};
@@ -13,12 +13,18 @@ use super::model::ApiFailure;
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
use crate::config::ProxyConfig;
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
use crate::web::manager::{ControlError, SessionDetail, WebProcessRuntime};
use crate::web::manager::{ControlError, OperatorLifecycleError, SessionDetail, WebProcessRuntime};
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
mod request;
// Ingress, capacity, and decoy telemetry remain separate availability planes.
mod observability;
use observability::{
WebCapacityStatus, WebCarrierNegotiationStatus, WebDecoyFastTrackStatus,
WebDecoyUpstreamStatus, WebIngressStatus, WebLifecycleCountersStatus,
};
use request::{
CloseRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
CloseRequest, DrainRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
valid_runtime_instance,
};
@@ -27,6 +33,9 @@ const SESSIONS_PATH: &str = "/v1/runtime/web/sessions";
const CLOSE_PATH: &str = "/v1/runtime/web/sessions/close";
const DEBUG_CLEAR_PATH: &str = "/v1/runtime/web/debug/clear";
const LEARNING_RESET_PATH: &str = "/v1/runtime/web/carrier-learning/reset";
const LIFECYCLE_PAUSE_PATH: &str = "/v1/runtime/web/lifecycle/pause";
const LIFECYCLE_DRAIN_PATH: &str = "/v1/runtime/web/lifecycle/drain";
const LIFECYCLE_RESUME_PATH: &str = "/v1/runtime/web/lifecycle/resume";
const SESSION_DETAIL_PREFIX: &str = "/v1/runtime/web/sessions/";
const OPERATION_PREFIX: &str = "/v1/runtime/web/operations/";
const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
@@ -35,7 +44,12 @@ const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
pub(super) fn allowed_methods(path: &str) -> Option<&'static str> {
match path {
STATUS_PATH | SESSIONS_PATH => Some(ALLOW_GET),
CLOSE_PATH | DEBUG_CLEAR_PATH | LEARNING_RESET_PATH => Some(ALLOW_POST),
CLOSE_PATH
| DEBUG_CLEAR_PATH
| LEARNING_RESET_PATH
| LIFECYCLE_PAUSE_PATH
| LIFECYCLE_DRAIN_PATH
| LIFECYCLE_RESUME_PATH => Some(ALLOW_POST),
_ if detail_ref(path).is_some() || operation_ref(path).is_some() => Some(ALLOW_GET),
_ => None,
}
@@ -63,7 +77,7 @@ pub(super) async fn handle(
reject_query(query)?;
let publication = shared.web_runtime_rx.borrow().clone();
let runtime = publication.runtime.upgrade();
let data = WebStatusData::new(publication, runtime.as_deref(), config.web.enabled);
let data = WebStatusData::new(publication, runtime.as_deref(), config);
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", SESSIONS_PATH) => {
@@ -79,12 +93,20 @@ pub(super) async fn handle(
let trace_session_id = parse_session_ref(&runtime, session_ref)?;
match runtime.session_detail(trace_session_id) {
SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
SessionDetail::Gone { attempt } => Ok(success_response(
SessionDetail::Gone {
attempt,
carrier,
reason,
closed_age_ms,
} => Ok(success_response(
StatusCode::GONE,
GoneSessionData {
session_ref: session_ref.to_string(),
state: "closed",
attempt,
carrier,
reason,
closed_age_ms,
},
revision,
)),
@@ -105,6 +127,67 @@ pub(super) async fn handle(
.map_err(control_failure)?;
Ok(success_response(StatusCode::OK, status, revision))
}
("POST", LIFECYCLE_PAUSE_PATH) => {
require_mutable(config)?;
reject_query(query)?;
require_json_content_type(&request)?;
let request = read_json::<RuntimeInstanceRequest>(
request.into_body(),
body_limit.min(MAX_CONTROL_BODY_BYTES),
)
.await?;
let runtime = control_runtime(shared)?;
require_runtime_instance(&runtime, &request.runtime_instance)?;
let status = runtime.pause_operator().await.map_err(lifecycle_failure)?;
shared.runtime_events.record(
"api.web.lifecycle.pause.ok",
format!("epoch={}", status.epoch),
);
Ok(success_response(StatusCode::OK, status, revision))
}
("POST", LIFECYCLE_DRAIN_PATH) => {
require_mutable(config)?;
reject_query(query)?;
require_json_content_type(&request)?;
let request = read_json::<DrainRequest>(
request.into_body(),
body_limit.min(MAX_CONTROL_BODY_BYTES),
)
.await?;
let timeout = drain_timeout(request.timeout_secs)?;
let runtime = control_runtime(shared)?;
require_runtime_instance(&runtime, &request.runtime_instance)?;
let status = runtime
.drain_operator(timeout)
.await
.map_err(lifecycle_failure)?;
shared.runtime_events.record(
"api.web.lifecycle.drain.accepted",
format!(
"epoch={} timeout_secs={}",
status.epoch, request.timeout_secs
),
);
Ok(success_response(StatusCode::ACCEPTED, status, revision))
}
("POST", LIFECYCLE_RESUME_PATH) => {
require_mutable(config)?;
reject_query(query)?;
require_json_content_type(&request)?;
let request = read_json::<RuntimeInstanceRequest>(
request.into_body(),
body_limit.min(MAX_CONTROL_BODY_BYTES),
)
.await?;
let runtime = control_runtime(shared)?;
require_runtime_instance(&runtime, &request.runtime_instance)?;
let status = runtime.resume_operator().await.map_err(lifecycle_failure)?;
shared.runtime_events.record(
"api.web.lifecycle.resume.ok",
format!("epoch={}", status.epoch),
);
Ok(success_response(StatusCode::OK, status, revision))
}
("POST", CLOSE_PATH) => {
require_mutable(config)?;
reject_query(query)?;
@@ -194,6 +277,14 @@ struct WebStatusData {
reason: Option<&'static str>,
listeners: Vec<String>,
effective_config_enabled: bool,
ingress: WebIngressStatus,
capacity: WebCapacityStatus,
decoy_upstream: WebDecoyUpstreamStatus,
decoy_fasttrack: WebDecoyFastTrackStatus,
carrier_negotiation: WebCarrierNegotiationStatus,
lifecycle_counters: WebLifecycleCountersStatus,
#[serde(skip_serializing_if = "Option::is_none")]
operator_lifecycle: Option<crate::web::manager::OperatorLifecycleStatus>,
#[serde(skip_serializing_if = "Option::is_none")]
runtime: Option<crate::web::manager::WebRuntimeStatus>,
}
@@ -202,7 +293,7 @@ impl WebStatusData {
fn new(
publication: WebRuntimePublication,
runtime: Option<&WebProcessRuntime>,
effective_config_enabled: bool,
config: &ProxyConfig,
) -> Self {
let available = runtime.is_some()
&& matches!(
@@ -221,6 +312,13 @@ impl WebStatusData {
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
})
};
let operator_lifecycle = runtime.map(WebProcessRuntime::operator_lifecycle_status);
let ingress = WebIngressStatus::new(&publication, runtime.is_some());
let capacity = WebCapacityStatus::new(&publication, runtime, config);
let decoy_upstream = WebDecoyUpstreamStatus::new(&publication);
let decoy_fasttrack = WebDecoyFastTrackStatus::new(&publication, config);
let carrier_negotiation = WebCarrierNegotiationStatus::new(&publication);
let lifecycle_counters = WebLifecycleCountersStatus::new(&publication, config);
Self {
lifecycle: publication.lifecycle.as_str(),
lifecycle_epoch: publication.epoch,
@@ -232,7 +330,14 @@ impl WebStatusData {
.iter()
.map(ToString::to_string)
.collect(),
effective_config_enabled,
effective_config_enabled: config.web.enabled,
ingress,
capacity,
decoy_upstream,
decoy_fasttrack,
carrier_negotiation,
lifecycle_counters,
operator_lifecycle,
runtime: runtime.map(WebProcessRuntime::try_status),
}
}
@@ -243,6 +348,9 @@ struct GoneSessionData {
session_ref: String,
state: &'static str,
attempt: u8,
carrier: crate::config::WebCarrier,
reason: &'static str,
closed_age_ms: u64,
}
#[derive(Serialize)]
@@ -369,6 +477,17 @@ fn control_failure(error: ControlError) -> ApiFailure {
}
}
fn lifecycle_failure(error: OperatorLifecycleError) -> ApiFailure {
match error {
OperatorLifecycleError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
OperatorLifecycleError::OperationInProgress => ApiFailure::new(
StatusCode::CONFLICT,
"web_lifecycle_in_progress",
"Another WEB drain operation is active",
),
}
}
fn snapshot_busy() -> ApiFailure {
ApiFailure::new(
StatusCode::SERVICE_UNAVAILABLE,
@@ -377,6 +496,15 @@ fn snapshot_busy() -> ApiFailure {
)
}
fn drain_timeout(timeout_secs: u64) -> Result<Duration, ApiFailure> {
if !(1..=3600).contains(&timeout_secs) {
return Err(ApiFailure::bad_request(
"timeout_secs must be within 1..=3600",
));
}
Ok(Duration::from_secs(timeout_secs))
}
fn reject_query(query: Option<&str>) -> Result<(), ApiFailure> {
if query.is_some_and(|query| !query.is_empty()) {
return Err(ApiFailure::bad_request(
@@ -401,44 +529,5 @@ fn millis(duration: std::time::Duration) -> u64 {
}
#[cfg(test)]
mod tests {
use super::*;
use hyper::header::HeaderValue;
#[test]
fn route_table_keeps_status_read_only_and_controls_post_only() {
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
assert_eq!(
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
Some(ALLOW_GET)
);
}
#[test]
fn control_content_type_is_exact_and_single() {
let exact = Request::builder()
.header(CONTENT_TYPE, "application/json")
.body(())
.unwrap();
assert!(require_json_content_type(&exact).is_ok());
let parameterized = Request::builder()
.header(CONTENT_TYPE, "application/json; charset=utf-8")
.body(())
.unwrap();
assert!(require_json_content_type(&parameterized).is_err());
let mut duplicated = Request::builder()
.header(CONTENT_TYPE, "application/json")
.body(())
.unwrap();
duplicated
.headers_mut()
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
assert!(require_json_content_type(&duplicated).is_err());
}
}
#[path = "web_runtime/tests.rs"]
mod tests;
+274
View File
@@ -0,0 +1,274 @@
use serde::Serialize;
use crate::config::{ProxyConfig, WebDecoyFastTrackMode, WebHttpConnectionCapacityAction};
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
use crate::web::manager::{WebCapacityResourceStatus, WebCapacitySnapshot, WebProcessRuntime};
use crate::web::telemetry::{
WebBridgeRecoveryCounter, WebCarrierFailureCounter, WebCarrierLearningCounter,
WebCarrierSelectionCounter, WebDecoyFastTrackCounter, WebSessionCloseCounter,
WebSessionLifecycleObservationCounter,
};
use crate::web::telemetry::{WebOutcomeCounter, WebRejectionCounter};
/// Private WEB ingress state owned by this Telemt process.
#[derive(Serialize)]
pub(super) struct WebIngressStatus {
configured_listeners: usize,
live_acceptors: usize,
accepting_connections: bool,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
tcp_accept_total: u64,
tcp_accept_error_total: u64,
}
impl WebIngressStatus {
/// Builds a process-ingress snapshot without probing external TLS termination.
pub(super) fn new(publication: &WebRuntimePublication, runtime_available: bool) -> Self {
let configured_listeners = publication.listeners.len();
let live_acceptors = publication.telemetry.live_acceptors();
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
&& runtime_available
&& configured_listeners != 0
&& live_acceptors == configured_listeners;
let reason = if accepting_connections {
None
} else {
Some(match publication.lifecycle {
WebRuntimeLifecycle::Starting => "starting",
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
WebRuntimeLifecycle::Draining => "ingress_draining",
WebRuntimeLifecycle::Drained => "ingress_drained",
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
WebRuntimeLifecycle::Running if !runtime_available => "runtime_released",
WebRuntimeLifecycle::Running if configured_listeners == 0 => "no_web_listener",
WebRuntimeLifecycle::Running => "acceptor_unavailable",
})
};
Self {
configured_listeners,
live_acceptors,
accepting_connections,
reason,
tcp_accept_total: publication.telemetry.accepted(),
tcp_accept_error_total: publication.telemetry.accept_errors(),
}
}
}
/// Bounded process-wide WEB capacity and terminal rejection view.
#[derive(Serialize)]
pub(super) struct WebCapacityStatus {
http_connection_capacity_action: WebHttpConnectionCapacityAction,
max_http_overload_connections: usize,
http_overload_timeout_ms: u64,
resources: Vec<WebCapacityResourceStatus>,
saturated_resources: Vec<&'static str>,
partial: Vec<&'static str>,
rejections: Vec<WebRejectionCounter>,
http_connection_overload_outcomes: Vec<WebOutcomeCounter>,
}
impl WebCapacityStatus {
/// Builds a bounded capacity snapshot from non-blocking runtime observations.
pub(super) fn new(
publication: &WebRuntimePublication,
runtime: Option<&WebProcessRuntime>,
config: &ProxyConfig,
) -> Self {
let snapshot = runtime
.map(WebProcessRuntime::capacity_snapshot)
.unwrap_or_else(runtime_unavailable_snapshot);
Self {
http_connection_capacity_action: config.web.http_connection_capacity_action,
max_http_overload_connections: config.web.limits.max_http_overload_connections,
http_overload_timeout_ms: config.web.timeouts.http_overload_timeout_ms,
resources: snapshot.resources,
saturated_resources: snapshot.saturated_resources,
partial: snapshot.partial,
rejections: publication.telemetry.rejection_counters(),
http_connection_overload_outcomes: publication.telemetry.overload_counters(),
}
}
}
fn runtime_unavailable_snapshot() -> WebCapacitySnapshot {
WebCapacitySnapshot {
resources: Vec::new(),
saturated_resources: Vec::new(),
partial: vec!["runtime"],
}
}
/// Passive health of Telemt's internal plain-HTTP decoy origin hop.
#[derive(Serialize)]
pub(super) struct WebDecoyUpstreamStatus {
outcomes: Vec<WebOutcomeCounter>,
#[serde(skip_serializing_if = "Option::is_none")]
last_outcome: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
last_outcome_age_ms: Option<u64>,
}
impl WebDecoyUpstreamStatus {
/// Builds the fixed internal decoy-origin outcome snapshot.
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
let last = publication.telemetry.last_decoy();
Self {
outcomes: publication.telemetry.decoy_counters(),
last_outcome: last.map(|value| value.0),
last_outcome_age_ms: last.map(|value| value.1),
}
}
}
/// Fixed-cardinality process-lifetime decoy capability-routing counters.
#[derive(Serialize)]
pub(super) struct WebDecoyFastTrackStatus {
mode: WebDecoyFastTrackMode,
requests: Vec<WebDecoyFastTrackCounter>,
}
impl WebDecoyFastTrackStatus {
/// Builds effective policy and counters without requiring the runtime manager.
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
Self {
mode: config.web.decoy_fasttrack_mode,
requests: publication.telemetry.decoy_fasttrack_counters(),
}
}
}
/// Fixed-cardinality process-lifetime carrier negotiation counters.
#[derive(Serialize)]
pub(super) struct WebCarrierNegotiationStatus {
selections: Vec<WebCarrierSelectionCounter>,
reported_failures: Vec<WebCarrierFailureCounter>,
learning_outcomes: Vec<WebCarrierLearningCounter>,
}
impl WebCarrierNegotiationStatus {
/// Builds counters from publication ownership even when runtime state is unavailable.
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
Self {
selections: publication.telemetry.carrier_selection_counters(),
reported_failures: publication.telemetry.carrier_failure_counters(),
learning_outcomes: publication.telemetry.carrier_learning_counters(),
}
}
}
/// Fixed-cardinality process-lifetime WEB lifecycle counters.
#[derive(Serialize)]
pub(super) struct WebLifecycleCountersStatus {
bridge_recovery_secs: u64,
session_closures: Vec<WebSessionCloseCounter>,
session_observations: Vec<WebSessionLifecycleObservationCounter>,
bridge_recovery_events: Vec<WebBridgeRecoveryCounter>,
}
impl WebLifecycleCountersStatus {
/// Builds a complete counter set from process-owned telemetry.
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
Self {
bridge_recovery_secs: config.web.timeouts.bridge_recovery_secs,
session_closures: publication.telemetry.session_close_counters(),
session_observations: publication.telemetry.session_observation_counters(),
bridge_recovery_events: publication.telemetry.bridge_recovery_counters(),
}
}
}
#[cfg(test)]
mod tests {
use crate::config::ProxyConfig;
use crate::web::control::WebRuntimeControl;
#[test]
fn starting_ingress_does_not_claim_external_availability() {
let control = WebRuntimeControl::new();
let publication = control.subscribe().borrow().clone();
let value =
serde_json::to_value(super::WebIngressStatus::new(&publication, false)).unwrap();
assert_eq!(value["configured_listeners"], 0);
assert_eq!(value["live_acceptors"], 0);
assert_eq!(value["accepting_connections"], false);
assert_eq!(value["reason"], "starting");
}
#[test]
fn unavailable_runtime_keeps_fixed_counter_sets_visible() {
let control = WebRuntimeControl::new();
let publication = control.subscribe().borrow().clone();
let config = ProxyConfig::default();
let capacity =
serde_json::to_value(super::WebCapacityStatus::new(&publication, None, &config))
.unwrap();
let decoy = serde_json::to_value(super::WebDecoyUpstreamStatus::new(&publication)).unwrap();
let fasttrack =
serde_json::to_value(super::WebDecoyFastTrackStatus::new(&publication, &config))
.unwrap();
let carrier =
serde_json::to_value(super::WebCarrierNegotiationStatus::new(&publication)).unwrap();
let lifecycle = serde_json::to_value(super::WebLifecycleCountersStatus::new(
&publication,
&config,
))
.unwrap();
assert_eq!(
capacity["rejections"].as_array().unwrap().len(),
crate::web::telemetry::WebRejectionReason::ALL.len()
);
assert_eq!(
capacity["http_connection_overload_outcomes"]
.as_array()
.unwrap()
.len(),
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
);
assert_eq!(
decoy["outcomes"].as_array().unwrap().len(),
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
);
assert_eq!(fasttrack["mode"], "off");
assert_eq!(
fasttrack["requests"].as_array().unwrap().len(),
crate::web::telemetry::WebDecoyFastTrackDisposition::ALL.len()
);
assert_eq!(capacity["partial"][0], "runtime");
assert_eq!(
carrier["selections"].as_array().unwrap().len(),
crate::config::WebCarrier::ALL.len()
* crate::web::telemetry::WebCarrierSelectionDisposition::ALL.len()
);
assert_eq!(
carrier["reported_failures"].as_array().unwrap().len(),
crate::config::WebCarrier::ALL.len()
* crate::web::telemetry::WebCarrierFailurePhase::ALL.len()
* crate::web::manager::CarrierFailure::ALL.len()
);
assert_eq!(
carrier["learning_outcomes"].as_array().unwrap().len(),
crate::config::WebCarrier::ALL.len()
* crate::web::telemetry::WebCarrierLearningOutcome::ALL.len()
);
assert_eq!(
lifecycle["session_closures"].as_array().unwrap().len(),
crate::config::WebCarrier::ALL.len()
* crate::web::session::SessionCloseReason::ALL.len()
);
assert_eq!(
lifecycle["session_observations"].as_array().unwrap().len(),
crate::config::WebCarrier::ALL.len()
* crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
);
assert_eq!(
lifecycle["bridge_recovery_events"]
.as_array()
.unwrap()
.len(),
crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
);
}
}
+18
View File
@@ -22,6 +22,16 @@ pub(super) struct RuntimeInstanceRequest {
pub(super) runtime_instance: String,
}
/// Process-fenced graceful drain request with one bounded relative deadline.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct DrainRequest {
/// Random process identifier copied from WEB runtime status.
pub(super) runtime_instance: String,
/// Relative drain deadline frozen into one monotonic server deadline.
pub(super) timeout_secs: u64,
}
/// One process-fenced asynchronous close request.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
@@ -331,6 +341,14 @@ mod tests {
}))
.is_err()
);
assert!(
serde_json::from_value::<DrainRequest>(serde_json::json!({
"runtime_instance": runtime_instance,
"timeout_secs": 30,
"extra": true,
}))
.is_err()
);
}
#[test]
+51
View File
@@ -0,0 +1,51 @@
use super::*;
use hyper::header::HeaderValue;
#[test]
fn route_table_keeps_status_read_only_and_controls_post_only() {
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(LIFECYCLE_PAUSE_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(LIFECYCLE_DRAIN_PATH), Some(ALLOW_POST));
assert_eq!(allowed_methods(LIFECYCLE_RESUME_PATH), Some(ALLOW_POST));
assert_eq!(
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
Some(ALLOW_GET)
);
}
#[test]
fn control_content_type_is_exact_and_single() {
let exact = Request::builder()
.header(CONTENT_TYPE, "application/json")
.body(())
.unwrap();
assert!(require_json_content_type(&exact).is_ok());
let parameterized = Request::builder()
.header(CONTENT_TYPE, "application/json; charset=utf-8")
.body(())
.unwrap();
assert!(require_json_content_type(&parameterized).is_err());
let mut duplicated = Request::builder()
.header(CONTENT_TYPE, "application/json")
.body(())
.unwrap();
duplicated
.headers_mut()
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
assert!(require_json_content_type(&duplicated).is_err());
}
#[test]
fn drain_timeout_is_bounded_to_the_public_contract() {
assert_eq!(drain_timeout(1).unwrap(), Duration::from_secs(1));
assert_eq!(drain_timeout(3600).unwrap(), Duration::from_secs(3600));
assert!(drain_timeout(0).is_err());
assert!(drain_timeout(3601).is_err());
}
+5
View File
@@ -67,6 +67,11 @@ pub(super) async fn render(
push_filter_form(&mut html, &query);
html.push_str("<section><h2>Store</h2><table><tbody>");
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
summary_row(
&mut html,
"sideband",
yes_no(status.policy.bridge_diagnostics_enabled()),
);
summary_row(&mut html, "body capture", body_mode(&status.policy));
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
summary_row(
+14
View File
@@ -97,6 +97,20 @@ pub(super) fn push_lifecycle(html: &mut String, event: &crate::web::trace::Trace
);
html.push_str("\nreason: ");
html.push_str(event.reason.unwrap_or("-"));
html.push_str("\npeer gap ms: ");
html.push_str(
&event
.peer_gap_ms
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string()),
);
html.push_str("\npredecessor session: ");
html.push_str(
&event
.predecessor_session_id
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string()),
);
if let Some(carrier) = &event.carrier {
html.push_str("\nclient class: ");
html.push_str(carrier.client_class);
+2
View File
@@ -22,6 +22,7 @@ fn html_escaping_covers_active_markup_characters() {
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
let policy = WebDebugConfig {
enabled: true,
sideband: true,
..Default::default()
};
let limits = crate::config::WebLimitsConfig {
@@ -58,6 +59,7 @@ async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
let body = response.into_body().collect().await.unwrap().to_bytes();
let body = std::str::from_utf8(&body).unwrap();
assert!(body.contains("session_created"));
assert!(body.contains("<th>sideband</th><td>yes</td>"));
assert!(body.contains("0123456789abcdef"));
assert!(body.contains("192.0.2.40"));
}
+23 -496
View File
@@ -8,15 +8,22 @@
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
use rand::RngExt;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::path::PathBuf;
use crate::healthcheck::{self, HealthcheckMode};
#[cfg(unix)]
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
use crate::daemon::{DEFAULT_PID_FILE, DaemonOptions};
// Unix daemon control and argument parsing.
#[cfg(unix)]
mod daemon_commands;
// Fire-and-forget installation workflow.
mod init;
#[cfg(unix)]
pub use daemon_commands::parse_daemon_args;
pub use init::{InitOptions, parse_init_args, run_init};
/// CLI subcommand to execute.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -40,13 +47,20 @@ pub enum Subcommand {
/// Parsed subcommand with its options.
#[derive(Debug)]
pub struct ParsedCommand {
/// Selected command mode.
pub subcommand: Subcommand,
/// PID file used by daemon-control commands.
pub pid_file: PathBuf,
/// Configuration file passed to runtime or healthcheck.
pub config_path: String,
/// Requested healthcheck mode.
pub healthcheck_mode: HealthcheckMode,
/// Invalid healthcheck mode retained for command diagnostics.
pub healthcheck_mode_invalid: Option<String>,
#[cfg(unix)]
/// Unix daemon lifecycle options.
pub daemon_opts: DaemonOptions,
/// Fire-and-forget initialization options.
pub init_opts: Option<InitOptions>,
}
@@ -79,7 +93,6 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
return cmd;
}
// Check for subcommand as first argument
if let Some(first) = args.first() {
match first.as_str() {
"start" => {
@@ -120,11 +133,9 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
}
}
// Parse remaining options
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
// Skip subcommand names
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
"--mode" => {
i += 1;
@@ -154,7 +165,6 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
}
}
}
// PID file option (for stop/reload/status)
"--pid-file" => {
i += 1;
if i < args.len() {
@@ -189,9 +199,9 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
#[cfg(unix)]
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
match cmd.subcommand {
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
Subcommand::Stop => Some(daemon_commands::stop(&cmd.pid_file)),
Subcommand::Reload => Some(daemon_commands::reload(&cmd.pid_file)),
Subcommand::Status => Some(daemon_commands::status(&cmd.pid_file)),
Subcommand::Healthcheck => {
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
if invalid_mode.is_empty() {
@@ -224,6 +234,7 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
}
}
/// Executes a non-server subcommand on platforms without daemon support.
#[cfg(not(unix))]
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
match cmd.subcommand {
@@ -261,487 +272,3 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
Subcommand::Run | Subcommand::Start => None,
}
}
/// Stop command: send SIGTERM to the running daemon.
#[cfg(unix)]
fn cmd_stop(pid_file: &Path) -> i32 {
use nix::sys::signal::Signal;
println!("Stopping telemt daemon...");
match daemon::signal_pid_file(pid_file, Signal::SIGTERM) {
Ok(()) => {
println!("Stop signal sent successfully");
// Wait for process to exit (up to 10 seconds)
for _ in 0..20 {
std::thread::sleep(std::time::Duration::from_millis(500));
if let daemon::DaemonStatus::NotRunning = daemon::check_status(pid_file) {
println!("Daemon stopped");
return 0;
}
}
println!("Daemon may still be shutting down");
0
}
Err(e) => {
eprintln!("Failed to stop daemon: {}", e);
1
}
}
}
/// Reload command: send SIGHUP to trigger config reload.
#[cfg(unix)]
fn cmd_reload(pid_file: &Path) -> i32 {
use nix::sys::signal::Signal;
println!("Reloading telemt configuration...");
match daemon::signal_pid_file(pid_file, Signal::SIGHUP) {
Ok(()) => {
println!("Reload signal sent successfully");
0
}
Err(e) => {
eprintln!("Failed to reload daemon: {}", e);
1
}
}
}
/// Status command: check if daemon is running.
#[cfg(unix)]
fn cmd_status(pid_file: &Path) -> i32 {
match daemon::check_status(pid_file) {
daemon::DaemonStatus::Running(pid) => {
println!("telemt is running (pid {})", pid);
0
}
daemon::DaemonStatus::Stale(pid) => {
println!("telemt is not running (stale pid file, was pid {})", pid);
// Clean up stale PID file
let _ = std::fs::remove_file(pid_file);
1
}
daemon::DaemonStatus::NotRunning => {
println!("telemt is not running");
1
}
}
}
/// Options for the init command
#[derive(Debug, Clone)]
pub struct InitOptions {
pub port: u16,
pub domain: String,
pub secret: Option<String>,
pub username: String,
pub config_dir: PathBuf,
pub no_start: bool,
}
/// Parse daemon-related options from CLI args.
#[cfg(unix)]
pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
let mut opts = DaemonOptions::default();
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--daemon" | "-d" => {
opts.daemonize = true;
}
"--foreground" | "-f" => {
opts.foreground = true;
}
"--pid-file" => {
i += 1;
if i < args.len() {
opts.pid_file = Some(PathBuf::from(&args[i]));
}
}
s if s.starts_with("--pid-file=") => {
opts.pid_file = Some(PathBuf::from(s.trim_start_matches("--pid-file=")));
}
"--run-as-user" => {
i += 1;
if i < args.len() {
opts.user = Some(args[i].clone());
}
}
s if s.starts_with("--run-as-user=") => {
opts.user = Some(s.trim_start_matches("--run-as-user=").to_string());
}
"--run-as-group" => {
i += 1;
if i < args.len() {
opts.group = Some(args[i].clone());
}
}
s if s.starts_with("--run-as-group=") => {
opts.group = Some(s.trim_start_matches("--run-as-group=").to_string());
}
"--working-dir" => {
i += 1;
if i < args.len() {
opts.working_dir = Some(PathBuf::from(&args[i]));
}
}
s if s.starts_with("--working-dir=") => {
opts.working_dir = Some(PathBuf::from(s.trim_start_matches("--working-dir=")));
}
_ => {}
}
i += 1;
}
opts
}
impl Default for InitOptions {
fn default() -> Self {
Self {
port: 443,
domain: "www.google.com".to_string(),
secret: None,
username: "user".to_string(),
config_dir: PathBuf::from("/etc/telemt"),
no_start: false,
}
}
}
/// Parse --init subcommand options from CLI args.
///
/// Returns `Some(InitOptions)` if `--init` was found, `None` otherwise.
pub fn parse_init_args(args: &[String]) -> Option<InitOptions> {
if !args.iter().any(|a| a == "--init") {
return None;
}
let mut opts = InitOptions::default();
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--port" => {
i += 1;
if i < args.len() {
opts.port = args[i].parse().unwrap_or(443);
}
}
"--domain" => {
i += 1;
if i < args.len() {
opts.domain = args[i].clone();
}
}
"--secret" => {
i += 1;
if i < args.len() {
opts.secret = Some(args[i].clone());
}
}
"--user" => {
i += 1;
if i < args.len() {
opts.username = args[i].clone();
}
}
"--config-dir" => {
i += 1;
if i < args.len() {
opts.config_dir = PathBuf::from(&args[i]);
}
}
"--no-start" => {
opts.no_start = true;
}
_ => {}
}
i += 1;
}
Some(opts)
}
/// Run the fire-and-forget setup.
pub fn run_init(opts: InitOptions) -> Result<(), Box<dyn std::error::Error>> {
use crate::service::{self, InitSystem, ServiceOptions};
eprintln!("[telemt] Fire-and-forget setup");
eprintln!();
// 1. Detect init system
let init_system = service::detect_init_system();
eprintln!("[+] Detected init system: {}", init_system);
// 2. Generate or validate secret
let secret = match opts.secret {
Some(s) => {
if s.len() != 32 || !s.chars().all(|c| c.is_ascii_hexdigit()) {
eprintln!("[error] Secret must be exactly 32 hex characters");
std::process::exit(1);
}
s
}
None => generate_secret(),
};
eprintln!("[+] Secret: {}", secret);
eprintln!("[+] User: {}", opts.username);
eprintln!("[+] Port: {}", opts.port);
eprintln!("[+] Domain: {}", opts.domain);
// 3. Create config directory
fs::create_dir_all(&opts.config_dir)?;
let config_path = opts.config_dir.join("config.toml");
// 4. Write config
let config_content = generate_config(&opts.username, &secret, opts.port, &opts.domain);
fs::write(&config_path, &config_content)?;
eprintln!("[+] Config written to {}", config_path.display());
// 5. Generate and write service file
let exe_path =
std::env::current_exe().unwrap_or_else(|_| PathBuf::from("/usr/local/bin/telemt"));
let service_opts = ServiceOptions {
exe_path: &exe_path,
config_path: &config_path,
user: None, // Let systemd/init handle user
group: None,
pid_file: "/var/run/telemt.pid",
working_dir: Some("/var/lib/telemt"),
description: "Telemt MTProxy - Telegram MTProto Proxy",
};
let service_path = service::service_file_path(init_system);
let service_content = service::generate_service_file(init_system, &service_opts);
// Ensure parent directory exists
if let Some(parent) = Path::new(service_path).parent() {
let _ = fs::create_dir_all(parent);
}
match fs::write(service_path, &service_content) {
Ok(()) => {
eprintln!("[+] Service file written to {}", service_path);
// Make script executable for OpenRC/FreeBSD
#[cfg(unix)]
if init_system == InitSystem::OpenRC || init_system == InitSystem::FreeBSDRc {
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(service_path)?.permissions();
perms.set_mode(0o755);
fs::set_permissions(service_path, perms)?;
}
}
Err(e) => {
eprintln!("[!] Cannot write service file (run as root?): {}", e);
eprintln!("[!] Manual service file content:");
eprintln!("{}", service_content);
// Still print links and installation instructions
eprintln!();
eprintln!("{}", service::installation_instructions(init_system));
print_links(&opts.username, &secret, opts.port, &opts.domain);
return Ok(());
}
}
// 6. Install and enable service based on init system
match init_system {
InitSystem::Systemd => {
run_cmd("systemctl", &["daemon-reload"]);
run_cmd("systemctl", &["enable", "telemt.service"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("systemctl", &["start", "telemt.service"]);
eprintln!("[+] Service started");
std::thread::sleep(std::time::Duration::from_secs(1));
let status = Command::new("systemctl")
.args(["is-active", "telemt.service"])
.output();
match status {
Ok(out) if out.status.success() => {
eprintln!("[+] Service is running");
}
_ => {
eprintln!("[!] Service may not have started correctly");
eprintln!("[!] Check: journalctl -u telemt.service -n 20");
}
}
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: systemctl start telemt.service");
}
}
InitSystem::OpenRC => {
run_cmd("rc-update", &["add", "telemt", "default"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("rc-service", &["telemt", "start"]);
eprintln!("[+] Service started");
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: rc-service telemt start");
}
}
InitSystem::FreeBSDRc => {
run_cmd("sysrc", &["telemt_enable=YES"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("service", &["telemt", "start"]);
eprintln!("[+] Service started");
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: service telemt start");
}
}
InitSystem::Unknown => {
eprintln!("[!] Unknown init system - service file written but not installed");
eprintln!("[!] You may need to install it manually");
}
}
eprintln!();
// 7. Print links
print_links(&opts.username, &secret, opts.port, &opts.domain);
Ok(())
}
fn generate_secret() -> String {
let mut rng = rand::rng();
let bytes: Vec<u8> = (0..16).map(|_| rng.random::<u8>()).collect();
hex::encode(bytes)
}
fn generate_config(username: &str, secret: &str, port: u16, domain: &str) -> String {
format!(
r#"# Telemt MTProxy — auto-generated config
# Re-run `telemt --init` to regenerate
show_link = ["{username}"]
[general]
# prefer_ipv6 is deprecated; use [network].prefer
prefer_ipv6 = false
fast_mode = true
use_middle_proxy = false
log_level = "normal"
desync_all_full = false
update_every = 43200
hardswap = false
me_pool_drain_ttl_secs = 90
me_instadrain = false
me_pool_drain_threshold = 32
me_pool_drain_soft_evict_grace_secs = 10
me_pool_drain_soft_evict_per_writer = 2
me_pool_drain_soft_evict_budget_per_core = 16
me_pool_drain_soft_evict_cooldown_ms = 1000
me_bind_stale_mode = "never"
me_pool_min_fresh_ratio = 0.8
me_reinit_drain_timeout_secs = 90
tg_connect = 10
[network]
ipv4 = true
ipv6 = true
prefer = 4
multipath = false
[general.modes]
classic = false
secure = false
tls = true
[server]
listen_addr_ipv4 = "0.0.0.0"
listen_addr_ipv6 = "::"
[[server.listeners]]
ip = "0.0.0.0"
port = {port}
# reuse_allow = false # Set true only when intentionally running multiple telemt instances on same port
[[server.listeners]]
ip = "::"
port = {port}
[timeouts]
client_first_byte_idle_secs = 300
client_handshake = 60
client_keepalive = 60
client_ack = 300
[censorship]
tls_domain = "{domain}"
mask = true
mask_port = 443
fake_cert_len = 2048
serverhello_compact = false
tls_full_cert_ttl_secs = 90
[access]
user_max_tcp_conns_global_each = 0
replay_check_len = 65536
replay_window_secs = 120
ignore_time_skew = false
[access.users]
{username} = "{secret}"
[[upstreams]]
type = "direct"
enabled = true
weight = 10
# Optional per-upstream DC family policy:
# ipv6 = true
# prefer = 6
"#,
username = username,
secret = secret,
port = port,
domain = domain,
)
}
fn run_cmd(cmd: &str, args: &[&str]) {
match Command::new(cmd).args(args).output() {
Ok(output) => {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
eprintln!("[!] {} {} failed: {}", cmd, args.join(" "), stderr.trim());
}
}
Err(e) => {
eprintln!("[!] Failed to run {} {}: {}", cmd, args.join(" "), e);
}
}
}
fn print_links(username: &str, secret: &str, port: u16, domain: &str) {
let domain_hex = hex::encode(domain);
println!("=== Proxy Links ===");
println!("[{}]", username);
println!(
" EE-TLS: tg://proxy?server=YOUR_SERVER_IP&port={}&secret=ee{}{}",
port, secret, domain_hex
);
println!();
println!("Replace YOUR_SERVER_IP with your server's public IP.");
println!("The proxy will auto-detect and display the correct link on startup.");
println!("Check: journalctl -u telemt.service | head -30");
println!("===================");
}
+141
View File
@@ -0,0 +1,141 @@
use std::path::{Path, PathBuf};
use crate::daemon::{self, DaemonOptions};
/// Parses daemon-related options from CLI arguments.
pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
let mut opts = DaemonOptions::default();
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--daemon" | "-d" => {
opts.daemonize = true;
}
"--foreground" | "-f" => {
opts.foreground = true;
}
"--pid-file" => {
i += 1;
if i < args.len() {
opts.pid_file = Some(PathBuf::from(&args[i]));
}
}
s if s.starts_with("--pid-file=") => {
opts.pid_file = Some(PathBuf::from(s.trim_start_matches("--pid-file=")));
}
"--run-as-user" => {
i += 1;
if i < args.len() {
opts.user = Some(args[i].clone());
}
}
s if s.starts_with("--run-as-user=") => {
opts.user = Some(s.trim_start_matches("--run-as-user=").to_string());
}
"--run-as-group" => {
i += 1;
if i < args.len() {
opts.group = Some(args[i].clone());
}
}
s if s.starts_with("--run-as-group=") => {
opts.group = Some(s.trim_start_matches("--run-as-group=").to_string());
}
"--working-dir" => {
i += 1;
if i < args.len() {
opts.working_dir = Some(PathBuf::from(&args[i]));
}
}
s if s.starts_with("--working-dir=") => {
opts.working_dir = Some(PathBuf::from(s.trim_start_matches("--working-dir=")));
}
_ => {}
}
i += 1;
}
opts
}
/// Sends SIGTERM and waits briefly for graceful PID-file cleanup.
pub(super) fn stop(pid_file: &Path) -> i32 {
use nix::sys::signal::Signal;
println!("Stopping telemt daemon...");
match daemon::signal_pid_file(pid_file, Signal::SIGTERM) {
Ok(()) => {
println!("Stop signal sent successfully");
// Wait for process to exit for up to ten seconds.
for _ in 0..20 {
std::thread::sleep(std::time::Duration::from_millis(500));
if let daemon::DaemonStatus::NotRunning = daemon::check_status(pid_file) {
println!("Daemon stopped");
return 0;
}
}
println!("Daemon may still be shutting down");
0
}
Err(e) => {
eprintln!("Failed to stop daemon: {}", e);
1
}
}
}
/// Sends SIGHUP to trigger configuration reload.
pub(super) fn reload(pid_file: &Path) -> i32 {
use nix::sys::signal::Signal;
println!("Reloading telemt configuration...");
match daemon::signal_pid_file(pid_file, Signal::SIGHUP) {
Ok(()) => {
println!("Reload signal sent successfully");
0
}
Err(e) => {
eprintln!("Failed to reload daemon: {}", e);
1
}
}
}
/// Reports daemon status without mutating PID lifecycle state.
pub(super) fn status(pid_file: &Path) -> i32 {
match daemon::check_status(pid_file) {
daemon::DaemonStatus::Running(pid) => {
println!("telemt is running (pid {})", pid);
0
}
daemon::DaemonStatus::Stale(pid) => {
println!("telemt is not running (stale pid file, was pid {})", pid);
1
}
daemon::DaemonStatus::NotRunning => {
println!("telemt is not running");
1
}
}
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn status_does_not_remove_stale_pid_file() {
let directory = tempfile::tempdir().unwrap();
let pid_file = directory.path().join("telemt.pid");
fs::write(&pid_file, b"2000000000\n").unwrap();
assert_eq!(status(&pid_file), 1);
assert!(pid_file.exists());
}
}
+368
View File
@@ -0,0 +1,368 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use rand::RngExt;
use crate::util::trusted_command::resolve_trusted_helper;
/// Options for the fire-and-forget init command.
#[derive(Debug, Clone)]
pub struct InitOptions {
/// Public listener port.
pub port: u16,
/// TLS camouflage domain.
pub domain: String,
/// Optional pre-generated proxy secret.
pub secret: Option<String>,
/// Initial access username.
pub username: String,
/// Destination directory for generated configuration.
pub config_dir: PathBuf,
/// Generate service files without starting the service.
pub no_start: bool,
}
impl Default for InitOptions {
fn default() -> Self {
Self {
port: 443,
domain: "www.google.com".to_string(),
secret: None,
username: "user".to_string(),
config_dir: PathBuf::from("/etc/telemt"),
no_start: false,
}
}
}
/// Parse --init subcommand options from CLI args.
///
/// Returns `Some(InitOptions)` if `--init` was found, `None` otherwise.
pub fn parse_init_args(args: &[String]) -> Option<InitOptions> {
if !args.iter().any(|a| a == "--init") {
return None;
}
let mut opts = InitOptions::default();
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--port" => {
i += 1;
if i < args.len() {
opts.port = args[i].parse().unwrap_or(443);
}
}
"--domain" => {
i += 1;
if i < args.len() {
opts.domain = args[i].clone();
}
}
"--secret" => {
i += 1;
if i < args.len() {
opts.secret = Some(args[i].clone());
}
}
"--user" => {
i += 1;
if i < args.len() {
opts.username = args[i].clone();
}
}
"--config-dir" => {
i += 1;
if i < args.len() {
opts.config_dir = PathBuf::from(&args[i]);
}
}
"--no-start" => {
opts.no_start = true;
}
_ => {}
}
i += 1;
}
Some(opts)
}
/// Run the fire-and-forget setup.
pub fn run_init(opts: InitOptions) -> Result<(), Box<dyn std::error::Error>> {
use crate::service::{self, InitSystem, ServiceOptions};
eprintln!("[telemt] Fire-and-forget setup");
eprintln!();
let init_system = service::detect_init_system();
eprintln!("[+] Detected init system: {}", init_system);
let secret = match opts.secret {
Some(s) => {
if s.len() != 32 || !s.chars().all(|c| c.is_ascii_hexdigit()) {
eprintln!("[error] Secret must be exactly 32 hex characters");
std::process::exit(1);
}
s
}
None => generate_secret(),
};
eprintln!("[+] Secret: {}", secret);
eprintln!("[+] User: {}", opts.username);
eprintln!("[+] Port: {}", opts.port);
eprintln!("[+] Domain: {}", opts.domain);
let config_path = opts.config_dir.join("config.toml");
let config_content = generate_config(&opts.username, &secret, opts.port, &opts.domain);
write_init_file(&config_path, &config_content, 0o600)?;
eprintln!("[+] Config written to {}", config_path.display());
let exe_path =
std::env::current_exe().unwrap_or_else(|_| PathBuf::from("/usr/local/bin/telemt"));
let service_opts = ServiceOptions {
exe_path: &exe_path,
config_path: &config_path,
// Let the selected init system manage process identity.
user: None,
group: None,
pid_file: "/var/run/telemt.pid",
working_dir: Some("/var/lib/telemt"),
description: "Telemt MTProxy - Telegram MTProto Proxy",
};
let service_path = service::service_file_path(init_system);
let service_content = service::generate_service_file(init_system, &service_opts);
let service_mode = if init_system == InitSystem::OpenRC || init_system == InitSystem::FreeBSDRc
{
0o755
} else {
0o644
};
match write_init_file(Path::new(service_path), &service_content, service_mode) {
Ok(()) => {
eprintln!("[+] Service file written to {}", service_path);
}
Err(e) => {
eprintln!("[!] Cannot write service file (run as root?): {}", e);
eprintln!("[!] Manual service file content:");
eprintln!("{}", service_content);
eprintln!();
eprintln!("{}", service::installation_instructions(init_system));
print_links(&opts.username, &secret, opts.port, &opts.domain);
return Ok(());
}
}
match init_system {
InitSystem::Systemd => {
run_cmd("systemctl", &["daemon-reload"]);
run_cmd("systemctl", &["enable", "telemt.service"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("systemctl", &["start", "telemt.service"]);
eprintln!("[+] Service started");
std::thread::sleep(std::time::Duration::from_secs(1));
let status = resolve_trusted_helper("systemctl").and_then(|command_path| {
Command::new(command_path)
.args(["is-active", "telemt.service"])
.output()
.ok()
});
match status {
Some(out) if out.status.success() => {
eprintln!("[+] Service is running");
}
_ => {
eprintln!("[!] Service may not have started correctly");
eprintln!("[!] Check: journalctl -u telemt.service -n 20");
}
}
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: systemctl start telemt.service");
}
}
InitSystem::OpenRC => {
run_cmd("rc-update", &["add", "telemt", "default"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("rc-service", &["telemt", "start"]);
eprintln!("[+] Service started");
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: rc-service telemt start");
}
}
InitSystem::FreeBSDRc => {
run_cmd("sysrc", &["telemt_enable=YES"]);
eprintln!("[+] Service enabled");
if !opts.no_start {
run_cmd("service", &["telemt", "start"]);
eprintln!("[+] Service started");
} else {
eprintln!("[+] Service not started (--no-start)");
eprintln!("[+] Start manually: service telemt start");
}
}
InitSystem::Unknown => {
eprintln!("[!] Unknown init system - service file written but not installed");
eprintln!("[!] You may need to install it manually");
}
}
eprintln!();
print_links(&opts.username, &secret, opts.port, &opts.domain);
Ok(())
}
fn write_init_file(path: &Path, contents: &str, mode: u32) -> std::io::Result<()> {
#[cfg(unix)]
{
crate::util::secure_fs::atomic_replace(path, contents.as_bytes(), mode)
}
#[cfg(not(unix))]
{
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let _ = mode;
std::fs::write(path, contents)
}
}
fn generate_secret() -> String {
let mut rng = rand::rng();
let bytes: Vec<u8> = (0..16).map(|_| rng.random::<u8>()).collect();
hex::encode(bytes)
}
fn generate_config(username: &str, secret: &str, port: u16, domain: &str) -> String {
format!(
r#"# Telemt MTProxy — auto-generated config
# Re-run `telemt --init` to regenerate
show_link = ["{username}"]
[general]
# prefer_ipv6 is deprecated; use [network].prefer
prefer_ipv6 = false
fast_mode = true
use_middle_proxy = false
log_level = "normal"
desync_all_full = false
update_every = 43200
hardswap = false
me_pool_drain_ttl_secs = 90
me_instadrain = false
me_pool_drain_threshold = 32
me_pool_drain_soft_evict_grace_secs = 10
me_pool_drain_soft_evict_per_writer = 2
me_pool_drain_soft_evict_budget_per_core = 16
me_pool_drain_soft_evict_cooldown_ms = 1000
me_bind_stale_mode = "never"
me_pool_min_fresh_ratio = 0.8
me_reinit_drain_timeout_secs = 90
tg_connect = 10
[network]
ipv4 = true
ipv6 = true
prefer = 4
multipath = false
[general.modes]
classic = false
secure = false
tls = true
[server]
listen_addr_ipv4 = "0.0.0.0"
listen_addr_ipv6 = "::"
[[server.listeners]]
ip = "0.0.0.0"
port = {port}
# reuse_allow = false # Set true only when intentionally running multiple telemt instances on same port
[[server.listeners]]
ip = "::"
port = {port}
[timeouts]
client_first_byte_idle_secs = 300
client_handshake = 60
client_keepalive = 60
client_ack = 300
[censorship]
tls_domain = "{domain}"
mask = true
mask_port = 443
fake_cert_len = 2048
serverhello_compact = false
tls_full_cert_ttl_secs = 90
[access]
user_max_tcp_conns_global_each = 0
replay_check_len = 65536
replay_window_secs = 120
ignore_time_skew = false
[access.users]
{username} = "{secret}"
[[upstreams]]
type = "direct"
enabled = true
weight = 10
# Optional per-upstream DC family policy:
# ipv6 = true
# prefer = 6
"#,
username = username,
secret = secret,
port = port,
domain = domain,
)
}
fn run_cmd(cmd: &str, args: &[&str]) {
let Some(command_path) = resolve_trusted_helper(cmd) else {
eprintln!("[!] Refusing unavailable or untrusted command: {}", cmd);
return;
};
match Command::new(command_path).args(args).output() {
Ok(output) => {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
eprintln!("[!] {} {} failed: {}", cmd, args.join(" "), stderr.trim());
}
}
Err(e) => {
eprintln!("[!] Failed to run {} {}: {}", cmd, args.join(" "), e);
}
}
}
fn print_links(username: &str, secret: &str, port: u16, domain: &str) {
let domain_hex = hex::encode(domain);
println!("=== Proxy Links ===");
println!("[{}]", username);
println!(
" EE-TLS: tg://proxy?server=YOUR_SERVER_IP&port={}&secret=ee{}{}",
port, secret, domain_hex
);
println!();
println!("Replace YOUR_SERVER_IP with your server's public IP.");
println!("The proxy will auto-detect and display the correct link on startup.");
println!("Check: journalctl -u telemt.service | head -30");
println!("===================");
}
+5 -446
View File
@@ -1,6 +1,9 @@
use ipnetwork::IpNetwork;
use serde::Deserialize;
use std::collections::HashMap;
// Extended transport, masking, and ME default values.
mod extended;
pub(crate) use extended::*;
// Helper defaults kept private to the config module.
const DEFAULT_NETWORK_IPV6: Option<bool> = Some(false);
@@ -520,447 +523,3 @@ pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize {
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
}
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
}
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
}
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
}
pub(crate) fn default_me_admission_poll_ms() -> u64 {
DEFAULT_ME_ADMISSION_POLL_MS
}
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
DEFAULT_ME_WARN_RATE_LIMIT_MS
}
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
}
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
}
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
}
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
}
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
100
}
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
}
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
}
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
false
}
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
0
}
pub(crate) fn default_crypto_pending_buffer() -> usize {
256 * 1024
}
pub(crate) fn default_max_client_frame() -> usize {
16 * 1024 * 1024
}
pub(crate) fn default_desync_all_full() -> bool {
false
}
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
25
}
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
}
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
}
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
120
}
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
80
}
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
250
}
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
3
}
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
3000
}
pub(crate) fn default_beobachten_minutes() -> u64 {
10
}
pub(crate) fn default_beobachten_flush_secs() -> u64 {
15
}
pub(crate) fn default_beobachten_file() -> String {
"beobachten.txt".to_string()
}
pub(crate) fn default_tls_new_session_tickets() -> u8 {
0
}
pub(crate) fn default_serverhello_compact() -> bool {
false
}
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
90
}
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
8
}
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
24
}
pub(crate) fn default_alpn_enforce() -> bool {
true
}
pub(crate) fn default_mask_shape_hardening() -> bool {
true
}
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
false
}
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
512
}
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
4096
}
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
false
}
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
512
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_max_bytes() -> usize {
5 * 1024 * 1024
}
#[cfg(test)]
pub(crate) fn default_mask_relay_max_bytes() -> usize {
32 * 1024
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
60_000
}
#[cfg(test)]
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
200
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
5_000
}
#[cfg(test)]
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
100
}
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
5
}
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
false
}
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
0
}
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
0
}
pub(crate) fn default_stun_servers() -> Vec<String> {
vec![
"stun.l.google.com:5349".to_string(),
"stun1.l.google.com:3478".to_string(),
"stun.gmx.net:3478".to_string(),
"stun.l.google.com:19302".to_string(),
"stun.1und1.de:3478".to_string(),
"stun1.l.google.com:19302".to_string(),
"stun2.l.google.com:19302".to_string(),
"stun3.l.google.com:19302".to_string(),
"stun4.l.google.com:19302".to_string(),
"stun.services.mozilla.com:3478".to_string(),
"stun.stunprotocol.org:3478".to_string(),
"stun.nextcloud.com:3478".to_string(),
"stun.voip.eutelia.it:3478".to_string(),
]
}
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
vec![
"https://ifconfig.me/ip".to_string(),
"https://api.ipify.org".to_string(),
]
}
pub(crate) fn default_cache_public_ip_path() -> String {
"cache/public_ip.txt".to_string()
}
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
60 * 60
}
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
60 * 60
}
pub(crate) fn default_update_every_secs() -> u64 {
5 * 60
}
pub(crate) fn default_update_every() -> Option<u64> {
Some(default_update_every_secs())
}
pub(crate) fn default_me_reinit_every_secs() -> u64 {
15 * 60
}
pub(crate) fn default_me_reinit_singleflight() -> bool {
true
}
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
64
}
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
200
}
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
1000
}
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
2000
}
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
3
}
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
500
}
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
2
}
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
300
}
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
true
}
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
true
}
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
1
}
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
2
}
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
true
}
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
true
}
pub(crate) fn default_proxy_secret_len_max() -> usize {
256
}
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
90
}
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
90
}
pub(crate) fn default_me_instadrain() -> bool {
false
}
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
32
}
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
}
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
}
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
}
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
}
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
}
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
default_me_pool_drain_ttl_secs()
}
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
0.8
}
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
true
}
pub(crate) fn default_hardswap() -> bool {
true
}
pub(crate) fn default_ntp_check() -> bool {
true
}
pub(crate) fn default_ntp_servers() -> Vec<String> {
vec!["pool.ntp.org".to_string()]
}
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
0
}
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
2
}
pub(crate) fn default_listen_addr_ipv6() -> String {
DEFAULT_LISTEN_ADDR_IPV6.to_string()
}
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
Some(default_listen_addr_ipv6())
}
pub(crate) fn default_access_users() -> HashMap<String, String> {
HashMap::from([(
DEFAULT_ACCESS_USER.to_string(),
DEFAULT_ACCESS_SECRET.to_string(),
)])
}
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
}
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
0
}
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
0
}
// Custom deserializer helpers
#[derive(Deserialize)]
#[serde(untagged)]
pub(crate) enum OneOrMany {
One(String),
Many(Vec<String>),
}
pub(crate) fn deserialize_dc_overrides<'de, D>(
deserializer: D,
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
where
D: serde::de::Deserializer<'de>,
{
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
let mut out = HashMap::new();
for (dc, val) in raw {
let mut addrs = match val {
OneOrMany::One(s) => vec![s],
OneOrMany::Many(v) => v,
};
addrs.retain(|s| !s.trim().is_empty());
if !addrs.is_empty() {
out.insert(dc, addrs);
}
}
Ok(out)
}
+453
View File
@@ -0,0 +1,453 @@
use std::collections::HashMap;
use serde::Deserialize;
use super::*;
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
}
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
}
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
}
pub(crate) fn default_me_admission_poll_ms() -> u64 {
DEFAULT_ME_ADMISSION_POLL_MS
}
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
DEFAULT_ME_WARN_RATE_LIMIT_MS
}
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
}
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
}
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
}
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
}
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
100
}
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
}
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
}
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
false
}
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
0
}
pub(crate) fn default_crypto_pending_buffer() -> usize {
256 * 1024
}
pub(crate) fn default_max_client_frame() -> usize {
16 * 1024 * 1024
}
pub(crate) fn default_desync_all_full() -> bool {
false
}
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
25
}
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
}
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
}
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
120
}
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
80
}
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
250
}
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
3
}
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
3000
}
pub(crate) fn default_beobachten_minutes() -> u64 {
10
}
pub(crate) fn default_beobachten_flush_secs() -> u64 {
15
}
pub(crate) fn default_beobachten_file() -> String {
"beobachten.txt".to_string()
}
pub(crate) fn default_tls_new_session_tickets() -> u8 {
0
}
pub(crate) fn default_serverhello_compact() -> bool {
false
}
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
90
}
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
8
}
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
24
}
pub(crate) fn default_alpn_enforce() -> bool {
true
}
pub(crate) fn default_mask_shape_hardening() -> bool {
true
}
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
false
}
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
512
}
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
4096
}
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
false
}
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
512
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_max_bytes() -> usize {
5 * 1024 * 1024
}
#[cfg(test)]
pub(crate) fn default_mask_relay_max_bytes() -> usize {
32 * 1024
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
60_000
}
#[cfg(test)]
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
200
}
#[cfg(not(test))]
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
5_000
}
#[cfg(test)]
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
100
}
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
5
}
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
false
}
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
0
}
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
0
}
pub(crate) fn default_stun_servers() -> Vec<String> {
vec![
"stun.l.google.com:5349".to_string(),
"stun1.l.google.com:3478".to_string(),
"stun.gmx.net:3478".to_string(),
"stun.l.google.com:19302".to_string(),
"stun.1und1.de:3478".to_string(),
"stun1.l.google.com:19302".to_string(),
"stun2.l.google.com:19302".to_string(),
"stun3.l.google.com:19302".to_string(),
"stun4.l.google.com:19302".to_string(),
"stun.services.mozilla.com:3478".to_string(),
"stun.stunprotocol.org:3478".to_string(),
"stun.nextcloud.com:3478".to_string(),
"stun.voip.eutelia.it:3478".to_string(),
]
}
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
vec![
"https://ifconfig.me/ip".to_string(),
"https://api.ipify.org".to_string(),
]
}
pub(crate) fn default_cache_public_ip_path() -> String {
"cache/public_ip.txt".to_string()
}
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
60 * 60
}
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
60 * 60
}
pub(crate) fn default_update_every_secs() -> u64 {
5 * 60
}
pub(crate) fn default_update_every() -> Option<u64> {
Some(default_update_every_secs())
}
pub(crate) fn default_me_reinit_every_secs() -> u64 {
15 * 60
}
pub(crate) fn default_me_reinit_singleflight() -> bool {
true
}
pub(crate) fn default_me_reinit_max_concurrency() -> usize {
2
}
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
64
}
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
200
}
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
1000
}
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
2000
}
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
3
}
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
500
}
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
2
}
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
300
}
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
true
}
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
true
}
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
1
}
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
2
}
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
true
}
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
true
}
pub(crate) fn default_proxy_secret_len_max() -> usize {
256
}
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
90
}
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
90
}
pub(crate) fn default_me_instadrain() -> bool {
false
}
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
32
}
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
}
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
}
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
}
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
}
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
}
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
default_me_pool_drain_ttl_secs()
}
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
0.8
}
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
true
}
pub(crate) fn default_hardswap() -> bool {
true
}
pub(crate) fn default_ntp_check() -> bool {
true
}
pub(crate) fn default_ntp_servers() -> Vec<String> {
vec!["pool.ntp.org".to_string()]
}
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
0
}
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
2
}
pub(crate) fn default_listen_addr_ipv6() -> String {
DEFAULT_LISTEN_ADDR_IPV6.to_string()
}
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
Some(default_listen_addr_ipv6())
}
pub(crate) fn default_access_users() -> HashMap<String, String> {
HashMap::from([(
DEFAULT_ACCESS_USER.to_string(),
DEFAULT_ACCESS_SECRET.to_string(),
)])
}
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
}
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
0
}
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
0
}
// Custom deserializer helpers
#[derive(Deserialize)]
#[serde(untagged)]
pub(crate) enum OneOrMany {
One(String),
Many(Vec<String>),
}
pub(crate) fn deserialize_dc_overrides<'de, D>(
deserializer: D,
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
where
D: serde::de::Deserializer<'de>,
{
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
let mut out = HashMap::new();
for (dc, val) in raw {
let mut addrs = match val {
OneOrMany::One(s) => vec![s],
OneOrMany::Many(v) => v,
};
addrs.retain(|s| !s.trim().is_empty());
if !addrs.is_empty() {
out.insert(dc, addrs);
}
}
Ok(out)
}
+3 -1
View File
@@ -21,6 +21,8 @@
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
//! reconciled only during privileged startup.
//! `web.decoy_fasttrack_mode` is also restart-only so one process never mixes
//! capability timing policies or process-lifetime counter semantics.
//! Non-hot changes are never mixed into the runtime config snapshot.
use std::collections::BTreeSet;
@@ -37,7 +39,7 @@ use super::load::{LoadedConfig, ProxyConfig};
#[allow(unused_imports)]
use crate::config::{
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
MeWriterPickMode, WebDebugConfig, web_debug_fits_limits,
MeWriterPickMode, WEB_CARRIER_LEARNING_MIN_ENTRIES, WebDebugConfig, web_debug_fits_limits,
};
#[cfg(test)]
use crate::config::{ListenerConfig, SynLimitMode};
+4
View File
@@ -85,6 +85,10 @@ pub(super) fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot
warned = true;
warn!("config reload: server listener settings changed; restart required");
}
if old.web.decoy_fasttrack_mode != new.web.decoy_fasttrack_mode {
warned = true;
warn!("config reload: web.decoy_fasttrack_mode changed; restart required");
}
if old.censorship.tls_domain != new.censorship.tls_domain
|| old.censorship.tls_domains != new.censorship.tls_domains
|| old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope
+15 -3
View File
@@ -10,6 +10,7 @@ pub struct HotFields {
pub update_every_secs: u64,
pub me_reinit_every_secs: u64,
pub me_reinit_singleflight: bool,
pub me_reinit_max_concurrency: usize,
pub me_reinit_coalesce_window_ms: u64,
pub hardswap: bool,
pub me_pool_drain_ttl_secs: u64,
@@ -102,6 +103,7 @@ impl HotFields {
update_every_secs: cfg.general.effective_update_every_secs(),
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
me_reinit_max_concurrency: cfg.general.me_reinit_max_concurrency,
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
hardswap: cfg.general.hardswap,
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
@@ -236,6 +238,7 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
cfg.general.proxy_config_auto_reload_secs = new.general.proxy_config_auto_reload_secs;
cfg.general.me_reinit_every_secs = new.general.me_reinit_every_secs;
cfg.general.me_reinit_singleflight = new.general.me_reinit_singleflight;
cfg.general.me_reinit_max_concurrency = new.general.me_reinit_max_concurrency;
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
cfg.general.hardswap = new.general.hardswap;
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
@@ -340,17 +343,26 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs;
let process_limits = cfg.web.limits.clone();
let decoy_fasttrack_mode = cfg.web.decoy_fasttrack_mode;
cfg.web = new.web.clone();
cfg.web.limits = process_limits;
cfg.web.decoy_fasttrack_mode = decoy_fasttrack_mode;
if cfg.web.carrier_negotiation_enabled()
&& cfg.web.carrier_learning
&& cfg.web.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
{
if old.web.carrier_learning != new.web.carrier_learning {
cfg.web.carrier_learning = old.web.carrier_learning;
} else {
cfg.web.carriers = old.web.carriers.clone();
}
}
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
cfg.web.debug = old.web.debug.clone();
}
if cfg.rebuild_runtime_user_auth().is_err() {
cfg.runtime_user_auth = None;
}
if cfg.rebuild_runtime_web().is_err() {
cfg.web = old.web.clone();
}
cfg
}
+3 -1
View File
@@ -114,12 +114,14 @@ pub(super) fn log_changes(
}
if old_hot.me_reinit_every_secs != new_hot.me_reinit_every_secs
|| old_hot.me_reinit_singleflight != new_hot.me_reinit_singleflight
|| old_hot.me_reinit_max_concurrency != new_hot.me_reinit_max_concurrency
|| old_hot.me_reinit_coalesce_window_ms != new_hot.me_reinit_coalesce_window_ms
{
info!(
"config reload: me_reinit: interval={}s singleflight={} coalesce={}ms",
"config reload: me_reinit: interval={}s singleflight={} max_concurrency={} coalesce={}ms",
new_hot.me_reinit_every_secs,
new_hot.me_reinit_singleflight,
new_hot.me_reinit_max_concurrency,
new_hot.me_reinit_coalesce_window_ms
);
}
+132
View File
@@ -47,6 +47,43 @@ fn write_web_reload_config(path: &Path, carriers: &str, carrier_learning: bool)
std::fs::write(path, config).unwrap();
}
fn write_web_fasttrack_reload_config(path: &Path, mode: &str, ad_tag: &str) {
let config = format!(
r#"
[general]
ad_tag = "{ad_tag}"
[access.users]
alice = "000102030405060708090a0b0c0d0e0f"
[[server.listeners]]
ip = "127.0.0.1"
port = 18080
transport = "web"
proxy_protocol = false
web_client_ip_source = "x_forwarded_for"
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
[web]
enabled = true
decoy_fasttrack_mode = "{mode}"
[[web.vhosts]]
host = "proxy.example.com"
public_addr = "203.0.113.10:443"
[web.vhosts.decoy]
mode = "http_upstream"
upstream = "http://127.0.0.1:18081"
[[web.vhosts.profiles]]
user = "alice"
secret_mode = "plain"
"#,
);
std::fs::write(path, config).unwrap();
}
fn temp_config_path(prefix: &str) -> PathBuf {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -107,11 +144,13 @@ fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
let old = sample_config();
let mut new = old.clone();
new.web.debug.enabled = true;
new.web.debug.sideband = true;
new.web.debug.default_window_secs = 60;
new.web.limits.debug_records_capacity += 1;
let applied = overlay_hot_fields(&old, &new);
assert!(applied.web.debug.enabled);
assert!(applied.web.debug.sideband);
assert_eq!(applied.web.debug.default_window_secs, 60);
assert_eq!(
applied.web.limits.debug_records_capacity,
@@ -123,6 +162,65 @@ fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
);
}
#[test]
fn decoy_fasttrack_mode_is_deferred_until_restart() {
let old = sample_config();
let mut new = old.clone();
new.web.decoy_fasttrack_mode = crate::config::WebDecoyFastTrackMode::Enforce;
let applied = overlay_hot_fields(&old, &new);
assert_eq!(
applied.web.decoy_fasttrack_mode,
old.web.decoy_fasttrack_mode
);
assert_eq!(
HotFields::from_config(&old),
HotFields::from_config(&applied)
);
}
#[test]
fn hot_overlay_defers_learning_that_requires_new_process_capacity() {
let mut old = sample_config();
old.web.limits.max_carrier_learning_entries = 1;
old.web.carriers = crate::config::WebCarriers::Disabled;
old.web.carrier_learning = false;
let mut new = old.clone();
new.web.limits.max_carrier_learning_entries = 3;
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
crate::config::WebCarrier::Websocket,
crate::config::WebCarrier::Https,
]);
new.web.carrier_learning = true;
let applied = overlay_hot_fields(&old, &new);
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
assert!(applied.web.carrier_negotiation_enabled());
assert!(!applied.web.carrier_learning);
}
#[test]
fn hot_overlay_defers_carriers_for_dormant_learning_with_small_capacity() {
let mut old = sample_config();
old.web.limits.max_carrier_learning_entries = 1;
old.web.carriers = crate::config::WebCarriers::Disabled;
old.web.carrier_learning = true;
let mut new = old.clone();
new.web.limits.max_carrier_learning_entries = 3;
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
crate::config::WebCarrier::Websocket,
crate::config::WebCarrier::Https,
]);
let applied = overlay_hot_fields(&old, &new);
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
assert!(!applied.web.carrier_negotiation_enabled());
assert!(applied.web.carrier_learning);
}
#[test]
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
let old = sample_config();
@@ -272,6 +370,7 @@ async fn candidate_watcher_waits_for_activation_and_reconciles_disk() {
None,
None,
cancellation.clone(),
None,
Some(activation_rx),
);
let watcher = tokio::spawn(watcher);
@@ -321,6 +420,39 @@ fn reload_keeps_hot_apply_when_non_hot_fields_change() {
let _ = std::fs::remove_file(path);
}
#[test]
fn reload_rebuilds_vhosts_with_the_effective_fasttrack_mode() {
let initial_tag = "abababababababababababababababab";
let final_tag = "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd";
let path = temp_config_path("telemt_web_fasttrack_reload");
write_web_fasttrack_reload_config(&path, "off", initial_tag);
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
let initial_hash = ProxyConfig::load_with_metadata(&path)
.unwrap()
.rendered_hash;
let (config_tx, _config_rx) = watch::channel(Arc::clone(&initial_cfg));
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
let mut reload_state = ReloadState::new(Some(initial_hash));
write_web_fasttrack_reload_config(&path, "enforce", final_tag);
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
let applied = config_tx.borrow().clone();
assert_eq!(applied.general.ad_tag.as_deref(), Some(final_tag));
assert_eq!(
applied.web.decoy_fasttrack_mode,
crate::config::WebDecoyFastTrackMode::Off
);
let runtime = applied.web.runtime.as_ref().unwrap();
assert_eq!(
runtime.vhosts["proxy.example.com"].decoy_fasttrack_mode,
crate::config::WebDecoyFastTrackMode::Off
);
let _ = std::fs::remove_file(path);
}
#[test]
fn reload_publishes_web_negotiation_policy_outside_hot_field_reporting() {
let path = temp_config_path("telemt_web_negotiation_reload");
+75 -13
View File
@@ -52,15 +52,24 @@ impl ReloadState {
}
fn normalize_watch_path(path: &Path) -> PathBuf {
path.canonicalize().unwrap_or_else(|_| {
if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
};
let mut normalized = PathBuf::new();
for component in absolute.components() {
match component {
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
normalized.pop();
}
component => normalized.push(component.as_os_str()),
}
})
}
normalized
}
fn sync_watch_paths<W: Watcher>(
@@ -124,13 +133,14 @@ fn apply_watch_manifest<W1: Watcher, W2: Watcher>(
}
/// Load config, validate, diff against current, and broadcast if changed.
pub(super) fn reload_config(
fn reload_config_with_resolver(
config_path: &PathBuf,
config_tx: &watch::Sender<Arc<ProxyConfig>>,
log_tx: &watch::Sender<LogLevel>,
detected_ip_v4: Option<IpAddr>,
detected_ip_v6: Option<IpAddr>,
reload_state: &mut ReloadState,
dns_resolver: Option<&crate::network::dns_overrides::GenerationDnsResolver>,
) -> Option<WatchManifest> {
let loaded = match ProxyConfig::load_with_metadata(config_path) {
Ok(loaded) => loaded,
@@ -160,7 +170,17 @@ pub(super) fn reload_config(
}
let old_cfg = config_tx.borrow().clone();
let applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
let mut applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
if let Err(error) = applied_cfg
.validate_effective_web()
.and_then(|_| applied_cfg.rebuild_runtime_web())
{
error!(
"config reload: effective WEB validation failed: {}; keeping old config",
error
);
return Some(next_manifest);
}
let old_hot = HotFields::from_config(&old_cfg);
let applied_hot = HotFields::from_config(&applied_cfg);
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
@@ -176,7 +196,8 @@ pub(super) fn reload_config(
}
if old_hot.dns_overrides != applied_hot.dns_overrides
&& let Err(e) = crate::network::dns_overrides::install_entries(&applied_hot.dns_overrides)
&& let Some(dns_resolver) = dns_resolver
&& let Err(e) = dns_resolver.apply_entries(&applied_hot.dns_overrides)
{
error!(
"config reload: invalid network.dns_overrides: {}; keeping old config",
@@ -198,6 +219,26 @@ pub(super) fn reload_config(
Some(next_manifest)
}
#[cfg(test)]
pub(super) fn reload_config(
config_path: &PathBuf,
config_tx: &watch::Sender<Arc<ProxyConfig>>,
log_tx: &watch::Sender<LogLevel>,
detected_ip_v4: Option<IpAddr>,
detected_ip_v6: Option<IpAddr>,
reload_state: &mut ReloadState,
) -> Option<WatchManifest> {
reload_config_with_resolver(
config_path,
config_tx,
log_tx,
detected_ip_v4,
detected_ip_v6,
reload_state,
None,
)
}
/// Spawn the hot-reload watcher task.
///
/// Uses `notify` (inotify on Linux) to detect file changes instantly.
@@ -213,6 +254,7 @@ pub fn spawn_config_watcher(
detected_ip_v4: Option<IpAddr>,
detected_ip_v6: Option<IpAddr>,
cancellation: tokio_util::sync::CancellationToken,
dns_resolver: Option<Arc<crate::network::dns_overrides::GenerationDnsResolver>>,
mut activation: Option<watch::Receiver<bool>>,
) -> (
watch::Receiver<Arc<ProxyConfig>>,
@@ -364,24 +406,26 @@ pub fn spawn_config_watcher(
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
while notify_rx.try_recv().is_ok() {}
let mut next_manifest = reload_config(
let mut next_manifest = reload_config_with_resolver(
&config_path,
&config_tx,
&log_tx,
detected_ip_v4,
detected_ip_v6,
&mut reload_state,
dns_resolver.as_deref(),
);
if next_manifest.is_none() {
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
while notify_rx.try_recv().is_ok() {}
next_manifest = reload_config(
next_manifest = reload_config_with_resolver(
&config_path,
&config_tx,
&log_tx,
detected_ip_v4,
detected_ip_v6,
&mut reload_state,
dns_resolver.as_deref(),
);
}
@@ -398,3 +442,21 @@ pub fn spawn_config_watcher(
(config_rx, log_rx, task)
}
#[cfg(all(test, unix))]
mod path_tests {
use std::os::unix::fs::symlink;
use super::normalize_watch_path;
#[test]
fn watch_path_normalization_preserves_symlinks() {
let dir = tempfile::tempdir().unwrap();
let real = dir.path().join("real.toml");
let linked = dir.path().join("linked.toml");
std::fs::write(&real, "").unwrap();
symlink(&real, &linked).unwrap();
assert_eq!(normalize_watch_path(&linked), linked);
}
}
+63 -10
View File
@@ -9,6 +9,7 @@ use rand::RngExt;
use serde::{Deserialize, Serialize};
use tracing::warn;
use crate::crypto::sha256;
use crate::error::{ProxyError, Result};
use super::defaults::*;
@@ -35,7 +36,9 @@ mod validate_server;
mod validate_web;
mod validation;
use self::includes::{hash_rendered_snapshot, normalize_config_path, preprocess_includes};
use self::includes::{
hash_rendered_snapshot, normalize_config_path, preprocess_includes, read_config_source,
};
use self::normalize::{
is_valid_ad_tag, is_valid_tls_domain_name, normalize_domain_to_ascii,
normalize_exclusive_mask_target, normalize_mask_host_to_ascii, parse_exclusive_mask_target,
@@ -63,9 +66,9 @@ const MAX_API_REQUEST_BODY_LIMIT_BYTES: usize = 1024 * 1024;
pub(crate) struct LoadedConfig {
/// Validated and normalized effective configuration.
pub(crate) config: ProxyConfig,
/// Canonical paths participating in the recursive include graph.
/// Normalized absolute paths participating in the recursive include graph.
pub(crate) source_files: Vec<PathBuf>,
/// Raw source bytes keyed by canonical source path.
/// Raw source bytes keyed by normalized absolute source path.
pub(crate) source_contents: BTreeMap<PathBuf, String>,
/// Legacy hash of the include-expanded rendered snapshot.
pub(crate) rendered_hash: u64,
@@ -74,7 +77,7 @@ pub(crate) struct LoadedConfig {
/// Raw recursive source graph captured before typed deserialization.
#[derive(Debug, Clone)]
pub(crate) struct ConfigSourceGraph {
/// Raw source bytes keyed by canonical source path.
/// Raw source bytes keyed by normalized absolute source path.
pub(crate) source_contents: BTreeMap<PathBuf, String>,
/// Include-expanded TOML used for typed deserialization.
pub(crate) rendered: String,
@@ -174,21 +177,42 @@ impl ProxyConfig {
source_overrides: &BTreeMap<PathBuf, String>,
) -> Result<ConfigSourceGraph> {
let path = path.as_ref();
let normalized_path = normalize_config_path(path);
let mut previous = Self::capture_source_graph(path, source_overrides)?;
for _ in 0..2 {
let current = Self::capture_source_graph(path, source_overrides)?;
if current.source_contents == previous.source_contents
&& current.rendered == previous.rendered
{
return Ok(current);
}
previous = current;
}
Err(ProxyError::Config(
"config source graph changed repeatedly while it was read".to_string(),
))
}
fn capture_source_graph(
path: &Path,
source_overrides: &BTreeMap<PathBuf, String>,
) -> Result<ConfigSourceGraph> {
let path = path.as_ref();
let (normalized_path, disk_content) = read_config_source(path)?;
let content = source_overrides
.get(&normalized_path)
.cloned()
.map(Ok)
.unwrap_or_else(|| std::fs::read_to_string(path))
.map_err(|e| ProxyError::Config(e.to_string()))?;
let base_dir = path.parent().unwrap_or(Path::new("."));
.unwrap_or(disk_content);
let base_dir = normalized_path
.parent()
.unwrap_or(Path::new("."))
.to_path_buf();
let mut source_files = BTreeSet::new();
source_files.insert(normalized_path.clone());
let mut source_contents = BTreeMap::new();
source_contents.insert(normalized_path, content.clone());
let processed = preprocess_includes(
&content,
base_dir,
&base_dir,
0,
&mut source_files,
&mut source_contents,
@@ -216,10 +240,39 @@ impl ProxyConfig {
runtime_web::rebuild(self)
}
/// Validates the mixed effective WEB snapshot after restart fields are retained.
pub(crate) fn validate_effective_web(&mut self) -> Result<()> {
validate_web::validate(self)
}
/// Revalidates decoy separation after restart-only listener fields are resolved.
pub(crate) fn validate_web_decoy_listener_separation(&self) -> Result<()> {
validate_web::validate_decoy_listener_separation(self)
}
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
self.runtime_user_auth.as_deref()
}
/// Returns the credential identity frozen into this runtime snapshot.
pub(crate) fn runtime_user_credential_id(&self, user: &str) -> Option<[u8; 16]> {
self.runtime_user_auth()
.and_then(|snapshot| snapshot.credential_id_by_name(user))
.or_else(|| {
self.access
.users
.get(user)
.and_then(|secret| hex::decode(secret).ok())
.and_then(|secret| <[u8; 16]>::try_from(secret).ok())
.map(|secret| {
let digest = sha256(&secret);
let mut credential_id = [0; 16];
credential_id.copy_from_slice(&digest[..16]);
credential_id
})
})
}
/// Validates cross-field configuration invariants after deserialization.
pub fn validate(&self) -> Result<()> {
if self.access.users.is_empty() {
+50 -15
View File
@@ -4,16 +4,27 @@ use std::path::{Path, PathBuf};
use crate::error::{ProxyError, Result};
const MAX_CONFIG_SOURCE_BYTES: usize = 8 * 1024 * 1024;
pub(super) fn normalize_config_path(path: &Path) -> PathBuf {
path.canonicalize().unwrap_or_else(|_| {
if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(path))
.unwrap_or_else(|_| path.to_path_buf())
};
let mut normalized = PathBuf::new();
for component in absolute.components() {
match component {
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
normalized.pop();
}
component => normalized.push(component.as_os_str()),
}
})
}
normalized
}
pub(super) fn hash_rendered_snapshot(rendered: &str) -> u64 {
@@ -22,6 +33,29 @@ pub(super) fn hash_rendered_snapshot(rendered: &str) -> u64 {
hasher.finish()
}
pub(super) fn read_config_source(path: &Path) -> Result<(PathBuf, String)> {
#[cfg(unix)]
let bytes = crate::util::secure_fs::read_regular_limited(path, MAX_CONFIG_SOURCE_BYTES)
.map_err(|error| ProxyError::Config(error.to_string()))?;
#[cfg(not(unix))]
let bytes = std::fs::read(path).map_err(|error| ProxyError::Config(error.to_string()))?;
if bytes.len() > MAX_CONFIG_SOURCE_BYTES {
return Err(ProxyError::Config(format!(
"config source `{}` exceeds {} bytes",
path.display(),
MAX_CONFIG_SOURCE_BYTES
)));
}
let contents = String::from_utf8(bytes).map_err(|error| {
ProxyError::Config(format!(
"config source `{}` is not valid UTF-8: {error}",
path.display()
))
})?;
let normalized = normalize_config_path(path);
Ok((normalized, contents))
}
pub(super) fn preprocess_includes(
content: &str,
base_dir: &Path,
@@ -41,16 +75,17 @@ pub(super) fn preprocess_includes(
if let Some(rest) = rest.strip_prefix('=') {
let path_str = rest.trim().trim_matches('"');
let resolved = base_dir.join(path_str);
let normalized = normalize_config_path(&resolved);
source_files.insert(normalized.clone());
let (normalized, disk_contents) = read_config_source(&resolved)?;
let included = source_overrides
.get(&normalized)
.cloned()
.map(Ok)
.unwrap_or_else(|| std::fs::read_to_string(&resolved))
.map_err(|e| ProxyError::Config(e.to_string()))?;
source_contents.insert(normalized, included.clone());
let included_dir = resolved.parent().unwrap_or(base_dir);
.or_else(|| source_contents.get(&normalized).cloned())
.unwrap_or(disk_contents);
source_files.insert(normalized.clone());
source_contents
.entry(normalized.clone())
.or_insert_with(|| included.clone());
let included_dir = normalized.parent().unwrap_or(base_dir);
output.push_str(&preprocess_includes(
&included,
included_dir,
+87 -1
View File
@@ -2,6 +2,7 @@ use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use crate::crypto::sha256;
use crate::error::{ProxyError, Result};
const ACCESS_SECRET_BYTES: usize = 16;
@@ -11,6 +12,7 @@ const ACCESS_SECRET_BYTES: usize = 16;
pub(crate) struct UserAuthSnapshot {
entries: Vec<UserAuthEntry>,
by_name: HashMap<String, u32>,
by_hint_key: HashMap<u64, Vec<u32>>,
sni_index: HashMap<u64, Vec<u32>>,
sni_initial_index: HashMap<u8, Vec<u32>>,
}
@@ -19,16 +21,23 @@ pub(crate) struct UserAuthSnapshot {
pub(crate) struct UserAuthEntry {
pub(crate) user: String,
pub(crate) secret: [u8; ACCESS_SECRET_BYTES],
/// Stable secret identity used by process-wide admission fencing.
pub(crate) credential_id: [u8; 16],
/// Stable compact key used only to resolve bounded authentication hints.
pub(crate) hint_key: u64,
}
impl UserAuthSnapshot {
pub(super) fn from_users(users: &HashMap<String, String>) -> Result<Self> {
let mut entries = Vec::with_capacity(users.len());
let mut by_name = HashMap::with_capacity(users.len());
let mut by_hint_key = HashMap::with_capacity(users.len());
let mut sni_index = HashMap::with_capacity(users.len());
let mut sni_initial_index = HashMap::with_capacity(users.len());
for (user, secret_hex) in users {
let mut ordered_users = users.iter().collect::<Vec<_>>();
ordered_users.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
for (user, secret_hex) in ordered_users {
let decoded = hex::decode(secret_hex).map_err(|_| ProxyError::InvalidSecret {
user: user.clone(),
reason: "Must be 32 hex characters".to_string(),
@@ -46,11 +55,30 @@ impl UserAuthSnapshot {
let mut secret = [0u8; ACCESS_SECRET_BYTES];
secret.copy_from_slice(&decoded);
let digest = sha256(&secret);
let mut credential_id = [0; 16];
credential_id.copy_from_slice(&digest[..16]);
let hint_key = u64::from_le_bytes([
credential_id[0],
credential_id[1],
credential_id[2],
credential_id[3],
credential_id[4],
credential_id[5],
credential_id[6],
credential_id[7],
]) | 1;
entries.push(UserAuthEntry {
user: user.clone(),
secret,
credential_id,
hint_key,
});
by_name.insert(user.clone(), user_id);
by_hint_key
.entry(hint_key)
.or_insert_with(Vec::new)
.push(user_id);
sni_index
.entry(Self::sni_lookup_hash(user))
.or_insert_with(Vec::new)
@@ -70,6 +98,7 @@ impl UserAuthSnapshot {
Ok(Self {
entries,
by_name,
by_hint_key,
sni_index,
sni_initial_index,
})
@@ -88,6 +117,18 @@ impl UserAuthSnapshot {
self.entries.get(idx)
}
/// Returns the stable credential identity for an exact configured username.
pub(crate) fn credential_id_by_name(&self, user: &str) -> Option<[u8; 16]> {
self.user_id_by_name(user)
.and_then(|user_id| self.entry_by_id(user_id))
.map(|entry| entry.credential_id)
}
/// Returns every bounded authentication candidate sharing a stable hint key.
pub(crate) fn candidate_ids_by_hint_key(&self, hint_key: u64) -> Option<&[u32]> {
self.by_hint_key.get(&hint_key).map(Vec::as_slice)
}
pub(crate) fn sni_candidates(&self, sni: &str) -> Option<&[u32]> {
self.sni_index
.get(&Self::sni_lookup_hash(sni))
@@ -110,3 +151,48 @@ impl UserAuthSnapshot {
hasher.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn credential_hint_survives_positional_id_shift() {
let mut initial = HashMap::new();
initial.insert(
"alice".to_string(),
"11111111111111111111111111111111".to_string(),
);
initial.insert(
"bob".to_string(),
"22222222222222222222222222222222".to_string(),
);
let initial = UserAuthSnapshot::from_users(&initial).unwrap();
let initial_id = initial.user_id_by_name("alice").unwrap();
let hint_key = initial.entry_by_id(initial_id).unwrap().hint_key;
let mut reloaded = HashMap::new();
reloaded.insert(
"aaron".to_string(),
"33333333333333333333333333333333".to_string(),
);
reloaded.insert(
"alice".to_string(),
"11111111111111111111111111111111".to_string(),
);
reloaded.insert(
"bob".to_string(),
"22222222222222222222222222222222".to_string(),
);
let reloaded = UserAuthSnapshot::from_users(&reloaded).unwrap();
let reloaded_id = reloaded.user_id_by_name("alice").unwrap();
assert_ne!(initial_id, reloaded_id);
assert!(
reloaded
.candidate_ids_by_hint_key(hint_key)
.unwrap()
.contains(&reloaded_id)
);
}
}
+239 -119
View File
@@ -5,13 +5,30 @@ use std::path::Path;
use std::sync::Arc;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
use std::ffi::OsString;
#[cfg(unix)]
use std::os::unix::ffi::OsStringExt;
#[cfg(unix)]
use std::os::unix::fs::MetadataExt;
#[cfg(unix)]
use nix::dir::Dir;
#[cfg(unix)]
use nix::fcntl::{OFlag, openat};
#[cfg(unix)]
use nix::sys::stat::Mode;
use bytes::Bytes;
use hmac::{Hmac, Mac};
use sha2::{Digest, Sha256};
use super::*;
#[cfg(unix)]
use crate::util::secure_fs::open_dir_nofollow;
// Path-based static snapshot fallback for platforms without directory descriptors.
#[cfg(not(unix))]
mod static_site_fallback;
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
const WEB_DEBUG_FINGERPRINT_CONTEXT: &[u8] = b"telemt-web-debug-key-fingerprint-v1\0";
@@ -36,6 +53,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
&mut static_bytes,
)?;
let mut profiles = Vec::with_capacity(vhost.profiles.len());
let mut capability_table = Vec::with_capacity(vhost.profiles.len());
let mut capabilities = HashSet::with_capacity(vhost.profiles.len());
for profile in &vhost.profiles {
let user_id = auth.user_id_by_name(&profile.user).ok_or_else(|| {
@@ -62,6 +80,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
host: vhost.host.clone(),
public_addr: vhost.public_addr,
user: profile.user.clone(),
credential_id: auth_entry.credential_id,
secret_mode: profile.secret_mode,
carrier: config.web.carrier,
carrier_negotiation_enabled: config.web.carrier_negotiation_enabled(),
@@ -84,6 +103,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
.max_streams_per_session
.unwrap_or(config.web.limits.max_streams_per_session),
});
capability_table.push(capability);
profiles.push(Arc::clone(&runtime_profile));
runtime_profiles.push(runtime_profile);
}
@@ -91,9 +111,11 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
vhost.host.clone(),
Arc::new(WebRuntimeVhost {
host: vhost.host.clone(),
decoy_fasttrack_mode: config.web.decoy_fasttrack_mode,
decoy,
decoy_header_secs: config.web.timeouts.decoy_header_secs,
profiles,
capabilities: capability_table.into_boxed_slice(),
}),
);
}
@@ -188,34 +210,31 @@ fn load_static_site(
total_files: &mut usize,
total_bytes: &mut usize,
) -> Result<WebStaticSite> {
let root_metadata = fs::symlink_metadata(root).map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static directory `{}`: {error}",
root.display()
))
})?;
if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() {
return Err(ProxyError::Config(format!(
"WEB static directory `{}` must be a real directory, not a symlink",
root.display()
)));
}
let canonical_root = fs::canonicalize(root).map_err(|error| {
ProxyError::Config(format!(
"failed to canonicalize WEB static directory `{}`: {error}",
root.display()
))
})?;
let mut assets = BTreeMap::new();
load_static_directory(
&canonical_root,
&canonical_root,
&mut assets,
total_files,
total_bytes,
limits,
0,
)?;
#[cfg(unix)]
{
let directory = open_static_root(root)?;
load_static_directory(
directory,
Path::new(""),
root,
&mut assets,
total_files,
total_bytes,
limits,
0,
)?;
}
#[cfg(not(unix))]
{
static_site_fallback::load_static_site_by_path(
root,
limits,
&mut assets,
total_files,
total_bytes,
)?;
}
if !assets.contains_key(&format!("/{index}")) {
return Err(ProxyError::Config(format!(
"WEB static directory `{}` does not contain index `{index}`",
@@ -228,54 +247,95 @@ fn load_static_site(
})
}
#[cfg(unix)]
fn open_static_root(root: &Path) -> Result<Dir> {
let descriptor = open_dir_nofollow(root).map_err(|error| {
ProxyError::Config(format!(
"WEB static directory `{}` must be a real directory, not a symlink: {error}",
root.display()
))
})?;
Dir::from_fd(descriptor).map_err(|error| {
ProxyError::Config(format!(
"failed to read WEB static directory `{}`: {error}",
root.display()
))
})
}
#[cfg(unix)]
fn load_static_directory(
mut directory: Dir,
relative: &Path,
root: &Path,
directory: &Path,
assets: &mut BTreeMap<String, WebStaticAsset>,
total_files: &mut usize,
total_bytes: &mut usize,
limits: &WebLimitsConfig,
depth: usize,
) -> Result<()> {
let entries = fs::read_dir(directory).map_err(|error| {
ProxyError::Config(format!(
"failed to read WEB static directory `{}`: {error}",
directory.display()
))
})?;
for entry in entries {
let mut entries = Vec::new();
for entry in directory.iter() {
let entry = entry.map_err(|error| {
ProxyError::Config(format!("failed to read WEB static entry: {error}"))
ProxyError::Config(format!(
"failed to read WEB static directory `{}`: {error}",
root.join(relative).display()
))
})?;
let name = entry.file_name().to_bytes();
if name == b"." || name == b".." {
continue;
}
if *total_files >= limits.max_static_files {
return Err(ProxyError::Config(
"WEB static entries exceed process-wide web.limits.max_static_files".to_string(),
));
}
*total_files += 1;
let path = entry.path();
let file_type = entry.file_type().map_err(|error| {
entries.push(OsString::from_vec(name.to_vec()));
}
entries.sort_unstable();
for name in entries {
let relative_path = relative.join(&name);
let display_path = root.join(&relative_path);
let descriptor = openat(
&directory,
name.as_os_str(),
OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC,
Mode::empty(),
)
.map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static entry `{}`: {error}",
path.display()
"failed to open WEB static entry `{}` without following symlinks: {error}",
display_path.display()
))
})?;
if file_type.is_symlink() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` must not be a symlink",
path.display()
)));
}
if file_type.is_dir() {
let file = fs::File::from(descriptor);
let metadata = file.metadata().map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static entry `{}`: {error}",
display_path.display()
))
})?;
if metadata.is_dir() {
if depth >= MAX_WEB_STATIC_DEPTH {
return Err(ProxyError::Config(format!(
"WEB static directory `{}` exceeds the maximum nesting depth",
path.display()
display_path.display()
)));
}
let descriptor = file.into();
let child = Dir::from_fd(descriptor).map_err(|error| {
ProxyError::Config(format!(
"failed to open WEB static directory `{}`: {error}",
display_path.display()
))
})?;
load_static_directory(
child,
&relative_path,
root,
&path,
assets,
total_files,
total_bytes,
@@ -284,83 +344,107 @@ fn load_static_directory(
)?;
continue;
}
if !file_type.is_file() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` must be a regular file",
path.display()
)));
}
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW);
let file = options.open(&path).map_err(|error| {
ProxyError::Config(format!(
"failed to open WEB static file `{}`: {error}",
path.display()
))
})?;
let metadata = file.metadata().map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static file `{}`: {error}",
path.display()
))
})?;
if !metadata.is_file() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` changed before it was opened",
path.display()
"WEB static entry `{}` must be a regular file",
display_path.display()
)));
}
let file_len = usize::try_from(metadata.len()).map_err(|_| {
ProxyError::Config(format!("WEB static file `{}` is too large", path.display()))
})?;
if file_len > limits.max_static_file_bytes {
return Err(ProxyError::Config(format!(
"WEB static file `{}` exceeds web.limits.max_static_file_bytes",
path.display()
)));
}
*total_bytes = total_bytes.checked_add(file_len).ok_or_else(|| {
ProxyError::Config("WEB static snapshot byte count overflowed usize".to_string())
})?;
if *total_bytes > limits.max_static_bytes {
return Err(ProxyError::Config(
"WEB static snapshots exceed process-wide web.limits.max_static_bytes".to_string(),
));
}
let relative = path.strip_prefix(root).map_err(|_| {
ProxyError::Config("WEB static path escaped its configured root".to_string())
})?;
let route = static_route(relative)?;
let mut body = Vec::with_capacity(file_len);
file.take(limits.max_static_file_bytes as u64 + 1)
.read_to_end(&mut body)
.map_err(|error| {
ProxyError::Config(format!(
"failed to read WEB static file `{}`: {error}",
path.display()
))
})?;
if body.len() != file_len {
return Err(ProxyError::Config(format!(
"WEB static file `{}` changed while its snapshot was built",
path.display()
)));
}
let etag = format!("\"{}\"", hex::encode(Sha256::digest(&body)));
assets.insert(
route,
WebStaticAsset {
body: Bytes::from(body),
content_type: static_content_type(&path),
etag,
},
);
load_static_file(
file,
&metadata,
&relative_path,
&display_path,
assets,
total_bytes,
limits,
)?;
}
Ok(())
}
fn load_static_file(
mut file: fs::File,
metadata: &fs::Metadata,
relative: &Path,
display_path: &Path,
assets: &mut BTreeMap<String, WebStaticAsset>,
total_bytes: &mut usize,
limits: &WebLimitsConfig,
) -> Result<()> {
let file_len = usize::try_from(metadata.len()).map_err(|_| {
ProxyError::Config(format!(
"WEB static file `{}` is too large",
display_path.display()
))
})?;
if file_len > limits.max_static_file_bytes {
return Err(ProxyError::Config(format!(
"WEB static file `{}` exceeds web.limits.max_static_file_bytes",
display_path.display()
)));
}
*total_bytes = total_bytes.checked_add(file_len).ok_or_else(|| {
ProxyError::Config("WEB static snapshot byte count overflowed usize".to_string())
})?;
if *total_bytes > limits.max_static_bytes {
return Err(ProxyError::Config(
"WEB static snapshots exceed process-wide web.limits.max_static_bytes".to_string(),
));
}
let route = static_route(relative)?;
let mut body = Vec::with_capacity(file_len);
file.by_ref()
.take(limits.max_static_file_bytes as u64 + 1)
.read_to_end(&mut body)
.map_err(|error| {
ProxyError::Config(format!(
"failed to read WEB static file `{}`: {error}",
display_path.display()
))
})?;
let final_metadata = file.metadata().map_err(|error| {
ProxyError::Config(format!(
"failed to recheck WEB static file `{}`: {error}",
display_path.display()
))
})?;
if body.len() != file_len || !static_file_version_matches(metadata, &final_metadata) {
return Err(ProxyError::Config(format!(
"WEB static file `{}` changed while its snapshot was built",
display_path.display()
)));
}
let etag = format!("\"{}\"", hex::encode(Sha256::digest(&body)));
assets.insert(
route,
WebStaticAsset {
body: Bytes::from(body),
content_type: static_content_type(relative),
etag,
},
);
Ok(())
}
#[cfg(unix)]
fn static_file_version_matches(before: &fs::Metadata, after: &fs::Metadata) -> bool {
before.dev() == after.dev()
&& before.ino() == after.ino()
&& before.len() == after.len()
&& before.mtime() == after.mtime()
&& before.mtime_nsec() == after.mtime_nsec()
&& before.ctime() == after.ctime()
&& before.ctime_nsec() == after.ctime_nsec()
}
#[cfg(not(unix))]
fn static_file_version_matches(before: &fs::Metadata, after: &fs::Metadata) -> bool {
before.len() == after.len()
&& before.modified().ok() == after.modified().ok()
&& before.created().ok() == after.created().ok()
}
fn static_route(relative: &Path) -> Result<String> {
let mut route = String::new();
for component in relative.components() {
@@ -420,4 +504,40 @@ mod tests {
"IpJrt3e7sKtzPyoXy6w-Zj6GGEvsvclN66JzQEfPYLA"
);
}
#[cfg(unix)]
#[test]
fn static_snapshot_remains_anchored_after_root_path_replacement() {
use std::os::unix::fs::symlink;
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("site");
let detached = temp.path().join("detached");
let replacement = temp.path().join("replacement");
fs::create_dir(&root).unwrap();
fs::write(root.join("index.html"), b"original").unwrap();
fs::create_dir(&replacement).unwrap();
fs::write(replacement.join("index.html"), b"replacement").unwrap();
let directory = open_static_root(&root).unwrap();
fs::rename(&root, &detached).unwrap();
symlink(&replacement, &root).unwrap();
let mut assets = BTreeMap::new();
let mut total_files = 0;
let mut total_bytes = 0;
load_static_directory(
directory,
Path::new(""),
&root,
&mut assets,
&mut total_files,
&mut total_bytes,
&WebLimitsConfig::default(),
0,
)
.unwrap();
assert_eq!(assets["/index.html"].body.as_ref(), b"original");
}
}
@@ -0,0 +1,138 @@
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
use super::*;
/// Builds a bounded static-site snapshot on platforms without directory descriptors.
pub(super) fn load_static_site_by_path(
root: &Path,
limits: &WebLimitsConfig,
assets: &mut BTreeMap<String, WebStaticAsset>,
total_files: &mut usize,
total_bytes: &mut usize,
) -> Result<()> {
let root_metadata = fs::symlink_metadata(root).map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static directory `{}`: {error}",
root.display()
))
})?;
if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() {
return Err(ProxyError::Config(format!(
"WEB static directory `{}` must be a real directory, not a symlink",
root.display()
)));
}
let canonical_root = fs::canonicalize(root).map_err(|error| {
ProxyError::Config(format!(
"failed to canonicalize WEB static directory `{}`: {error}",
root.display()
))
})?;
load_static_directory(
&canonical_root,
&canonical_root,
assets,
total_files,
total_bytes,
limits,
0,
)
}
fn load_static_directory(
root: &Path,
directory: &Path,
assets: &mut BTreeMap<String, WebStaticAsset>,
total_files: &mut usize,
total_bytes: &mut usize,
limits: &WebLimitsConfig,
depth: usize,
) -> Result<()> {
let entries = fs::read_dir(directory).map_err(|error| {
ProxyError::Config(format!(
"failed to read WEB static directory `{}`: {error}",
directory.display()
))
})?;
for entry in entries {
let entry = entry.map_err(|error| {
ProxyError::Config(format!("failed to read WEB static entry: {error}"))
})?;
if *total_files >= limits.max_static_files {
return Err(ProxyError::Config(
"WEB static entries exceed process-wide web.limits.max_static_files".to_string(),
));
}
*total_files += 1;
let path = entry.path();
let file_type = entry.file_type().map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static entry `{}`: {error}",
path.display()
))
})?;
if file_type.is_symlink() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` must not be a symlink",
path.display()
)));
}
if file_type.is_dir() {
if depth >= MAX_WEB_STATIC_DEPTH {
return Err(ProxyError::Config(format!(
"WEB static directory `{}` exceeds the maximum nesting depth",
path.display()
)));
}
load_static_directory(
root,
&path,
assets,
total_files,
total_bytes,
limits,
depth + 1,
)?;
continue;
}
if !file_type.is_file() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` must be a regular file",
path.display()
)));
}
let file = fs::File::open(&path).map_err(|error| {
ProxyError::Config(format!(
"failed to open WEB static file `{}`: {error}",
path.display()
))
})?;
let metadata = file.metadata().map_err(|error| {
ProxyError::Config(format!(
"failed to inspect WEB static file `{}`: {error}",
path.display()
))
})?;
if !metadata.is_file() {
return Err(ProxyError::Config(format!(
"WEB static entry `{}` changed before it was opened",
path.display()
)));
}
let relative = path.strip_prefix(root).map_err(|_| {
ProxyError::Config("WEB static path escaped its configured root".to_string())
})?;
load_static_file(
file,
&metadata,
relative,
&path,
assets,
total_bytes,
limits,
)?;
}
Ok(())
}
+7
View File
@@ -161,6 +161,7 @@ const GENERAL_CONFIG_KEYS: &[&str] = &[
"proxy_secret_auto_reload_secs",
"proxy_config_auto_reload_secs",
"me_reinit_singleflight",
"me_reinit_max_concurrency",
"me_reinit_trigger_channel",
"me_reinit_coalesce_window_ms",
"me_deterministic_writer_sort",
@@ -265,6 +266,8 @@ const WEB_CONFIG_KEYS: &[&str] = &[
"carriers",
"carrier_learning",
"carrier_negotiation_aggressiveness",
"decoy_fasttrack_mode",
"http_connection_capacity_action",
"debug",
"limits",
"timeouts",
@@ -278,6 +281,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
"carrier_batch_bytes",
"max_frames_per_body",
"max_http_connections",
"max_http_overload_connections",
"max_http_handlers",
"max_lane_open_waits_per_session",
"pending_bytes_per_lane",
@@ -322,6 +326,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
"enabled",
"sideband",
"capture_lifecycle",
"capture_headers",
"capture_timings",
@@ -341,6 +346,7 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"long_poll_secs",
"bridge_request_secs",
"bridge_retry_secs",
"bridge_recovery_secs",
"carrier_probe_coalesce_ms",
"lane_open_wait_secs",
"carrier_health_secs",
@@ -354,6 +360,7 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"bootstrap_lifetime_secs",
"reconnect_grace_secs",
"http_idle_secs",
"http_overload_timeout_ms",
"shutdown_secs",
"decoy_header_secs",
];
+8 -2
View File
@@ -143,9 +143,15 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
));
}
if config.general.me_reinit_trigger_channel == 0 {
if !(1..=8).contains(&config.general.me_reinit_max_concurrency) {
return Err(ProxyError::Config(
"general.me_reinit_trigger_channel must be > 0".to_string(),
"general.me_reinit_max_concurrency must be within [1, 8]".to_string(),
));
}
if !(1..=4096).contains(&config.general.me_reinit_trigger_channel) {
return Err(ProxyError::Config(
"general.me_reinit_trigger_channel must be within [1, 4096]".to_string(),
));
}
+14
View File
@@ -196,6 +196,13 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
"access.user_rate_limits.{user} must set at least one non-zero direction"
)));
}
for (direction, value) in [("up_bps", limit.up_bps), ("down_bps", limit.down_bps)] {
if value > MAX_RATE_LIMIT_BPS {
return Err(ProxyError::Config(format!(
"access.user_rate_limits.{user}.{direction} must be within [0, {MAX_RATE_LIMIT_BPS}]"
)));
}
}
}
for (cidr, limit) in &config.access.cidr_rate_limits {
@@ -204,6 +211,13 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
"access.cidr_rate_limits.{cidr} must set at least one non-zero direction"
)));
}
for (direction, value) in [("up_bps", limit.up_bps), ("down_bps", limit.down_bps)] {
if value > MAX_RATE_LIMIT_BPS {
return Err(ProxyError::Config(format!(
"access.cidr_rate_limits.{cidr}.{direction} must be within [0, {MAX_RATE_LIMIT_BPS}]"
)));
}
}
}
let mut cidr_auto_templates = HashSet::new();
for cidr in config.access.cidr_rate_limits.keys() {
+61 -190
View File
@@ -83,9 +83,59 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
timeouts::validate(&config.web.timeouts)?;
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
validate_vhosts(config)?;
validate_decoy_listener_separation(config)?;
Ok(())
}
/// Rejects a direct decoy recursion into an effective WEB listener.
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
let web_listeners = config
.server
.listeners
.iter()
.filter(|listener| listener.transport == ListenerTransport::Web)
.filter(|listener| {
(listener.ip.is_ipv4() && config.network.ipv4)
|| (listener.ip.is_ipv6() && config.network.ipv6 != Some(false))
})
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
.collect::<Vec<_>>();
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
let WebDecoyConfig::HttpUpstream { upstream } = &vhost.decoy else {
continue;
};
let parsed = url::Url::parse(upstream).map_err(|error| {
ProxyError::Config(format!(
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
))
})?;
let Some(port) = parsed.port_or_known_default() else {
continue;
};
let upstream_ip = match parsed.host() {
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
_ => continue,
};
let upstream_addr = SocketAddr::new(upstream_ip, port);
if web_listeners
.iter()
.any(|listener| listener_covers(*listener, upstream_addr))
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
));
}
}
Ok(())
}
fn listener_covers(listener: SocketAddr, target: SocketAddr) -> bool {
listener.port() == target.port()
&& (listener.ip() == target.ip()
|| (listener.ip().is_unspecified() && listener.is_ipv4() == target.is_ipv4()))
}
fn validate_web_listener(
config: &ProxyConfig,
idx: usize,
@@ -169,6 +219,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
let positive = [
("max_http_connections", limits.max_http_connections),
(
"max_http_overload_connections",
limits.max_http_overload_connections,
),
("max_http_handlers", limits.max_http_handlers),
(
"max_lane_open_waits_per_session",
@@ -222,6 +276,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
}
for (field, value) in [
("max_http_connections", limits.max_http_connections),
(
"max_http_overload_connections",
limits.max_http_overload_connections,
),
("max_http_handlers", limits.max_http_handlers),
("max_body_readers", limits.max_body_readers),
("max_body_bytes_global", limits.max_body_bytes_global),
@@ -356,196 +414,9 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
Ok(())
}
fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
let limits = &config.web.limits;
if config.web.vhosts.len() > limits.max_vhosts {
return config_error("web.vhosts exceeds web.limits.max_vhosts");
}
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
let mut profile_count = 0usize;
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
if !hosts.insert(vhost.host.clone()) {
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
}
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
));
}
if config.web.enabled && vhost.profiles.is_empty() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
));
}
validate_decoy(vhost_idx, &vhost.decoy)?;
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
if profile.user.is_empty() || profile.user.len() > 64 {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
));
}
if !config.access.users.contains_key(&profile.user) {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
profile.user
));
}
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
return config_error(&format!(
"duplicate WEB profile for user `{}` in vhost `{}`",
profile.user, vhost.host
));
}
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
let max_streams_per_session = profile
.max_streams_per_session
.unwrap_or(limits.max_streams_per_session);
if profile.max_sessions == Some(0)
|| profile
.max_sessions
.is_some_and(|value| value > limits.max_sessions_global)
|| profile.max_streams == Some(0)
|| profile
.max_streams
.is_some_and(|value| value > limits.max_streams_global)
|| profile.max_streams_per_session == Some(0)
|| profile
.max_streams_per_session
.is_some_and(|value| value > limits.max_streams_per_session)
|| max_streams_per_session > max_streams
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
));
}
profile_count = profile_count.checked_add(1).ok_or_else(|| {
ProxyError::Config("WEB profile count overflowed usize".to_string())
})?;
}
}
if profile_count > limits.max_profiles {
return config_error("WEB profiles exceed web.limits.max_profiles");
}
Ok(())
}
fn normalize_web_host(value: &str, field: &str) -> Result<String> {
let input = value.trim();
if input.is_empty()
|| input.ends_with('.')
|| input
.chars()
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
{
return config_error(&format!(
"{field} must be a hostname without a port, path, credentials, or trailing dot"
));
}
let host = normalize_domain_to_ascii(input, field)?;
if host.len() > 253
|| !host.contains('.')
|| host.parse::<IpAddr>().is_ok()
|| web_host_last_label_is_numeric(&host)
{
return config_error(&format!(
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
));
}
for label in host.split('.') {
if label.is_empty()
|| label.len() > 63
|| label.starts_with('-')
|| label.ends_with('-')
|| !label
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
{
return config_error(&format!(
"{field} contains a hostname label rejected by Telegram Desktop"
));
}
}
Ok(host)
}
fn web_host_last_label_is_numeric(host: &str) -> bool {
let label = host.rsplit('.').next().unwrap_or_default();
let digits = label
.strip_prefix("0x")
.or_else(|| label.strip_prefix("0X"));
if let Some(digits) = digits {
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
}
label.bytes().all(|byte| byte.is_ascii_digit())
}
fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
match decoy {
WebDecoyConfig::HttpUpstream { upstream } => {
let parsed = url::Url::parse(upstream).map_err(|error| {
ProxyError::Config(format!(
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
))
})?;
if parsed.scheme() != "http"
|| parsed.host_str().is_none()
|| !parsed.username().is_empty()
|| parsed.password().is_some()
|| parsed.query().is_some()
|| parsed.fragment().is_some()
|| parsed.path() != "/"
|| parsed.port() == Some(0)
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
));
}
let ip = match parsed.host() {
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
_ => {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
));
}
};
let private = match ip {
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
IpAddr::V6(ip) => {
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
}
};
if !private {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
));
}
}
WebDecoyConfig::StaticDirectory { directory, index } => {
if !directory.is_absolute() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
));
}
if index.is_empty()
|| index.contains('\\')
|| std::path::Path::new(index).components().count() != 1
|| matches!(index.as_str(), "." | "..")
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
));
}
}
}
Ok(())
}
fn config_error<T>(message: &str) -> Result<T> {
Err(ProxyError::Config(message.to_string()))
}
// Virtual-host, hostname, and decoy validation.
mod vhosts;
use vhosts::*;
#[cfg(test)]
mod tests;
+26
View File
@@ -5,6 +5,8 @@ const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
const WEB_LANE_STATE_BYTES: usize = 512;
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 32;
/// Validates process-wide body, header, queue, static, and debug reservations.
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
@@ -30,6 +32,12 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
})?;
let overload_connection_reservation = limits
.max_http_overload_connections
.checked_mul(WEB_OVERLOAD_CONNECTION_BYTES)
.ok_or_else(|| {
ProxyError::Config("web.limits HTTP overload reservations overflow usize".to_string())
})?;
let debug_ring_index = limits
.debug_records_capacity
.checked_mul(std::mem::size_of::<usize>())
@@ -58,6 +66,12 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
.ok_or_else(|| {
ProxyError::Config("web.carrier learning reservation overflowed usize".to_string())
})?;
let capability_index_reservation = limits
.max_profiles
.checked_mul(WEB_CAPABILITY_INDEX_ENTRY_BYTES)
.ok_or_else(|| {
ProxyError::Config("web capability index reservation overflowed usize".to_string())
})?;
let lane_state_reservation = limits
.max_streams_per_session
.checked_add(limits.max_tombstones_per_session)
@@ -75,8 +89,10 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
.and_then(|value| value.checked_add(status_pages))
.and_then(|value| value.checked_add(debug_reservation))
.and_then(|value| value.checked_add(carrier_learning_reservation))
.and_then(|value| value.checked_add(capability_index_reservation))
.and_then(|value| value.checked_add(lane_state_reservation))
.and_then(|value| value.checked_add(http_header_reservation))
.and_then(|value| value.checked_add(overload_connection_reservation))
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
if reserved > limits.memory_envelope_bytes
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
@@ -103,4 +119,14 @@ mod tests {
};
assert!(validate(&previous_envelope).is_err());
}
#[test]
fn capability_index_reservation_rejects_size_overflow() {
let limits = WebLimitsConfig {
max_profiles: usize::MAX,
..WebLimitsConfig::default()
};
let error = validate(&limits).unwrap_err().to_string();
assert!(error.contains("web capability index reservation overflowed usize"));
}
}
+1 -1
View File
@@ -16,7 +16,7 @@ pub(super) fn validate(config: &WebConfig) -> Result<Vec<WebCarrier>> {
let candidates = config.carrier_candidates();
if config.carrier_negotiation_enabled()
&& config.carrier_learning
&& config.limits.max_carrier_learning_entries < 3
&& config.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
{
return config_error(
"web.limits.max_carrier_learning_entries must be >= 3 when carrier learning is enabled",
+6
View File
@@ -2,6 +2,9 @@ use super::*;
/// Validates WEB request, learning, and lifecycle timeouts.
pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
if !(1..=60_000).contains(&timeouts.http_overload_timeout_ms) {
return config_error("web.timeouts.http_overload_timeout_ms must be within [1, 60000]");
}
let values = [
("header_secs", timeouts.header_secs),
("body_secs", timeouts.body_secs),
@@ -39,6 +42,9 @@ pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
if !(1..=300).contains(&timeouts.bridge_retry_secs) {
return config_error("web.timeouts.bridge_retry_secs must be within [1, 300]");
}
if !(1..=60).contains(&timeouts.bridge_recovery_secs) {
return config_error("web.timeouts.bridge_recovery_secs must be within [1, 60]");
}
if timeouts.bridge_request_secs > timeouts.bridge_retry_secs {
return config_error("web.timeouts.bridge_request_secs must not exceed bridge_retry_secs");
}
+192
View File
@@ -0,0 +1,192 @@
use super::*;
pub(super) fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
let limits = &config.web.limits;
if config.web.vhosts.len() > limits.max_vhosts {
return config_error("web.vhosts exceeds web.limits.max_vhosts");
}
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
let mut profile_count = 0usize;
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
if !hosts.insert(vhost.host.clone()) {
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
}
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
));
}
if config.web.enabled && vhost.profiles.is_empty() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
));
}
validate_decoy(vhost_idx, &vhost.decoy)?;
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
if profile.user.is_empty() || profile.user.len() > 64 {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
));
}
if !config.access.users.contains_key(&profile.user) {
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
profile.user
));
}
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
return config_error(&format!(
"duplicate WEB profile for user `{}` in vhost `{}`",
profile.user, vhost.host
));
}
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
let max_streams_per_session = profile
.max_streams_per_session
.unwrap_or(limits.max_streams_per_session);
if profile.max_sessions == Some(0)
|| profile
.max_sessions
.is_some_and(|value| value > limits.max_sessions_global)
|| profile.max_streams == Some(0)
|| profile
.max_streams
.is_some_and(|value| value > limits.max_streams_global)
|| profile.max_streams_per_session == Some(0)
|| profile
.max_streams_per_session
.is_some_and(|value| value > limits.max_streams_per_session)
|| max_streams_per_session > max_streams
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
));
}
profile_count = profile_count.checked_add(1).ok_or_else(|| {
ProxyError::Config("WEB profile count overflowed usize".to_string())
})?;
}
}
if profile_count > limits.max_profiles {
return config_error("WEB profiles exceed web.limits.max_profiles");
}
Ok(())
}
pub(super) fn normalize_web_host(value: &str, field: &str) -> Result<String> {
let input = value.trim();
if input.is_empty()
|| input.ends_with('.')
|| input
.chars()
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
{
return config_error(&format!(
"{field} must be a hostname without a port, path, credentials, or trailing dot"
));
}
let host = normalize_domain_to_ascii(input, field)?;
if host.len() > 253
|| !host.contains('.')
|| host.parse::<IpAddr>().is_ok()
|| web_host_last_label_is_numeric(&host)
{
return config_error(&format!(
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
));
}
for label in host.split('.') {
if label.is_empty()
|| label.len() > 63
|| label.starts_with('-')
|| label.ends_with('-')
|| !label
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
{
return config_error(&format!(
"{field} contains a hostname label rejected by Telegram Desktop"
));
}
}
Ok(host)
}
pub(super) fn web_host_last_label_is_numeric(host: &str) -> bool {
let label = host.rsplit('.').next().unwrap_or_default();
let digits = label
.strip_prefix("0x")
.or_else(|| label.strip_prefix("0X"));
if let Some(digits) = digits {
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
}
label.bytes().all(|byte| byte.is_ascii_digit())
}
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
match decoy {
WebDecoyConfig::HttpUpstream { upstream } => {
let parsed = url::Url::parse(upstream).map_err(|error| {
ProxyError::Config(format!(
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
))
})?;
if parsed.scheme() != "http"
|| parsed.host_str().is_none()
|| !parsed.username().is_empty()
|| parsed.password().is_some()
|| parsed.query().is_some()
|| parsed.fragment().is_some()
|| parsed.path() != "/"
|| parsed.port() == Some(0)
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
));
}
let ip = match parsed.host() {
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
_ => {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
));
}
};
let private = match ip {
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
IpAddr::V6(ip) => {
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
}
};
if !private {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
));
}
}
WebDecoyConfig::StaticDirectory { directory, index } => {
if !directory.is_absolute() {
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
));
}
if index.is_empty()
|| index.contains('\\')
|| std::path::Path::new(index).components().count() != 1
|| matches!(index.as_str(), "." | "..")
{
return config_error(&format!(
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
));
}
}
}
Ok(())
}
pub(super) fn config_error<T>(message: &str) -> Result<T> {
Err(ProxyError::Config(message.to_string()))
}
+2
View File
@@ -46,6 +46,8 @@ mod legacy_policy_tests;
mod me_route_tests;
#[path = "load_basic_tests/me_startup_tests.rs"]
mod me_startup_tests;
#[path = "load_basic_tests/source_security_tests.rs"]
mod source_security_tests;
#[path = "load_basic_tests/synlimit_mss_tests.rs"]
mod synlimit_mss_tests;
#[path = "load_basic_tests/tls_fetch_tests.rs"]
@@ -288,6 +288,68 @@ fn cidr_rate_limits_reject_duplicate_normalized_auto_templates() {
assert!(error.contains("duplicates normalized auto-template *6/128"));
}
#[test]
fn rate_limits_accept_the_packed_counter_maximum() {
let cfg = load_config_from_temp_toml(
r#"
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
[access.user_rate_limits]
user = { up_bps = 100000000000, down_bps = 0 }
[access.cidr_rate_limits]
"203.0.113.0/24" = { up_bps = 0, down_bps = 100000000000 }
"#,
);
assert_eq!(cfg.access.user_rate_limits["user"].up_bps, 100_000_000_000);
assert_eq!(
cfg.access.cidr_rate_limits[&CidrRateLimitKey::Network("203.0.113.0/24".parse().unwrap())]
.down_bps,
100_000_000_000
);
}
#[test]
fn user_rate_limits_reject_values_above_the_packed_counter_maximum() {
let error = load_config_error_from_temp_toml(
r#"
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
[access.user_rate_limits]
user = { up_bps = 100000000001, down_bps = 0 }
"#,
);
assert!(error.contains("access.user_rate_limits.user.up_bps must be within"));
}
#[test]
fn cidr_rate_limits_reject_values_above_the_packed_counter_maximum() {
let error = load_config_error_from_temp_toml(
r#"
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
[access.cidr_rate_limits]
"203.0.113.0/24" = { up_bps = 0, down_bps = 100000000001 }
"#,
);
assert!(error.contains("access.cidr_rate_limits.203.0.113.0/24.down_bps must be within"));
}
#[test]
fn file_logging_requires_path() {
let error = load_config_error_from_temp_toml(
@@ -0,0 +1,42 @@
#[cfg(unix)]
use std::os::unix::fs::symlink;
use super::*;
#[cfg(unix)]
#[test]
fn config_loader_rejects_final_and_intermediate_symlinks() {
let directory = tempfile::tempdir().unwrap();
let real_directory = directory.path().join("real");
let linked_directory = directory.path().join("linked");
std::fs::create_dir(&real_directory).unwrap();
let real_config = real_directory.join("config.toml");
let final_link = directory.path().join("config.toml");
std::fs::write(&real_config, "[general]\n").unwrap();
symlink(&real_config, &final_link).unwrap();
symlink(&real_directory, &linked_directory).unwrap();
assert!(ProxyConfig::load(&final_link).is_err());
assert!(ProxyConfig::load(linked_directory.join("config.toml")).is_err());
}
#[test]
fn config_loader_rejects_oversized_source() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("config.toml");
std::fs::write(&path, vec![b' '; 8 * 1024 * 1024 + 1]).unwrap();
let error = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(error.contains("size limit") || error.contains("exceeds"));
}
#[cfg(unix)]
#[test]
fn config_loader_rejects_fifo_without_blocking() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("config.toml");
nix::unistd::mkfifo(&path, nix::sys::stat::Mode::S_IRUSR).unwrap();
assert!(ProxyConfig::load(&path).is_err());
}
+137 -3
View File
@@ -41,6 +41,9 @@ fn web_config_builds_canonical_runtime_snapshot() {
.get("proxy.example.com")
.expect("canonical WEB vhost");
assert_eq!(vhost.profiles.len(), 1);
assert_eq!(vhost.capabilities.len(), vhost.profiles.len());
assert_eq!(vhost.capabilities[0], vhost.profiles[0].capability);
assert_eq!(vhost.decoy_fasttrack_mode, WebDecoyFastTrackMode::Off);
assert_eq!(vhost.profiles[0].user, "alice");
assert_eq!(vhost.profiles[0].secret_mode, WebSecretMode::Dd);
assert_eq!(vhost.profiles[0].carrier, WebCarrier::HttpsLanes);
@@ -56,6 +59,109 @@ fn web_config_builds_canonical_runtime_snapshot() {
);
}
#[test]
fn web_decoy_fasttrack_mode_is_typed_and_defaults_off() {
let defaults = ProxyConfig::default();
assert_eq!(
defaults.web.decoy_fasttrack_mode,
WebDecoyFastTrackMode::Off
);
for (token, expected) in [
("shadow", WebDecoyFastTrackMode::Shadow),
("enforce", WebDecoyFastTrackMode::Enforce),
] {
let configured = WEB_CONFIG.replace(
"carrier = \"https-lanes\"",
&format!("carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"{token}\""),
);
let config = load_config_from_temp_toml(&configured);
assert_eq!(config.web.decoy_fasttrack_mode, expected);
assert_eq!(
config.web.runtime.as_ref().unwrap().vhosts["proxy.example.com"].decoy_fasttrack_mode,
expected
);
}
let invalid = WEB_CONFIG.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"automatic\"",
);
assert!(load_config_error_from_temp_toml(&invalid).contains("decoy_fasttrack_mode"));
}
#[test]
fn web_http_connection_capacity_policy_is_bounded_and_configurable() {
let configured = WEB_CONFIG
.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"wait\"",
)
.replace(
"[[web.vhosts]]",
"[web.limits]\nmax_http_overload_connections = 23\n\n[web.timeouts]\nhttp_overload_timeout_ms = 731\n\n[[web.vhosts]]",
);
let config = load_config_from_temp_toml(&configured);
assert_eq!(
config.web.http_connection_capacity_action,
WebHttpConnectionCapacityAction::Wait
);
assert_eq!(config.web.limits.max_http_overload_connections, 23);
assert_eq!(config.web.timeouts.http_overload_timeout_ms, 731);
let defaults = ProxyConfig::default();
assert_eq!(
defaults.web.http_connection_capacity_action,
WebHttpConnectionCapacityAction::Drop
);
assert_eq!(defaults.web.limits.max_http_overload_connections, 64);
assert_eq!(defaults.web.timeouts.http_overload_timeout_ms, 250);
}
#[test]
fn web_http_connection_capacity_policy_rejects_unknown_or_unbounded_values() {
let unknown = WEB_CONFIG.replace(
"carrier = \"https-lanes\"",
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"queue\"",
);
assert!(load_config_error_from_temp_toml(&unknown).contains("http_connection_capacity_action"));
for timeout in [0, 60_001] {
let invalid = WEB_CONFIG.replace(
"[[web.vhosts]]",
&format!("[web.timeouts]\nhttp_overload_timeout_ms = {timeout}\n\n[[web.vhosts]]"),
);
assert!(
load_config_error_from_temp_toml(&invalid)
.contains("web.timeouts.http_overload_timeout_ms")
);
}
let no_overload_slots = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.limits]\nmax_http_overload_connections = 0\n\n[[web.vhosts]]",
);
assert!(
load_config_error_from_temp_toml(&no_overload_slots)
.contains("web.limits.max_http_overload_connections")
);
}
#[test]
fn web_decoy_rejects_direct_and_wildcard_listener_loops() {
let direct = WEB_CONFIG.replace("http://127.0.0.1:18081", "http://127.0.0.1:18080");
assert!(
load_config_error_from_temp_toml(&direct).contains("decoy upstream overlaps WEB listener")
);
let wildcard = direct.replace("ip = \"127.0.0.1\"", "ip = \"0.0.0.0\"");
assert!(
load_config_error_from_temp_toml(&wildcard)
.contains("decoy upstream overlaps WEB listener")
);
}
#[test]
fn web_profile_user_labels_are_bounded_for_runtime_status() {
let user = "a".repeat(65);
@@ -122,7 +228,7 @@ fn web_carriers_reject_true_empty_and_duplicates() {
fn web_carrier_and_bridge_deadlines_are_configurable() {
let configured = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
);
let config = load_config_from_temp_toml(&configured);
assert_eq!(
@@ -132,6 +238,7 @@ fn web_carrier_and_bridge_deadlines_are_configurable() {
assert_eq!(config.web.timeouts.carrier_learning_secs, 30);
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
}
@@ -139,13 +246,14 @@ fn web_carrier_and_bridge_deadlines_are_configurable() {
fn web_bridge_deadlines_are_known_in_strict_mode() {
let configured = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
);
let configured = format!("[general]\nconfig_strict = true\n{configured}");
let config = load_config_from_temp_toml(&configured);
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
}
@@ -156,6 +264,8 @@ fn web_bridge_deadlines_are_bounded_and_ordered() {
("bridge_request_secs", "61"),
("bridge_retry_secs", "0"),
("bridge_retry_secs", "301"),
("bridge_recovery_secs", "0"),
("bridge_recovery_secs", "61"),
("carrier_probe_coalesce_ms", "11"),
] {
let invalid = WEB_CONFIG.replace(
@@ -191,17 +301,31 @@ fn web_carrier_learning_capacity_must_remain_nonzero() {
#[test]
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
assert!(!crate::config::WebDebugConfig::default().sideband);
let mut ineffective = crate::config::WebDebugConfig {
enabled: true,
sideband: true,
..Default::default()
};
ineffective.capture_lifecycle = false;
assert!(!ineffective.bridge_diagnostics_enabled());
let configured = WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
"[web.debug]\nenabled = true\nsideband = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
);
let config = load_config_from_temp_toml(&configured);
assert!(config.web.debug.enabled);
assert!(config.web.debug.sideband);
assert!(config.web.debug.bridge_diagnostics_enabled());
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
assert_eq!(config.web.debug.default_window_secs, 180);
assert_eq!(config.web.debug.max_window_secs, 900);
let strict = format!("[general]\nconfig_strict = true\n{configured}");
assert!(load_config_from_temp_toml(&strict).web.debug.sideband);
let old_name = format!(
"[general]\nconfig_strict = true\n{}",
WEB_CONFIG.replace(
@@ -211,6 +335,16 @@ fn web_debug_table_uses_debug_name_and_bounded_defaults() {
);
let error = load_config_error_from_temp_toml(&old_name);
assert!(error.contains("web.trace"));
let old_parameter = format!(
"[general]\nconfig_strict = true\n{}",
WEB_CONFIG.replace(
"[[web.vhosts]]",
"[web.debug]\nbridge_diagnostics = true\n\n[[web.vhosts]]",
)
);
let error = load_config_error_from_temp_toml(&old_parameter);
assert!(error.contains("web.debug.bridge_diagnostics"));
}
#[test]
+5 -3
View File
@@ -31,7 +31,7 @@ mod web_debug;
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
#[allow(unused_imports)]
pub(crate) use access::{CidrAutoTemplate, CidrAutoTemplateFamily};
pub(crate) use access::{CidrAutoTemplate, CidrAutoTemplateFamily, MAX_RATE_LIMIT_BPS};
pub use api::{ApiConfig, ApiGrayAction};
pub use censorship::{
AntiCensorshipConfig, ExclusiveMaskTarget, TlsFetchConfig, TlsFetchProfile, UnknownSniAction,
@@ -53,13 +53,15 @@ pub use server::{
};
#[allow(unused_imports)]
pub use web::{
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebLimitsConfig,
WebProfileConfig, WebSecretMode, WebTimeoutsConfig, WebVhostConfig,
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebDecoyFastTrackMode,
WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig, WebSecretMode,
WebTimeoutsConfig, WebVhostConfig,
};
pub(crate) use web::{
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
WebStaticSite,
};
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
#[allow(unused_imports)]
pub use web_carrier::{WebCarrier, WebCarriers};
pub(crate) use web_debug::web_debug_fits_limits;
+5 -2
View File
@@ -1,5 +1,8 @@
use super::*;
/// Highest rate that fits one packed 20 ms shaping epoch.
pub(crate) const MAX_RATE_LIMIT_BPS: u64 = 100_000_000_000;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AccessConfig {
#[serde(default = "default_access_users")]
@@ -260,10 +263,10 @@ fn parse_cidr_auto_prefix(
/// Transport rate limit in bits-per-second.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RateLimitBps {
/// Upload direction limit in bits-per-second; `0` means unlimited.
/// Upload limit in bits-per-second within `0..=100_000_000_000`; `0` means unlimited.
#[serde(default)]
pub up_bps: u64,
/// Download direction limit in bits-per-second; `0` means unlimited.
/// Download limit in bits-per-second within `0..=100_000_000_000`; `0` means unlimited.
#[serde(default)]
pub down_bps: u64,
}
+3
View File
@@ -441,6 +441,9 @@ pub struct GeneralConfig {
/// Serialize ME reinit cycles across all trigger sources.
#[serde(default = "default_me_reinit_singleflight")]
pub me_reinit_singleflight: bool,
/// Maximum concurrent ME reinit warmups when single-flight mode is disabled.
#[serde(default = "default_me_reinit_max_concurrency")]
pub me_reinit_max_concurrency: usize,
/// Trigger queue capacity for reinit scheduler.
#[serde(default = "default_me_reinit_trigger_channel")]
pub me_reinit_trigger_channel: usize,
+1
View File
@@ -159,6 +159,7 @@ impl Default for GeneralConfig {
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
me_reinit_singleflight: default_me_reinit_singleflight(),
me_reinit_max_concurrency: default_me_reinit_max_concurrency(),
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
+34 -83
View File
@@ -12,6 +12,12 @@ use super::web_debug::WebDebugConfig;
// Serialized WEB defaults remain separate from the runtime data model.
mod defaults;
use defaults::*;
// Decoy fast-track policy remains isolated from the bulky WEB data model.
mod fasttrack;
pub use fasttrack::WebDecoyFastTrackMode;
// Accepted-socket overload policy remains separate from the bulky WEB data model.
mod overload;
pub use overload::WebHttpConnectionCapacityAction;
/// Client-facing secret representation used to derive a WEB capability.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -95,6 +101,9 @@ pub struct WebLimitsConfig {
/// Process-wide accepted WEB HTTP connection ceiling.
#[serde(default = "default_web_max_http_connections")]
pub max_http_connections: usize,
/// Accepted overload sockets allowed to wait or emit a retryable response.
#[serde(default = "default_web_max_http_overload_connections")]
pub max_http_overload_connections: usize,
/// Process-wide concurrently executing HTTP handler ceiling.
#[serde(default = "default_web_max_http_handlers")]
pub max_http_handlers: usize,
@@ -194,7 +203,7 @@ pub struct WebLimitsConfig {
/// Process-wide retained and in-flight WEB debug byte ceiling.
#[serde(default = "default_web_debug_bytes_global")]
pub debug_bytes_global: usize,
/// Declared process envelope for HTTP, queues, lane state, learning, and static snapshots.
/// Declared process envelope for HTTP, queues, capabilities, learning, and static snapshots.
#[serde(default = "default_web_memory_envelope_bytes")]
pub memory_envelope_bytes: usize,
/// Sustained process-wide bootstrap issuance rate.
@@ -226,6 +235,7 @@ impl Default for WebLimitsConfig {
carrier_batch_bytes: default_web_carrier_batch_bytes(),
max_frames_per_body: default_web_max_frames_per_body(),
max_http_connections: default_web_max_http_connections(),
max_http_overload_connections: default_web_max_http_overload_connections(),
max_http_handlers: default_web_max_http_handlers(),
max_lane_open_waits_per_session: default_web_max_lane_open_waits_per_session(),
pending_bytes_per_lane: default_web_pending_bytes_per_lane(),
@@ -294,6 +304,9 @@ pub struct WebTimeoutsConfig {
/// Absolute generated-bridge budget for one retryable HTTP operation.
#[serde(default = "default_web_bridge_retry_secs")]
pub bridge_retry_secs: u64,
/// Absolute post-commit budget for one surviving bridge recovery epoch.
#[serde(default = "default_web_bridge_recovery_secs")]
pub bridge_recovery_secs: u64,
/// Optional delay for coalescing the first OPEN with immediate DATA.
#[serde(default = "default_web_carrier_probe_coalesce_ms")]
pub carrier_probe_coalesce_ms: u64,
@@ -327,12 +340,15 @@ pub struct WebTimeoutsConfig {
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
#[serde(default = "default_web_bootstrap_lifetime_secs")]
pub bootstrap_lifetime_secs: u64,
/// Maximum carrier inactivity before a session is closed.
/// Maximum validated peer inactivity before a session is closed.
#[serde(default = "default_web_reconnect_grace_secs")]
pub reconnect_grace_secs: u64,
/// Maximum idle lifetime of a WEB HTTP keep-alive connection.
#[serde(default = "default_web_http_idle_secs")]
pub http_idle_secs: u64,
/// Per-phase wait or response deadline for accepted HTTP overload sockets.
#[serde(default = "default_web_http_overload_timeout_ms")]
pub http_overload_timeout_ms: u64,
/// Maximum graceful wait for WEB connections and process-owned tasks.
#[serde(default = "default_web_shutdown_secs")]
pub shutdown_secs: u64,
@@ -351,6 +367,7 @@ impl Default for WebTimeoutsConfig {
long_poll_secs: default_web_long_poll_timeout_secs(),
bridge_request_secs: default_web_bridge_request_secs(),
bridge_retry_secs: default_web_bridge_retry_secs(),
bridge_recovery_secs: default_web_bridge_recovery_secs(),
carrier_probe_coalesce_ms: default_web_carrier_probe_coalesce_ms(),
lane_open_wait_secs: default_web_lane_open_wait_secs(),
carrier_health_secs: default_web_carrier_health_secs(),
@@ -364,6 +381,7 @@ impl Default for WebTimeoutsConfig {
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
reconnect_grace_secs: default_web_reconnect_grace_secs(),
http_idle_secs: default_web_http_idle_secs(),
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
shutdown_secs: default_web_shutdown_secs(),
decoy_header_secs: default_web_decoy_header_timeout_secs(),
}
@@ -401,6 +419,12 @@ pub struct WebConfig {
/// Controls the evidence thresholds used by automatic carrier ranking.
#[serde(default)]
pub carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness,
/// Restart-only capability-scan policy for structurally impossible bridge requests.
#[serde(default)]
pub decoy_fasttrack_mode: WebDecoyFastTrackMode,
/// Action applied when accepted HTTP connection capacity is exhausted.
#[serde(default)]
pub http_connection_capacity_action: WebHttpConnectionCapacityAction,
/// Hard process and protocol limits.
#[serde(default)]
pub limits: WebLimitsConfig,
@@ -447,6 +471,8 @@ impl Default for WebConfig {
carriers: WebCarriers::default(),
carrier_learning: default_web_carrier_learning(),
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
decoy_fasttrack_mode: WebDecoyFastTrackMode::default(),
http_connection_capacity_action: WebHttpConnectionCapacityAction::default(),
limits: WebLimitsConfig::default(),
debug: WebDebugConfig::default(),
timeouts: WebTimeoutsConfig::default(),
@@ -456,84 +482,9 @@ impl Default for WebConfig {
}
}
/// Precomputed WEB configuration consumed by listener hot paths.
#[derive(Debug)]
pub(crate) struct WebRuntimeConfig {
/// Canonical host lookup used by HTTP request routing.
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
/// Flat profile inventory used by startup link emission.
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
}
/// Precomputed immutable virtual-host data.
#[derive(Debug)]
pub(crate) struct WebRuntimeVhost {
/// Canonical lowercase ACE hostname.
pub(crate) host: String,
/// Immutable ordinary-site fallback snapshot.
pub(crate) decoy: WebRuntimeDecoy,
/// Upstream connect and response-head deadline.
pub(crate) decoy_header_secs: u64,
/// Exact capability profiles accepted by this host.
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
}
/// Precomputed exact-user capability entry.
#[derive(Debug)]
pub(crate) struct WebRuntimeProfile {
/// Canonical host that owns this profile.
pub(crate) host: String,
/// Stable public destination tuple supplied to relay routing.
pub(crate) public_addr: SocketAddr,
/// Exact access user authenticated by logical streams.
pub(crate) user: String,
/// Client secret representation and inner protocol policy.
pub(crate) secret_mode: WebSecretMode,
/// Sole carrier or final fallback frozen into the issued bridge policy.
pub(crate) carrier: WebCarrier,
/// Whether an explicit carrier list enabled automatic negotiation.
pub(crate) carrier_negotiation_enabled: bool,
/// Whether automatic outcomes consult and update process-local evidence.
pub(crate) carrier_learning: bool,
/// Ordered negotiation candidates including the fallback carrier exactly once.
pub(crate) carriers: Arc<[WebCarrier]>,
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
/// HMAC-derived bridge capability.
pub(crate) capability: [u8; 32],
/// Non-secret domain-separated client-secret fingerprint for debugging.
pub(crate) key_fingerprint: String,
/// Per-profile live session ceiling.
pub(crate) max_sessions: usize,
/// Per-profile live logical-stream ceiling.
pub(crate) max_streams: usize,
/// Per-session live relay-task ceiling.
pub(crate) max_streams_per_session: usize,
}
/// Runtime-ready ordinary-site fallback.
#[derive(Debug)]
pub(crate) enum WebRuntimeDecoy {
HttpUpstream { addr: SocketAddr, authority: String },
StaticDirectory(Arc<WebStaticSite>),
}
/// Immutable bounded static-site snapshot.
#[derive(Debug)]
pub(crate) struct WebStaticSite {
/// Canonical URL-path to immutable response asset mapping.
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
/// Configured root index file name.
pub(crate) index: String,
}
/// One immutable static response body and metadata.
#[derive(Debug)]
pub(crate) struct WebStaticAsset {
/// Immutable response body retained by the runtime snapshot.
pub(crate) body: Bytes,
/// Extension-derived static content type.
pub(crate) content_type: &'static str,
/// Strong SHA-256 entity tag.
pub(crate) etag: String,
}
// Immutable runtime WEB configuration consumed by hot paths.
mod runtime;
pub(crate) use runtime::{
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
WebStaticSite,
};
+3
View File
@@ -40,6 +40,7 @@ usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
usize_default!(default_web_max_frames_per_body, 4096);
usize_default!(default_web_max_http_connections, 1024);
usize_default!(default_web_max_http_overload_connections, 64);
usize_default!(default_web_max_http_handlers, 512);
usize_default!(default_web_max_lane_open_waits_per_session, 16);
usize_default!(default_web_pending_bytes_per_lane, 8 * 1024 * 1024);
@@ -87,6 +88,7 @@ u64_default!(default_web_stream_first_byte_secs, 30);
u64_default!(default_web_long_poll_timeout_secs, 25);
u64_default!(default_web_bridge_request_secs, 10);
u64_default!(default_web_bridge_retry_secs, 90);
u64_default!(default_web_bridge_recovery_secs, 15);
u64_default!(default_web_carrier_probe_coalesce_ms, 0);
u64_default!(default_web_lane_open_wait_secs, 2);
u64_default!(default_web_carrier_health_secs, 30);
@@ -105,5 +107,6 @@ pub(super) fn default_web_carrier_learning() -> bool {
u64_default!(default_web_bootstrap_lifetime_secs, 120);
u64_default!(default_web_reconnect_grace_secs, 120);
u64_default!(default_web_http_idle_secs, 75);
u64_default!(default_web_http_overload_timeout_ms, 250);
u64_default!(default_web_shutdown_secs, 15);
u64_default!(default_web_decoy_header_timeout_secs, 30);
+28
View File
@@ -0,0 +1,28 @@
use serde::{Deserialize, Serialize};
/// Capability-scan policy for structurally impossible WEB bridge requests.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WebDecoyFastTrackMode {
/// Preserve the legacy full scan without collecting fast-track decisions.
#[default]
Off,
/// Record eligible requests while preserving the legacy full scan.
Shadow,
/// Skip the scan only when the public request shape cannot open a bridge.
Enforce,
}
impl WebDecoyFastTrackMode {
/// Complete fixed mode set in stable API and metric order.
pub const ALL: [Self; 3] = [Self::Off, Self::Shadow, Self::Enforce];
/// Returns the stable serialized mode token.
pub const fn as_str(self) -> &'static str {
match self {
Self::Off => "off",
Self::Shadow => "shadow",
Self::Enforce => "enforce",
}
}
}
+14
View File
@@ -0,0 +1,14 @@
use serde::{Deserialize, Serialize};
/// Action applied after an accepted WEB socket finds HTTP connection capacity exhausted.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WebHttpConnectionCapacityAction {
/// Close the accepted socket without emitting an HTTP response.
#[default]
Drop,
/// Wait for ordinary HTTP connection capacity under the overload deadline.
Wait,
/// Emit a bounded retryable HTTP response without parsing the request.
Respond,
}
+89
View File
@@ -0,0 +1,89 @@
use super::*;
/// Precomputed WEB configuration consumed by listener hot paths.
#[derive(Debug)]
pub(crate) struct WebRuntimeConfig {
/// Canonical host lookup used by HTTP request routing.
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
/// Flat profile inventory used by startup link emission.
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
}
/// Precomputed immutable virtual-host data.
#[derive(Debug)]
pub(crate) struct WebRuntimeVhost {
/// Canonical lowercase ACE hostname.
pub(crate) host: String,
/// Restart-frozen decoy capability-scan policy.
pub(crate) decoy_fasttrack_mode: WebDecoyFastTrackMode,
/// Immutable ordinary-site fallback snapshot.
pub(crate) decoy: WebRuntimeDecoy,
/// Upstream connect and response-head deadline.
pub(crate) decoy_header_secs: u64,
/// Exact capability profiles accepted by this host.
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
/// Contiguous capability table aligned one-to-one with `profiles`.
pub(crate) capabilities: Box<[[u8; 32]]>,
}
/// Precomputed exact-user capability entry.
#[derive(Debug)]
pub(crate) struct WebRuntimeProfile {
/// Canonical host that owns this profile.
pub(crate) host: String,
/// Stable public destination tuple supplied to relay routing.
pub(crate) public_addr: SocketAddr,
/// Exact access user authenticated by logical streams.
pub(crate) user: String,
/// Stable credential identity used by process-wide admission fencing.
pub(crate) credential_id: [u8; 16],
/// Client secret representation and inner protocol policy.
pub(crate) secret_mode: WebSecretMode,
/// Sole carrier or final fallback frozen into the issued bridge policy.
pub(crate) carrier: WebCarrier,
/// Whether an explicit carrier list enabled automatic negotiation.
pub(crate) carrier_negotiation_enabled: bool,
/// Whether automatic outcomes consult and update process-local evidence.
pub(crate) carrier_learning: bool,
/// Ordered negotiation candidates including the fallback carrier exactly once.
pub(crate) carriers: Arc<[WebCarrier]>,
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
/// HMAC-derived bridge capability.
pub(crate) capability: [u8; 32],
/// Non-secret domain-separated client-secret fingerprint for debugging.
pub(crate) key_fingerprint: String,
/// Per-profile live session ceiling.
pub(crate) max_sessions: usize,
/// Per-profile live logical-stream ceiling.
pub(crate) max_streams: usize,
/// Per-session live relay-task ceiling.
pub(crate) max_streams_per_session: usize,
}
/// Runtime-ready ordinary-site fallback.
#[derive(Debug)]
pub(crate) enum WebRuntimeDecoy {
HttpUpstream { addr: SocketAddr, authority: String },
StaticDirectory(Arc<WebStaticSite>),
}
/// Immutable bounded static-site snapshot.
#[derive(Debug)]
pub(crate) struct WebStaticSite {
/// Canonical URL-path to immutable response asset mapping.
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
/// Configured root index file name.
pub(crate) index: String,
}
/// One immutable static response body and metadata.
#[derive(Debug)]
pub(crate) struct WebStaticAsset {
/// Immutable response body retained by the runtime snapshot.
pub(crate) body: Bytes,
/// Extension-derived static content type.
pub(crate) content_type: &'static str,
/// Strong SHA-256 entity tag.
pub(crate) etag: String,
}
+3
View File
@@ -1,5 +1,8 @@
use serde::{Deserialize, Serialize};
/// Minimum restart-owned entries required for one complete learning sample.
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
/// Carrier selected for one newly issued WEB relay session.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
+11
View File
@@ -23,6 +23,9 @@ pub struct WebDebugConfig {
/// Enables process-owned WEB debug collection.
#[serde(default)]
pub enabled: bool,
/// Enables generated-bridge diagnostic reports over the HTTPS sideband.
#[serde(default)]
pub sideband: bool,
/// Records typed bridge, session, stream, handshake, and relay events.
#[serde(default = "default_true")]
pub capture_lifecycle: bool,
@@ -56,6 +59,7 @@ impl Default for WebDebugConfig {
fn default() -> Self {
Self {
enabled: false,
sideband: false,
capture_lifecycle: true,
capture_headers: true,
capture_timings: true,
@@ -69,6 +73,13 @@ impl Default for WebDebugConfig {
}
}
impl WebDebugConfig {
/// Returns whether newly issued bridges may report diagnostic lifecycle events.
pub(crate) const fn bridge_diagnostics_enabled(&self) -> bool {
self.enabled && self.sideband && self.capture_lifecycle
}
}
fn default_true() -> bool {
true
}
+10 -409
View File
@@ -1,20 +1,23 @@
use std::collections::BTreeSet;
use std::net::IpAddr;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncWriteExt;
use tokio::process::Command;
use tokio::sync::{mpsc, watch};
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use tracing::{info, warn};
use crate::config::{ConntrackBackend, ConntrackMode, ProxyConfig};
use crate::config::ProxyConfig;
use crate::proxy::middle_relay::note_global_relay_pressure;
use crate::proxy::shared_state::{ConntrackCloseEvent, ConntrackCloseReason, ProxySharedState};
use crate::stats::Stats;
// Privileged netfilter rule and conntrack helper execution.
mod firewall;
pub(crate) use firewall::FirewallAuthority;
use firewall::{
DeleteOutcome, delete_conntrack_entry, effective_conntrack_enabled, probe_runtime_support,
};
const CONNTRACK_EVENT_QUEUE_CAPACITY: usize = 32_768;
const PRESSURE_RELEASE_TICKS: u8 = 3;
const PRESSURE_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);
@@ -112,7 +115,6 @@ async fn run_conntrack_controller_worker(
runtime_support,
false,
);
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
loop {
tokio::select! {
@@ -126,7 +128,6 @@ async fn run_conntrack_controller_worker(
effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, runtime_support, pressure_state.active);
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
}
event = close_rx.recv() => {
let Some(event) = event else {
@@ -310,383 +311,6 @@ fn update_pressure_state(
state.low_streak = 0;
}
async fn reconcile_rules(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
stats: &Stats,
) {
if !cfg.server.conntrack_control.inline_conntrack_control {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(true);
return;
}
if !effective_conntrack_enabled(cfg, runtime_support) {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(false);
return;
}
let backend = runtime_support
.netfilter_backend
.expect("netfilter backend must be available for effective conntrack control");
let apply_result = match backend {
NetfilterBackend::Nftables => apply_nft_rules(cfg).await,
NetfilterBackend::Iptables => apply_iptables_rules(cfg).await,
};
if let Err(error) = apply_result {
warn!(error = %error, "Failed to reconcile conntrack/notrack rules");
stats.set_conntrack_rule_apply_ok(false);
} else {
stats.set_conntrack_rule_apply_ok(true);
}
}
fn probe_runtime_support(configured_backend: ConntrackBackend) -> ConntrackRuntimeSupport {
ConntrackRuntimeSupport {
netfilter_backend: pick_backend(configured_backend),
has_cap_net_admin: has_cap_net_admin(),
has_conntrack_binary: command_exists("conntrack"),
}
}
fn effective_conntrack_enabled(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
) -> bool {
cfg.server.conntrack_control.inline_conntrack_control
&& runtime_support.has_cap_net_admin
&& runtime_support.netfilter_backend.is_some()
&& runtime_support.has_conntrack_binary
}
fn pick_backend(configured: ConntrackBackend) -> Option<NetfilterBackend> {
match configured {
ConntrackBackend::Auto => {
if command_exists("nft") {
Some(NetfilterBackend::Nftables)
} else if command_exists("iptables") {
Some(NetfilterBackend::Iptables)
} else {
None
}
}
ConntrackBackend::Nftables => command_exists("nft").then_some(NetfilterBackend::Nftables),
ConntrackBackend::Iptables => {
command_exists("iptables").then_some(NetfilterBackend::Iptables)
}
}
}
fn command_exists(binary: &str) -> bool {
let Some(path_var) = std::env::var_os("PATH") else {
return false;
};
std::env::split_paths(&path_var).any(|dir| {
let candidate: PathBuf = dir.join(binary);
candidate.exists() && candidate.is_file()
})
}
fn listener_port_set(cfg: &ProxyConfig) -> Vec<u16> {
let mut ports: BTreeSet<u16> = BTreeSet::new();
if cfg.server.listeners.is_empty() {
ports.insert(cfg.server.port);
} else {
for listener in &cfg.server.listeners {
ports.insert(listener.port.unwrap_or(cfg.server.port));
}
}
ports.into_iter().collect()
}
fn notrack_targets(cfg: &ProxyConfig) -> (Vec<(Option<IpAddr>, u16)>, Vec<(Option<IpAddr>, u16)>) {
let mode = cfg.server.conntrack_control.mode;
let mut v4_targets: BTreeSet<(Option<IpAddr>, u16)> = BTreeSet::new();
let mut v6_targets: BTreeSet<(Option<IpAddr>, u16)> = BTreeSet::new();
match mode {
ConntrackMode::Tracked => {}
ConntrackMode::Notrack => {
if cfg.server.listeners.is_empty() {
let port = cfg.server.port;
if let Some(ipv4) = cfg
.server
.listen_addr_ipv4
.as_ref()
.and_then(|s| s.parse::<IpAddr>().ok())
{
if ipv4.is_unspecified() {
v4_targets.insert((None, port));
} else {
v4_targets.insert((Some(ipv4), port));
}
}
if let Some(ipv6) = cfg
.server
.listen_addr_ipv6
.as_ref()
.and_then(|s| s.parse::<IpAddr>().ok())
{
if ipv6.is_unspecified() {
v6_targets.insert((None, port));
} else {
v6_targets.insert((Some(ipv6), port));
}
}
} else {
for listener in &cfg.server.listeners {
let port = listener.port.unwrap_or(cfg.server.port);
if listener.ip.is_ipv4() {
if listener.ip.is_unspecified() {
v4_targets.insert((None, port));
} else {
v4_targets.insert((Some(listener.ip), port));
}
} else if listener.ip.is_unspecified() {
v6_targets.insert((None, port));
} else {
v6_targets.insert((Some(listener.ip), port));
}
}
}
}
ConntrackMode::Hybrid => {
let ports = listener_port_set(cfg);
for ip in &cfg.server.conntrack_control.hybrid_listener_ips {
if ip.is_ipv4() {
for port in &ports {
v4_targets.insert((Some(*ip), *port));
}
} else {
for port in &ports {
v6_targets.insert((Some(*ip), *port));
}
}
}
}
}
(
v4_targets.into_iter().collect(),
v6_targets.into_iter().collect(),
)
}
async fn apply_nft_rules(cfg: &ProxyConfig) -> Result<(), String> {
let _ = run_command(
"nft",
&["delete", "table", "inet", "telemt_conntrack"],
None,
)
.await;
if matches!(cfg.server.conntrack_control.mode, ConntrackMode::Tracked) {
return Ok(());
}
let (v4_targets, v6_targets) = notrack_targets(cfg);
let mut rules = Vec::new();
for (ip, port) in v4_targets {
let rule = if let Some(ip) = ip {
format!("tcp dport {} ip daddr {} notrack", port, ip)
} else {
format!("tcp dport {} notrack", port)
};
rules.push(rule);
}
for (ip, port) in v6_targets {
let rule = if let Some(ip) = ip {
format!("tcp dport {} ip6 daddr {} notrack", port, ip)
} else {
format!("tcp dport {} notrack", port)
};
rules.push(rule);
}
let rule_blob = if rules.is_empty() {
String::new()
} else {
format!(" {}\n", rules.join("\n "))
};
let script = format!(
"table inet telemt_conntrack {{\n chain preraw {{\n type filter hook prerouting priority raw; policy accept;\n{rule_blob} }}\n}}\n"
);
run_command("nft", &["-f", "-"], Some(script)).await
}
async fn apply_iptables_rules(cfg: &ProxyConfig) -> Result<(), String> {
apply_iptables_rules_for_binary("iptables", cfg, true).await?;
apply_iptables_rules_for_binary("ip6tables", cfg, false).await?;
Ok(())
}
async fn apply_iptables_rules_for_binary(
binary: &str,
cfg: &ProxyConfig,
ipv4: bool,
) -> Result<(), String> {
if !command_exists(binary) {
return Ok(());
}
let chain = "TELEMT_NOTRACK";
let _ = run_command(
binary,
&["-t", "raw", "-D", "PREROUTING", "-j", chain],
None,
)
.await;
let _ = run_command(binary, &["-t", "raw", "-F", chain], None).await;
let _ = run_command(binary, &["-t", "raw", "-X", chain], None).await;
if matches!(cfg.server.conntrack_control.mode, ConntrackMode::Tracked) {
return Ok(());
}
run_command(binary, &["-t", "raw", "-N", chain], None).await?;
run_command(binary, &["-t", "raw", "-F", chain], None).await?;
if run_command(
binary,
&["-t", "raw", "-C", "PREROUTING", "-j", chain],
None,
)
.await
.is_err()
{
run_command(
binary,
&["-t", "raw", "-I", "PREROUTING", "1", "-j", chain],
None,
)
.await?;
}
let (v4_targets, v6_targets) = notrack_targets(cfg);
let selected = if ipv4 { v4_targets } else { v6_targets };
for (ip, port) in selected {
let mut args = vec![
"-t".to_string(),
"raw".to_string(),
"-A".to_string(),
chain.to_string(),
"-p".to_string(),
"tcp".to_string(),
"--dport".to_string(),
port.to_string(),
];
if let Some(ip) = ip {
args.push("-d".to_string());
args.push(ip.to_string());
}
args.push("-j".to_string());
args.push("CT".to_string());
args.push("--notrack".to_string());
let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect();
run_command(binary, &arg_refs, None).await?;
}
Ok(())
}
async fn clear_notrack_rules_all_backends() {
let _ = run_command(
"nft",
&["delete", "table", "inet", "telemt_conntrack"],
None,
)
.await;
let _ = run_command(
"iptables",
&["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
None,
)
.await;
let _ = run_command("iptables", &["-t", "raw", "-F", "TELEMT_NOTRACK"], None).await;
let _ = run_command("iptables", &["-t", "raw", "-X", "TELEMT_NOTRACK"], None).await;
let _ = run_command(
"ip6tables",
&["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
None,
)
.await;
let _ = run_command("ip6tables", &["-t", "raw", "-F", "TELEMT_NOTRACK"], None).await;
let _ = run_command("ip6tables", &["-t", "raw", "-X", "TELEMT_NOTRACK"], None).await;
}
enum DeleteOutcome {
Deleted,
NotFound,
Error,
}
async fn delete_conntrack_entry(event: ConntrackCloseEvent) -> DeleteOutcome {
if !command_exists("conntrack") {
return DeleteOutcome::Error;
}
let args = vec![
"-D".to_string(),
"-p".to_string(),
"tcp".to_string(),
"-s".to_string(),
event.src.ip().to_string(),
"--sport".to_string(),
event.src.port().to_string(),
"-d".to_string(),
event.dst.ip().to_string(),
"--dport".to_string(),
event.dst.port().to_string(),
];
let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect();
match run_command("conntrack", &arg_refs, None).await {
Ok(()) => DeleteOutcome::Deleted,
Err(error) => {
if error.contains("0 flow entries have been deleted") {
DeleteOutcome::NotFound
} else {
debug!(error = %error, "conntrack delete failed");
DeleteOutcome::Error
}
}
}
}
async fn run_command(binary: &str, args: &[&str], stdin: Option<String>) -> Result<(), String> {
if !command_exists(binary) {
return Err(format!("{binary} is not available"));
}
let mut command = Command::new(binary);
command.args(args);
if stdin.is_some() {
command.stdin(std::process::Stdio::piped());
}
command.stdout(std::process::Stdio::null());
command.stderr(std::process::Stdio::piped());
let mut child = command
.spawn()
.map_err(|e| format!("spawn {binary} failed: {e}"))?;
if let Some(blob) = stdin
&& let Some(mut writer) = child.stdin.take()
{
writer
.write_all(blob.as_bytes())
.await
.map_err(|e| format!("stdin write {binary} failed: {e}"))?;
}
let output = child
.wait_with_output()
.await
.map_err(|e| format!("wait {binary} failed: {e}"))?;
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
Err(if stderr.is_empty() {
format!("{binary} exited with status {}", output.status)
} else {
stderr
})
}
fn fd_usage_pct() -> Option<u8> {
let soft_limit = nofile_soft_limit()?;
if soft_limit == 0 {
@@ -715,29 +339,6 @@ fn nofile_soft_limit() -> Option<u64> {
}
}
fn has_cap_net_admin() -> bool {
#[cfg(target_os = "linux")]
{
let Ok(status) = std::fs::read_to_string("/proc/self/status") else {
return false;
};
for line in status.lines() {
if let Some(raw) = line.strip_prefix("CapEff:") {
let caps = raw.trim();
if let Ok(bits) = u64::from_str_radix(caps, 16) {
const CAP_NET_ADMIN_BIT: u64 = 12;
return (bits & (1u64 << CAP_NET_ADMIN_BIT)) != 0;
}
}
}
false
}
#[cfg(not(target_os = "linux"))]
{
false
}
}
#[cfg(test)]
mod tests {
use super::*;
+18
View File
@@ -0,0 +1,18 @@
// Process-owned firewall reconciliation and privileged conntrack helpers.
mod actor;
mod command;
mod iptables;
mod model;
mod nftables;
mod runtime;
mod transaction;
pub(crate) use actor::FirewallAuthority;
pub(super) use runtime::{
DeleteOutcome, delete_conntrack_entry, effective_conntrack_enabled, probe_runtime_support,
};
#[cfg(test)]
#[path = "firewall/tests.rs"]
mod tests;
+371
View File
@@ -0,0 +1,371 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::sync::{Notify, watch};
use tokio_util::sync::CancellationToken;
use tracing::warn;
use crate::config::ProxyConfig;
use crate::maestro::control_plane::ProcessControlPlane;
use crate::stats::Stats;
use super::command::{CommandError, FirewallCommandRunner, SystemCommandRunner};
use super::model::{AppliedPlan, AppliedState, DesiredPolicy, DesiredState};
use super::transaction::{InterruptibleRunner, reconcile_once, recover_to_empty};
const SHUTDOWN_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30);
const SHUTDOWN_WAIT_TIMEOUT: Duration = Duration::from_secs(35);
const INITIAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(65);
const RETRY_DELAYS: [Duration; 6] = [
Duration::from_secs(1),
Duration::from_secs(2),
Duration::from_secs(4),
Duration::from_secs(8),
Duration::from_secs(16),
Duration::from_secs(30),
];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum ReconcileOutcome {
Applied,
Failed,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) struct ReconcileStatus {
pub(super) generation: u64,
pub(super) outcome: ReconcileOutcome,
}
/// Process-owned publisher and shutdown owner for conntrack firewall policy.
#[derive(Clone)]
pub(crate) struct FirewallAuthority {
desired_tx: watch::Sender<Option<DesiredState>>,
status_rx: watch::Receiver<Option<ReconcileStatus>>,
terminal: CancellationToken,
closed: Arc<AtomicBool>,
completed_flag: Arc<AtomicBool>,
cleanup_succeeded: Arc<AtomicBool>,
completed: Arc<Notify>,
}
impl FirewallAuthority {
/// Starts the single process-owned firewall reconciler.
pub(crate) fn spawn(control_plane: &ProcessControlPlane) -> Result<Self, String> {
let (desired_tx, desired_rx) = watch::channel(None);
let (status_tx, status_rx) = watch::channel(None);
let terminal = CancellationToken::new();
let closed = Arc::new(AtomicBool::new(false));
let completed_flag = Arc::new(AtomicBool::new(false));
let cleanup_succeeded = Arc::new(AtomicBool::new(false));
let completed = Arc::new(Notify::new());
let actor = FirewallReconciler::new(
SystemCommandRunner,
desired_rx,
status_tx,
terminal.clone(),
Arc::clone(&closed),
Arc::clone(&completed_flag),
Arc::clone(&cleanup_succeeded),
Arc::clone(&completed),
);
control_plane
.spawn_cooperative(move |process_cancellation| async move {
actor.run(process_cancellation).await;
})
.map_err(|_| {
"process control-plane admission closed before conntrack firewall startup"
.to_string()
})?;
Ok(Self {
desired_tx,
status_rx,
terminal,
closed,
completed_flag,
cleanup_succeeded,
completed,
})
}
/// Publishes policy only after its runtime generation becomes active.
pub(crate) fn publish(
&self,
generation: u64,
config: Arc<ProxyConfig>,
stats: Arc<Stats>,
) -> bool {
if self.closed.load(Ordering::Acquire) {
stats.set_conntrack_rule_apply_ok(false);
return false;
}
stats.set_conntrack_rule_apply_ok(false);
self.desired_tx.send_replace(Some(DesiredState {
generation,
policy: DesiredPolicy::from_config(config.as_ref()),
stats,
}));
true
}
/// Publishes startup policy and waits for its first bounded attempt.
pub(crate) async fn publish_initial(
&self,
generation: u64,
config: Arc<ProxyConfig>,
stats: Arc<Stats>,
) -> bool {
let mut status_rx = self.status_rx.clone();
if !self.publish(generation, config, stats) {
return false;
}
tokio::time::timeout(INITIAL_RECONCILE_TIMEOUT, async move {
loop {
if let Some(status) = *status_rx.borrow_and_update()
&& status.generation == generation
{
return status.outcome == ReconcileOutcome::Applied;
}
if status_rx.changed().await.is_err() {
return false;
}
}
})
.await
.unwrap_or(false)
}
/// Stops policy admission and waits for bounded terminal cleanup.
pub(crate) async fn shutdown_and_clear(&self) -> bool {
let completed = self.completed.notified();
tokio::pin!(completed);
completed.as_mut().enable();
if !self.closed.swap(true, Ordering::AcqRel) {
if let Some(desired) = self.desired_tx.borrow().as_ref() {
desired.stats.set_conntrack_rule_apply_ok(false);
}
self.terminal.cancel();
}
let finished = self.completed_flag.load(Ordering::Acquire)
|| tokio::time::timeout(SHUTDOWN_WAIT_TIMEOUT, completed)
.await
.is_ok();
finished && self.cleanup_succeeded.load(Ordering::Acquire)
}
}
struct CompletionGuard {
closed: Arc<AtomicBool>,
completed_flag: Arc<AtomicBool>,
completed: Arc<Notify>,
}
impl Drop for CompletionGuard {
fn drop(&mut self) {
self.closed.store(true, Ordering::Release);
self.completed_flag.store(true, Ordering::Release);
self.completed.notify_waiters();
}
}
pub(super) struct FirewallReconciler<R> {
runner: R,
desired_rx: watch::Receiver<Option<DesiredState>>,
status_tx: watch::Sender<Option<ReconcileStatus>>,
terminal: CancellationToken,
completion: CompletionGuard,
cleanup_succeeded: Arc<AtomicBool>,
applied: AppliedState,
last_generation: u64,
last_policy: Option<DesiredPolicy>,
last_stats: Option<Arc<Stats>>,
}
impl<R> FirewallReconciler<R>
where
R: FirewallCommandRunner + 'static,
{
pub(super) fn new(
runner: R,
desired_rx: watch::Receiver<Option<DesiredState>>,
status_tx: watch::Sender<Option<ReconcileStatus>>,
terminal: CancellationToken,
closed: Arc<AtomicBool>,
completed_flag: Arc<AtomicBool>,
cleanup_succeeded: Arc<AtomicBool>,
completed: Arc<Notify>,
) -> Self {
Self {
runner,
desired_rx,
status_tx,
terminal,
completion: CompletionGuard {
closed,
completed_flag,
completed,
},
cleanup_succeeded,
applied: AppliedState::Unknown,
last_generation: 0,
last_policy: None,
last_stats: None,
}
}
pub(super) async fn run(mut self, process_cancellation: CancellationToken) {
let mut current = None;
let mut retry_index = 0usize;
'run: loop {
if current.is_none() {
let changed = tokio::select! {
biased;
_ = self.terminal.cancelled() => false,
_ = process_cancellation.cancelled() => false,
changed = self.desired_rx.changed() => changed.is_ok(),
};
if !changed {
break;
}
current = self.take_latest_desired();
retry_index = 0;
if current.is_none() {
continue;
}
}
let desired = current.as_ref().expect("desired state is present").clone();
let interruptible =
InterruptibleRunner::new(&self.runner, &self.terminal, &process_cancellation);
let result =
reconcile_once(&interruptible, &interruptible, &mut self.applied, &desired).await;
match result {
Ok(()) => {
desired
.stats
.increment_conntrack_rule_reconcile_success_total();
desired.stats.set_conntrack_rule_apply_ok(true);
self.status_tx.send_replace(Some(ReconcileStatus {
generation: desired.generation,
outcome: ReconcileOutcome::Applied,
}));
if self.terminal.is_cancelled() || process_cancellation.is_cancelled() {
desired.stats.set_conntrack_rule_apply_ok(false);
break;
}
current = None;
retry_index = 0;
}
Err(failure) if failure.cancelled => break,
Err(failure) => {
desired
.stats
.increment_conntrack_rule_reconcile_error_total();
desired.stats.set_conntrack_rule_apply_ok(false);
if let Some(rollback_succeeded) = failure.rollback_succeeded {
if rollback_succeeded {
desired
.stats
.increment_conntrack_rule_rollback_success_total();
} else {
desired
.stats
.increment_conntrack_rule_rollback_error_total();
}
}
self.status_tx.send_replace(Some(ReconcileStatus {
generation: desired.generation,
outcome: ReconcileOutcome::Failed,
}));
warn!(
generation = desired.generation,
error = %failure.message,
"Failed to reconcile conntrack firewall policy"
);
let delay = RETRY_DELAYS[retry_index.min(RETRY_DELAYS.len() - 1)];
retry_index = retry_index.saturating_add(1);
let retry_deadline = tokio::time::Instant::now() + delay;
loop {
tokio::select! {
biased;
_ = self.terminal.cancelled() => break 'run,
_ = process_cancellation.cancelled() => break 'run,
changed = self.desired_rx.changed() => {
if changed.is_err() {
break 'run;
}
if let Some(next) = self.take_latest_desired() {
current = Some(next);
retry_index = 0;
break;
}
}
_ = tokio::time::sleep_until(retry_deadline) => break,
}
}
}
}
if self.terminal.is_cancelled() || process_cancellation.is_cancelled() {
break;
}
if self.desired_rx.has_changed().unwrap_or(false) {
if let Some(next) = self.take_latest_desired() {
current = Some(next);
retry_index = 0;
}
}
}
if let Some(stats) = &self.last_stats {
stats.set_conntrack_rule_apply_ok(false);
}
if let Err(error) =
tokio::time::timeout(SHUTDOWN_CLEANUP_TIMEOUT, recover_to_empty(&self.runner))
.await
.unwrap_or_else(|_| {
Err(CommandError::failed("firewall shutdown cleanup timed out"))
})
{
warn!(error = %error, "Failed to clear conntrack firewall policy during shutdown");
} else {
self.applied = AppliedState::Known(AppliedPlan::Empty);
self.cleanup_succeeded.store(true, Ordering::Release);
}
let _completion = &self.completion;
}
pub(super) fn take_latest_desired(&mut self) -> Option<DesiredState> {
let next = self.desired_rx.borrow_and_update().clone()?;
if next.generation < self.last_generation {
warn!(
generation = next.generation,
active_generation = self.last_generation,
"Ignored stale conntrack firewall policy publication"
);
return None;
}
if next.generation == self.last_generation {
if self.last_policy.as_ref() != Some(&next.policy) {
warn!(
generation = next.generation,
"Ignored conflicting conntrack firewall policy for active generation"
);
return None;
}
self.last_stats = Some(next.stats.clone());
if let AppliedState::Known(applied) = &self.applied
&& applied.matches_policy(&next.policy)
{
next.stats.set_conntrack_rule_apply_ok(true);
}
return Some(next);
}
self.last_generation = next.generation;
self.last_policy = Some(next.policy.clone());
self.last_stats = Some(next.stats.clone());
Some(next)
}
}
+204
View File
@@ -0,0 +1,204 @@
use std::time::Duration;
#[cfg(unix)]
use tokio::io::AsyncWriteExt;
#[cfg(unix)]
use tokio::process::Command;
#[cfg(unix)]
use crate::util::trusted_command::resolve_trusted_helper;
const COMMAND_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct CommandSpec {
pub(super) binary: &'static str,
pub(super) args: Vec<String>,
pub(super) stdin: Option<String>,
}
impl CommandSpec {
pub(super) fn new(binary: &'static str, args: impl IntoIterator<Item = &'static str>) -> Self {
Self {
binary,
args: args.into_iter().map(str::to_string).collect(),
stdin: None,
}
}
pub(super) fn with_stdin(
binary: &'static str,
args: impl IntoIterator<Item = &'static str>,
stdin: String,
) -> Self {
Self {
binary,
args: args.into_iter().map(str::to_string).collect(),
stdin: Some(stdin),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum CommandErrorKind {
Missing,
NotFound,
Cancelled,
Timeout,
Failed,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct CommandError {
pub(super) kind: CommandErrorKind,
pub(super) message: String,
}
impl CommandError {
pub(super) fn cancelled() -> Self {
Self {
kind: CommandErrorKind::Cancelled,
message: "firewall transaction cancelled".to_string(),
}
}
pub(super) fn failed(message: impl Into<String>) -> Self {
Self {
kind: CommandErrorKind::Failed,
message: message.into(),
}
}
}
impl std::fmt::Display for CommandError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
pub(super) trait FirewallCommandRunner: Send + Sync {
fn available(&self, binary: &str) -> bool;
fn has_cap_net_admin(&self) -> bool;
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError>;
}
#[derive(Clone, Copy, Default)]
pub(super) struct SystemCommandRunner;
impl FirewallCommandRunner for SystemCommandRunner {
fn available(&self, binary: &str) -> bool {
#[cfg(unix)]
{
resolve_trusted_helper(binary).is_some()
}
#[cfg(not(unix))]
{
let _ = binary;
false
}
}
fn has_cap_net_admin(&self) -> bool {
#[cfg(target_os = "linux")]
{
let Ok(status) = std::fs::read_to_string("/proc/self/status") else {
return false;
};
for line in status.lines() {
if let Some(raw) = line.strip_prefix("CapEff:") {
let caps = raw.trim();
if let Ok(bits) = u64::from_str_radix(caps, 16) {
const CAP_NET_ADMIN_BIT: u64 = 12;
return (bits & (1u64 << CAP_NET_ADMIN_BIT)) != 0;
}
}
}
false
}
#[cfg(not(target_os = "linux"))]
{
false
}
}
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
#[cfg(not(unix))]
{
Err(CommandError {
kind: CommandErrorKind::Missing,
message: format!("{} is not available", spec.binary),
})
}
#[cfg(unix)]
{
let Some(command_path) = resolve_trusted_helper(spec.binary) else {
return Err(CommandError {
kind: CommandErrorKind::Missing,
message: format!("{} is not available", spec.binary),
});
};
let mut command = Command::new(command_path);
command.args(&spec.args);
command.env("LC_ALL", "C");
if spec.stdin.is_some() {
command.stdin(std::process::Stdio::piped());
}
command.stdout(std::process::Stdio::null());
command.stderr(std::process::Stdio::piped());
command.kill_on_drop(true);
let mut child = command.spawn().map_err(|error| CommandError {
kind: CommandErrorKind::Failed,
message: format!("spawn {} failed: {error}", spec.binary),
})?;
let binary = spec.binary;
let output = tokio::time::timeout(COMMAND_TIMEOUT, async move {
if let Some(blob) = spec.stdin
&& let Some(mut writer) = child.stdin.take()
{
writer
.write_all(blob.as_bytes())
.await
.map_err(|error| CommandError {
kind: CommandErrorKind::Failed,
message: format!("stdin write {binary} failed: {error}"),
})?;
}
child
.wait_with_output()
.await
.map_err(|error| CommandError {
kind: CommandErrorKind::Failed,
message: format!("wait {binary} failed: {error}"),
})
})
.await
.map_err(|_| CommandError {
kind: CommandErrorKind::Timeout,
message: format!("{binary} timed out after {}s", COMMAND_TIMEOUT.as_secs()),
})??;
if output.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
let message = if stderr.is_empty() {
format!("{binary} exited with status {}", output.status)
} else {
stderr
};
let kind = if is_not_found_error(&message) {
CommandErrorKind::NotFound
} else {
CommandErrorKind::Failed
};
Err(CommandError { kind, message })
}
}
}
pub(super) fn is_not_found_error(message: &str) -> bool {
message.contains("No chain/target/match by that name")
|| message.contains("Bad rule (does a matching rule exist in that chain?)")
|| message.contains("Could not process rule: No such file or directory")
}
+240
View File
@@ -0,0 +1,240 @@
use super::command::{CommandError, CommandErrorKind, CommandSpec, FirewallCommandRunner};
use super::model::{NotrackTarget, ShadowSlot};
const DISPATCH_CHAIN: &str = "TELEMT_NOTRACK";
const SHADOW_CHAIN_A: &str = "TELEMT_NT_A";
const SHADOW_CHAIN_B: &str = "TELEMT_NT_B";
const MAX_OWNED_JUMPS: usize = 8;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum IpFamily {
V4,
V6,
}
impl IpFamily {
fn command_binary(self) -> &'static str {
match self {
Self::V4 => "iptables",
Self::V6 => "ip6tables",
}
}
fn restore_binary(self) -> &'static str {
match self {
Self::V4 => "iptables-restore",
Self::V6 => "ip6tables-restore",
}
}
}
pub(super) fn family_available<R: FirewallCommandRunner>(runner: &R, family: IpFamily) -> bool {
runner.available(family.command_binary()) && runner.available(family.restore_binary())
}
fn shadow_chain(slot: ShadowSlot) -> &'static str {
match slot {
ShadowSlot::A => SHADOW_CHAIN_A,
ShadowSlot::B => SHADOW_CHAIN_B,
}
}
pub(super) async fn stage_family<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
slot: ShadowSlot,
targets: &[NotrackTarget],
) -> Result<(), CommandError> {
if targets.is_empty() {
return Ok(());
}
require_family(runner, family)?;
ensure_owned_chains(runner, family).await?;
let script = render_stage_script(slot, targets);
runner
.run(CommandSpec::with_stdin(
family.restore_binary(),
["--noflush"],
script,
))
.await
}
pub(super) async fn activate_family<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
slot: Option<ShadowSlot>,
) -> Result<(), CommandError> {
require_family(runner, family)?;
if slot.is_some() {
ensure_prerouting_jump(runner, family).await?;
}
runner
.run(CommandSpec::with_stdin(
family.restore_binary(),
["--noflush"],
render_dispatch_script(slot),
))
.await
}
pub(super) async fn cleanup_all<R: FirewallCommandRunner>(runner: &R) -> Result<(), CommandError> {
let mut errors = Vec::new();
for family in [IpFamily::V4, IpFamily::V6] {
if !runner.available(family.command_binary()) {
continue;
}
if let Err(error) = cleanup_family(runner, family).await {
errors.push(error.message);
}
}
if errors.is_empty() {
Ok(())
} else {
Err(CommandError::failed(errors.join("; ")))
}
}
async fn cleanup_family<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
) -> Result<(), CommandError> {
let binary = family.command_binary();
let mut errors = Vec::new();
for _ in 0..MAX_OWNED_JUMPS {
let result = runner
.run(CommandSpec::new(
binary,
["-t", "raw", "-D", "PREROUTING", "-j", DISPATCH_CHAIN],
))
.await;
match result {
Ok(()) => {}
Err(error)
if matches!(
error.kind,
CommandErrorKind::NotFound | CommandErrorKind::Missing
) =>
{
break;
}
Err(error) => {
errors.push(error.message);
break;
}
}
}
for chain in [DISPATCH_CHAIN, SHADOW_CHAIN_A, SHADOW_CHAIN_B] {
for operation in ["-F", "-X"] {
let result = runner
.run(CommandSpec::new(binary, ["-t", "raw", operation, chain]))
.await;
if let Err(error) = result
&& !matches!(
error.kind,
CommandErrorKind::NotFound | CommandErrorKind::Missing
)
{
errors.push(error.message);
}
}
}
if errors.is_empty() {
Ok(())
} else {
Err(CommandError::failed(errors.join("; ")))
}
}
async fn ensure_prerouting_jump<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
) -> Result<(), CommandError> {
let binary = family.command_binary();
match runner
.run(CommandSpec::new(
binary,
["-t", "raw", "-C", "PREROUTING", "-j", DISPATCH_CHAIN],
))
.await
{
Ok(()) => Ok(()),
Err(error) if error.kind == CommandErrorKind::NotFound => {
runner
.run(CommandSpec::new(
binary,
["-t", "raw", "-I", "PREROUTING", "1", "-j", DISPATCH_CHAIN],
))
.await
}
Err(error) => Err(error),
}
}
async fn ensure_owned_chains<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
) -> Result<(), CommandError> {
let binary = family.command_binary();
for chain in [DISPATCH_CHAIN, SHADOW_CHAIN_A, SHADOW_CHAIN_B] {
match runner
.run(CommandSpec::new(binary, ["-t", "raw", "-N", chain]))
.await
{
Ok(()) => {}
Err(error) if is_chain_exists_error(&error.message) => {}
Err(error) => return Err(error),
}
}
Ok(())
}
pub(super) fn is_chain_exists_error(message: &str) -> bool {
message.contains("Chain already exists")
}
fn require_family<R: FirewallCommandRunner>(
runner: &R,
family: IpFamily,
) -> Result<(), CommandError> {
for binary in [family.command_binary(), family.restore_binary()] {
if !runner.available(binary) {
return Err(CommandError {
kind: CommandErrorKind::Missing,
message: format!("{binary} is required for conntrack firewall reconciliation"),
});
}
}
Ok(())
}
pub(super) fn render_stage_script(slot: ShadowSlot, targets: &[NotrackTarget]) -> String {
let chain = shadow_chain(slot);
let mut script = format!("*raw\n-F {chain}\n");
for target in targets {
script.push_str("-A ");
script.push_str(chain);
script.push_str(" -p tcp --dport ");
script.push_str(&target.port.to_string());
if let Some(ip) = target.ip {
script.push_str(" -d ");
script.push_str(&ip.to_string());
}
script.push_str(" -j CT --notrack\n");
}
script.push_str("COMMIT\n");
script
}
pub(super) fn render_dispatch_script(slot: Option<ShadowSlot>) -> String {
let mut script = format!("*raw\n-F {DISPATCH_CHAIN}\n");
if let Some(slot) = slot {
script.push_str("-A ");
script.push_str(DISPATCH_CHAIN);
script.push_str(" -j ");
script.push_str(shadow_chain(slot));
script.push('\n');
}
script.push_str("COMMIT\n");
script
}
+204
View File
@@ -0,0 +1,204 @@
use std::collections::BTreeSet;
use std::net::IpAddr;
use std::sync::Arc;
use crate::config::{ConntrackBackend, ConntrackMode, ProxyConfig};
use crate::stats::Stats;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum ShadowSlot {
A,
B,
}
impl ShadowSlot {
pub(super) fn other(self) -> Self {
match self {
Self::A => Self::B,
Self::B => Self::A,
}
}
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub(super) struct NotrackTarget {
pub(super) ip: Option<IpAddr>,
pub(super) port: u16,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) enum DesiredPolicy {
Empty,
Rules {
configured_backend: ConntrackBackend,
v4: Vec<NotrackTarget>,
v6: Vec<NotrackTarget>,
},
}
impl DesiredPolicy {
pub(super) fn from_config(cfg: &ProxyConfig) -> Self {
if !cfg.server.conntrack_control.inline_conntrack_control
|| matches!(cfg.server.conntrack_control.mode, ConntrackMode::Tracked)
{
return Self::Empty;
}
let (v4, v6) = notrack_targets(cfg);
if v4.is_empty() && v6.is_empty() {
Self::Empty
} else {
Self::Rules {
configured_backend: cfg.server.conntrack_control.backend,
v4,
v6,
}
}
}
}
#[derive(Clone)]
pub(super) struct DesiredState {
pub(super) generation: u64,
pub(super) policy: DesiredPolicy,
pub(super) stats: Arc<Stats>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) enum AppliedPlan {
Empty,
Iptables {
slot: ShadowSlot,
v4: Vec<NotrackTarget>,
v6: Vec<NotrackTarget>,
},
Nftables {
slot: ShadowSlot,
v4: Vec<NotrackTarget>,
v6: Vec<NotrackTarget>,
},
}
impl AppliedPlan {
pub(super) fn slot(&self) -> Option<ShadowSlot> {
match self {
Self::Empty => None,
Self::Iptables { slot, .. } | Self::Nftables { slot, .. } => Some(*slot),
}
}
pub(super) fn matches_policy(&self, policy: &DesiredPolicy) -> bool {
match (self, policy) {
(Self::Empty, DesiredPolicy::Empty) => true,
(
Self::Iptables { v4, v6, .. },
DesiredPolicy::Rules {
configured_backend,
v4: desired_v4,
v6: desired_v6,
},
) => {
matches!(
configured_backend,
ConntrackBackend::Auto | ConntrackBackend::Iptables
) && v4 == desired_v4
&& v6 == desired_v6
}
(
Self::Nftables { v4, v6, .. },
DesiredPolicy::Rules {
configured_backend,
v4: desired_v4,
v6: desired_v6,
},
) => {
matches!(
configured_backend,
ConntrackBackend::Auto | ConntrackBackend::Nftables
) && v4 == desired_v4
&& v6 == desired_v6
}
_ => false,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) enum AppliedState {
Known(AppliedPlan),
Unknown,
}
fn listener_port_set(cfg: &ProxyConfig) -> Vec<u16> {
let mut ports = BTreeSet::new();
if cfg.server.listeners.is_empty() {
ports.insert(cfg.server.port);
} else {
for listener in &cfg.server.listeners {
ports.insert(listener.port.unwrap_or(cfg.server.port));
}
}
ports.into_iter().collect()
}
fn notrack_targets(cfg: &ProxyConfig) -> (Vec<NotrackTarget>, Vec<NotrackTarget>) {
let mut v4_targets = BTreeSet::new();
let mut v6_targets = BTreeSet::new();
match cfg.server.conntrack_control.mode {
ConntrackMode::Tracked => {}
ConntrackMode::Notrack => {
if cfg.server.listeners.is_empty() {
let port = cfg.server.port;
for raw in [
cfg.server.listen_addr_ipv4.as_deref(),
cfg.server.listen_addr_ipv6.as_deref(),
]
.into_iter()
.flatten()
{
if let Ok(ip) = raw.parse::<IpAddr>() {
let target = NotrackTarget {
ip: (!ip.is_unspecified()).then_some(ip),
port,
};
if ip.is_ipv4() {
v4_targets.insert(target);
} else {
v6_targets.insert(target);
}
}
}
} else {
for listener in &cfg.server.listeners {
let target = NotrackTarget {
ip: (!listener.ip.is_unspecified()).then_some(listener.ip),
port: listener.port.unwrap_or(cfg.server.port),
};
if listener.ip.is_ipv4() {
v4_targets.insert(target);
} else {
v6_targets.insert(target);
}
}
}
}
ConntrackMode::Hybrid => {
for ip in &cfg.server.conntrack_control.hybrid_listener_ips {
for port in listener_port_set(cfg) {
let target = NotrackTarget {
ip: Some(*ip),
port,
};
if ip.is_ipv4() {
v4_targets.insert(target);
} else {
v6_targets.insert(target);
}
}
}
}
}
(
v4_targets.into_iter().collect(),
v6_targets.into_iter().collect(),
)
}
+146
View File
@@ -0,0 +1,146 @@
use super::command::{CommandError, CommandErrorKind, CommandSpec, FirewallCommandRunner};
use super::model::{NotrackTarget, ShadowSlot};
const LEGACY_TABLE: &str = "telemt_conntrack";
const TABLE_A: &str = "telemt_conntrack_a";
const TABLE_B: &str = "telemt_conntrack_b";
pub(super) fn available<R: FirewallCommandRunner>(runner: &R) -> bool {
runner.available("nft")
}
fn table(slot: ShadowSlot) -> &'static str {
match slot {
ShadowSlot::A => TABLE_A,
ShadowSlot::B => TABLE_B,
}
}
pub(super) async fn stage<R: FirewallCommandRunner>(
runner: &R,
slot: ShadowSlot,
v4: &[NotrackTarget],
v6: &[NotrackTarget],
) -> Result<(), CommandError> {
require_nft(runner)?;
delete_table_if_present(runner, table(slot)).await?;
runner
.run(CommandSpec::with_stdin(
"nft",
["-f", "-"],
render_stage_script(slot, v4, v6),
))
.await
}
pub(super) async fn activate<R: FirewallCommandRunner>(
runner: &R,
slot: ShadowSlot,
) -> Result<(), CommandError> {
require_nft(runner)?;
runner
.run(CommandSpec::with_stdin(
"nft",
["-f", "-"],
render_activate_script(slot),
))
.await
}
pub(super) async fn deactivate<R: FirewallCommandRunner>(
runner: &R,
slot: ShadowSlot,
) -> Result<(), CommandError> {
delete_table_if_present(runner, table(slot)).await
}
pub(super) async fn cleanup_all<R: FirewallCommandRunner>(runner: &R) -> Result<(), CommandError> {
if !runner.available("nft") {
return Ok(());
}
let mut errors = Vec::new();
for table_name in [LEGACY_TABLE, TABLE_A, TABLE_B] {
if let Err(error) = delete_table_if_present(runner, table_name).await {
errors.push(error.message);
}
}
if errors.is_empty() {
Ok(())
} else {
Err(CommandError::failed(errors.join("; ")))
}
}
async fn delete_table_if_present<R: FirewallCommandRunner>(
runner: &R,
table_name: &'static str,
) -> Result<(), CommandError> {
match runner
.run(CommandSpec::new(
"nft",
["delete", "table", "inet", table_name],
))
.await
{
Ok(()) => Ok(()),
Err(error)
if matches!(
error.kind,
CommandErrorKind::NotFound | CommandErrorKind::Missing
) =>
{
Ok(())
}
Err(error) => Err(error),
}
}
fn require_nft<R: FirewallCommandRunner>(runner: &R) -> Result<(), CommandError> {
if runner.available("nft") {
Ok(())
} else {
Err(CommandError {
kind: CommandErrorKind::Missing,
message: "nft is required for conntrack firewall reconciliation".to_string(),
})
}
}
pub(super) fn render_stage_script(
slot: ShadowSlot,
v4: &[NotrackTarget],
v6: &[NotrackTarget],
) -> String {
let table = table(slot);
let mut script = format!("add table inet {table}\nadd chain inet {table} rules\n");
for target in v4 {
script.push_str("add rule inet ");
script.push_str(table);
script.push_str(" rules tcp dport ");
script.push_str(&target.port.to_string());
if let Some(ip) = target.ip {
script.push_str(" ip daddr ");
script.push_str(&ip.to_string());
}
script.push_str(" notrack\n");
}
for target in v6 {
script.push_str("add rule inet ");
script.push_str(table);
script.push_str(" rules tcp dport ");
script.push_str(&target.port.to_string());
if let Some(ip) = target.ip {
script.push_str(" ip6 daddr ");
script.push_str(&ip.to_string());
}
script.push_str(" notrack\n");
}
script
}
pub(super) fn render_activate_script(slot: ShadowSlot) -> String {
let table = table(slot);
format!(
"add chain inet {table} preraw {{ type filter hook prerouting priority raw; policy accept; }}\nadd rule inet {table} preraw jump rules\n"
)
}
+91
View File
@@ -0,0 +1,91 @@
use tracing::debug;
use crate::config::{ConntrackBackend, ProxyConfig};
use crate::conntrack_control::{ConntrackRuntimeSupport, NetfilterBackend};
use crate::proxy::shared_state::ConntrackCloseEvent;
use super::command::{CommandSpec, FirewallCommandRunner, SystemCommandRunner};
use super::nftables;
/// Probes the effective firewall backend and conntrack deletion capability.
pub(in crate::conntrack_control) fn probe_runtime_support(
configured_backend: ConntrackBackend,
) -> ConntrackRuntimeSupport {
let runner = SystemCommandRunner;
let iptables_available = runner.available("iptables");
let netfilter_backend = match configured_backend {
ConntrackBackend::Auto if nftables::available(&runner) => Some(NetfilterBackend::Nftables),
ConntrackBackend::Auto if iptables_available => Some(NetfilterBackend::Iptables),
ConntrackBackend::Nftables if nftables::available(&runner) => {
Some(NetfilterBackend::Nftables)
}
ConntrackBackend::Iptables if iptables_available => Some(NetfilterBackend::Iptables),
_ => None,
};
ConntrackRuntimeSupport {
netfilter_backend,
has_cap_net_admin: runner.has_cap_net_admin(),
has_conntrack_binary: runner.available("conntrack"),
}
}
/// Resolves whether conntrack close publication is usable for this runtime.
pub(in crate::conntrack_control) fn effective_conntrack_enabled(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
) -> bool {
cfg.server.conntrack_control.inline_conntrack_control
&& runtime_support.has_cap_net_admin
&& runtime_support.netfilter_backend.is_some()
&& runtime_support.has_conntrack_binary
}
/// Result of one best-effort kernel conntrack deletion.
pub(in crate::conntrack_control) enum DeleteOutcome {
/// The kernel reported successful deletion.
Deleted,
/// No matching conntrack entry existed.
NotFound,
/// The helper was unavailable or returned an unexpected failure.
Error,
}
/// Deletes the exact TCP tuple represented by one close event.
pub(in crate::conntrack_control) async fn delete_conntrack_entry(
event: ConntrackCloseEvent,
) -> DeleteOutcome {
let runner = SystemCommandRunner;
if !runner.available("conntrack") {
return DeleteOutcome::Error;
}
let args = vec![
"-D".to_string(),
"-p".to_string(),
"tcp".to_string(),
"-s".to_string(),
event.src.ip().to_string(),
"--sport".to_string(),
event.src.port().to_string(),
"-d".to_string(),
event.dst.ip().to_string(),
"--dport".to_string(),
event.dst.port().to_string(),
];
match runner
.run(CommandSpec {
binary: "conntrack",
args,
stdin: None,
})
.await
{
Ok(()) => DeleteOutcome::Deleted,
Err(error) if error.message.contains("0 flow entries have been deleted") => {
DeleteOutcome::NotFound
}
Err(error) => {
debug!(error = %error, "conntrack delete failed");
DeleteOutcome::Error
}
}
}
+488
View File
@@ -0,0 +1,488 @@
use std::collections::{BTreeMap, BTreeSet};
use std::future::pending;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::sync::{Notify, watch};
use tokio_util::sync::CancellationToken;
use crate::config::ConntrackBackend;
use crate::stats::Stats;
use super::actor::{FirewallReconciler, ReconcileOutcome};
use super::command::{CommandError, CommandErrorKind, CommandSpec, FirewallCommandRunner};
use super::model::{
AppliedPlan, AppliedState, DesiredPolicy, DesiredState, NotrackTarget, ShadowSlot,
};
use super::transaction::{InterruptibleRunner, reconcile_once};
#[path = "tests/model_tests.rs"]
mod model_tests;
#[derive(Clone)]
struct FailureRule {
binary: &'static str,
occurrence: usize,
}
#[derive(Default)]
struct FakeState {
calls: Vec<CommandSpec>,
binary_calls: BTreeMap<&'static str, usize>,
failures: Vec<FailureRule>,
}
#[derive(Clone)]
struct FakeRunner {
available: Arc<BTreeSet<&'static str>>,
has_cap_net_admin: bool,
state: Arc<Mutex<FakeState>>,
}
impl FakeRunner {
fn all_available() -> Self {
Self {
available: Arc::new(BTreeSet::from([
"conntrack",
"ip6tables",
"ip6tables-restore",
"iptables",
"iptables-restore",
"nft",
])),
has_cap_net_admin: true,
state: Arc::new(Mutex::new(FakeState::default())),
}
}
fn with_failure(self, binary: &'static str, occurrence: usize) -> Self {
self.state
.lock()
.unwrap()
.failures
.push(FailureRule { binary, occurrence });
self
}
fn calls(&self) -> Vec<CommandSpec> {
self.state.lock().unwrap().calls.clone()
}
}
impl FirewallCommandRunner for FakeRunner {
fn available(&self, binary: &str) -> bool {
self.available.contains(binary)
}
fn has_cap_net_admin(&self) -> bool {
self.has_cap_net_admin
}
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
let mut state = self.state.lock().unwrap();
let occurrence = {
let count = state.binary_calls.entry(spec.binary).or_default();
*count += 1;
*count
};
state.calls.push(spec.clone());
if state
.failures
.iter()
.any(|failure| failure.binary == spec.binary && failure.occurrence == occurrence)
{
return Err(CommandError::failed(format!(
"injected {} failure at occurrence {}",
spec.binary, occurrence
)));
}
drop(state);
let operation = spec.args.get(2).map(String::as_str);
if (matches!(spec.binary, "iptables" | "ip6tables")
&& matches!(operation, Some("-C" | "-D" | "-F" | "-X")))
|| (spec.binary == "nft" && spec.args.first().map(String::as_str) == Some("delete"))
{
return Err(CommandError {
kind: CommandErrorKind::NotFound,
message: "injected object not found".to_string(),
});
}
Ok(())
}
}
struct BlockingRunner {
entered: Arc<Notify>,
}
impl FirewallCommandRunner for BlockingRunner {
fn available(&self, _binary: &str) -> bool {
true
}
fn has_cap_net_admin(&self) -> bool {
true
}
async fn run(&self, _spec: CommandSpec) -> Result<(), CommandError> {
self.entered.notify_one();
pending().await
}
}
fn target(ip: Option<&str>, port: u16) -> NotrackTarget {
NotrackTarget {
ip: ip.map(|value| value.parse().unwrap()),
port,
}
}
fn desired(generation: u64, policy: DesiredPolicy) -> DesiredState {
DesiredState {
generation,
policy,
stats: Arc::new(Stats::new()),
}
}
fn dual_stack_policy(port: u16) -> DesiredPolicy {
DesiredPolicy::Rules {
configured_backend: ConntrackBackend::Iptables,
v4: vec![target(Some("192.0.2.10"), port)],
v6: vec![target(Some("2001:db8::10"), port)],
}
}
fn nft_dual_stack_policy(port: u16) -> DesiredPolicy {
DesiredPolicy::Rules {
configured_backend: ConntrackBackend::Nftables,
v4: vec![target(Some("192.0.2.10"), port)],
v6: vec![target(Some("2001:db8::10"), port)],
}
}
#[tokio::test]
async fn successful_reconcile_flips_between_shadow_slots() {
let runner = FakeRunner::all_available();
let mut applied = AppliedState::Known(AppliedPlan::Empty);
let first = desired(1, dual_stack_policy(443));
reconcile_once(&runner, &runner, &mut applied, &first)
.await
.unwrap();
assert!(matches!(
applied,
AppliedState::Known(AppliedPlan::Iptables {
slot: ShadowSlot::A,
..
})
));
let second = desired(2, dual_stack_policy(8443));
reconcile_once(&runner, &runner, &mut applied, &second)
.await
.unwrap();
assert!(matches!(
applied,
AppliedState::Known(AppliedPlan::Iptables {
slot: ShadowSlot::B,
..
})
));
assert!(runner.calls().iter().any(|call| {
call.stdin
.as_deref()
.is_some_and(|script| script.contains("-A TELEMT_NOTRACK -j TELEMT_NT_B"))
}));
}
#[tokio::test]
async fn identical_policy_is_command_free_after_convergence() {
let runner = FakeRunner::all_available();
let mut applied = AppliedState::Known(AppliedPlan::Empty);
reconcile_once(
&runner,
&runner,
&mut applied,
&desired(1, dual_stack_policy(443)),
)
.await
.unwrap();
let calls_after_convergence = runner.calls().len();
reconcile_once(
&runner,
&runner,
&mut applied,
&desired(2, dual_stack_policy(443)),
)
.await
.unwrap();
assert_eq!(runner.calls().len(), calls_after_convergence);
}
#[tokio::test]
async fn backend_migration_failure_restores_previous_backend() {
let runner = FakeRunner::all_available().with_failure("ip6tables-restore", 3);
let mut applied = AppliedState::Known(AppliedPlan::Empty);
reconcile_once(
&runner,
&runner,
&mut applied,
&desired(1, dual_stack_policy(443)),
)
.await
.unwrap();
let previous = applied.clone();
let failure = reconcile_once(
&runner,
&runner,
&mut applied,
&desired(2, nft_dual_stack_policy(8443)),
)
.await
.unwrap_err();
assert_eq!(failure.rollback_succeeded, Some(true));
assert_eq!(applied, previous);
}
#[tokio::test]
async fn unavailable_target_does_not_clear_confirmed_applied_policy() {
let runner = FakeRunner::all_available();
let mut applied = AppliedState::Known(AppliedPlan::Empty);
reconcile_once(
&runner,
&runner,
&mut applied,
&desired(1, dual_stack_policy(443)),
)
.await
.unwrap();
let previous = applied.clone();
let unavailable = FakeRunner {
available: Arc::new(BTreeSet::new()),
has_cap_net_admin: false,
state: Arc::new(Mutex::new(FakeState::default())),
};
let failure = reconcile_once(
&unavailable,
&unavailable,
&mut applied,
&desired(2, nft_dual_stack_policy(8443)),
)
.await
.unwrap_err();
assert_eq!(failure.rollback_succeeded, None);
assert_eq!(applied, previous);
assert!(unavailable.calls().is_empty());
}
#[tokio::test]
async fn partial_dual_stack_failure_restores_previous_applied_plan() {
let runner = FakeRunner::all_available().with_failure("ip6tables-restore", 4);
let mut applied = AppliedState::Known(AppliedPlan::Empty);
let first = desired(1, dual_stack_policy(443));
reconcile_once(&runner, &runner, &mut applied, &first)
.await
.unwrap();
let previous = applied.clone();
let failure = reconcile_once(
&runner,
&runner,
&mut applied,
&desired(2, dual_stack_policy(8443)),
)
.await
.unwrap_err();
assert_eq!(failure.rollback_succeeded, Some(true));
assert_eq!(applied, previous);
}
#[tokio::test]
async fn rollback_failure_marks_applied_state_unknown() {
let runner = FakeRunner::all_available()
.with_failure("ip6tables-restore", 4)
.with_failure("nft", 2);
let mut applied = AppliedState::Known(AppliedPlan::Empty);
reconcile_once(
&runner,
&runner,
&mut applied,
&desired(1, dual_stack_policy(443)),
)
.await
.unwrap();
let failure = reconcile_once(
&runner,
&runner,
&mut applied,
&desired(2, dual_stack_policy(8443)),
)
.await
.unwrap_err();
assert_eq!(failure.rollback_succeeded, Some(false));
assert_eq!(applied, AppliedState::Unknown);
}
#[tokio::test]
async fn transaction_cancellation_does_not_claim_a_new_applied_plan() {
let entered = Arc::new(Notify::new());
let runner = BlockingRunner {
entered: entered.clone(),
};
let terminal = CancellationToken::new();
let process_cancellation = CancellationToken::new();
let interruptible = InterruptibleRunner::new(&runner, &terminal, &process_cancellation);
let mut applied = AppliedState::Known(AppliedPlan::Empty);
let desired = desired(1, dual_stack_policy(443));
let failure = {
let transaction = reconcile_once(&interruptible, &runner, &mut applied, &desired);
tokio::pin!(transaction);
tokio::select! {
_ = entered.notified() => terminal.cancel(),
_ = &mut transaction => panic!("transaction completed before injected cancellation"),
}
transaction.await.unwrap_err()
};
assert!(failure.cancelled);
assert_eq!(applied, AppliedState::Known(AppliedPlan::Empty));
}
#[test]
fn desired_watch_coalesces_and_rejects_stale_or_conflicting_generations() {
let runner = FakeRunner::all_available();
let (desired_tx, desired_rx) = watch::channel(None);
let (status_tx, _status_rx) = watch::channel(None);
let terminal = CancellationToken::new();
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
let completed = Arc::new(Notify::new());
let mut reconciler = FirewallReconciler::new(
runner,
desired_rx,
status_tx,
terminal,
closed,
Arc::new(std::sync::atomic::AtomicBool::new(false)),
Arc::new(std::sync::atomic::AtomicBool::new(false)),
completed,
);
desired_tx.send_replace(Some(desired(1, dual_stack_policy(443))));
desired_tx.send_replace(Some(desired(2, dual_stack_policy(8443))));
assert_eq!(reconciler.take_latest_desired().unwrap().generation, 2);
desired_tx.send_replace(Some(desired(1, dual_stack_policy(443))));
assert!(reconciler.take_latest_desired().is_none());
desired_tx.send_replace(Some(desired(2, dual_stack_policy(9443))));
assert!(reconciler.take_latest_desired().is_none());
}
#[tokio::test(start_paused = true)]
async fn actor_retries_with_backoff_and_cleans_owned_rules_on_shutdown() {
let runner = FakeRunner::all_available().with_failure("iptables-restore", 1);
let observed_runner = runner.clone();
let (desired_tx, desired_rx) = watch::channel(None);
let (status_tx, mut status_rx) = watch::channel(None);
let terminal = CancellationToken::new();
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
let completed_flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
let cleanup_succeeded = Arc::new(std::sync::atomic::AtomicBool::new(false));
let completed = Arc::new(Notify::new());
let reconciler = FirewallReconciler::new(
runner,
desired_rx,
status_tx,
terminal.clone(),
closed.clone(),
completed_flag.clone(),
cleanup_succeeded.clone(),
completed,
);
let process_cancellation = CancellationToken::new();
let task = tokio::spawn(reconciler.run(process_cancellation));
let initial_desired = desired(2, dual_stack_policy(443));
let desired_stats = initial_desired.stats.clone();
desired_tx.send_replace(Some(initial_desired));
status_rx.changed().await.unwrap();
assert_eq!(
status_rx.borrow().as_ref().unwrap().outcome,
ReconcileOutcome::Failed
);
let calls_after_failure = observed_runner.calls().len();
desired_tx.send_replace(Some(desired(1, dual_stack_policy(7443))));
tokio::task::yield_now().await;
desired_tx.send_replace(Some(desired(2, dual_stack_policy(9443))));
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_millis(999)).await;
tokio::task::yield_now().await;
assert_eq!(observed_runner.calls().len(), calls_after_failure);
tokio::time::advance(Duration::from_millis(1)).await;
status_rx.changed().await.unwrap();
assert_eq!(
status_rx.borrow().as_ref().unwrap().outcome,
ReconcileOutcome::Applied
);
assert_eq!(status_rx.borrow().as_ref().unwrap().generation, 2);
assert!(observed_runner.calls().iter().all(|call| {
call.stdin
.as_ref()
.is_none_or(|script| !script.contains("7443") && !script.contains("9443"))
}));
let calls_before_shutdown = observed_runner.calls().len();
terminal.cancel();
tokio::time::timeout(Duration::from_secs(1), task)
.await
.unwrap()
.unwrap();
assert!(closed.load(Ordering::Acquire));
assert!(completed_flag.load(Ordering::Acquire));
assert!(cleanup_succeeded.load(Ordering::Acquire));
assert!(!desired_stats.get_conntrack_rule_apply_ok());
assert!(observed_runner.calls().len() > calls_before_shutdown);
}
#[tokio::test]
async fn actor_reports_terminal_cleanup_failure_separately_from_completion() {
let runner = FakeRunner::all_available().with_failure("nft", 1);
let (_desired_tx, desired_rx) = watch::channel(None);
let (status_tx, _status_rx) = watch::channel(None);
let terminal = CancellationToken::new();
terminal.cancel();
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
let completed_flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
let cleanup_succeeded = Arc::new(std::sync::atomic::AtomicBool::new(false));
let reconciler = FirewallReconciler::new(
runner,
desired_rx,
status_tx,
terminal,
closed.clone(),
completed_flag.clone(),
cleanup_succeeded.clone(),
Arc::new(Notify::new()),
);
reconciler.run(CancellationToken::new()).await;
assert!(closed.load(Ordering::Acquire));
assert!(completed_flag.load(Ordering::Acquire));
assert!(!cleanup_succeeded.load(Ordering::Acquire));
}
@@ -0,0 +1,118 @@
use crate::config::{ConntrackBackend, ConntrackMode, ProxyConfig};
use super::super::command::is_not_found_error;
use super::super::iptables::{self, is_chain_exists_error};
use super::super::model::{DesiredPolicy, ShadowSlot};
use super::super::nftables;
use super::target;
#[test]
fn desired_policy_derives_exact_listener_targets() {
let mut config = ProxyConfig::default();
config.server.port = 8443;
config.server.listen_addr_ipv4 = Some("0.0.0.0".to_string());
config.server.listen_addr_ipv6 = Some("2001:db8::10".to_string());
config.server.conntrack_control.inline_conntrack_control = true;
config.server.conntrack_control.mode = ConntrackMode::Notrack;
config.server.conntrack_control.backend = ConntrackBackend::Iptables;
assert_eq!(
DesiredPolicy::from_config(&config),
DesiredPolicy::Rules {
configured_backend: ConntrackBackend::Iptables,
v4: vec![target(None, 8443)],
v6: vec![target(Some("2001:db8::10"), 8443)],
}
);
config.server.conntrack_control.mode = ConntrackMode::Tracked;
assert_eq!(DesiredPolicy::from_config(&config), DesiredPolicy::Empty);
config.server.conntrack_control.mode = ConntrackMode::Notrack;
config.server.conntrack_control.inline_conntrack_control = false;
assert_eq!(DesiredPolicy::from_config(&config), DesiredPolicy::Empty);
}
#[test]
fn hybrid_policy_is_a_sorted_deduplicated_address_port_product() {
let mut config = ProxyConfig::default();
config.server.conntrack_control.inline_conntrack_control = true;
config.server.conntrack_control.mode = ConntrackMode::Hybrid;
config.server.conntrack_control.hybrid_listener_ips = vec![
"2001:db8::10".parse().unwrap(),
"192.0.2.10".parse().unwrap(),
"192.0.2.10".parse().unwrap(),
];
config.server.listeners = vec![
serde_json::from_value(serde_json::json!({
"ip": "0.0.0.0",
"port": 8443
}))
.unwrap(),
serde_json::from_value(serde_json::json!({
"ip": "::",
"port": 443
}))
.unwrap(),
];
assert_eq!(
DesiredPolicy::from_config(&config),
DesiredPolicy::Rules {
configured_backend: ConntrackBackend::Auto,
v4: vec![
target(Some("192.0.2.10"), 443),
target(Some("192.0.2.10"), 8443),
],
v6: vec![
target(Some("2001:db8::10"), 443),
target(Some("2001:db8::10"), 8443),
],
}
);
}
#[test]
fn restore_renderers_keep_staging_detached_from_activation() {
let stage = iptables::render_stage_script(ShadowSlot::B, &[target(Some("192.0.2.20"), 443)]);
assert!(stage.contains("-F TELEMT_NT_B\n"));
assert!(stage.contains("-A TELEMT_NT_B -p tcp --dport 443 -d 192.0.2.20 -j CT --notrack\n"));
assert!(!stage.contains("-A TELEMT_NOTRACK -j TELEMT_NT_B"));
assert!(!stage.contains(":TELEMT_"));
let activation = iptables::render_dispatch_script(Some(ShadowSlot::B));
assert!(activation.contains("-F TELEMT_NOTRACK\n"));
assert!(activation.contains("-A TELEMT_NOTRACK -j TELEMT_NT_B\n"));
assert!(!activation.contains(":TELEMT_"));
let nft_stage = nftables::render_stage_script(
ShadowSlot::A,
&[target(None, 443)],
&[target(Some("2001:db8::20"), 8443)],
);
assert!(!nft_stage.contains("hook prerouting"));
assert!(nft_stage.contains("tcp dport 443 notrack\n"));
assert!(nft_stage.contains("tcp dport 8443 ip6 daddr 2001:db8::20 notrack\n"));
assert!(nftables::render_activate_script(ShadowSlot::A).contains("hook prerouting"));
}
#[test]
fn command_error_classification_only_accepts_absent_owned_objects() {
assert!(is_not_found_error(
"iptables: No chain/target/match by that name."
));
assert!(is_not_found_error(
"Bad rule (does a matching rule exist in that chain?)."
));
assert!(is_not_found_error(
"Error: Could not process rule: No such file or directory"
));
assert!(!is_not_found_error("Permission denied"));
assert!(!is_not_found_error(
"can't initialize iptables table `raw': Table does not exist"
));
assert!(!is_not_found_error(
"Another app is currently holding the xtables lock"
));
assert!(is_chain_exists_error("iptables: Chain already exists."));
assert!(!is_chain_exists_error("Permission denied"));
}
@@ -0,0 +1,335 @@
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use crate::config::ConntrackBackend;
use super::command::{CommandError, CommandErrorKind, CommandSpec, FirewallCommandRunner};
use super::iptables::{self, IpFamily};
use super::model::{AppliedPlan, AppliedState, DesiredPolicy, DesiredState, ShadowSlot};
use super::nftables;
const TRANSACTION_TIMEOUT: Duration = Duration::from_secs(30);
pub(super) struct InterruptibleRunner<'a, R> {
inner: &'a R,
terminal: &'a CancellationToken,
process_cancellation: &'a CancellationToken,
}
impl<'a, R> InterruptibleRunner<'a, R> {
pub(super) fn new(
inner: &'a R,
terminal: &'a CancellationToken,
process_cancellation: &'a CancellationToken,
) -> Self {
Self {
inner,
terminal,
process_cancellation,
}
}
}
impl<R: FirewallCommandRunner> FirewallCommandRunner for InterruptibleRunner<'_, R> {
fn available(&self, binary: &str) -> bool {
self.inner.available(binary)
}
fn has_cap_net_admin(&self) -> bool {
self.inner.has_cap_net_admin()
}
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
tokio::select! {
biased;
_ = self.terminal.cancelled() => Err(CommandError::cancelled()),
_ = self.process_cancellation.cancelled() => Err(CommandError::cancelled()),
result = self.inner.run(spec) => result,
}
}
}
#[derive(Debug)]
pub(super) struct ReconcileFailure {
pub(super) message: String,
pub(super) rollback_succeeded: Option<bool>,
pub(super) cancelled: bool,
}
pub(super) async fn reconcile_once<I, R>(
interruptible: &I,
recovery_runner: &R,
applied: &mut AppliedState,
desired: &DesiredState,
) -> Result<(), ReconcileFailure>
where
I: FirewallCommandRunner,
R: FirewallCommandRunner,
{
if let AppliedState::Known(current) = applied
&& current.matches_policy(&desired.policy)
{
return Ok(());
}
if matches!(applied, AppliedState::Unknown) {
match tokio::time::timeout(TRANSACTION_TIMEOUT, recover_to_empty(interruptible)).await {
Ok(Ok(())) => *applied = AppliedState::Known(AppliedPlan::Empty),
Ok(Err(error)) if error.kind == CommandErrorKind::Cancelled => {
return Err(cancelled_failure());
}
Ok(Err(error)) => {
return Err(ReconcileFailure {
message: format!("startup recovery failed: {error}"),
rollback_succeeded: None,
cancelled: false,
});
}
Err(_) => {
return Err(ReconcileFailure {
message: "startup recovery timed out".to_string(),
rollback_succeeded: None,
cancelled: false,
});
}
}
}
let previous = match applied {
AppliedState::Known(plan) => plan.clone(),
AppliedState::Unknown => unreachable!("unknown state was recovered above"),
};
let target = match resolve_target(interruptible, &desired.policy, &previous) {
Ok(target) => target,
Err(error) => {
return Err(ReconcileFailure {
message: error.message,
rollback_succeeded: None,
cancelled: false,
});
}
};
let transition = tokio::time::timeout(
TRANSACTION_TIMEOUT,
transition_plan(interruptible, &previous, &target),
)
.await;
match transition {
Ok(Ok(())) => {
*applied = AppliedState::Known(target);
Ok(())
}
Ok(Err(error)) if error.kind == CommandErrorKind::Cancelled => Err(cancelled_failure()),
Ok(Err(error)) => {
rollback_after_failure(recovery_runner, applied, previous, error.message).await
}
Err(_) => {
rollback_after_failure(
recovery_runner,
applied,
previous,
"firewall apply transaction timed out".to_string(),
)
.await
}
}
}
async fn rollback_after_failure<R: FirewallCommandRunner>(
runner: &R,
applied: &mut AppliedState,
previous: AppliedPlan,
apply_error: String,
) -> Result<(), ReconcileFailure> {
let rollback = tokio::time::timeout(TRANSACTION_TIMEOUT, restore_plan(runner, &previous)).await;
let rollback_result = match rollback {
Ok(result) => result,
Err(_) => Err(CommandError::failed("firewall rollback timed out")),
};
match rollback_result {
Ok(()) => {
*applied = AppliedState::Known(previous);
Err(ReconcileFailure {
message: apply_error,
rollback_succeeded: Some(true),
cancelled: false,
})
}
Err(rollback_error) => {
*applied = AppliedState::Unknown;
if rollback_error.kind == CommandErrorKind::Cancelled {
return Err(cancelled_failure());
}
Err(ReconcileFailure {
message: format!("{apply_error}; rollback failed: {rollback_error}"),
rollback_succeeded: Some(false),
cancelled: false,
})
}
}
}
fn cancelled_failure() -> ReconcileFailure {
ReconcileFailure {
message: "firewall transaction cancelled for process shutdown".to_string(),
rollback_succeeded: None,
cancelled: true,
}
}
pub(super) fn resolve_target<R: FirewallCommandRunner>(
runner: &R,
desired: &DesiredPolicy,
previous: &AppliedPlan,
) -> Result<AppliedPlan, CommandError> {
let DesiredPolicy::Rules {
configured_backend,
v4,
v6,
} = desired
else {
return Ok(AppliedPlan::Empty);
};
if !runner.has_cap_net_admin() {
return Err(CommandError::failed(
"CAP_NET_ADMIN is required for conntrack firewall reconciliation",
));
}
let next_slot = previous.slot().map_or(ShadowSlot::A, ShadowSlot::other);
let iptables_available = (v4.is_empty() || iptables::family_available(runner, IpFamily::V4))
&& (v6.is_empty() || iptables::family_available(runner, IpFamily::V6));
match configured_backend {
ConntrackBackend::Nftables if nftables::available(runner) => Ok(AppliedPlan::Nftables {
slot: next_slot,
v4: v4.clone(),
v6: v6.clone(),
}),
ConntrackBackend::Iptables if iptables_available => Ok(AppliedPlan::Iptables {
slot: next_slot,
v4: v4.clone(),
v6: v6.clone(),
}),
ConntrackBackend::Auto if nftables::available(runner) => Ok(AppliedPlan::Nftables {
slot: next_slot,
v4: v4.clone(),
v6: v6.clone(),
}),
ConntrackBackend::Auto if iptables_available => Ok(AppliedPlan::Iptables {
slot: next_slot,
v4: v4.clone(),
v6: v6.clone(),
}),
backend => Err(CommandError::failed(format!(
"configured conntrack firewall backend {backend:?} is unavailable"
))),
}
}
pub(super) async fn transition_plan<R: FirewallCommandRunner>(
runner: &R,
previous: &AppliedPlan,
target: &AppliedPlan,
) -> Result<(), CommandError> {
install_plan(runner, target).await?;
match (previous, target) {
(
AppliedPlan::Iptables {
v4: previous_v4,
v6: previous_v6,
..
},
AppliedPlan::Iptables { v4, v6, .. },
) => {
if !previous_v4.is_empty() && v4.is_empty() {
iptables::activate_family(runner, IpFamily::V4, None).await?;
}
if !previous_v6.is_empty() && v6.is_empty() {
iptables::activate_family(runner, IpFamily::V6, None).await?;
}
}
(
AppliedPlan::Nftables {
slot: previous_slot,
..
},
AppliedPlan::Nftables { slot, .. },
) if previous_slot != slot => {
nftables::deactivate(runner, *previous_slot).await?;
}
(_, _) if previous != target => clear_plan(runner, previous).await?,
_ => {}
}
Ok(())
}
async fn install_plan<R: FirewallCommandRunner>(
runner: &R,
plan: &AppliedPlan,
) -> Result<(), CommandError> {
match plan {
AppliedPlan::Empty => Ok(()),
AppliedPlan::Iptables { slot, v4, v6 } => {
if !v4.is_empty() {
iptables::stage_family(runner, IpFamily::V4, *slot, v4).await?;
}
if !v6.is_empty() {
iptables::stage_family(runner, IpFamily::V6, *slot, v6).await?;
}
if !v4.is_empty() {
iptables::activate_family(runner, IpFamily::V4, Some(*slot)).await?;
}
if !v6.is_empty() {
iptables::activate_family(runner, IpFamily::V6, Some(*slot)).await?;
}
Ok(())
}
AppliedPlan::Nftables { slot, v4, v6 } => {
nftables::stage(runner, *slot, v4, v6).await?;
nftables::activate(runner, *slot).await
}
}
}
async fn clear_plan<R: FirewallCommandRunner>(
runner: &R,
plan: &AppliedPlan,
) -> Result<(), CommandError> {
match plan {
AppliedPlan::Empty => Ok(()),
AppliedPlan::Iptables { v4, v6, .. } => {
if !v4.is_empty() {
iptables::activate_family(runner, IpFamily::V4, None).await?;
}
if !v6.is_empty() {
iptables::activate_family(runner, IpFamily::V6, None).await?;
}
Ok(())
}
AppliedPlan::Nftables { slot, .. } => nftables::deactivate(runner, *slot).await,
}
}
async fn restore_plan<R: FirewallCommandRunner>(
runner: &R,
plan: &AppliedPlan,
) -> Result<(), CommandError> {
recover_to_empty(runner).await?;
install_plan(runner, plan).await
}
pub(super) async fn recover_to_empty<R: FirewallCommandRunner>(
runner: &R,
) -> Result<(), CommandError> {
let nft_result = nftables::cleanup_all(runner).await;
let iptables_result = iptables::cleanup_all(runner).await;
match (nft_result, iptables_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(first), Ok(())) | (Ok(()), Err(first)) => Err(first),
(Err(first), Err(second)) => Err(CommandError::failed(format!(
"{}; {}",
first.message, second.message
))),
}
}
+28 -279
View File
@@ -4,14 +4,20 @@
//! and privilege dropping for running telemt as a background service.
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::io;
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use nix::errno::Errno;
use nix::fcntl::{Flock, FlockArg};
use nix::unistd::{self, ForkResult, Gid, Pid, Uid, chdir, close, fork, getpid, setsid};
use tracing::{debug, info, warn};
use nix::unistd::{self, ForkResult, Gid, Uid, chdir, close, fork, getpid, setsid};
use tracing::info;
// PID file ownership and process-control helpers.
mod pid_file;
pub use pid_file::DaemonStatus;
#[allow(unused_imports)]
pub use pid_file::{PidFile, check_status, read_pid_file, signal_pid_file};
/// Default PID file location.
pub const DEFAULT_PID_FILE: &str = "/var/run/telemt.pid";
@@ -51,36 +57,47 @@ impl DaemonOptions {
/// Error types for daemon operations.
#[derive(Debug, thiserror::Error)]
pub enum DaemonError {
/// A daemonization fork failed.
#[error("fork failed: {0}")]
ForkFailed(#[source] nix::Error),
/// Creation of the detached session failed.
#[error("setsid failed: {0}")]
SetsidFailed(#[source] nix::Error),
/// Switching to the configured working directory failed.
#[error("chdir failed: {0}")]
ChdirFailed(#[source] nix::Error),
/// Opening `/dev/null` for standard-stream redirection failed.
#[error("failed to open /dev/null: {0}")]
DevNullFailed(#[source] io::Error),
/// Redirecting a standard file descriptor failed.
#[error("failed to redirect stdio: {0}")]
RedirectFailed(#[source] nix::Error),
/// A PID lifecycle operation failed.
#[error("PID file error: {0}")]
PidFile(String),
/// Another process owns the daemon PID lifecycle.
#[error("another instance is already running (pid {0})")]
AlreadyRunning(i32),
/// The configured runtime user does not exist.
#[error("user '{0}' not found")]
UserNotFound(String),
/// The configured runtime group does not exist.
#[error("group '{0}' not found")]
GroupNotFound(String),
/// Applying the configured runtime identity failed.
#[error("failed to set uid/gid: {0}")]
PrivilegeDrop(#[source] nix::Error),
/// An underlying filesystem operation failed.
#[error("io error: {0}")]
Io(#[from] io::Error),
}
@@ -106,38 +123,28 @@ pub enum DaemonizeResult {
/// Returns `DaemonizeResult::Parent` in the original parent (which should exit),
/// or `DaemonizeResult::Child` in the final daemon child.
pub fn daemonize(working_dir: Option<&Path>) -> Result<DaemonizeResult, DaemonError> {
// First fork
match unsafe { fork() } {
Ok(ForkResult::Parent { .. }) => {
// Parent exits
return Ok(DaemonizeResult::Parent);
}
Ok(ForkResult::Child) => {
// Child continues
}
Ok(ForkResult::Child) => {}
Err(e) => return Err(DaemonError::ForkFailed(e)),
}
// Create new session, become session leader
setsid().map_err(DaemonError::SetsidFailed)?;
// Second fork to ensure we can never acquire a controlling terminal
match unsafe { fork() } {
Ok(ForkResult::Parent { .. }) => {
// Intermediate parent exits
std::process::exit(0);
}
Ok(ForkResult::Child) => {
// Final daemon child continues
}
Ok(ForkResult::Child) => {}
Err(e) => return Err(DaemonError::ForkFailed(e)),
}
// Change working directory
let target_dir = working_dir.unwrap_or(Path::new("/"));
chdir(target_dir).map_err(DaemonError::ChdirFailed)?;
// Redirect stdin, stdout, stderr to /dev/null
redirect_stdio_to_devnull()?;
Ok(DaemonizeResult::Child)
@@ -156,21 +163,17 @@ fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
// Use libc::dup2 directly for redirecting standard file descriptors
// nix 0.31's dup2 requires OwnedFd which doesn't work well with stdio fds
unsafe {
// Redirect stdin (fd 0)
if libc::dup2(devnull_fd, 0) < 0 {
return Err(DaemonError::RedirectFailed(Errno::last()));
}
// Redirect stdout (fd 1)
if libc::dup2(devnull_fd, 1) < 0 {
return Err(DaemonError::RedirectFailed(Errno::last()));
}
// Redirect stderr (fd 2)
if libc::dup2(devnull_fd, 2) < 0 {
return Err(DaemonError::RedirectFailed(Errno::last()));
}
}
// Close original devnull fd if it's not one of the standard fds
if devnull_fd > 2 {
let _ = close(devnull_fd);
}
@@ -178,166 +181,6 @@ fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
Ok(())
}
/// PID file manager with flock-based locking.
pub struct PidFile {
path: PathBuf,
file: Option<File>,
locked: bool,
}
impl PidFile {
/// Creates a new PID file manager for the given path.
pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self {
path: path.as_ref().to_path_buf(),
file: None,
locked: false,
}
}
/// Checks if another instance is already running.
///
/// Returns the PID of the running instance if one exists.
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
if !self.path.exists() {
return Ok(None);
}
// Try to read existing PID
let mut contents = String::new();
File::open(&self.path)
.and_then(|mut f| f.read_to_string(&mut contents))
.map_err(|e| {
DaemonError::PidFile(format!("cannot read {}: {}", self.path.display(), e))
})?;
let pid: i32 = contents
.trim()
.parse()
.map_err(|_| DaemonError::PidFile(format!("invalid PID in {}", self.path.display())))?;
// Check if process is still running
if is_process_running(pid) {
Ok(Some(pid))
} else {
// Stale PID file
debug!(pid, path = %self.path.display(), "Removing stale PID file");
let _ = fs::remove_file(&self.path);
Ok(None)
}
}
/// Acquires the PID file lock and writes the current PID.
///
/// Fails if another instance is already running.
pub fn acquire(&mut self) -> Result<(), DaemonError> {
// Check for running instance first
if let Some(pid) = self.check_running()? {
return Err(DaemonError::AlreadyRunning(pid));
}
// Ensure parent directory exists
if let Some(parent) = self.path.parent() {
if !parent.exists() {
fs::create_dir_all(parent).map_err(|e| {
DaemonError::PidFile(format!(
"cannot create directory {}: {}",
parent.display(),
e
))
})?;
}
}
// Open/create PID file with exclusive lock
let file = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o644)
.open(&self.path)
.map_err(|e| {
DaemonError::PidFile(format!("cannot open {}: {}", self.path.display(), e))
})?;
// Try to acquire exclusive lock (non-blocking)
let flock = Flock::lock(file, FlockArg::LockExclusiveNonblock).map_err(|(_, errno)| {
// Check if another instance grabbed the lock
if let Some(pid) = self.check_running().ok().flatten() {
DaemonError::AlreadyRunning(pid)
} else {
DaemonError::PidFile(format!("cannot lock {}: {}", self.path.display(), errno))
}
})?;
// Write our PID
let pid = getpid();
let mut file = flock
.unlock()
.map_err(|(_, errno)| DaemonError::PidFile(format!("unlock failed: {}", errno)))?;
writeln!(file, "{}", pid).map_err(|e| {
DaemonError::PidFile(format!(
"cannot write PID to {}: {}",
self.path.display(),
e
))
})?;
// Re-acquire lock and keep it
let flock = Flock::lock(file, FlockArg::LockExclusiveNonblock).map_err(|(_, errno)| {
DaemonError::PidFile(format!("cannot re-lock {}: {}", self.path.display(), errno))
})?;
self.file = Some(flock.unlock().map_err(|(_, errno)| {
DaemonError::PidFile(format!("unlock for storage failed: {}", errno))
})?);
self.locked = true;
info!(pid = pid.as_raw(), path = %self.path.display(), "PID file created");
Ok(())
}
/// Releases the PID file lock and removes the file.
pub fn release(&mut self) -> Result<(), DaemonError> {
if let Some(file) = self.file.take() {
drop(file);
}
self.locked = false;
if self.path.exists() {
fs::remove_file(&self.path).map_err(|e| {
DaemonError::PidFile(format!("cannot remove {}: {}", self.path.display(), e))
})?;
debug!(path = %self.path.display(), "PID file removed");
}
Ok(())
}
/// Returns the path to this PID file.
#[allow(dead_code)]
pub fn path(&self) -> &Path {
&self.path
}
}
impl Drop for PidFile {
fn drop(&mut self) {
if self.locked {
if let Err(e) = self.release() {
warn!(error = %e, "Failed to clean up PID file on drop");
}
}
}
}
/// Checks if a process with the given PID is running.
fn is_process_running(pid: i32) -> bool {
// kill(pid, 0) checks if process exists without sending a signal
nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok()
}
// macOS gates nix::unistd::setgroups differently in the current dependency set,
// so call libc directly there while preserving the original nix path elsewhere.
fn set_supplementary_groups(gid: Gid) -> Result<(), nix::Error> {
@@ -383,9 +226,11 @@ pub fn drop_privileges(
};
if (target_uid.is_some() || target_gid.is_some())
&& let Some(file) = pid_file.and_then(|pid| pid.file.as_ref())
&& let Some(pid_file) = pid_file
{
unistd::fchown(file, target_uid, target_gid).map_err(DaemonError::PrivilegeDrop)?;
for file in pid_file.ownership_file_handles().into_iter().flatten() {
unistd::fchown(file, target_uid, target_gid).map_err(DaemonError::PrivilegeDrop)?;
}
}
if let Some(gid) = target_gid {
@@ -401,7 +246,7 @@ pub fn drop_privileges(
if uid.as_raw() != 0
&& let Some(pid) = pid_file
{
let parent = pid.path.parent().unwrap_or(Path::new("."));
let parent = pid.path().parent().unwrap_or(Path::new("."));
let probe_path = parent.join(format!(
".telemt_pid_probe_{}_{}",
std::process::id(),
@@ -436,7 +281,6 @@ pub fn drop_privileges(
/// Looks up a user by name and returns their UID.
fn lookup_user(name: &str) -> Result<Uid, DaemonError> {
// Use libc getpwnam
let c_name =
std::ffi::CString::new(name).map_err(|_| DaemonError::UserNotFound(name.to_string()))?;
@@ -480,76 +324,6 @@ fn lookup_group(name: &str) -> Result<Gid, DaemonError> {
}
}
/// Reads PID from a PID file.
#[allow(dead_code)]
pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
let path = path.as_ref();
let mut contents = String::new();
File::open(path)
.and_then(|mut f| f.read_to_string(&mut contents))
.map_err(|e| DaemonError::PidFile(format!("cannot read {}: {}", path.display(), e)))?;
contents
.trim()
.parse()
.map_err(|_| DaemonError::PidFile(format!("invalid PID in {}", path.display())))
}
/// Sends a signal to the process specified in a PID file.
#[allow(dead_code)]
pub fn signal_pid_file<P: AsRef<Path>>(
path: P,
signal: nix::sys::signal::Signal,
) -> Result<(), DaemonError> {
let pid = read_pid_file(&path)?;
if !is_process_running(pid) {
return Err(DaemonError::PidFile(format!(
"process {} from {} is not running",
pid,
path.as_ref().display()
)));
}
nix::sys::signal::kill(Pid::from_raw(pid), signal)
.map_err(|e| DaemonError::PidFile(format!("cannot signal process {}: {}", pid, e)))?;
Ok(())
}
/// Returns the status of the daemon based on PID file.
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DaemonStatus {
/// Daemon is running with the given PID.
Running(i32),
/// PID file exists but process is not running.
Stale(i32),
/// No PID file exists.
NotRunning,
}
/// Checks the daemon status from a PID file.
#[allow(dead_code)]
pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
let path = path.as_ref();
if !path.exists() {
return DaemonStatus::NotRunning;
}
match read_pid_file(path) {
Ok(pid) => {
if is_process_running(pid) {
DaemonStatus::Running(pid)
} else {
DaemonStatus::Stale(pid)
}
}
Err(_) => DaemonStatus::NotRunning,
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -571,29 +345,4 @@ mod tests {
};
assert!(!opts.should_daemonize());
}
#[test]
fn test_check_status_not_running() {
let path = "/tmp/telemt_test_nonexistent.pid";
assert_eq!(check_status(path), DaemonStatus::NotRunning);
}
#[test]
fn test_pid_file_basic() {
let path = "/tmp/telemt_test_pidfile.pid";
let _ = fs::remove_file(path);
let mut pf = PidFile::new(path);
assert!(pf.check_running().unwrap().is_none());
pf.acquire().unwrap();
assert!(Path::new(path).exists());
// Read it back
let pid = read_pid_file(path).unwrap();
assert_eq!(pid, std::process::id() as i32);
pf.release().unwrap();
assert!(!Path::new(path).exists());
}
}
+511
View File
@@ -0,0 +1,511 @@
use std::ffi::OsStr;
use std::fs::{self, File};
use std::io::{self, ErrorKind, Read, Write};
#[cfg(target_os = "linux")]
use std::os::fd::{FromRawFd, OwnedFd};
use std::os::unix::fs::{MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use nix::fcntl::{Flock, FlockArg, OFlag, openat};
use nix::sys::stat::Mode;
use nix::unistd::{Pid, UnlinkatFlags, getpid, unlinkat};
use tracing::{debug, info, warn};
use super::DaemonError;
use crate::util::secure_fs::AnchoredPath;
/// PID file manager backed by a persistent sibling lock file.
pub struct PidFile {
path: PathBuf,
lock_path: PathBuf,
pid_file: Option<File>,
pid_identity: Option<FileIdentity>,
lock_file: Option<Flock<File>>,
anchor: Option<AnchoredPath>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct FileIdentity {
device: u64,
inode: u64,
}
impl FileIdentity {
fn from_metadata(metadata: &fs::Metadata) -> Self {
Self {
device: metadata.dev(),
inode: metadata.ino(),
}
}
}
impl PidFile {
/// Creates a new PID file manager for the given path.
pub fn new<P: AsRef<Path>>(path: P) -> Self {
let path = normalize_pid_path(path.as_ref());
let lock_path = sibling_lock_path(&path);
Self {
path,
lock_path,
pid_file: None,
pid_identity: None,
lock_file: None,
anchor: None,
}
}
/// Checks whether the PID file names a running process without modifying either file.
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
let Some(pid) = read_pid_file_if_exists(&self.path)? else {
return Ok(None);
};
Ok(is_process_running(pid).then_some(pid))
}
/// Acquires the persistent sibling lock and writes the current PID.
///
/// Fails if another owner holds the lock or the existing PID names a running process.
pub fn acquire(&mut self) -> Result<(), DaemonError> {
let anchor =
AnchoredPath::open_trusted_parent_or_create(&self.path, 0o755).map_err(|error| {
DaemonError::PidFile(format!(
"cannot open trusted parent for {}: {}",
self.path.display(),
error
))
})?;
let lock_name = self.lock_path.file_name().ok_or_else(|| {
DaemonError::PidFile(format!(
"lock path {} has no file name",
self.lock_path.display()
))
})?;
let lock_file = open_file_at(
&anchor,
lock_name,
OFlag::O_RDWR | OFlag::O_CREAT | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
0o644,
)
.map_err(|error| {
DaemonError::PidFile(format!(
"cannot open lock file {}: {}",
self.lock_path.display(),
error
))
})?;
validate_regular_single_link(&lock_file, &self.lock_path)?;
let lock_file =
Flock::lock(lock_file, FlockArg::LockExclusiveNonblock).map_err(|(_, errno)| {
if let Some(pid) = read_pid_file_at(&anchor, &self.path)
.ok()
.flatten()
.filter(|pid| is_process_running(*pid))
{
DaemonError::AlreadyRunning(pid)
} else {
DaemonError::PidFile(format!(
"cannot lock {}: {}",
self.lock_path.display(),
errno
))
}
})?;
if let Some(pid) = read_pid_file_at(&anchor, &self.path)?
&& is_process_running(pid)
{
return Err(DaemonError::AlreadyRunning(pid));
}
let mut pid_file = open_file_at(
&anchor,
anchor.name(),
OFlag::O_RDWR | OFlag::O_CREAT | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
0o644,
)
.map_err(|error| {
DaemonError::PidFile(format!("cannot open {}: {}", self.path.display(), error))
})?;
let pid_metadata = validate_regular_single_link(&pid_file, &self.path)?;
let pid_identity = FileIdentity::from_metadata(&pid_metadata);
// Validate the opened inode before modifying it so a hard-link substitution
// cannot turn PID publication into truncation of an unrelated file.
pid_file.set_len(0).map_err(|error| {
DaemonError::PidFile(format!(
"cannot truncate {}: {}",
self.path.display(),
error
))
})?;
let pid = getpid();
writeln!(pid_file, "{}", pid).map_err(|error| {
DaemonError::PidFile(format!(
"cannot write PID to {}: {}",
self.path.display(),
error
))
})?;
pid_file.sync_data().map_err(|error| {
DaemonError::PidFile(format!(
"cannot sync PID file {}: {}",
self.path.display(),
error
))
})?;
self.pid_file = Some(pid_file);
self.pid_identity = Some(pid_identity);
self.lock_file = Some(lock_file);
self.anchor = Some(anchor);
info!(pid = pid.as_raw(), path = %self.path.display(), "PID file created");
Ok(())
}
/// Removes the PID file while retaining exclusive lock ownership until cleanup completes.
pub fn release(&mut self) -> Result<(), DaemonError> {
if self.lock_file.is_none() {
self.pid_file = None;
self.pid_identity = None;
self.anchor = None;
return Ok(());
}
let removal = match self.anchor.as_ref() {
Some(anchor) => remove_owned_pid_file(anchor, &self.path, self.pid_identity),
None => Err(DaemonError::PidFile(
"PID file lock is held without a directory anchor".to_string(),
)),
};
self.pid_file = None;
self.pid_identity = None;
self.lock_file = None;
self.anchor = None;
removal?;
debug!(path = %self.path.display(), "PID file removed");
Ok(())
}
/// Returns the path to this PID file.
pub fn path(&self) -> &Path {
&self.path
}
/// Returns open files whose ownership must follow the target runtime identity.
pub(super) fn ownership_file_handles(&self) -> [Option<&File>; 2] {
[self.pid_file.as_ref(), self.lock_file.as_deref()]
}
}
impl Drop for PidFile {
fn drop(&mut self) {
if self.lock_file.is_some()
&& let Err(error) = self.release()
{
warn!(error = %error, "Failed to clean up PID file on drop");
}
}
}
fn sibling_lock_path(path: &Path) -> PathBuf {
let mut lock_path = path.as_os_str().to_os_string();
lock_path.push(".lock");
lock_path.into()
}
fn normalize_pid_path(path: &Path) -> PathBuf {
let legacy_run = Path::new("/var/run");
let Ok(remainder) = path.strip_prefix(legacy_run) else {
return path.to_path_buf();
};
let Ok(var_metadata) = fs::metadata("/var") else {
return path.to_path_buf();
};
let Ok(link_metadata) = fs::symlink_metadata(legacy_run) else {
return path.to_path_buf();
};
let Ok(target) = fs::read_link(legacy_run) else {
return path.to_path_buf();
};
let trusted_var = var_metadata.is_dir()
&& var_metadata.uid() == 0
&& var_metadata.permissions().mode() & 0o022 == 0;
let trusted_alias = link_metadata.file_type().is_symlink()
&& link_metadata.uid() == 0
&& (target == Path::new("/run") || target == Path::new("../run"));
if trusted_var && trusted_alias {
Path::new("/run").join(remainder)
} else {
path.to_path_buf()
}
}
fn open_file_at(anchor: &AnchoredPath, name: &OsStr, flags: OFlag, mode: u32) -> io::Result<File> {
let descriptor = openat(anchor.parent(), name, flags, Mode::from_bits_truncate(mode))
.map_err(|error| io::Error::from_raw_os_error(error as i32))?;
Ok(File::from(descriptor))
}
fn read_pid_file_if_exists(path: &Path) -> Result<Option<i32>, DaemonError> {
let anchor = match AnchoredPath::open_trusted_parent(path) {
Ok(anchor) => anchor,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
Err(error) => {
return Err(DaemonError::PidFile(format!(
"cannot open trusted parent for {}: {}",
path.display(),
error
)));
}
};
read_pid_file_at(&anchor, path)
}
fn read_pid_file_at(anchor: &AnchoredPath, path: &Path) -> Result<Option<i32>, DaemonError> {
let mut file = match open_file_at(
anchor,
anchor.name(),
OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
0,
) {
Ok(file) => file,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
Err(error) => {
return Err(DaemonError::PidFile(format!(
"cannot read {}: {}",
path.display(),
error
)));
}
};
let metadata = validate_regular_single_link(&file, path)?;
if metadata.len() > 64 {
return Err(DaemonError::PidFile(format!(
"invalid PID in {}",
path.display()
)));
}
let mut contents = String::new();
file.read_to_string(&mut contents).map_err(|error| {
DaemonError::PidFile(format!("cannot read {}: {}", path.display(), error))
})?;
let pid: i32 = contents
.trim()
.parse()
.map_err(|_| DaemonError::PidFile(format!("invalid PID in {}", path.display())))?;
if pid <= 1 {
return Err(DaemonError::PidFile(format!(
"invalid PID in {}",
path.display()
)));
}
Ok(Some(pid))
}
fn remove_owned_pid_file(
anchor: &AnchoredPath,
path: &Path,
expected: Option<FileIdentity>,
) -> Result<(), DaemonError> {
let file = match open_file_at(
anchor,
anchor.name(),
OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
0,
) {
Ok(file) => file,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
Err(error) => {
return Err(DaemonError::PidFile(format!(
"cannot inspect {} before removal: {}",
path.display(),
error
)));
}
};
let metadata = validate_regular_single_link(&file, path)?;
if expected != Some(FileIdentity::from_metadata(&metadata)) {
return Err(DaemonError::PidFile(format!(
"refusing to remove replaced PID file {}",
path.display()
)));
}
drop(file);
unlinkat(anchor.parent(), anchor.name(), UnlinkatFlags::NoRemoveDir).map_err(|error| {
DaemonError::PidFile(format!(
"cannot remove {}: {}",
path.display(),
io::Error::from_raw_os_error(error as i32)
))
})
}
fn validate_regular_single_link(file: &File, path: &Path) -> Result<fs::Metadata, DaemonError> {
let metadata = file.metadata().map_err(|error| {
DaemonError::PidFile(format!("cannot inspect {}: {}", path.display(), error))
})?;
if !metadata.is_file() || metadata.nlink() != 1 {
return Err(DaemonError::PidFile(format!(
"{} must be a regular file with one directory entry",
path.display()
)));
}
Ok(metadata)
}
/// Reads a PID from a PID file.
#[allow(dead_code)]
pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
let path = normalize_pid_path(path.as_ref());
read_pid_file_if_exists(&path)?.ok_or_else(|| {
DaemonError::PidFile(format!(
"cannot read {}: file does not exist",
path.display()
))
})
}
/// Sends a signal to the process specified in a PID file.
#[allow(dead_code)]
pub fn signal_pid_file<P: AsRef<Path>>(
path: P,
signal: nix::sys::signal::Signal,
) -> Result<(), DaemonError> {
let path = normalize_pid_path(path.as_ref());
let pid = read_pid_file(&path)?;
#[cfg(target_os = "linux")]
let pidfd = open_pidfd(pid)?;
if !daemon_lock_is_held(&path)? {
return Err(DaemonError::PidFile(format!(
"refusing to signal unlocked or stale PID file {}",
path.display()
)));
}
#[cfg(target_os = "linux")]
return signal_pidfd(&pidfd, pid, signal);
#[cfg(not(target_os = "linux"))]
nix::sys::signal::kill(Pid::from_raw(pid), signal)
.map_err(|error| DaemonError::PidFile(format!("cannot signal process {}: {}", pid, error)))
}
/// Daemon state derived from the PID file.
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DaemonStatus {
/// Daemon is running with the given PID.
Running(i32),
/// PID file exists but the named process is not running.
Stale(i32),
/// No readable PID file exists.
NotRunning,
}
/// Checks daemon status without modifying the PID or lock file.
#[allow(dead_code)]
pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
let path = normalize_pid_path(path.as_ref());
match read_pid_file_if_exists(&path) {
Ok(Some(pid)) if daemon_lock_is_held(&path).unwrap_or(false) && is_process_running(pid) => {
DaemonStatus::Running(pid)
}
Ok(Some(pid)) => DaemonStatus::Stale(pid),
Ok(None) | Err(_) => DaemonStatus::NotRunning,
}
}
fn daemon_lock_is_held(path: &Path) -> Result<bool, DaemonError> {
let lock_path = sibling_lock_path(path);
let anchor = match AnchoredPath::open_trusted_parent(path) {
Ok(anchor) => anchor,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
Err(error) => {
return Err(DaemonError::PidFile(format!(
"cannot open trusted parent for {}: {}",
path.display(),
error
)));
}
};
let lock_name = lock_path.file_name().ok_or_else(|| {
DaemonError::PidFile(format!(
"lock path {} has no file name",
lock_path.display()
))
})?;
let file = match open_file_at(
&anchor,
lock_name,
OFlag::O_RDWR | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
0,
) {
Ok(file) => file,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
Err(error) => {
return Err(DaemonError::PidFile(format!(
"cannot inspect lock {}: {}",
lock_path.display(),
error
)));
}
};
validate_regular_single_link(&file, &lock_path)?;
match Flock::lock(file, FlockArg::LockExclusiveNonblock) {
Ok(_available) => Ok(false),
Err((_file, nix::errno::Errno::EWOULDBLOCK)) => Ok(true),
Err((_file, error)) => Err(DaemonError::PidFile(format!(
"cannot inspect lock ownership for {}: {}",
lock_path.display(),
error
))),
}
}
#[cfg(target_os = "linux")]
fn open_pidfd(pid: i32) -> Result<OwnedFd, DaemonError> {
// SAFETY: `pidfd_open` receives a validated positive PID and no pointer arguments.
let descriptor = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) };
if descriptor < 0 {
return Err(DaemonError::PidFile(format!(
"cannot open stable process handle for {}: {}",
pid,
std::io::Error::last_os_error()
)));
}
// SAFETY: a successful `pidfd_open` returns one newly owned descriptor.
Ok(unsafe { OwnedFd::from_raw_fd(descriptor as i32) })
}
#[cfg(target_os = "linux")]
fn signal_pidfd(
pidfd: &OwnedFd,
pid: i32,
signal: nix::sys::signal::Signal,
) -> Result<(), DaemonError> {
use std::os::fd::AsRawFd;
// SAFETY: the pidfd is owned and valid, and both optional pointer arguments are null.
let result = unsafe {
libc::syscall(
libc::SYS_pidfd_send_signal,
pidfd.as_raw_fd(),
signal as libc::c_int,
std::ptr::null::<libc::siginfo_t>(),
0,
)
};
if result == 0 {
Ok(())
} else {
Err(DaemonError::PidFile(format!(
"cannot signal process {} through stable handle: {}",
pid,
std::io::Error::last_os_error()
)))
}
}
fn is_process_running(pid: i32) -> bool {
nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok()
}
#[cfg(test)]
mod tests;
+283
View File
@@ -0,0 +1,283 @@
use std::os::unix::fs::{MetadataExt, symlink};
use std::process::{Child, Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use super::*;
const HELPER_PID_PATH: &str = "TELEMT_PID_LOCK_HELPER_PATH";
const HELPER_READY_PATH: &str = "TELEMT_PID_LOCK_HELPER_READY";
const HELPER_STOP_PATH: &str = "TELEMT_PID_LOCK_HELPER_STOP";
fn wait_for_path(path: &Path, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if path.exists() {
return true;
}
thread::sleep(Duration::from_millis(10));
}
false
}
fn wait_for_child(child: &mut Child, timeout: Duration) -> Option<std::process::ExitStatus> {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if let Some(status) = child.try_wait().unwrap() {
return Some(status);
}
thread::sleep(Duration::from_millis(10));
}
None
}
#[test]
fn pid_file_remains_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<PidFile>();
}
#[test]
fn system_var_run_alias_keeps_the_default_pid_path_usable() {
let Ok(metadata) = fs::symlink_metadata("/var/run") else {
return;
};
let Ok(target) = fs::read_link("/var/run") else {
return;
};
if !metadata.file_type().is_symlink()
|| (target != Path::new("/run") && target != Path::new("../run"))
{
return;
}
let pid_file = PidFile::new("/var/run/telemt.pid");
assert_eq!(pid_file.path(), Path::new("/run/telemt.pid"));
}
#[test]
fn lock_holder_subprocess() {
let Some(pid_path) = std::env::var_os(HELPER_PID_PATH) else {
return;
};
let ready_path = PathBuf::from(std::env::var_os(HELPER_READY_PATH).unwrap());
let stop_path = PathBuf::from(std::env::var_os(HELPER_STOP_PATH).unwrap());
let mut pid_file = PidFile::new(PathBuf::from(pid_path));
pid_file.acquire().unwrap();
fs::write(&ready_path, b"ready").unwrap();
assert!(wait_for_path(&stop_path, Duration::from_secs(10)));
pid_file.release().unwrap();
}
#[test]
fn persistent_sibling_lock_serializes_processes_after_pid_unlink() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
let lock_path = sibling_lock_path(&pid_path);
let ready_path = directory.path().join("ready");
let stop_path = directory.path().join("stop");
let mut child = Command::new(std::env::current_exe().unwrap())
.args([
"--exact",
"daemon::pid_file::tests::lock_holder_subprocess",
"--nocapture",
])
.env(HELPER_PID_PATH, &pid_path)
.env(HELPER_READY_PATH, &ready_path)
.env(HELPER_STOP_PATH, &stop_path)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.unwrap();
if !wait_for_path(&ready_path, Duration::from_secs(5)) {
let _ = child.kill();
let _ = child.wait();
panic!("PID lock holder did not become ready");
}
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
fs::remove_file(&pid_path).unwrap();
let mut contender = PidFile::new(&pid_path);
assert!(contender.acquire().is_err());
fs::write(&stop_path, b"stop").unwrap();
let status = wait_for_child(&mut child, Duration::from_secs(5)).unwrap_or_else(|| {
let _ = child.kill();
child.wait().unwrap()
});
assert!(status.success());
assert_eq!(fs::metadata(&lock_path).unwrap().ino(), lock_inode);
contender.acquire().unwrap();
assert_eq!(fs::metadata(&lock_path).unwrap().ino(), lock_inode);
contender.release().unwrap();
assert!(!pid_path.exists());
assert!(lock_path.exists());
}
#[test]
fn stale_pid_checks_are_read_only() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
fs::write(&pid_path, b"2000000000\n").unwrap();
let pid_file = PidFile::new(&pid_path);
assert_eq!(pid_file.check_running().unwrap(), None);
assert_eq!(check_status(&pid_path), DaemonStatus::Stale(2_000_000_000));
assert!(pid_path.exists());
}
#[test]
fn status_requires_live_lock_ownership() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
fs::write(&pid_path, format!("{}\n", std::process::id())).unwrap();
assert_eq!(
check_status(&pid_path),
DaemonStatus::Stale(std::process::id() as i32)
);
fs::remove_file(&pid_path).unwrap();
let mut owner = PidFile::new(&pid_path);
owner.acquire().unwrap();
assert_eq!(
check_status(&pid_path),
DaemonStatus::Running(std::process::id() as i32)
);
owner.release().unwrap();
}
#[test]
fn unowned_release_does_not_remove_pid_file() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
fs::write(&pid_path, b"2000000000\n").unwrap();
let mut pid_file = PidFile::new(&pid_path);
pid_file.release().unwrap();
assert!(pid_path.exists());
}
#[test]
fn acquire_rejects_pid_symlink_without_truncating_target() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
let target_path = directory.path().join("target");
fs::write(&target_path, b"preserve\n").unwrap();
symlink(&target_path, &pid_path).unwrap();
let mut pid_file = PidFile::new(&pid_path);
assert!(pid_file.acquire().is_err());
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
}
#[test]
fn acquire_rejects_pid_hard_link_without_truncating_target() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
let target_path = directory.path().join("target");
fs::write(&target_path, b"preserve\n").unwrap();
fs::hard_link(&target_path, &pid_path).unwrap();
let mut pid_file = PidFile::new(&pid_path);
assert!(pid_file.acquire().is_err());
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
}
#[test]
fn acquire_rejects_symlinked_parent_without_publishing_outside() {
let directory = tempfile::tempdir().unwrap();
let real_parent = directory.path().join("real");
let linked_parent = directory.path().join("linked");
fs::create_dir(&real_parent).unwrap();
symlink(&real_parent, &linked_parent).unwrap();
let pid_path = linked_parent.join("telemt.pid");
let mut pid_file = PidFile::new(&pid_path);
assert!(pid_file.acquire().is_err());
assert!(!real_parent.join("telemt.pid").exists());
assert!(!real_parent.join("telemt.pid.lock").exists());
}
#[test]
fn release_remains_anchored_after_parent_path_replacement() {
let directory = tempfile::tempdir().unwrap();
let active_parent = directory.path().join("active");
let moved_parent = directory.path().join("moved");
fs::create_dir(&active_parent).unwrap();
let pid_path = active_parent.join("telemt.pid");
let mut pid_file = PidFile::new(&pid_path);
pid_file.acquire().unwrap();
fs::rename(&active_parent, &moved_parent).unwrap();
fs::create_dir(&active_parent).unwrap();
fs::write(active_parent.join("telemt.pid"), b"replacement\n").unwrap();
pid_file.release().unwrap();
assert!(!moved_parent.join("telemt.pid").exists());
assert_eq!(
fs::read(active_parent.join("telemt.pid")).unwrap(),
b"replacement\n"
);
}
#[test]
fn release_does_not_remove_replacement_path() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
let owned_path = directory.path().join("owned.pid");
let mut pid_file = PidFile::new(&pid_path);
pid_file.acquire().unwrap();
fs::rename(&pid_path, &owned_path).unwrap();
fs::write(&pid_path, b"replacement\n").unwrap();
let error = pid_file.release().unwrap_err();
assert!(
error
.to_string()
.contains("refusing to remove replaced PID file")
);
assert_eq!(fs::read(&pid_path).unwrap(), b"replacement\n");
}
#[test]
fn pid_parser_rejects_process_group_values() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
for value in ["-1\n", "0\n", "1\n"] {
fs::write(&pid_path, value).unwrap();
assert!(read_pid_file(&pid_path).is_err());
}
}
#[test]
fn pid_file_release_keeps_lock_inode() {
let directory = tempfile::tempdir().unwrap();
let pid_path = directory.path().join("telemt.pid");
let lock_path = sibling_lock_path(&pid_path);
let mut pid_file = PidFile::new(&pid_path);
pid_file.acquire().unwrap();
assert!(
pid_file
.ownership_file_handles()
.into_iter()
.all(|file| file.is_some())
);
assert_eq!(read_pid_file(&pid_path).unwrap(), std::process::id() as i32);
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
pid_file.release().unwrap();
assert!(!pid_path.exists());
assert!(lock_path.exists());
pid_file.acquire().unwrap();
assert_eq!(fs::metadata(&lock_path).unwrap().ino(), lock_inode);
pid_file.release().unwrap();
}
+26 -5
View File
@@ -6,6 +6,8 @@ use serde_json::Value;
use crate::config::ProxyConfig;
const HEALTHCHECK_RESPONSE_MAX_BYTES: u64 = 64 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HealthcheckMode {
Liveness,
@@ -73,10 +75,7 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
.flush()
.map_err(|error| format!("request flush failed: {error}"))?;
let mut raw_response = Vec::new();
stream
.read_to_end(&mut raw_response)
.map_err(|error| format!("response read failed: {error}"))?;
let raw_response = read_response_bounded(&mut stream)?;
let response =
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
@@ -89,6 +88,18 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
Ok(())
}
fn read_response_bounded(reader: &mut impl Read) -> Result<Vec<u8>, String> {
let mut raw_response = Vec::new();
reader
.take(HEALTHCHECK_RESPONSE_MAX_BYTES.saturating_add(1))
.read_to_end(&mut raw_response)
.map_err(|error| format!("response read failed: {error}"))?;
if raw_response.len() as u64 > HEALTHCHECK_RESPONSE_MAX_BYTES {
return Err("response exceeds the 64 KiB healthcheck limit".to_string());
}
Ok(raw_response)
}
fn probe_target(listen: SocketAddr) -> SocketAddr {
match listen {
SocketAddr::V4(addr) => {
@@ -180,7 +191,10 @@ fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
#[cfg(test)]
mod tests {
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
use super::{
HEALTHCHECK_RESPONSE_MAX_BYTES, HealthcheckMode, parse_status_code, read_response_bounded,
split_response, validate_payload,
};
#[test]
fn parse_status_code_reads_http_200() {
@@ -208,4 +222,11 @@ mod tests {
let result = validate_payload(HealthcheckMode::Ready, body);
assert!(result.is_err());
}
#[test]
fn bounded_reader_rejects_oversized_health_response() {
let payload = vec![b'x'; HEALTHCHECK_RESPONSE_MAX_BYTES as usize + 1];
assert!(read_response_bounded(&mut payload.as_slice()).is_err());
}
}
+70 -48
View File
@@ -8,13 +8,14 @@ use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU8, AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use dashmap::DashMap;
use arc_swap::ArcSwap;
use tokio::sync::{Mutex as AsyncMutex, RwLock};
use crate::config::UserMaxUniqueIpsMode;
use crate::proxy::user_admission::UserIncarnation;
const CLEANUP_DRAIN_BATCH_LIMIT: usize = 1024;
const MAX_ACTIVE_IP_ENTRIES: u64 = 131_072;
@@ -32,11 +33,36 @@ mod tests;
struct UserIpShard {
active_ips: HashMap<String, HashMap<IpAddr, usize>>,
recent_ips: HashMap<String, HashMap<IpAddr, Instant>>,
incarnations: HashMap<String, UserIncarnation>,
}
#[derive(Debug, Default)]
struct CleanupShard {
queue: Mutex<HashMap<String, HashMap<IpAddr, usize>>>,
queue: Mutex<CleanupQueue>,
}
type CleanupQueue = HashMap<String, HashMap<UserIncarnation, HashMap<IpAddr, usize>>>;
type CleanupBatch = HashMap<(String, UserIncarnation, IpAddr), usize>;
#[derive(Debug, Clone)]
struct UserIpLimitPolicy {
source_generation: u64,
max_ips: Arc<HashMap<String, usize>>,
default_max_ips: usize,
mode: UserMaxUniqueIpsMode,
window_secs: u64,
}
impl Default for UserIpLimitPolicy {
fn default() -> Self {
Self {
source_generation: 0,
max_ips: Arc::new(HashMap::new()),
default_max_ips: 0,
mode: UserMaxUniqueIpsMode::ActiveWindow,
window_secs: 30,
}
}
}
/// Tracks active and recent client IPs for per-user admission control.
@@ -48,10 +74,8 @@ pub struct UserIpTracker {
active_cap_rejects: Arc<AtomicU64>,
recent_cap_rejects: Arc<AtomicU64>,
cleanup_deferred_releases: Arc<AtomicU64>,
max_ips: Arc<DashMap<String, usize>>,
default_max_ips: Arc<AtomicUsize>,
limit_mode: Arc<AtomicU8>,
limit_window_secs: Arc<AtomicU64>,
limit_policy: Arc<ArcSwap<UserIpLimitPolicy>>,
policy_update: Arc<Mutex<()>>,
last_compact_epoch_secs: Arc<AtomicU64>,
cleanup_queue_len: Arc<AtomicU64>,
cleanup_shards: Arc<Box<[CleanupShard]>>,
@@ -102,12 +126,8 @@ impl UserIpTracker {
active_cap_rejects: Arc::new(AtomicU64::new(0)),
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
max_ips: Arc::new(DashMap::new()),
default_max_ips: Arc::new(AtomicUsize::new(0)),
limit_mode: Arc::new(AtomicU8::new(Self::mode_to_u8(
UserMaxUniqueIpsMode::ActiveWindow,
))),
limit_window_secs: Arc::new(AtomicU64::new(30)),
limit_policy: Arc::new(ArcSwap::from_pointee(UserIpLimitPolicy::default())),
policy_update: Arc::new(Mutex::new(())),
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
cleanup_shards: Arc::new(cleanup_shards),
@@ -117,41 +137,23 @@ impl UserIpTracker {
}
}
pub(super) fn mode_to_u8(mode: UserMaxUniqueIpsMode) -> u8 {
match mode {
UserMaxUniqueIpsMode::ActiveWindow => 0,
UserMaxUniqueIpsMode::TimeWindow => 1,
UserMaxUniqueIpsMode::Combined => 2,
}
}
pub(super) fn mode_from_u8(raw: u8) -> UserMaxUniqueIpsMode {
match raw {
1 => UserMaxUniqueIpsMode::TimeWindow,
2 => UserMaxUniqueIpsMode::Combined,
_ => UserMaxUniqueIpsMode::ActiveWindow,
}
}
pub(super) fn shard_idx(username: &str) -> usize {
let mut hasher = DefaultHasher::new();
username.hash(&mut hasher);
(hasher.finish() as usize) & USER_IP_TRACKER_SHARD_MASK
}
pub(super) fn limit_window(&self) -> Duration {
Duration::from_secs(self.limit_window_secs.load(Ordering::Relaxed).max(1))
fn limit_window(policy: &UserIpLimitPolicy) -> Duration {
Duration::from_secs(policy.window_secs)
}
pub(super) fn user_limit(&self, username: &str) -> Option<usize> {
self.max_ips
fn user_limit(policy: &UserIpLimitPolicy, username: &str) -> Option<usize> {
policy
.max_ips
.get(username)
.map(|limit| *limit)
.copied()
.filter(|limit| *limit > 0)
.or_else(|| {
let default_limit = self.default_max_ips.load(Ordering::Relaxed);
(default_limit > 0).then_some(default_limit)
})
.or_else(|| (policy.default_max_ips > 0).then_some(policy.default_max_ips))
}
pub(super) fn decrement_counter(counter: &AtomicU64, amount: usize) {
@@ -201,19 +203,26 @@ impl UserIpTracker {
}
pub(super) fn pop_one_cleanup(
queue: &mut HashMap<String, HashMap<IpAddr, usize>>,
) -> Option<(String, IpAddr, usize)> {
queue: &mut CleanupQueue,
) -> Option<(String, UserIncarnation, IpAddr, usize)> {
let user = queue.keys().next().cloned()?;
let ip = queue.get(&user)?.keys().next().copied()?;
let count = queue.get_mut(&user)?.remove(&ip)?;
let remove_user = queue
.get(&user)
.map(|user_queue| user_queue.is_empty())
.unwrap_or(false);
if remove_user {
let incarnation = queue.get(&user)?.keys().next().copied()?;
let ip = queue
.get(&user)?
.get(&incarnation)?
.keys()
.next()
.copied()?;
let incarnations = queue.get_mut(&user)?;
let ips = incarnations.get_mut(&incarnation)?;
let count = ips.remove(&ip)?;
if ips.is_empty() {
incarnations.remove(&incarnation);
}
if incarnations.is_empty() {
queue.remove(&user);
}
Some((user, ip, count))
Some((user, incarnation, ip, count))
}
#[cfg(test)]
@@ -229,6 +238,19 @@ impl UserIpTracker {
#[cfg(not(test))]
pub(super) fn observe_cleanup_poison_for_tests(&self) {}
#[cfg(test)]
pub(crate) async fn hold_user_shard_for_tests(
&self,
user: &str,
entered: tokio::sync::oneshot::Sender<()>,
release: tokio::sync::oneshot::Receiver<()>,
) {
let shard_idx = Self::shard_idx(user);
let _guard = self.shards[shard_idx].write().await;
let _ = entered.send(());
let _ = release.await;
}
pub(super) fn now_epoch_secs() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)

Some files were not shown because too many files have changed in this diff Show More