mirror of
https://github.com/telemt/telemt.git
synced 2026-09-17 15:58:31 +03:00
Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 021ad1fe68 | |||
| 4ca7418442 | |||
| 464bc180e1 | |||
| 8b0ee9bf3d | |||
| 24c4cefc74 | |||
| 97c046757a | |||
| 4e3d560a88 | |||
| 3693d1e2a8 | |||
| f857aefd06 | |||
| 3cb123fbf0 | |||
| 597b6b0226 | |||
| 63b9cce25f | |||
| 20a4d50524 | |||
| 106b26a5b7 | |||
| 9908e04e04 | |||
| 9f023ff9c7 | |||
| 0d044c7372 | |||
| e796cb112c | |||
| 7de6edda98 | |||
| 7f4e637b99 | |||
| 50faaba8c5 | |||
| 9170e347f5 | |||
| 0d8d331c7c | |||
| 01ffca5d34 | |||
| aec7d1619a | |||
| 718ce0847e | |||
| 66f2b8889f | |||
| 281f63f940 | |||
| 1bb6b0bdda | |||
| 084834f5ec | |||
| 012dc07a98 |
@@ -19,13 +19,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Install latest stable Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry & build artifacts
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
run: cargo build --release --verbose
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7.0.1
|
||||
with:
|
||||
name: telemt
|
||||
path: target/release/telemt
|
||||
path: target/release/telemt
|
||||
|
||||
+40
-24
@@ -14,9 +14,7 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ==========================
|
||||
# Formatting
|
||||
# ==========================
|
||||
# Rust formatting validation
|
||||
fmt:
|
||||
name: Fmt
|
||||
runs-on: ubuntu-latest
|
||||
@@ -25,7 +23,7 @@ jobs:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
@@ -33,9 +31,23 @@ jobs:
|
||||
|
||||
- run: cargo fmt -- --check
|
||||
|
||||
# ==========================
|
||||
# Tests
|
||||
# ==========================
|
||||
# Minimum supported Rust version validation
|
||||
msrv:
|
||||
name: MSRV
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@1.88.0
|
||||
|
||||
- name: Check all targets on MSRV
|
||||
run: cargo check --all-targets --locked
|
||||
|
||||
# Rust test suite
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
@@ -46,32 +58,34 @@ jobs:
|
||||
checks: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-cargo-nextest-0.9.143-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-nextest-0.9.143-
|
||||
${{ runner.os }}-cargo-nextest-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Install cargo-nextest
|
||||
run: cargo install --locked cargo-nextest || true
|
||||
run: |
|
||||
if ! cargo-nextest --version 2>/dev/null | grep -Fq "cargo-nextest 0.9.143"; then
|
||||
cargo install --locked --version 0.9.143 --force cargo-nextest
|
||||
fi
|
||||
|
||||
- name: Run tests with nextest
|
||||
run: cargo nextest run -j "$(nproc)"
|
||||
|
||||
# ==========================
|
||||
# Clippy
|
||||
# ==========================
|
||||
# Rust lint validation
|
||||
clippy:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
@@ -81,14 +95,14 @@ jobs:
|
||||
checks: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -102,9 +116,7 @@ jobs:
|
||||
- name: Run clippy
|
||||
run: cargo clippy -j "$(nproc)" -- --cap-lints warn
|
||||
|
||||
# ==========================
|
||||
# Udeps
|
||||
# ==========================
|
||||
# Unused dependency validation
|
||||
udeps:
|
||||
name: Udeps
|
||||
runs-on: ubuntu-latest
|
||||
@@ -113,27 +125,31 @@ jobs:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: rust-src
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-udeps-${{ hashFiles('**/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-cargo-udeps-0.1.61-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-udeps-0.1.61-
|
||||
${{ runner.os }}-cargo-udeps-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Install cargo-udeps
|
||||
run: cargo install --locked cargo-udeps || true
|
||||
run: |
|
||||
if ! cargo-udeps --version 2>/dev/null | grep -Fq "cargo-udeps 0.1.61"; then
|
||||
cargo install --locked --version 0.1.61 --force cargo-udeps
|
||||
fi
|
||||
|
||||
- name: Run udeps
|
||||
run: cargo udeps -j "$(nproc)" || true
|
||||
|
||||
@@ -30,16 +30,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@v4.37.9
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@v4.37.9
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
||||
@@ -17,33 +17,37 @@ jobs:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: llvm-tools-preview
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-cargo-llvm-cov-0.9.0-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-llvm-cov-0.9.0-
|
||||
${{ runner.os }}-cargo-llvm-cov-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Install cargo-llvm-cov
|
||||
run: cargo install --locked cargo-llvm-cov || true
|
||||
run: |
|
||||
if ! cargo-llvm-cov --version 2>/dev/null | grep -Fq "cargo-llvm-cov 0.9.0"; then
|
||||
cargo install --locked --version 0.9.0 --force cargo-llvm-cov
|
||||
fi
|
||||
|
||||
- name: Generate LCOV report
|
||||
run: cargo llvm-cov --locked --lcov --output-path lcov.info
|
||||
|
||||
- name: Upload LCOV report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7.0.1
|
||||
with:
|
||||
name: telemt-lcov
|
||||
path: lcov.info
|
||||
|
||||
@@ -88,9 +88,9 @@ jobs:
|
||||
cpu: generic
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: dtolnay/rust-toolchain@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: stable
|
||||
targets: |
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
gcc-aarch64-linux-gnu \
|
||||
g++-aarch64-linux-gnu
|
||||
|
||||
- uses: actions/cache@v4
|
||||
- uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
/usr/local/cargo/registry
|
||||
@@ -166,7 +166,7 @@ jobs:
|
||||
|
||||
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.asset }}
|
||||
path: dist/*
|
||||
@@ -199,7 +199,7 @@ jobs:
|
||||
cpu: generic
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Install deps
|
||||
run: |
|
||||
@@ -209,7 +209,7 @@ jobs:
|
||||
pkg-config \
|
||||
curl
|
||||
|
||||
- uses: actions/cache@v4
|
||||
- uses: actions/cache@v6.1.0
|
||||
if: matrix.target == 'aarch64-unknown-linux-musl'
|
||||
with:
|
||||
path: ~/.musl-aarch64
|
||||
@@ -244,7 +244,7 @@ jobs:
|
||||
- name: Add rust target
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
|
||||
- uses: actions/cache@v4
|
||||
- uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: |
|
||||
/usr/local/cargo/registry
|
||||
@@ -302,7 +302,7 @@ jobs:
|
||||
|
||||
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.asset }}
|
||||
path: dist/*
|
||||
@@ -319,7 +319,7 @@ jobs:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@v8.0.1
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -331,7 +331,7 @@ jobs:
|
||||
find artifacts -type f -exec cp {} dist/ \;
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v3.0.3
|
||||
with:
|
||||
tag_name: ${{ needs.prepare.outputs.version }}
|
||||
target_commitish: ${{ github.sha }}
|
||||
@@ -353,13 +353,13 @@ jobs:
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-qemu-action@v4.3.0
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/setup-buildx-action@v4.3.0
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
- uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -407,7 +407,7 @@ jobs:
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Build & Push
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7.3.0
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
|
||||
Generated
+321
-468
File diff suppressed because it is too large
Load Diff
+35
-27
@@ -1,28 +1,29 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.5.5"
|
||||
version = "3.5.7"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
|
||||
[features]
|
||||
redteam_offline_expected_fail = []
|
||||
|
||||
[dependencies]
|
||||
# C
|
||||
libc = "0.2.186"
|
||||
libc = "0.2.189"
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1.52.3", features = ["full", "tracing"] }
|
||||
tokio-util = { version = "0.7.18", features = ["full"] }
|
||||
tokio = { version = "1.53.1", features = ["full", "tracing"] }
|
||||
tokio-util = { version = "0.7.19", features = ["full"] }
|
||||
|
||||
# Crypto
|
||||
aes = { version = "0.8.4", features = ["zeroize"] }
|
||||
ctr = { version = "0.9.2", features = ["zeroize"] }
|
||||
cbc = "0.1.2"
|
||||
sha2 = "0.10.9"
|
||||
sha1 = "0.10.6"
|
||||
sha1 = "0.10.7"
|
||||
md-5 = "0.10.6"
|
||||
hmac = "0.12.1"
|
||||
crc32fast = "1.5.0"
|
||||
crc32fast = "1.5.1"
|
||||
crc32c = "0.6.8"
|
||||
zeroize = { version = "1.9.0", features = ["derive"] }
|
||||
subtle = "2.6.1"
|
||||
@@ -30,7 +31,7 @@ static_assertions = "1.1.0"
|
||||
ml-kem = { version = "0.3.2", default-features = false, features = ["alloc", "zeroize"] }
|
||||
|
||||
# Network
|
||||
socket2 = { version = "0.6.4", features = ["all"] }
|
||||
socket2 = { version = "0.6.5", features = ["all"] }
|
||||
nix = { version = "0.31.3", default-features = false, features = [
|
||||
"net",
|
||||
"user",
|
||||
@@ -38,67 +39,74 @@ nix = { version = "0.31.3", default-features = false, features = [
|
||||
"fs",
|
||||
"signal",
|
||||
] }
|
||||
shadowsocks = { version = "1.24.0", features = ["aead-cipher-2022"] }
|
||||
shadowsocks = { version = "1.24.0", default-features = false, features = [
|
||||
"aead-cipher",
|
||||
"aead-cipher-2022",
|
||||
] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
toml = "1.1"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
serde_json = "1.0.151"
|
||||
toml = "1.1.5"
|
||||
x509-parser = "0.18.1"
|
||||
|
||||
# Utils
|
||||
bytes = "1.12.0"
|
||||
thiserror = "2.0.18"
|
||||
bytes = "1.12.1"
|
||||
thiserror = "2.0.20"
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
||||
tracing-appender = "0.2.5"
|
||||
parking_lot = "0.12.5"
|
||||
dashmap = "6.2.1"
|
||||
arc-swap = "1.9.1"
|
||||
lru = "0.16.4"
|
||||
rand = "0.10.1"
|
||||
arc-swap = "1.9.2"
|
||||
lru = "0.18.4"
|
||||
rand = "0.10.2"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
hex = "0.4.3"
|
||||
base64 = "0.22.1"
|
||||
url = "2.5.8"
|
||||
regex = "1.12.4"
|
||||
crossbeam-queue = "0.3.12"
|
||||
num-bigint = "0.4.6"
|
||||
regex = "1.13.1"
|
||||
crossbeam-queue = "0.3.14"
|
||||
num-bigint = "0.4.8"
|
||||
num-traits = "0.2.19"
|
||||
x25519-dalek = "2.0.1"
|
||||
anyhow = "1.0.102"
|
||||
anyhow = "1.0.104"
|
||||
|
||||
# HTTP
|
||||
reqwest = { version = "0.13.4", features = ["rustls"], default-features = false }
|
||||
notify = "8.2.0"
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
hyper = { version = "1.10.1", features = ["client", "server", "http1"] }
|
||||
hyper = { version = "1.11.1", features = ["client", "server", "http1"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
|
||||
http-body-util = "0.1.3"
|
||||
http-body-util = "0.1.5"
|
||||
httpdate = "1.0.3"
|
||||
tokio-tungstenite = { version = "0.30.0", default-features = false }
|
||||
futures-util = { version = "0.3.32", default-features = false, features = ["sink", "std"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
futures-util = { version = "0.3.34", default-features = false, features = ["sink", "std"] }
|
||||
tokio-rustls = { version = "0.26.5", default-features = false, features = [
|
||||
"tls12",
|
||||
] }
|
||||
rustls = { version = "0.23.41", default-features = false, features = [
|
||||
rustls = { version = "0.23.43", default-features = false, features = [
|
||||
"std",
|
||||
"tls12",
|
||||
"ring",
|
||||
] }
|
||||
webpki-roots = "1.0.8"
|
||||
webpki-roots = "1.0.9"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4.5"
|
||||
criterion = "0.8.2"
|
||||
proptest = "1.11.0"
|
||||
futures = "0.3.32"
|
||||
futures = "0.3.34"
|
||||
tempfile = "3.27.0"
|
||||
|
||||
[[bench]]
|
||||
name = "crypto_bench"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "web_decoy_fasttrack"
|
||||
harness = false
|
||||
|
||||
[profile.release]
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
use std::hint::black_box;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use base64::Engine as _;
|
||||
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[path = "../src/web/http/capability.rs"]
|
||||
mod capability;
|
||||
|
||||
fn capability_at(index: usize) -> [u8; 32] {
|
||||
let mut capability = [0xa5u8; 32];
|
||||
capability[..8].copy_from_slice(&(index as u64).to_le_bytes());
|
||||
capability
|
||||
}
|
||||
|
||||
fn consume_scan(scan: capability::CapabilityScan) {
|
||||
black_box(scan.matched.unwrap_u8());
|
||||
black_box(scan.matched_index);
|
||||
}
|
||||
|
||||
fn bench_decoy_fasttrack(c: &mut Criterion) {
|
||||
for profile_count in [1usize, 32, 256, 1024] {
|
||||
let capabilities = (0..profile_count).map(capability_at).collect::<Vec<_>>();
|
||||
let miss = [0x5au8; 32];
|
||||
let first = capabilities[0];
|
||||
let middle = capabilities[profile_count / 2];
|
||||
let last = capabilities[profile_count - 1];
|
||||
let telemetry = AtomicU64::new(0);
|
||||
let mut group = c.benchmark_group(format!("web_decoy_fasttrack/{profile_count}"));
|
||||
|
||||
group.bench_function(BenchmarkId::new("ordinary_enforce", profile_count), |b| {
|
||||
b.iter(|| {
|
||||
let candidate = capability::bridge_candidate(black_box(None));
|
||||
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||
black_box(candidate.is_canonical());
|
||||
});
|
||||
});
|
||||
group.bench_function(BenchmarkId::new("ordinary_shadow", profile_count), |b| {
|
||||
b.iter(|| {
|
||||
let candidate = capability::bridge_candidate(black_box(None));
|
||||
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||
consume_scan(capability::scan_capabilities(
|
||||
black_box(&capabilities),
|
||||
candidate.scan_bytes(),
|
||||
));
|
||||
});
|
||||
});
|
||||
for (name, candidate) in [
|
||||
("canonical_miss", miss),
|
||||
("canonical_hit_first", first),
|
||||
("canonical_hit_middle", middle),
|
||||
("canonical_hit_last", last),
|
||||
] {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(candidate);
|
||||
let query = format!("bridge={token}");
|
||||
group.bench_function(BenchmarkId::new(name, profile_count), |b| {
|
||||
b.iter(|| {
|
||||
let candidate = capability::bridge_candidate(black_box(Some(&query)));
|
||||
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||
consume_scan(capability::scan_capabilities(
|
||||
black_box(&capabilities),
|
||||
candidate.scan_bytes(),
|
||||
));
|
||||
});
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_decoy_fasttrack);
|
||||
criterion_main!(benches);
|
||||
@@ -111,6 +111,9 @@ Notes:
|
||||
| `GET` | `/v1/runtime/web/operations/{operation_id}` | none | `200` | `ControlOperationStatus` |
|
||||
| `POST` | `/v1/runtime/web/debug/clear` | `RuntimeInstanceRequest` | `200` | `DebugClearData` |
|
||||
| `POST` | `/v1/runtime/web/carrier-learning/reset` | `RuntimeInstanceRequest` | `200` | `LearningResetData` |
|
||||
| `POST` | `/v1/runtime/web/lifecycle/pause` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
|
||||
| `POST` | `/v1/runtime/web/lifecycle/drain` | `DrainRequest` | `202` | `OperatorLifecycleStatus` |
|
||||
| `POST` | `/v1/runtime/web/lifecycle/resume` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
|
||||
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
|
||||
@@ -159,6 +162,9 @@ Notes:
|
||||
| `GET /v1/runtime/web/operations/{operation_id}` | Returns one of the 32 most recently retained WEB close-operation states. |
|
||||
| `POST /v1/runtime/web/debug/clear` | Clears the bounded WEB debug ring under an epoch fence. |
|
||||
| `POST /v1/runtime/web/carrier-learning/reset` | Clears process-local carrier-learning evidence without changing live attempt chains. |
|
||||
| `POST /v1/runtime/web/lifecycle/pause` | Ephemerally closes new WEB work admission without closing existing sessions or streams. |
|
||||
| `POST /v1/runtime/web/lifecycle/drain` | Starts one asynchronous graceful WEB drain under a bounded monotonic deadline. |
|
||||
| `POST /v1/runtime/web/lifecycle/resume` | Cancels an active drain, if any, and reopens only the operator-owned admission fence. |
|
||||
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
||||
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
||||
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
|
||||
@@ -193,6 +199,7 @@ Notes:
|
||||
| `409` | `web_runtime_mismatch` | A runtime instance, session reference, or operation reference belongs to another WEB process instance. |
|
||||
| `409` | `web_issuance_enabled` | A WEB close-all operation was requested while effective issuance remained enabled. |
|
||||
| `409` | `web_operation_in_progress` | Another bounded WEB close operation is active. |
|
||||
| `409` | `web_lifecycle_in_progress` | Another WEB drain operation is active. |
|
||||
| `409` | `user_exists` | User already exists on create. |
|
||||
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
|
||||
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
|
||||
@@ -328,7 +335,7 @@ Sections absent from the config file are absent from the response (not `null`).
|
||||
|
||||
### WEB runtime identity and lifecycle
|
||||
|
||||
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>` and close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all three as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
|
||||
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>`, close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`, and drain references use `wd1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all references as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
|
||||
|
||||
`GET /v1/config` is the desired on-disk configuration view. `GET /v1/runtime/web/status` is the effective process view. Its envelope `revision` still identifies the current source graph and can therefore be newer than the active runtime generation while a reload is pending.
|
||||
|
||||
@@ -339,14 +346,37 @@ The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit
|
||||
| `lifecycle` | `string` | `starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`. |
|
||||
| `lifecycle_epoch` | `u64` | Monotonic publication epoch. |
|
||||
| `lifecycle_age_ms` | `u64` | Monotonic age of the current lifecycle publication. |
|
||||
| `available` | `bool` | Whether a readable process runtime is currently published. |
|
||||
| `available` | `bool` | Backward-compatible readable-runtime flag; it is not public TLS or private acceptor readiness. |
|
||||
| `reason` | `string?` | Stable unavailability reason when `available=false`. |
|
||||
| `listeners` | `string[]` | Effective bound WEB listener addresses. |
|
||||
| `effective_config_enabled` | `bool` | `web.enabled` in the API request's active runtime generation. |
|
||||
| `ingress` | `WebIngressStatus` | Process-owned listener/acceptor liveness and TCP accept counters. |
|
||||
| `capacity` | `WebCapacityStatus` | Effective accepted-socket policy, fixed global resources, and typed rejection counters. |
|
||||
| `decoy_upstream` | `WebDecoyUpstreamStatus` | Passive outcomes for Telemt's internal plain-HTTP decoy origin hop. |
|
||||
| `decoy_fasttrack` | `WebDecoyFastTrackStatus` | Effective restart-frozen capability policy and fixed process-lifetime routing counters. |
|
||||
| `carrier_negotiation` | `WebCarrierNegotiationStatus` | Fixed process-lifetime selection, reported-failure, and health/learning outcome counters. |
|
||||
| `lifecycle_counters` | `WebLifecycleCountersStatus` | Fixed close-reason, post-gap observation, and bridge-recovery counters plus the effective recovery deadline. |
|
||||
| `operator_lifecycle` | `OperatorLifecycleStatus?` | Process-local reversible admission and active/latest drain status while a runtime is published. |
|
||||
| `runtime` | `WebRuntimeStatus?` | Present while the weak process-runtime publication can be upgraded. |
|
||||
|
||||
`WebIngressStatus` contains `configured_listeners`, `live_acceptors`, `accepting_connections`, optional `reason`, `tcp_accept_total`, and `tcp_accept_error_total`. Accepting requires lifecycle `running`, a readable runtime, at least one effective WEB listener, and one live accept loop per listener. Stable non-accepting reasons are `starting`, `no_web_listener`, `ingress_draining`, `ingress_drained`, `deadline_exceeded`, `runtime_released`, and `acceptor_unavailable`. Accept errors are `accept(2)` failures observed by Telemt; they are not kernel backlog drops or failed connection attempts that never reached the process.
|
||||
|
||||
`WebCapacityStatus` contains `http_connection_capacity_action`, `max_http_overload_connections`, `http_overload_timeout_ms`, fixed `resources`, `saturated_resources`, `partial`, `rejections`, and `http_connection_overload_outcomes`. Each resource has a closed-set `resource`, `unit`, `used`, `available`, `limit`, and terminal `closed` flag. Saturation is an instantaneous plane-local observation and never changes `available` or ingress readiness. Rejections are monotonic admission decisions indexed only by a closed reason enum; an internally retried queue or byte-budget decision may later make progress. Accepted-socket outcomes are `dropped`, `wait_admitted`, `wait_timeout_503`, `responded_503`, `overflow_capacity_drop`, `response_error_drop`, and `shutdown_drop`; `wait_admitted` is not a rejection.
|
||||
|
||||
`WebDecoyUpstreamStatus` contains the complete fixed outcome set plus optional `last_outcome` and `last_outcome_age_ms`. Outcomes distinguish `success`, `deadline_exhausted`, `connect_refused`, `connect_timeout`, `connect_error`, `http_handshake_timeout`, `http_handshake_error`, `response_head_timeout`, and `request_error`. This describes only Telemt to the configured decoy origin. A public client to NGINX refusal, or an NGINX to Telemt refusal before `accept(2)`, is outside this counter plane.
|
||||
|
||||
`WebDecoyFastTrackStatus` contains effective `mode` and the complete fixed `requests` disposition array. Dispositions are `shadow_would_fasttrack`, `shadow_candidate_full_scan`, `enforce_fasttrack`, and `enforce_candidate_full_scan`. `off` performs no fast-track counter writes. The complete set remains visible and retains its process-lifetime totals after runtime release because telemetry is process-owned.
|
||||
|
||||
`WebCarrierNegotiationStatus` remains present when the process runtime is unavailable because its counters belong to the WEB publication. `selections` is the complete carrier x disposition matrix (`profile_disabled`, `policy_disabled`, `policy_pending`, `epoch_exhausted`, `cold`, `applied`). `reported_failures` is the complete carrier x phase x canonical reason matrix, where phase is `provisional` or `committed` and reason is `timeout`, `network`, `upgrade`, `http`, or `protocol`. `learning_outcomes` distinguishes `recorded`, `not_eligible`, `policy_disabled`, `stale_epoch`, `capacity_rejected`, `sequence_exhausted`, `missing_chain`, `phase_mismatch`, `session_mismatch`, `owner_not_live`, and `closed_before_health`. Reported failures and rejection outcomes are diagnostic only and never create negative ranking evidence.
|
||||
|
||||
`WebLifecycleCountersStatus` always contains `bridge_recovery_secs`, the complete carrier x close-reason matrix, the complete carrier x lifecycle-observation matrix, and all recovery milestones. Close reasons are `client_delete`, `bridge_recovery`, `peer_idle`, `negotiation_timeout`, `carrier_superseded`, `protocol`, `backpressure`, `websocket_ended`, `api_close`, `operator_force`, and `runtime_shutdown`. Observations are `http_activity_after_gap`, `websocket_activity_after_gap`, and `request_after_close`. Recovery events are `bootstrap_issued`, `session_created`, `committed`, `expired_unused`, and `closed_before_commit`. All counters are process-owned, monotonic, fixed-cardinality, and remain present at zero while the runtime is unavailable.
|
||||
|
||||
`WebRuntimeStatus` includes `runtime_instance`, `generation_id`, immutable effective `limits`, manager/stream/budget/WebSocket/learning/debug planes, permit usage, task/counter totals, and `partial`. Plane locks are read with `try_lock`; a contended plane is omitted and named in `partial`. Status collection performs no cleanup, waits, or data-plane mutation, so fields are plane-local observations rather than one globally atomic snapshot. `runtime.manager.issuance_enabled` is the authority to check before close-all.
|
||||
|
||||
`OperatorLifecycleStatus` is a lock-free process snapshot with `state`, monotonic `epoch`, `age_ms`, `admission_open`, `effective_new_work_admission`, and the active or latest `drain`. States are `running`, `paused`, `draining`, `force_closing`, and `drained`. Drain status contains its opaque id, phase/outcome, frozen timeout, wall-clock correlation timestamps, latest session/stream/WebSocket remainder, and `force_close_signalled`. The response envelope `revision` remains a config source-graph revision and is not a lifecycle version.
|
||||
|
||||
The Prometheus endpoint exports the same process-owned observations through fixed-cardinality `telemt_web_*` families: ingress/operator lifecycle states, independent ingress flags, listener and TCP accept counts, resource usage/closure/saturation, typed rejection totals, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier aggregate totals. Decoy routing adds `telemt_web_decoy_fasttrack_mode{mode}` and `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation adds `telemt_web_carrier_selections_total{carrier,disposition}`, `telemt_web_carrier_reported_failures_total{carrier,phase,reason}`, `telemt_web_carrier_learning_outcomes_total{carrier,outcome}`, one-hot `telemt_web_carrier_learning_state{state}`, `telemt_web_carrier_learning_entries{kind}`, and one-hot `telemt_web_carrier_learning_policy{aggressiveness}`. Lifecycle recovery adds `telemt_web_session_closures_total{carrier,reason}`, `telemt_web_session_lifecycle_observations_total{carrier,observation}`, `telemt_web_bridge_recovery_events_total{event}`, and `telemt_web_bridge_recovery_seconds`. The learning states are `unavailable`, `partial`, `pending`, `exhausted`, `disabled`, and `enabled`; `pending` explicitly exposes a generation/policy publication mismatch instead of silently treating it as cold evidence. WEB labels never contain a host, user, client IP, listener address, token, session reference, profile key, runtime instance, or generation ID. Telemt does not claim health for the externally owned NGINX or HAProxy TLS endpoint; that boundary requires terminator telemetry and an external TCP/TLS probe.
|
||||
|
||||
### WEB session enumeration
|
||||
|
||||
`GET /v1/runtime/web/sessions` defaults to `limit=50`, permits `1..=200`, and scans at most 1000 ordered candidates. `next_cursor` continues after the last scanned opaque session reference. `scan_truncated` reports the scan bound, `partial_sessions` counts contended per-session snapshots, and `partial` names an unavailable manager plane. The complete serialized page remains below the API response envelope because every string and row count is bounded.
|
||||
@@ -364,12 +394,32 @@ Filters are exact unless stated otherwise:
|
||||
| `carrier` | `https`, `https-lanes`, `websocket`, or `websocket-lanes`. |
|
||||
| `state` | `provisional`, `replacing`, `committed`, `healthy`, `closing`, `superseded`, or transient live-index `closed`. |
|
||||
|
||||
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, stream/task/lane/WebSocket counts, pending/control usage, age/idle timing, and optional negotiation time remaining. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `{state:"closed", attempt}` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
|
||||
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, health publication, stream/task/lane/WebSocket counts, pending/control usage, progress idle time, authenticated peer idle time, frozen reconnect grace, remaining peer deadline, and optional negotiation time remaining. Server-only progress and empty long polls do not extend the authenticated peer deadline. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `state`, `attempt`, `carrier`, `reason`, and `closed_age_ms` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
|
||||
|
||||
### WEB runtime mutations
|
||||
|
||||
Every WEB runtime POST requires the currently published `runtime_instance`, exactly one `Content-Type: application/json` header, no query parameters, and a JSON object with no unknown fields. All mutations inherit API authentication, direct-peer whitelist, body limit, audit recording, and `read_only` enforcement.
|
||||
|
||||
Operator lifecycle requests are:
|
||||
|
||||
```json
|
||||
{"runtime_instance":"0123456789abcdef0123456789abcdef"}
|
||||
```
|
||||
|
||||
for `POST /v1/runtime/web/lifecycle/pause` and `/resume`, and:
|
||||
|
||||
```json
|
||||
{"runtime_instance":"0123456789abcdef0123456789abcdef","timeout_secs":30}
|
||||
```
|
||||
|
||||
for `POST /v1/runtime/web/lifecycle/drain`, where `timeout_secs` is bounded to `1..=3600`. Pause and resume return `200`; drain freezes one monotonic absolute deadline and returns `202` without waiting for completion. A second drain while one is `draining` or `force_closing` returns `409 web_lifecycle_in_progress` and cannot alter the first deadline. Repeated pause/resume requests already satisfied by the current state are idempotent and do not advance the lifecycle epoch. Pause during an active drain leaves that drain running. Resume cancels an active drain and opens admission; if the deadline already committed its forced-close snapshot, those old session close signals remain effective.
|
||||
|
||||
Pause and drain block bootstrap issuance, initial/replacement session creation, and logical-stream admission. Exact session-creation replay, existing DATA/WINDOW/CLOSE, carrier polling/WebSocket exchanges, and explicit session DELETE remain available. Rejection does not consume bootstrap/session/stream rate or quota state: authenticated session creation returns retryable `503` with `Retry-After: 1`, while bridge issuance preserves the decoy route and a rejected logical `OPEN` receives a stream-local close.
|
||||
|
||||
Drain remains graceful until either all live sessions, logical-stream ownership, and session-owned WebSockets reach zero or its deadline fires. The deadline is the latest time to commit close signals, not a claim that cooperative task teardown is already complete. At the deadline every remaining live session receives an idempotent close signal outside manager locks, status becomes `force_closing`, and only confirmed zero publishes `drained` with outcome `forced`. Natural zero publishes outcome `graceful`. Both outcomes keep operator admission closed until explicit resume.
|
||||
|
||||
This lifecycle is ephemeral: it survives in-process generation reload because its authority is process-owned, is not written to configuration, and starts as `running` after process restart. Resume never overrides `web.enabled=false`, disabled-user policy, generation health admission, or terminal process shutdown. The global health/readiness and native TCP/Unix admission contracts are unchanged.
|
||||
|
||||
`POST /v1/runtime/web/sessions/close` accepts:
|
||||
|
||||
```json
|
||||
@@ -416,8 +466,8 @@ Returned by `PATCH /v1/config` on success (`200`, or `202` when a reload was acc
|
||||
| `revision` | `string` | SHA-256 hex of the config file after the patch was written. |
|
||||
| `restart_required` | `bool` | Legacy classifier result: `true` when the old file watcher alone cannot apply every changed field. Use `runtime_reload_required` and `process_restart_required` for new integrations. |
|
||||
| `runtime_reload_required` | `bool` | `true` when full effect requires a Maestro runtime-generation reload rather than the legacy hot-field overlay. |
|
||||
| `process_restart_required` | `bool` | `true` when process-owned sockets or paths changed and remain deferred after an in-process reload. |
|
||||
| `deferred_process_fields` | `string[]` | Process-owned fields that the active process cannot rebind during generation activation. |
|
||||
| `process_restart_required` | `bool` | `true` when a process-owned field changed and remains deferred after an in-process reload. |
|
||||
| `deferred_process_fields` | `string[]` | Process-owned sockets, paths, capacities, or policies retained by the active process. |
|
||||
| `changed` | `string[]` | Top-level section names that differed between the old and new config (e.g. `["censorship"]`). |
|
||||
| `reload` | `ReloadAccepted?` | Present only when the patch included a valid reload query and Maestro accepted the operation. |
|
||||
|
||||
@@ -668,8 +718,11 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
|
||||
| --- | --- | --- |
|
||||
| `active_generation` | `u64` | Active pool generation id. |
|
||||
| `warm_generation` | `u64` | Warm pool generation id. |
|
||||
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
|
||||
| `pending_hardswap_generation` | `u64` | Pending hardswap generation id (`0` when none). |
|
||||
| `pending_hardswap_age_secs` | `u64?` | Age of pending hardswap generation in seconds. |
|
||||
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
|
||||
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
|
||||
| `draining_generations` | `u64[]` | Distinct generation ids currently draining. |
|
||||
|
||||
#### `RuntimeMePoolStateHardswapData`
|
||||
@@ -707,6 +760,8 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
|
||||
| --- | --- | --- |
|
||||
| `inflight_endpoints_total` | `usize` | Total in-flight endpoint refill operations. |
|
||||
| `inflight_dc_total` | `usize` | Number of distinct DC+family keys with refill in flight. |
|
||||
| `running_dc_total` | `usize` | DC+family refill workers currently running. |
|
||||
| `pending_dc_total` | `usize` | Running DC+family workers with one coalesced pending endpoint. |
|
||||
| `by_dc` | `RuntimeMePoolStateRefillDcData[]` | Per-DC refill rows. |
|
||||
|
||||
#### `RuntimeMePoolStateRefillDcData`
|
||||
@@ -1264,8 +1319,11 @@ JA3 follows the Salesforce ClientHello field order. JA4 follows the FoxIO TLS-cl
|
||||
| --- | --- | --- |
|
||||
| `active_generation` | `u64` | Active pool generation. |
|
||||
| `warm_generation` | `u64` | Warm pool generation. |
|
||||
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
|
||||
| `pending_hardswap_generation` | `u64` | Pending hardswap generation. |
|
||||
| `pending_hardswap_age_secs` | `u64?` | Pending hardswap age in seconds. |
|
||||
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
|
||||
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
|
||||
| `hardswap_enabled` | `bool` | Hardswap mode toggle. |
|
||||
| `floor_mode` | `string` | Writer floor mode. |
|
||||
| `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. |
|
||||
@@ -1566,7 +1624,7 @@ Without a `reload` query parameter, the endpoint writes the patch and the file w
|
||||
- `revision` — SHA-256 hex of the canonical source manifest after the write, including every recursive include path and its raw bytes.
|
||||
- `restart_required` — legacy file-watcher classification retained for compatibility.
|
||||
- `runtime_reload_required` — reports whether a full Maestro generation reload is needed for runtime effect.
|
||||
- `process_restart_required` and `deferred_process_fields` — report socket policies or process-owned paths that remain unchanged by an in-process reload. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
|
||||
- `process_restart_required` and `deferred_process_fields` — report process-owned sockets, paths, capacities, or policies that remain unchanged by an in-process reload, including `web.decoy_fasttrack_mode`. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
|
||||
- `changed` — list of top-level section names that differed.
|
||||
- `reload` — accepted operation metadata; omitted without a reload query and for process-only patches that cannot change the active generation.
|
||||
|
||||
@@ -1632,10 +1690,10 @@ The API exposes WEB desired configuration through the common config resource, pr
|
||||
|
||||
| Operation | Current contract |
|
||||
| --- | --- |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale, and `web.limits` remains process-deferred. |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale; `web.limits` and `web.decoy_fasttrack_mode` remain process-deferred. |
|
||||
| Persist `server.listeners` | Supported through `PATCH /v1/config`. Arrays replace wholesale. A changed WEB listener is process-owned and remains deferred until process restart. |
|
||||
| Apply an externally edited WEB config | Update the owning TOML source, call `POST /v1/system/reload`, then poll `GET /v1/system/reload/{id}`. |
|
||||
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners` and `web.limits` require process restart. |
|
||||
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners`, `web.limits`, and `web.decoy_fasttrack_mode` require process restart. |
|
||||
| Inspect WEB lifecycle, capacity, sessions, operations, learning, and debug state | Use the authenticated `GET /v1/runtime/web/*` routes documented above. |
|
||||
| Close selected or all point-in-time sessions | Use `POST /v1/runtime/web/sessions/close`; close-all first requires effective issuance to be disabled. |
|
||||
| Clear debug records or reset carrier learning | Use `POST /v1/runtime/web/debug/clear` or `/carrier-learning/reset` with the current `runtime_instance`. |
|
||||
|
||||
@@ -308,6 +308,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
| [`proxy_secret_auto_reload_secs`](#proxy_secret_auto_reload_secs) | `u64` | `3600` | `✔` |
|
||||
| [`proxy_config_auto_reload_secs`](#proxy_config_auto_reload_secs) | `u64` | `3600` | `✔` |
|
||||
| [`me_reinit_singleflight`](#me_reinit_singleflight) | `bool` | `true` | `✔` |
|
||||
| [`me_reinit_max_concurrency`](#me_reinit_max_concurrency) | `usize` | `2` | `✔` |
|
||||
| [`me_reinit_trigger_channel`](#me_reinit_trigger_channel) | `usize` | `64` | `✘` |
|
||||
| [`me_reinit_coalesce_window_ms`](#me_reinit_coalesce_window_ms) | `u64` | `200` | `✔` |
|
||||
| [`me_deterministic_writer_sort`](#me_deterministic_writer_sort) | `bool` | `true` | `✔` |
|
||||
@@ -1547,8 +1548,17 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
[general]
|
||||
me_reinit_singleflight = true
|
||||
```
|
||||
## me_reinit_max_concurrency
|
||||
- **Constraints / validation**: Must be within `[1, 8]`. The effective value is `1` while `me_reinit_singleflight = true`.
|
||||
- **Description**: Bounds concurrent ME generation warmups. Excess triggers are coalesced into one pending rerun.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[general]
|
||||
me_reinit_max_concurrency = 2
|
||||
```
|
||||
## me_reinit_trigger_channel
|
||||
- **Constraints / validation**: Must be `> 0`.
|
||||
- **Constraints / validation**: Must be within `[1, 4096]`.
|
||||
- **Description**: Trigger queue capacity for reinit scheduler.
|
||||
- **Example**:
|
||||
|
||||
@@ -2561,6 +2571,8 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
|
||||
| `carriers` | `false` or a non-empty array of unique carriers | `false` | `✔` |
|
||||
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"`, or `"aggressive"` | `"conservative"` | `✔` |
|
||||
| `decoy_fasttrack_mode` | `"off"`, `"shadow"`, or `"enforce"` | `"off"` | `✘` |
|
||||
| `http_connection_capacity_action` | `"drop"`, `"wait"`, or `"respond"` | `"drop"` | `✔` |
|
||||
| `debug` | table | disabled, bounded defaults | `✔` |
|
||||
| `limits` | table | bounded defaults | `✘` |
|
||||
| `timeouts` | table | bounded defaults | `✔` |
|
||||
@@ -2570,7 +2582,11 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
|
||||
|
||||
When `carriers` is missing or `false`, auto-negotiation and learning are disabled and `carrier` is the only mode. A non-empty `carriers` array enables startup-only negotiation in its configured order; `carrier` is appended exactly once as the final fallback. Empty arrays, duplicates, and `true` are rejected. The client advances candidates only before carrier commit and must create a new session to change carrier after commit. A metadata-free native client, including Telegram iOS, always uses the configured fixed `carrier`, even when negotiation is enabled. Current iOS supports only `https`, so such deployments must configure `carrier = "https"`. CFNetwork and Darwin User-Agent classification does not infer carrier support. Explicit native iOS capabilities are intersected with `{https}`; other explicit client capabilities participate as reported.
|
||||
|
||||
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload applies the policy to new negotiation chains and invalidates incompatible retained evidence. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
`http_connection_capacity_action` applies only after Telemt has accepted a private WEB TCP connection and `max_http_connections` is exhausted. `drop` preserves the legacy immediate close. `respond` emits an empty `503 Service Unavailable` with `Retry-After: 1`, `Cache-Control: no-store`, and `Connection: close`. `wait` waits for ordinary connection capacity for at most `http_overload_timeout_ms`, then enters normal HTTP handling; timeout emits the same bounded `503`. At most `max_http_overload_connections` accepted sockets may wait or respond outside ordinary connection capacity. This policy cannot observe or cause a TCP connect refusal before Telemt accepts the socket.
|
||||
|
||||
`decoy_fasttrack_mode` is restart-only. `off` preserves legacy root-request scanning and collects no fast-track decisions. `shadow` records eligible requests while preserving the full scan. `enforce` skips scans only for `HEAD` or absent/noncanonical bridge queries. A canonical bridge-shaped `GET`, including an unknown capability, always scans every profile in the selected vhost. The optimization does not bound hostile canonical probes and enforce mode must be validated for request-shape timing distinguishability behind the production TLS terminator.
|
||||
|
||||
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload preserves evidence across a generation change only when enabled state, aggressiveness, evidence lifetime, and health window are identical; any semantic change advances the evidence epoch and fences stale outcomes. Because `[web.limits]` is process-owned, a reload that enables learning or negotiation using only a desired larger `max_carrier_learning_entries` atomically defers the dependent learning/carrier field rather than publishing an invalid effective combination. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
@@ -2595,7 +2611,7 @@ Authenticated JSON control may clear the ring explicitly with `POST /v1/runtime/
|
||||
|
||||
# [web.limits]
|
||||
|
||||
These process-wide ceilings make every WEB registry, queue, request body, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
|
||||
These process-wide ceilings make every WEB registry, queue, request body, capability index, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
@@ -2605,6 +2621,7 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `carrier_batch_bytes` | `usize` | `2097152` | Maximum encoded downlink batch. |
|
||||
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
|
||||
| `max_http_overload_connections` | `usize` | `64` | Accepted saturated sockets allowed to wait or emit the bounded retryable response outside ordinary HTTP capacity. |
|
||||
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
|
||||
| `max_lane_open_waits_per_session` | `usize` | `16` | Canonical cursor-zero downlink polls allowed to wait for a racing lane `OPEN` in one session. |
|
||||
| `pending_bytes_per_lane` | `usize` | `8388608` | Queued and resident `DATA` bytes allowed for one independent HTTPS or WebSocket lane. |
|
||||
@@ -2638,7 +2655,7 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Maximum retained WEB debug record count. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Retained plus in-flight WEB debug byte ceiling; minimum 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, capability indexes, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
|
||||
@@ -2659,6 +2676,7 @@ Unless a row states otherwise, timeouts are measured in seconds and must be with
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
|
||||
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-side deadline for one HTTP attempt through complete response-body consumption; `/down` additionally allows `long_poll_secs`. Validated within `1..=60`. |
|
||||
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolute bridge retry window including attempts and backoff; validated within `1..=300` and no lower than `bridge_request_secs`. |
|
||||
| `bridge_recovery_secs` | `u64` | `15` | `✔` | Absolute post-commit recovery window for a surviving bridge document; validated within `1..=60` and frozen when recovery starts. |
|
||||
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optional bridge wait after `OPEN` for matching `DATA`; milliseconds within `0..=10`, where `0` preserves immediate probing. |
|
||||
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wait for a canonical cursor-zero downlink that races its lane `OPEN`; no greater than `long_poll_secs`. |
|
||||
| `carrier_health_secs` | `u64` | `30` | `✔` | Post-commit observation interval required before a carrier can contribute learning evidence. |
|
||||
@@ -2670,8 +2688,9 @@ Unless a row states otherwise, timeouts are measured in seconds and must be with
|
||||
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Strictly increasing cumulative offsets used by the bridge before its first `/session` request and by the server when accepting the first automatic attempt. Checkpoints for one through four candidates are `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always uses `d3`. |
|
||||
| `carrier_learning_secs` | `u64` | `600` | `✔` | Fixed two-window process-local evidence lifetime; validated within `2..=86400`. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum carrier inactivity before session closure. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum validated peer inactivity before session closure; empty polls and backend-only progress do not renew this lease. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle limit between HTTP exchanges and while an emitted response body makes no progress. Explicitly bounded request-body, long-poll, decoy, and pending-Upgrade phases keep their own deadlines instead of being truncated by this timer. The value is frozen when the connection is accepted. |
|
||||
| `http_overload_timeout_ms` | `u64` | `250` | `✔` | Per-phase deadline in milliseconds for an accepted saturated socket to wait for capacity or write its retryable response; validated within `1..=60000`. A timed-out wait and its response write each receive at most one phase budget. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | One absolute process-shutdown budget shared by all listener acceptors and connections plus WEB session and auxiliary-task drains. The active value is captured once when shutdown starts. |
|
||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Connect and response-head deadline for an HTTP decoy. |
|
||||
|
||||
|
||||
+42
-10
@@ -76,6 +76,8 @@ web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||
[web]
|
||||
enabled = true
|
||||
carrier = "https-lanes"
|
||||
decoy_fasttrack_mode = "off"
|
||||
http_connection_capacity_action = "drop"
|
||||
|
||||
[[web.vhosts]]
|
||||
host = "proxy.example.com"
|
||||
@@ -93,6 +95,10 @@ max_streams = 512
|
||||
max_streams_per_session = 64
|
||||
```
|
||||
|
||||
Accepted-socket overload handling is independently configurable. `drop` preserves the legacy close after `accept(2)`. `respond` writes an empty retryable `503` without parsing a request. `wait` waits outside the accept loop for ordinary connection capacity and then enters normal HTTP handling; timeout writes the same `503`. Both waiting and response writing use `web.timeouts.http_overload_timeout_ms` per phase. `web.limits.max_http_overload_connections` bounds sockets outside ordinary capacity and requires a process restart when changed; the action and timeout are hot-reloadable.
|
||||
|
||||
`decoy_fasttrack_mode` controls only capability work for `GET/HEAD /`. `off` is the default and preserves the legacy full scan without fast-track counters. `shadow` records which structurally impossible requests could bypass the scan but still performs the complete legacy scan. `enforce` bypasses capability work only for `HEAD` or an absent/noncanonical `bridge` query. Every exact canonical `GET /?bridge=<43-character-base64url>` performs a complete scan across all profiles of the selected vhost, for both matches and misses. The setting requires a process restart; reload persists the desired value but reports `web.decoy_fasttrack_mode` as deferred. Fast-track does not protect against adversarial CPU load because a scanner can always submit canonical candidates, and enforce mode may expose a public request-shape timing class, especially with a static decoy. Do not enable enforce without external timing measurements through the production TLS terminator.
|
||||
|
||||
## Server-side carrier negotiation
|
||||
|
||||
Auto-negotiation is optional and disabled unless `carriers` is an explicit non-empty array. The configured `carrier` remains the final fallback and is appended exactly once, even when it also appears in the array:
|
||||
@@ -111,20 +117,29 @@ carrier_health_secs = 30
|
||||
carrier_learning_secs = 600
|
||||
bridge_request_secs = 10
|
||||
bridge_retry_secs = 90
|
||||
bridge_recovery_secs = 15
|
||||
carrier_probe_coalesce_ms = 0
|
||||
```
|
||||
|
||||
The generated bridge sends canonical `X-Carrier-Capabilities`, `X-Carrier-Attempt`, and, after the first attempt, `X-Carrier-Failure` headers on `/session`. Every successful automatic response returns `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline`, and `X-Carrier-State`. The bridge starts its local cumulative clock immediately before the first `/session` request; the server freezes its separate absolute chain deadline when it accepts the first automatic attempt. Both use the configured offsets, and neither resets across replacement attempts. For one through four effective candidates, the attempt checkpoints are respectively `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always owns `d3`. A successor remains admissible until its own checkpoint. The states are `provisional`, `committed`, and `healthy`.
|
||||
|
||||
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit dynamic switching is deliberately unsupported: reconnect with a new session instead.
|
||||
The bridge emits additive v1 status objects with `state`, `phase`, `reason`, and `deadline_ms`. `phase=provisional` follows the authenticated `WELCOME`; `state=connected,phase=committed` is emitted only after the selected transport acknowledges real `OPEN` or `DATA` progress. The initialization port has its own `bridge_request_secs` pre-`HELLO` deadline, and page navigation is terminal for that document instance. A later initialization message cannot resurrect a closed or BFCache-retained bridge.
|
||||
|
||||
Each bridge HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
|
||||
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the pre-commit replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit in-place carrier switching remains unsupported; a surviving bridge recovers by creating a fresh server session.
|
||||
|
||||
After commit, an HTTP failure first replays the exact frozen request against the current bearer. A successful replay keeps the current session. WebSocket loss, or a foreground/online/native event after at least `reconnect_grace_secs` of scheduler gap, starts one recovery epoch. The bridge performs exactly one `GET /?bridge=<capability>` with `Accept: application/vnd.telemt.web-recovery+json` and optional current bearer authorization. A positive response is an uncacheable JSON document of at most 1024 bytes containing a fresh bootstrap plus current limits, timeouts, and negotiation policy. Telemt issues that bootstrap before synchronously retiring a matching current session, so recreation remains possible with a one-session capacity. Unknown or already retired bearer authorization receives the same positive representation; malformed recovery headers, disabled admission, pause, drain, and capacity rejection follow the sanitized decoy path.
|
||||
|
||||
The recovery epoch has one dual wall/monotonic absolute `bridge_recovery_secs` deadline, a single recovery-document request, and bounded carrier retries with 250 ms through 2 s backoff. Recovery status is repeated at most every 2.5 seconds while active. A fresh incarnation aborts and releases old requests, sockets, lanes, and queues, sends one synthetic `CLOSE` for each still-active native stream, suppresses a second `WELCOME`, and commits only after real carrier progress. Retired stream IDs are retained in a bounded set so valid late frames cannot enter a new stream; the native side must allocate a new stream ID. Frequent native reconnect attempts are valid, but they neither extend the recovery epoch nor retain old incarnation state. Destroying the WebView destroys this recovery owner; a native supervisor must then create a new bridge document.
|
||||
|
||||
Each ordinary bridge carrier HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
|
||||
|
||||
Response bodies are streamed into explicit endpoint bounds: `/session` is exactly eight bytes, a successful `/down` is at most `carrier_batch_bytes`, and bodyless responses accept zero bytes. Declared overflow is rejected before reading, streamed overflow or excessive chunk count cancels the reader, and retryable response bodies are canceled before backoff. Terminal bridge cleanup sends at most one authenticated `DELETE`; canonical transport failures are copied to `X-Carrier-Failure` for diagnostics, while navigation and explicit close remain non-learning reasons.
|
||||
|
||||
Automatic WebSockets use `tproxy-auto-v1.<session-token>` or `tproxy-auto-lane-v1.<session-token>.<stream-id>`. The first accepted binary message containing real `OPEN` or `DATA` progress commits the carrier; the server then writes an empty binary commit acknowledgement to that exact connection. Ping/Pong does not commit a carrier and does not count as learning evidence.
|
||||
|
||||
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Closing earlier is neutral and records no learning result.
|
||||
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Health publication, owner eviction, and close have one terminal winner. Closing earlier remains neutral for ranking evidence but is visible as the diagnostic `closed_before_health` outcome.
|
||||
|
||||
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. Disabling learning or changing its policy on reload clears incompatible evidence without changing in-flight sessions.
|
||||
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. A generation change with identical learning semantics preserves evidence and atomically republishes its generation fence. Disabling learning or changing aggressiveness, evidence lifetime, or health window advances the evidence epoch and detaches incompatible state; stale outcomes cannot repopulate it.
|
||||
|
||||
`https` remains the default and preserves the original serialized behavior. `https-lanes` assigns lane zero to session control and one lane to every non-zero logical stream. Each lane has its own uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, queue, and newest-poll-wins lifecycle. A slow stream therefore does not block another stream at the WEB protocol layer.
|
||||
|
||||
@@ -144,7 +159,7 @@ Every pre-Upgrade authentication, shape, lane-reservation, or capacity failure f
|
||||
|
||||
The WEB listener must use `proxy_protocol = false` and `reuse_allow = false`. It cannot use `client_mss`, `synlimit`, `announce`, or `announce_ip`. `web_trusted_proxy_cidrs` must be non-empty and must contain only the immediate NGINX or HAProxy peers; `/0` networks are rejected.
|
||||
|
||||
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy.
|
||||
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy. A literal decoy endpoint that exactly matches an effective WEB listener, or is covered by its same-family wildcard address on the same port, is rejected. Indirect loops through DNS, NGINX, HAProxy, or another forwarding layer cannot be proven from Telemt configuration and must be excluded operationally.
|
||||
|
||||
An immutable static-site snapshot can be used instead:
|
||||
|
||||
@@ -205,6 +220,14 @@ Place the `map` in NGINX's `http` context. `client_max_body_size` must be at lea
|
||||
|
||||
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. WebSocket Upgrade requires HTTP/1.1, so the public endpoint must also permit HTTP/1.1 and the private NGINX-to-Telemt hop remains HTTP/1.1. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*` exactly as shown. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls or WebSocket lanes; `keepalive` controls the idle pool and is not a concurrency limit.
|
||||
|
||||
### Distinguishing refusal from WEB capacity
|
||||
|
||||
`connect() failed (111: Connection refused) while connecting to upstream` is a TCP-connect failure before Telemt accepts a socket. Check that the Telemt process is running, the effective WEB listener address and port match the NGINX upstream, both processes share the expected network namespace and address family, and no local firewall actively rejects the connection. Startup bind failure, terminal listener removal, or switching NGINX to a desired port before a restart-only listener change becomes effective can produce this symptom. Kernel listen-backlog pressure is separate and normally requires host `ListenOverflows`/`ListenDrops` telemetry.
|
||||
|
||||
WEB capacity is enforced after successful `accept(2)`. Exhausting `max_http_connections` therefore produces the configured `drop`, `wait`, or `respond` outcome; it does not produce an upstream connect refusal. Handler, body, lane, stream, queue, and WebSocket limits have their own HTTP, decoy, or stream-local failure boundaries. Operator pause and drain also leave the WEB listener bound, so they cannot by themselves cause a refusal.
|
||||
|
||||
Use `GET /v1/runtime/web/status` to correlate only Telemt-owned state. `ingress.accepting_connections` requires a running publication, a readable runtime, and one live acceptor for every effective WEB listener. `capacity.saturated_resources`, typed rejection totals, and overload outcomes identify failures after acceptance. `decoy_upstream` describes only Telemt's outgoing plain-HTTP decoy hop. None of these fields claims that the public NGINX TLS endpoint is reachable; use an external TCP/TLS probe and NGINX or HAProxy telemetry for that boundary.
|
||||
|
||||
## HAProxy TLS termination
|
||||
|
||||
```haproxy
|
||||
@@ -236,7 +259,8 @@ The frontend or `defaults` section must also set `timeout client 65s` or longer
|
||||
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
|
||||
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
|
||||
| `web.enabled`, carrier/negotiation policy, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, and probe-coalescing values. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
|
||||
| Operator pause/drain state | Process-owned and ephemeral; survives generation reload, never writes config, and resets to `running` after process restart. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, recovery, and probe-coalescing values. A recovery epoch freezes its current bridge budget, while a successful recovery representation refreshes the policy used by later epochs and the fresh session. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
|
||||
| Process shutdown | Captures the latest reloaded `web.timeouts.shutdown_secs` once and shares that single absolute deadline across listener acceptors and connections plus WEB sessions and auxiliary tasks. The waits do not receive sequential per-component budgets. |
|
||||
|
||||
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
|
||||
@@ -257,6 +281,7 @@ WEB configuration, runtime status, and bounded runtime controls share the authen
|
||||
| Inspect bounded server-side WEB request and lifecycle details | Yes, through authenticated `GET /web-status`. |
|
||||
| Inspect lifecycle, capacity planes, learning/debug state, and live sessions | Yes, through `GET /v1/runtime/web/status` and `/v1/runtime/web/sessions`. |
|
||||
| Close selected live WEB sessions | Yes, through the asynchronous `POST /v1/runtime/web/sessions/close` operation. |
|
||||
| Pause, deadline-drain, or resume new WEB work | Yes, through `/v1/runtime/web/lifecycle/{pause,drain,resume}`. |
|
||||
| Clear debug records or reset carrier learning | Yes, through the corresponding runtime POST endpoints. |
|
||||
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
|
||||
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
|
||||
@@ -276,18 +301,25 @@ The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For
|
||||
|
||||
### Runtime status and control
|
||||
|
||||
`GET /v1/runtime/web/status` always returns the published lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and availability. When the process-owned WEB runtime is alive, `runtime` adds its random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Status collection uses non-blocking plane reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
|
||||
`GET /v1/runtime/web/status` always returns the published ingress lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and backward-compatible runtime availability. `ingress` independently reports configured listeners, live acceptors, accepting state, accept totals, and a stable reason. `capacity` reports effective accepted-socket overload policy, fixed resource usage, instantaneous saturation, partial planes, typed rejection decisions, and overload outcomes. `decoy_upstream` reports fixed outcomes and the age of the latest internal origin result. `decoy_fasttrack` reports the effective restart-frozen mode and the complete fixed disposition set even while the runtime manager is unavailable. `carrier_negotiation` always reports fixed selection, client-failure, and terminal health/learning outcome matrices from publication ownership. When the process-owned WEB runtime is alive, `operator_lifecycle` independently exposes `running`, `paused`, `draining`, `force_closing`, or `drained`, its own epoch/admission flags, and the active or latest drain. `runtime` adds the random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Runtime plane collection uses non-blocking reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
|
||||
|
||||
Prometheus exports the same process-owned planes as fixed-cardinality `telemt_web_*` families: ingress and operator one-hot states, listener/accept counters, capacity usage and saturation, typed terminal rejections, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier totals. Decoy routing adds one-hot `telemt_web_decoy_fasttrack_mode` and fixed `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation uses `telemt_web_carrier_selections_total`, `telemt_web_carrier_reported_failures_total`, `telemt_web_carrier_learning_outcomes_total`, one-hot learning state/policy gauges, and used/limit entry gauges. Labels are closed enums or fixed resource names; user, host, client IP, token, profile key, runtime instance, listener address, and generation ID are never labels. A successful `wait` outcome does not increment a rejection counter.
|
||||
|
||||
`GET /v1/runtime/web/sessions` returns at most 50 sessions by default and at most 200 when `limit` is supplied. Its ordered scan is capped at 1000 candidates. `cursor` and `session_ref` use the opaque canonical form `ws1.<runtime-instance>.<lowercase-hex-id>`; exact `session_ref` is mutually exclusive with `cursor` and `limit`. Filters are `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, and `state`; duplicate or unknown query fields are rejected. The detail route is `GET /v1/runtime/web/sessions/{session_ref}`. A retained closed-session tombstone returns `410`; a contended exact snapshot returns `503 web_snapshot_busy`. Responses expose bounded non-secret metadata and never expose bootstrap/session bearers, capabilities, secret hashes, or synthetic/KDF ports.
|
||||
|
||||
Every runtime POST requires `Content-Type: application/json` exactly, rejects unknown JSON fields, obeys API authentication, whitelist, and `read_only`, and carries the current `runtime_instance` as an ABA fence. Available controls are:
|
||||
|
||||
- `POST /v1/runtime/web/lifecycle/pause` with `{"runtime_instance":"..."}`. It blocks new bootstrap, session incarnation, replacement, and logical-stream admission after a linearizable fence. Existing carrier exchanges and streams continue, exact session replay remains available, and bridge rejection stays on the decoy route.
|
||||
- `POST /v1/runtime/web/lifecycle/drain` with `{"runtime_instance":"...","timeout_secs":30}`. It returns `202`, keeps the same admission fence closed, and waits asynchronously for sessions, streams, and session-owned WebSockets. At the monotonic deadline it signals close to every remaining live session and reports `force_closing` until zero is confirmed. Natural and forced completion both remain closed until resume. A concurrent second drain returns `409 web_lifecycle_in_progress`.
|
||||
- `POST /v1/runtime/web/lifecycle/resume` with `{"runtime_instance":"..."}`. It cancels an active drain and reopens only operator admission. If forced close already committed, old session cancellation cannot be undone. Config, user, generation, and terminal shutdown gates still dominate.
|
||||
- `POST /v1/runtime/web/sessions/close` with one selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}`, or `{"kind":"all"}`. Exact refs are limited to 200, a filter must be non-empty, only one close operation may run, and `all` is rejected while effective issuance remains enabled. The `202` response returns `operation_id`; poll `GET /v1/runtime/web/operations/{operation_id}`. The operation scans only sessions at or below its submission high-water mark in chunks of 128.
|
||||
- `POST /v1/runtime/web/debug/clear` with `{"runtime_instance":"..."}`. The response reports cleared records, bytes still leased by already rendered snapshots, and the new epoch. In-flight writers from the old epoch cannot repopulate the ring.
|
||||
- `POST /v1/runtime/web/carrier-learning/reset` with the same body shape. It clears retained process-local evidence and advances the learning epoch; already frozen attempt chains and live sessions are unchanged.
|
||||
|
||||
For a deterministic close-all, patch `{"web":{"enabled":false}}` with runtime reload enabled, wait until `runtime.manager.issuance_enabled` is `false`, submit the `all` selector using that same `runtime_instance`, and poll the operation to a terminal state. Disabling WEB stops new bootstrap/session issuance but never implicitly closes existing sessions.
|
||||
|
||||
Operator lifecycle is WEB-only and does not change global readiness, liveness, native TCP/Unix listeners, TLS-fronting, or fallback behavior. A pre-pause WebSocket lane reservation is already admitted logical work: it may finish opening and remains included in drain accounting. Lifecycle rejection consumes no rate/quota tokens and adds no hot-path relay lock.
|
||||
|
||||
### Server-side WEB debug view
|
||||
|
||||
Enable bounded collection in the owned configuration file:
|
||||
@@ -306,7 +338,7 @@ default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, and reported-failure transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, reported-failure, exact close reason, peer gap, and recovered-session predecessor transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||
|
||||
The process-owned ring survives runtime generation replacement. Capture-policy changes clear incompatible retained records; window-only changes do not. The ring defaults to 65536 records and 64 MiB retained plus in-flight bytes, the HTML response is capped at 8 MiB, grouping is capped at 1024 groups, and no more than two response bodies retain page permits concurrently. Change `web.limits.debug_records_capacity` or `web.limits.debug_bytes_global` only with a process restart. A hot prefix that fits only a simultaneously increased restart-only capacity is deferred until that restart.
|
||||
|
||||
@@ -348,7 +380,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
- Never expose the plain HTTP WEB listener to an untrusted network. Enforce the restriction with host firewall rules even when it binds to loopback.
|
||||
- Disable request-target and authorization logging at the TLS terminator, or use a verified redacted format. Raw queries contain bridge capabilities and `Authorization` contains bootstrap or session bearer credentials.
|
||||
- Keep one stable public address per vhost. If DNS returns several ingress addresses, each deployment must use the address matching its external path.
|
||||
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: bridge GET, session creation, uplink, downlink, and DELETE. A single Telemt process needs no extra affinity.
|
||||
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: initial and recovery root GET, session creation, uplink, downlink, WebSocket Upgrade, and DELETE. A single Telemt process needs no extra affinity.
|
||||
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
|
||||
- The decoy is part of the anti-probing contract. Verify its ordinary 404 behavior and response timing through the public TLS endpoint before distributing links.
|
||||
|
||||
@@ -360,7 +392,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
4. Import the printed `tg://webproxy` link in the intended Telegram Desktop build and establish a proxy connection.
|
||||
5. For `https-lanes`, confirm that the public connection negotiated HTTP/2 and exercise at least two simultaneous logical streams; the private Telemt hop remains HTTP/1.1.
|
||||
6. For `websocket`, confirm one `101` response, binary relay traffic, and RFC 6455 Ping/Pong beyond 25 seconds. For `websocket-lanes`, exercise at least two simultaneous stream sockets and verify that closing or corrupting one lane does not close its sibling or parent session.
|
||||
7. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||
7. Exercise one HTTP replay and one fresh-session recovery after a scheduler gap, then keep a long poll open beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||
8. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
||||
9. When auto-negotiation is enabled, verify the configured sequence, exact-attempt replay after an intentionally lost response, terminal behavior after commit, and `carrier_committed`/`carrier_healthy` lifecycle rows in `/web-status`. Verify that a metadata-free native client uses the fixed `carrier` without automatic response headers and that explicit capabilities remain unchanged.
|
||||
|
||||
|
||||
+2
-2
@@ -447,9 +447,9 @@ check_port_availability() {
|
||||
port_info=""
|
||||
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
port_info=$($SUDO ss -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||
port_info=$($SUDO ss -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||
elif command -v netstat >/dev/null 2>&1; then
|
||||
port_info=$($SUDO netstat -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||
port_info=$($SUDO netstat -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||
elif command -v lsof >/dev/null 2>&1; then
|
||||
port_info=$($SUDO lsof -i :${SERVER_PORT} 2>/dev/null | grep LISTEN || true)
|
||||
else
|
||||
|
||||
+32
-7
@@ -15,20 +15,30 @@ use super::model::ApiFailure;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::config::hot_reload::classify_config_changes;
|
||||
use crate::maestro::reload::{ReloadAccepted, ReloadRequest, ReloadSubmitError};
|
||||
use crate::maestro::runtime_build::{deferred_process_fields, resolve_reload_config};
|
||||
use crate::maestro::runtime_build::{
|
||||
ResolvedReloadConfig, deferred_process_fields, resolve_reload_config,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Result of one validated managed-config mutation.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct PatchConfigResponse {
|
||||
/// Revision of the persisted desired configuration.
|
||||
pub revision: String,
|
||||
/// Whether any changed field is not hot-reloadable.
|
||||
pub restart_required: bool,
|
||||
/// Whether the effective runtime snapshot must be reloaded.
|
||||
pub runtime_reload_required: bool,
|
||||
/// Whether any desired field remains deferred until process restart.
|
||||
pub process_restart_required: bool,
|
||||
/// Stable paths of desired fields retained from the active process.
|
||||
pub deferred_process_fields: Vec<String>,
|
||||
/// Top-level managed sections changed by the mutation.
|
||||
pub changed: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
/// Accepted runtime reload when one was requested and required.
|
||||
pub reload: Option<ReloadAccepted>,
|
||||
}
|
||||
|
||||
@@ -51,10 +61,11 @@ pub(super) async fn patch_config(
|
||||
let active_config = shared.active_runtime.load_full().config();
|
||||
let mut prepared =
|
||||
prepare_patch_to_path(&shared.config_path, &patch_json, expected_revision).await?;
|
||||
let resolved = resolve_reload_config(&active_config, &prepared.desired_config);
|
||||
prepared.response.runtime_reload_required = resolved.runtime_changed;
|
||||
prepared.response.process_restart_required = !resolved.deferred_process_fields.is_empty();
|
||||
prepared.response.deferred_process_fields = resolved.deferred_process_fields;
|
||||
let resolved = reconcile_runtime_effect(
|
||||
&mut prepared.response,
|
||||
&active_config,
|
||||
&prepared.desired_config,
|
||||
)?;
|
||||
let reservation = if let Some(request) = reload_request.filter(|_| resolved.runtime_changed) {
|
||||
Some(
|
||||
shared
|
||||
@@ -78,6 +89,19 @@ pub(super) async fn patch_config(
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
fn reconcile_runtime_effect(
|
||||
response: &mut PatchConfigResponse,
|
||||
active_config: &ProxyConfig,
|
||||
desired_config: &ProxyConfig,
|
||||
) -> Result<ResolvedReloadConfig, ApiFailure> {
|
||||
let resolved =
|
||||
resolve_reload_config(active_config, desired_config).map_err(ApiFailure::bad_request)?;
|
||||
response.runtime_reload_required = resolved.runtime_changed;
|
||||
response.process_restart_required = !resolved.deferred_process_fields.is_empty();
|
||||
response.deferred_process_fields = resolved.deferred_process_fields.clone();
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
/// Core patch logic, decoupled from hyper/shared-state so it is unit-testable
|
||||
/// against a temp file. The route handler holds `mutation_lock` while calling this.
|
||||
#[cfg(test)]
|
||||
@@ -205,7 +229,8 @@ async fn prepare_patch_to_path(
|
||||
let revision = compute_snapshot_revision(&candidate);
|
||||
let new_cfg = candidate.config;
|
||||
let class = classify_config_changes(&old_cfg, &new_cfg);
|
||||
let deferred_process_fields = deferred_process_fields(&old_cfg, &new_cfg);
|
||||
let deferred_process_fields =
|
||||
deferred_process_fields(&old_cfg, &new_cfg).map_err(ApiFailure::bad_request)?;
|
||||
|
||||
Ok(PreparedConfigPatch {
|
||||
owner_path,
|
||||
@@ -238,7 +263,7 @@ fn reload_submit_failure(error: ReloadSubmitError) -> ApiFailure {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return only the editable config sections + current revision.
|
||||
/// Returns only the editable config sections and current revision.
|
||||
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
||||
let loaded = load_config_snapshot(config_path, false).await?;
|
||||
let revision = compute_snapshot_revision(&loaded);
|
||||
|
||||
@@ -107,14 +107,24 @@ async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() {
|
||||
#[tokio::test]
|
||||
async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
let active = ProxyConfig::load(&path).unwrap();
|
||||
let debug_patch: Json = serde_json::json!({
|
||||
"web": {"debug": {"enabled": true, "capture_headers": false}}
|
||||
"web": {"debug": {
|
||||
"enabled": true,
|
||||
"sideband": true,
|
||||
"capture_headers": false
|
||||
}}
|
||||
});
|
||||
let debug = apply_patch_to_path(&path, &debug_patch, None)
|
||||
let mut debug = apply_patch_to_path(&path, &debug_patch, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let desired = ProxyConfig::load(&path).unwrap();
|
||||
reconcile_runtime_effect(&mut debug, &active, &desired).unwrap();
|
||||
assert!(!debug.restart_required);
|
||||
assert!(debug.runtime_reload_required);
|
||||
assert!(!debug.process_restart_required);
|
||||
assert!(debug.changed.iter().any(|section| section == "web"));
|
||||
assert!(desired.web.debug.sideband);
|
||||
|
||||
let limits_patch: Json = serde_json::json!({
|
||||
"web": {"limits": {"max_http_connections": 2049}}
|
||||
@@ -131,6 +141,27 @@ async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_web_decoy_fasttrack_requires_only_process_restart() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
let active = ProxyConfig::load(&path).unwrap();
|
||||
let patch: Json = serde_json::json!({
|
||||
"web": {"decoy_fasttrack_mode": "shadow"}
|
||||
});
|
||||
|
||||
let mut prepared = prepare_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
reconcile_runtime_effect(&mut prepared.response, &active, &prepared.desired_config).unwrap();
|
||||
let response = prepared.response;
|
||||
|
||||
assert!(response.restart_required);
|
||||
assert!(!response.runtime_reload_required);
|
||||
assert!(response.process_restart_required);
|
||||
assert_eq!(
|
||||
response.deferred_process_fields,
|
||||
vec!["web.decoy_fasttrack_mode".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_web_patch_does_not_modify_the_source() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
use super::*;
|
||||
|
||||
// Read-only fixed API endpoints.
|
||||
mod read_routes;
|
||||
// Fixed configuration and lifecycle mutations.
|
||||
mod fixed_routes;
|
||||
// Dynamic reload and user-resource routes.
|
||||
mod user_routes;
|
||||
|
||||
pub(super) async fn handle(
|
||||
req: Request<Incoming>,
|
||||
peer: SocketAddr,
|
||||
shared: Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||
let runtime = shared.active_runtime.load_full();
|
||||
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||
if previous_cache_generation != runtime.id {
|
||||
*shared.minimal_cache.lock().await = None;
|
||||
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||
}
|
||||
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||
let config_rx = runtime.config_rx.clone();
|
||||
shared
|
||||
.runtime_state
|
||||
.admission_open
|
||||
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||
let request_id = shared.next_request_id();
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let api_cfg = &cfg.server.api;
|
||||
|
||||
if !api_cfg.enabled {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"api_disabled",
|
||||
"API is disabled",
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||
{
|
||||
return match api_cfg.gray_action {
|
||||
ApiGrayAction::Api => Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"forbidden",
|
||||
"Source IP is not allowed",
|
||||
),
|
||||
)),
|
||||
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/html; charset=utf-8")
|
||||
.body(Full::new(Bytes::new()))
|
||||
.unwrap()),
|
||||
ApiGrayAction::Drop => Err(IoError::new(
|
||||
ErrorKind::ConnectionAborted,
|
||||
"api request dropped by gray_action=drop",
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
if !api_cfg.auth_header.is_empty() {
|
||||
let auth_ok = req
|
||||
.headers()
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||
.unwrap_or(false);
|
||||
if !auth_ok {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"unauthorized",
|
||||
"Missing or invalid Authorization header",
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let normalized_path = if path.len() > 1 {
|
||||
path.trim_end_matches('/')
|
||||
} else {
|
||||
path.as_str()
|
||||
};
|
||||
let query = req.uri().query().map(str::to_string);
|
||||
let body_limit = api_cfg.request_body_limit_bytes;
|
||||
|
||||
let result = dispatch(
|
||||
req,
|
||||
method,
|
||||
&path,
|
||||
normalized_path,
|
||||
query.as_deref(),
|
||||
body_limit,
|
||||
&shared,
|
||||
cfg.as_ref(),
|
||||
&config_rx,
|
||||
request_id,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(resp) => Ok(resp),
|
||||
Err(error) => Ok(error_response(request_id, error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(
|
||||
req: Request<Incoming>,
|
||||
method: Method,
|
||||
path: &str,
|
||||
normalized_path: &str,
|
||||
query: Option<&str>,
|
||||
body_limit: usize,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
if web_runtime::is_route(normalized_path) {
|
||||
let web_mutation = method == Method::POST;
|
||||
let result = web_runtime::handle(
|
||||
method,
|
||||
normalized_path,
|
||||
query,
|
||||
req,
|
||||
shared.as_ref(),
|
||||
cfg,
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await;
|
||||
if web_mutation && let Err(error) = &result {
|
||||
shared.runtime_events.record(
|
||||
"api.web.control.failed",
|
||||
format!("path={} code={}", normalized_path, error.code),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if let Some(response) = read_routes::handle(
|
||||
&method,
|
||||
normalized_path,
|
||||
query,
|
||||
shared.as_ref(),
|
||||
cfg,
|
||||
config_rx,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
match (method.as_str(), normalized_path) {
|
||||
("POST", "/v1/users") => {
|
||||
fixed_routes::create_user_route(req, shared, cfg, config_rx, request_id, body_limit)
|
||||
.await
|
||||
}
|
||||
("GET", "/v1/config") => fixed_routes::get_config_route(shared).await,
|
||||
("POST", "/v1/system/reload") => {
|
||||
fixed_routes::reload_route(req, shared, cfg, request_id, body_limit).await
|
||||
}
|
||||
("PATCH", "/v1/config") => {
|
||||
fixed_routes::patch_config_route(req, shared, cfg, query, request_id, body_limit).await
|
||||
}
|
||||
_ => {
|
||||
user_routes::handle(
|
||||
req,
|
||||
&method,
|
||||
path,
|
||||
normalized_path,
|
||||
shared,
|
||||
cfg,
|
||||
config_rx,
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn create_user_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||
let requested_enabled = body.enabled;
|
||||
let result = create_user(body, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.create.failed", error.code);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
if let Some(enabled) = requested_enabled {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.user.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared
|
||||
.proxy_shared
|
||||
.cancel_user_sessions(&data.user.username);
|
||||
if cancelled > 0 {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.user.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.runtime_events.record(
|
||||
"api.user.create.ok",
|
||||
format!("username={}", data.user.username),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::CREATED
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
Ok(success_response(status, data, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn get_config_route(
|
||||
shared: &Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let (value, revision) = config_edit::read_managed_config(&shared.config_path).await?;
|
||||
Ok(success_response(StatusCode::OK, value, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn reload_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
request.validate().map_err(ApiFailure::bad_request)?;
|
||||
|
||||
let (accepted, revision) = submit_reload_from_disk(
|
||||
&shared.config_path,
|
||||
shared.mutation_lock.as_ref(),
|
||||
&shared.reload_control,
|
||||
expected_revision.as_deref(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn patch_config_route(
|
||||
req: Request<Incoming>,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
query: Option<&str>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let reload_request = ReloadRequest::from_query(query).map_err(ApiFailure::bad_request)?;
|
||||
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||
match config_edit::patch_config(body, expected_revision, reload_request, shared).await {
|
||||
Ok(resp) => {
|
||||
let revision = resp.revision.clone();
|
||||
let status = if resp.reload.is_some() {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(success_response(status, resp, revision))
|
||||
}
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.failed", error.code);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn handle(
|
||||
method: &Method,
|
||||
normalized_path: &str,
|
||||
query: Option<&str>,
|
||||
shared: &ApiShared,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => Ok(web_status::render(query, &shared.web_trace).await),
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = HealthData {
|
||||
status: "ok",
|
||||
read_only: api_cfg.read_only,
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/health/ready") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||
let reason = if ready {
|
||||
None
|
||||
} else if !admission_open {
|
||||
Some("admission_closed")
|
||||
} else {
|
||||
Some("no_healthy_upstreams")
|
||||
};
|
||||
let data = HealthReadyData {
|
||||
ready,
|
||||
status: if ready { "ready" } else { "not_ready" },
|
||||
reason,
|
||||
admission_open,
|
||||
healthy_upstreams: upstream_health.healthy_total,
|
||||
total_upstreams: upstream_health.configured_total,
|
||||
};
|
||||
let status_code = if ready {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
Ok(success_response(status_code, data, revision))
|
||||
}
|
||||
("GET", "/v1/system/info") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_system_info_data(shared, cfg, &revision);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/gates") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_gates_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/initialization") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_initialization_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/limits/effective") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_limits_effective_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/posture") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_posture_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/whitelist") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_whitelist_data(cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let connections_bad_by_class = shared
|
||||
.stats
|
||||
.get_connects_bad_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let handshake_failures_by_class = shared
|
||||
.stats
|
||||
.get_handshake_failure_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let data = SummaryData {
|
||||
uptime_seconds: shared.stats.uptime_secs(),
|
||||
connections_total: shared.stats.get_connects_all(),
|
||||
connections_bad_total: shared.stats.get_connects_bad(),
|
||||
connections_bad_by_class,
|
||||
handshake_failures_by_class,
|
||||
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
|
||||
configured_users: cfg.access.users.len(),
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/zero/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/upstreams") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_upstreams_data(shared, api_cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/minimal/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_minimal_all_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/me-writers") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_me_writers_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/dcs") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_dcs_data(shared, api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_pool_state_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_quality_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_upstream_quality_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_nat_stun_data(shared).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-selftest") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_selftest_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/connections/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_connections_summary_data(shared, cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/events/recent") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_events_recent_data(shared, cfg, query);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_tls_fingerprints_data(shared, cfg, query);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/active-ips") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
||||
let mut data: Vec<UserActiveIps> = active_ips_map
|
||||
.into_iter()
|
||||
.filter(|(_, ips)| !ips.is_empty())
|
||||
.map(|(username, active_ips)| UserActiveIps {
|
||||
username,
|
||||
active_ips,
|
||||
})
|
||||
.collect();
|
||||
data.sort_by(|a, b| a.username.cmp(&b.username));
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(success_response(StatusCode::OK, users, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/quota") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
|
||||
_ => return Ok(None),
|
||||
}
|
||||
.map(Some)
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn handle(
|
||||
req: Request<Incoming>,
|
||||
method: &Method,
|
||||
path: &str,
|
||||
normalized_path: &str,
|
||||
shared: &Arc<ApiShared>,
|
||||
cfg: &ProxyConfig,
|
||||
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||
request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let api_cfg = &cfg.server.api;
|
||||
if method == Method::GET
|
||||
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||
{
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let status = shared
|
||||
.reload_control
|
||||
.status(reload_id)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"reload_not_found",
|
||||
format!("Reload {} was not found", reload_id),
|
||||
)
|
||||
})?;
|
||||
return Ok(success_response(StatusCode::OK, status, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/enable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = set_user_enabled(base_user, true, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.enable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/disable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = set_user_enabled(base_user, false, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.ok",
|
||||
format!(
|
||||
"username={} newly_disabled={} cancelled_sessions={}",
|
||||
base_user, newly_disabled, cancelled
|
||||
),
|
||||
);
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let _mutation_guard = shared.mutation_lock.lock().await;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
|
||||
if !disk_cfg.access.users.contains_key(user) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
let configured_users = disk_cfg
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
let snapshot = match shared.quota_state.reset_user(&configured_users, user).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.reset_quota.failed",
|
||||
format!("username={} error={}", user, error),
|
||||
);
|
||||
return Err(ApiFailure::internal(format!(
|
||||
"Failed to reset user quota: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
return Ok(success_response(
|
||||
StatusCode::OK,
|
||||
ResetUserQuotaResponse {
|
||||
username: user.to_string(),
|
||||
used_bytes: snapshot.used_bytes,
|
||||
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||
},
|
||||
revision,
|
||||
));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit).await?;
|
||||
let result = rotate_secret(
|
||||
base_user,
|
||||
body.unwrap_or_default(),
|
||||
expected_revision,
|
||||
shared,
|
||||
)
|
||||
.await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.ok",
|
||||
format!("username={}", base_user),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if method == Method::GET {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) {
|
||||
return Ok(success_response(StatusCode::OK, user_info, revision));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
if method == Method::PATCH {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||
let enabled_update = match &body.enabled {
|
||||
Patch::Unchanged => None,
|
||||
Patch::Remove => Some(true),
|
||||
Patch::Set(enabled) => Some(*enabled),
|
||||
};
|
||||
let result = patch_user(user, body, expected_revision, shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.patch.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
if let Some(enabled) = enabled_update {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::DELETE {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = delete_user(user, expected_revision, shared).await;
|
||||
let (deleted_user, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.ok",
|
||||
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||
);
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||
let response = DeleteUserResponse {
|
||||
username: deleted_user,
|
||||
in_runtime,
|
||||
};
|
||||
let status = if response.in_runtime {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
return Ok(success_response(status, response, revision));
|
||||
}
|
||||
if method == Method::POST {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
if let Some(allow) = allowed_methods_for_path(normalized_path) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(allow),
|
||||
));
|
||||
}
|
||||
debug!(
|
||||
method = method.as_str(),
|
||||
path = %path,
|
||||
normalized_path = %normalized_path,
|
||||
"API route not found"
|
||||
);
|
||||
Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||
))
|
||||
}
|
||||
+19
-847
@@ -1,5 +1,6 @@
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::PathBuf;
|
||||
@@ -20,12 +21,14 @@ use tokio::sync::{Mutex, RwLock, Semaphore, watch};
|
||||
use tokio::time::timeout;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ApiGrayAction;
|
||||
use crate::config::{ApiGrayAction, ProxyConfig};
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::maestro::control_plane::ProcessControlPlane;
|
||||
use crate::maestro::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
use crate::maestro::reload::{ReloadAccepted, ReloadControl, ReloadRequest, ReloadSubmitError};
|
||||
use crate::proxy::route_mode::RouteRuntimeController;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::quota_state::QuotaStateOwner;
|
||||
use crate::startup::StartupTracker;
|
||||
use crate::stats::Stats;
|
||||
use crate::transport::UpstreamManager;
|
||||
@@ -37,6 +40,8 @@ mod config_edit;
|
||||
pub(crate) mod config_store;
|
||||
mod events;
|
||||
mod http_utils;
|
||||
// Authenticated request admission and route dispatch.
|
||||
mod handler;
|
||||
mod model;
|
||||
mod patch;
|
||||
#[cfg(test)]
|
||||
@@ -58,6 +63,7 @@ use config_store::{
|
||||
parse_if_match,
|
||||
};
|
||||
use events::ApiEventStore;
|
||||
use handler::handle;
|
||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||
use model::{
|
||||
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
|
||||
@@ -112,7 +118,7 @@ pub(super) struct ApiShared {
|
||||
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
pub(super) upstream_manager: Arc<UpstreamManager>,
|
||||
pub(super) config_path: PathBuf,
|
||||
pub(super) quota_state_path: PathBuf,
|
||||
pub(super) quota_state: Arc<QuotaStateOwner>,
|
||||
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||
pub(super) mutation_lock: Arc<Mutex<()>>,
|
||||
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
||||
@@ -147,7 +153,7 @@ impl ApiShared {
|
||||
me_pool: runtime.me_pool_runtime.clone(),
|
||||
upstream_manager: runtime.upstream_manager.clone(),
|
||||
config_path: self.config_path.clone(),
|
||||
quota_state_path: self.quota_state_path.clone(),
|
||||
quota_state: self.quota_state.clone(),
|
||||
detected_ips_rx: self.detected_ips_rx.clone(),
|
||||
mutation_lock: self.mutation_lock.clone(),
|
||||
minimal_cache: self.minimal_cache.clone(),
|
||||
@@ -282,8 +288,9 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn serve(
|
||||
listen: SocketAddr,
|
||||
/// Serves the API on a process-owned listener and task scope.
|
||||
pub(crate) async fn serve(
|
||||
listener: TcpListener,
|
||||
stats: Arc<Stats>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
@@ -291,7 +298,7 @@ pub async fn serve(
|
||||
proxy_shared: Arc<ProxySharedState>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
config_path: PathBuf,
|
||||
quota_state_path: PathBuf,
|
||||
quota_state: Arc<QuotaStateOwner>,
|
||||
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||
process_started_at_epoch_secs: u64,
|
||||
startup_tracker: Arc<StartupTracker>,
|
||||
@@ -300,6 +307,7 @@ pub async fn serve(
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
web_trace: Arc<WebTraceStore>,
|
||||
web_runtime_rx: watch::Receiver<WebRuntimePublication>,
|
||||
control_plane: ProcessControlPlane,
|
||||
) {
|
||||
let active_runtime = loop {
|
||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||
@@ -321,19 +329,9 @@ pub async fn serve(
|
||||
};
|
||||
let config_rx = initial_watch_state.config_rx.clone();
|
||||
let admission_rx = initial_watch_state.admission_rx.clone();
|
||||
let listener = match TcpListener::bind(listen).await {
|
||||
Ok(listener) => listener,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
listen = %listen,
|
||||
"Failed to bind API listener"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let listen = listener.local_addr().ok();
|
||||
|
||||
info!("API endpoint: http://{}/v1/* and /web-status", listen);
|
||||
info!(listen = ?listen, "API endpoint ready at /v1/* and /web-status");
|
||||
|
||||
let runtime_state = Arc::new(ApiRuntimeState {
|
||||
process_started_at_epoch_secs,
|
||||
@@ -348,7 +346,7 @@ pub async fn serve(
|
||||
me_pool,
|
||||
upstream_manager,
|
||||
config_path,
|
||||
quota_state_path,
|
||||
quota_state,
|
||||
detected_ips_rx,
|
||||
mutation_lock: Arc::new(Mutex::new(())),
|
||||
minimal_cache: Arc::new(Mutex::new(None)),
|
||||
@@ -373,6 +371,7 @@ pub async fn serve(
|
||||
runtime_watch_rx,
|
||||
runtime_state.clone(),
|
||||
shared.runtime_events.clone(),
|
||||
&control_plane,
|
||||
);
|
||||
|
||||
let connection_permits = Arc::new(Semaphore::new(API_MAX_CONTROL_CONNECTIONS));
|
||||
@@ -399,7 +398,7 @@ pub async fn serve(
|
||||
};
|
||||
|
||||
let shared_conn = shared.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = control_plane.spawn(async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let svc = service_fn(move |req: Request<Incoming>| {
|
||||
let shared_req = shared_conn.clone();
|
||||
@@ -428,830 +427,3 @@ pub async fn serve(
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
req: Request<Incoming>,
|
||||
peer: SocketAddr,
|
||||
shared: Arc<ApiShared>,
|
||||
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||
let runtime = shared.active_runtime.load_full();
|
||||
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||
if previous_cache_generation != runtime.id {
|
||||
*shared.minimal_cache.lock().await = None;
|
||||
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||
}
|
||||
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||
let config_rx = runtime.config_rx.clone();
|
||||
shared
|
||||
.runtime_state
|
||||
.admission_open
|
||||
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||
let request_id = shared.next_request_id();
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let api_cfg = &cfg.server.api;
|
||||
|
||||
if !api_cfg.enabled {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"api_disabled",
|
||||
"API is disabled",
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||
{
|
||||
return match api_cfg.gray_action {
|
||||
ApiGrayAction::Api => Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"forbidden",
|
||||
"Source IP is not allowed",
|
||||
),
|
||||
)),
|
||||
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/html; charset=utf-8")
|
||||
.body(Full::new(Bytes::new()))
|
||||
.unwrap()),
|
||||
ApiGrayAction::Drop => Err(IoError::new(
|
||||
ErrorKind::ConnectionAborted,
|
||||
"api request dropped by gray_action=drop",
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
if !api_cfg.auth_header.is_empty() {
|
||||
let auth_ok = req
|
||||
.headers()
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||
.unwrap_or(false);
|
||||
if !auth_ok {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"unauthorized",
|
||||
"Missing or invalid Authorization header",
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let normalized_path = if path.len() > 1 {
|
||||
path.trim_end_matches('/')
|
||||
} else {
|
||||
path.as_str()
|
||||
};
|
||||
let query = req.uri().query().map(str::to_string);
|
||||
let body_limit = api_cfg.request_body_limit_bytes;
|
||||
|
||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||
if web_runtime::is_route(normalized_path) {
|
||||
let web_mutation = method == Method::POST;
|
||||
let result = web_runtime::handle(
|
||||
method,
|
||||
normalized_path,
|
||||
query.as_deref(),
|
||||
req,
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await;
|
||||
if web_mutation && let Err(error) = &result {
|
||||
shared.runtime_events.record(
|
||||
"api.web.control.failed",
|
||||
format!("path={} code={}", normalized_path, error.code),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => {
|
||||
Ok(web_status::render(query.as_deref(), &shared.web_trace).await)
|
||||
}
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = HealthData {
|
||||
status: "ok",
|
||||
read_only: api_cfg.read_only,
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/health/ready") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||
let reason = if ready {
|
||||
None
|
||||
} else if !admission_open {
|
||||
Some("admission_closed")
|
||||
} else {
|
||||
Some("no_healthy_upstreams")
|
||||
};
|
||||
let data = HealthReadyData {
|
||||
ready,
|
||||
status: if ready { "ready" } else { "not_ready" },
|
||||
reason,
|
||||
admission_open,
|
||||
healthy_upstreams: upstream_health.healthy_total,
|
||||
total_upstreams: upstream_health.configured_total,
|
||||
};
|
||||
let status_code = if ready {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
};
|
||||
Ok(success_response(status_code, data, revision))
|
||||
}
|
||||
("GET", "/v1/system/info") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/gates") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/initialization") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_initialization_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/limits/effective") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_limits_effective_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/posture") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_posture_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/security/whitelist") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_security_whitelist_data(cfg.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let connections_bad_by_class = shared
|
||||
.stats
|
||||
.get_connects_bad_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let handshake_failures_by_class = shared
|
||||
.stats
|
||||
.get_handshake_failure_class_counts()
|
||||
.into_iter()
|
||||
.map(|(class, total)| ClassCount { class, total })
|
||||
.collect();
|
||||
let data = SummaryData {
|
||||
uptime_seconds: shared.stats.uptime_secs(),
|
||||
connections_total: shared.stats.get_connects_all(),
|
||||
connections_bad_total: shared.stats.get_connects_bad(),
|
||||
connections_bad_by_class,
|
||||
handshake_failures_by_class,
|
||||
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
|
||||
configured_users: cfg.access.users.len(),
|
||||
};
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/zero/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/upstreams") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_upstreams_data(shared.as_ref(), api_cfg);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/minimal/all") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_minimal_all_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/me-writers") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_me_writers_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/dcs") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_dcs_data(shared.as_ref(), api_cfg).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_pool_state_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_quality_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_upstream_quality_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_nat_stun_data(shared.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/me-selftest") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_me_selftest_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/connections/summary") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data =
|
||||
build_runtime_connections_summary_data(shared.as_ref(), cfg.as_ref()).await;
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/events/recent") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_events_recent_data(
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
query.as_deref(),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_tls_fingerprints_data(
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
query.as_deref(),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/active-ips") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
||||
let mut data: Vec<UserActiveIps> = active_ips_map
|
||||
.into_iter()
|
||||
.filter(|(_, ips)| !ips.is_empty())
|
||||
.map(|(username, active_ips)| UserActiveIps {
|
||||
username,
|
||||
active_ips,
|
||||
})
|
||||
.collect();
|
||||
data.sort_by(|a, b| a.username.cmp(&b.username));
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
Ok(success_response(StatusCode::OK, users, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/quota") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("POST", "/v1/users") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||
let requested_enabled = body.enabled;
|
||||
let result = create_user(body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.create.failed", error.code);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
if let Some(enabled) = requested_enabled {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.user.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared
|
||||
.proxy_shared
|
||||
.cancel_user_sessions(&data.user.username);
|
||||
if cancelled > 0 {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.user.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.runtime_events.record(
|
||||
"api.user.create.ok",
|
||||
format!("username={}", data.user.username),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::CREATED
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
Ok(success_response(status, data, revision))
|
||||
}
|
||||
("GET", "/v1/config") => {
|
||||
let (value, revision) =
|
||||
config_edit::read_managed_config(&shared.config_path).await?;
|
||||
Ok(success_response(StatusCode::OK, value, revision))
|
||||
}
|
||||
("POST", "/v1/system/reload") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
request.validate().map_err(ApiFailure::bad_request)?;
|
||||
|
||||
let (accepted, revision) = submit_reload_from_disk(
|
||||
&shared.config_path,
|
||||
shared.mutation_lock.as_ref(),
|
||||
&shared.reload_control,
|
||||
expected_revision.as_deref(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||
}
|
||||
("PATCH", "/v1/config") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let reload_request =
|
||||
ReloadRequest::from_query(query.as_deref()).map_err(ApiFailure::bad_request)?;
|
||||
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||
match config_edit::patch_config(body, expected_revision, reload_request, &shared)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => {
|
||||
let revision = resp.revision.clone();
|
||||
let status = if resp.reload.is_some() {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(success_response(status, resp, revision))
|
||||
}
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.failed", error.code);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if method == Method::GET
|
||||
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||
{
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let status =
|
||||
shared
|
||||
.reload_control
|
||||
.status(reload_id)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"reload_not_found",
|
||||
format!("Reload {} was not found", reload_id),
|
||||
)
|
||||
})?;
|
||||
return Ok(success_response(StatusCode::OK, status, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/enable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, true, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.enable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/disable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, false, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.ok",
|
||||
format!(
|
||||
"username={} newly_disabled={} cancelled_sessions={}",
|
||||
base_user, newly_disabled, cancelled
|
||||
),
|
||||
);
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
ensure_expected_revision(&shared.config_path, expected_revision.as_deref())
|
||||
.await?;
|
||||
if !disk_cfg.access.users.contains_key(user) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
let snapshot = match crate::quota_state::reset_user_quota(
|
||||
&shared.quota_state_path,
|
||||
shared.stats.as_ref(),
|
||||
user,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.reset_quota.failed",
|
||||
format!("username={} error={}", user, error),
|
||||
);
|
||||
return Err(ApiFailure::internal(format!(
|
||||
"Failed to reset user quota: {}",
|
||||
error
|
||||
)));
|
||||
}
|
||||
};
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
return Ok(success_response(
|
||||
StatusCode::OK,
|
||||
ResetUserQuotaResponse {
|
||||
username: user.to_string(),
|
||||
used_bytes: snapshot.used_bytes,
|
||||
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||
},
|
||||
revision,
|
||||
));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body =
|
||||
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
||||
.await?;
|
||||
let result = rotate_secret(
|
||||
base_user,
|
||||
body.unwrap_or_default(),
|
||||
expected_revision,
|
||||
&shared,
|
||||
)
|
||||
.await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime =
|
||||
runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.rotate_secret.ok",
|
||||
format!("username={}", base_user),
|
||||
);
|
||||
let status = if data.user.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||
&& !user.is_empty()
|
||||
&& !user.contains('/')
|
||||
{
|
||||
let user = parse_route_username(user)?;
|
||||
if method == Method::GET {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&disk_cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
Some(runtime_cfg.as_ref()),
|
||||
)
|
||||
.await;
|
||||
if let Some(user_info) =
|
||||
users.into_iter().find(|entry| entry.username == user)
|
||||
{
|
||||
return Ok(success_response(StatusCode::OK, user_info, revision));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||
));
|
||||
}
|
||||
if method == Method::PATCH {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body =
|
||||
read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||
let enabled_update = match &body.enabled {
|
||||
Patch::Unchanged => None,
|
||||
Patch::Remove => Some(true),
|
||||
Patch::Set(enabled) => Some(*enabled),
|
||||
};
|
||||
let result = patch_user(user, body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.patch.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
if let Some(enabled) = enabled_update {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled =
|
||||
shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::DELETE {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result = delete_user(user, expected_revision, &shared).await;
|
||||
let (deleted_user, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.failed",
|
||||
format!("username={} code={}", user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.ok",
|
||||
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||
);
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||
let response = DeleteUserResponse {
|
||||
username: deleted_user,
|
||||
in_runtime,
|
||||
};
|
||||
let status = if response.in_runtime {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
return Ok(success_response(status, response, revision));
|
||||
}
|
||||
if method == Method::POST {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||
));
|
||||
}
|
||||
if let Some(allow) = allowed_methods_for_path(normalized_path) {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::method_not_allowed(allow),
|
||||
));
|
||||
}
|
||||
debug!(
|
||||
method = method.as_str(),
|
||||
path = %path,
|
||||
normalized_path = %normalized_path,
|
||||
"API route not found"
|
||||
);
|
||||
Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(resp) => Ok(resp),
|
||||
Err(error) => Ok(error_response(request_id, error)),
|
||||
}
|
||||
}
|
||||
|
||||
+7
-161
@@ -211,6 +211,7 @@ pub(super) struct ZeroMiddleProxyData {
|
||||
pub(super) reconnect_success_total: u64,
|
||||
pub(super) handshake_reject_total: u64,
|
||||
pub(super) handshake_error_codes: Vec<ZeroCodeCount>,
|
||||
pub(super) handshake_error_code_overflow_total: u64,
|
||||
pub(super) reader_eof_total: u64,
|
||||
pub(super) idle_close_by_peer_total: u64,
|
||||
pub(super) route_drop_no_conn_total: u64,
|
||||
@@ -388,8 +389,11 @@ pub(super) struct MinimalDcPathData {
|
||||
pub(super) struct MinimalMeRuntimeData {
|
||||
pub(super) active_generation: u64,
|
||||
pub(super) warm_generation: u64,
|
||||
pub(super) warm_generations: Vec<u64>,
|
||||
pub(super) pending_hardswap_generation: u64,
|
||||
pub(super) pending_hardswap_age_secs: Option<u64>,
|
||||
pub(super) reinit_inflight: usize,
|
||||
pub(super) reinit_max_concurrency_effective: usize,
|
||||
pub(super) hardswap_enabled: bool,
|
||||
pub(super) floor_mode: &'static str,
|
||||
pub(super) adaptive_floor_idle_secs: u64,
|
||||
@@ -462,164 +466,6 @@ pub(super) struct MinimalAllData {
|
||||
pub(super) data: Option<MinimalAllPayload>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserLinks {
|
||||
pub(super) classic: Vec<String>,
|
||||
pub(super) secure: Vec<String>,
|
||||
pub(super) tls: Vec<String>,
|
||||
pub(super) tls_domains: Vec<TlsDomainLink>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct TlsDomainLink {
|
||||
pub(super) domain: String,
|
||||
pub(super) link: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserInfo {
|
||||
pub(super) username: String,
|
||||
pub(super) enabled: bool,
|
||||
pub(super) in_runtime: bool,
|
||||
pub(super) user_ad_tag: Option<String>,
|
||||
pub(super) max_tcp_conns: Option<usize>,
|
||||
pub(super) expiration_rfc3339: Option<String>,
|
||||
pub(super) data_quota_bytes: Option<u64>,
|
||||
pub(super) rate_limit_up_bps: Option<u64>,
|
||||
pub(super) rate_limit_down_bps: Option<u64>,
|
||||
pub(super) max_unique_ips: Option<usize>,
|
||||
pub(super) current_connections: u64,
|
||||
pub(super) active_unique_ips: usize,
|
||||
pub(super) active_unique_ips_list: Vec<IpAddr>,
|
||||
pub(super) recent_unique_ips: usize,
|
||||
pub(super) recent_unique_ips_list: Vec<IpAddr>,
|
||||
pub(super) total_octets: u64,
|
||||
pub(super) links: UserLinks,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserActiveIps {
|
||||
pub(super) username: String,
|
||||
pub(super) active_ips: Vec<IpAddr>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct CreateUserResponse {
|
||||
pub(super) user: UserInfo,
|
||||
pub(super) secret: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct DeleteUserResponse {
|
||||
pub(super) username: String,
|
||||
pub(super) in_runtime: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct ResetUserQuotaResponse {
|
||||
pub(super) username: String,
|
||||
pub(super) used_bytes: u64,
|
||||
pub(super) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserQuotaListData {
|
||||
pub(super) users: Vec<UserQuotaEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserQuotaEntry {
|
||||
pub(super) username: String,
|
||||
pub(super) data_quota_bytes: u64,
|
||||
pub(super) used_bytes: u64,
|
||||
pub(super) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct CreateUserRequest {
|
||||
pub(super) username: String,
|
||||
pub(super) secret: Option<String>,
|
||||
pub(super) user_ad_tag: Option<String>,
|
||||
pub(super) max_tcp_conns: Option<usize>,
|
||||
pub(super) expiration_rfc3339: Option<String>,
|
||||
pub(super) data_quota_bytes: Option<u64>,
|
||||
pub(super) rate_limit_up_bps: Option<u64>,
|
||||
pub(super) rate_limit_down_bps: Option<u64>,
|
||||
pub(super) max_unique_ips: Option<usize>,
|
||||
pub(super) enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct PatchUserRequest {
|
||||
pub(super) secret: Option<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) user_ad_tag: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) max_tcp_conns: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) expiration_rfc3339: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) data_quota_bytes: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) rate_limit_up_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) rate_limit_down_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) max_unique_ips: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) enabled: Patch<bool>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
pub(super) struct RotateSecretRequest {
|
||||
pub(super) secret: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn parse_optional_expiration(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
||||
let Some(raw) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Some(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
|
||||
pub(super) fn parse_patch_expiration(
|
||||
value: &Patch<String>,
|
||||
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
||||
match value {
|
||||
Patch::Unchanged => Ok(Patch::Unchanged),
|
||||
Patch::Remove => Ok(Patch::Remove),
|
||||
Patch::Set(raw) => {
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_user_secret(secret: &str) -> bool {
|
||||
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_username(user: &str) -> bool {
|
||||
!user.is_empty()
|
||||
&& user.len() <= MAX_USERNAME_LEN
|
||||
&& user
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
||||
}
|
||||
|
||||
pub(super) fn random_user_secret() -> String {
|
||||
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
||||
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
||||
let mut bytes = [0u8; 16];
|
||||
rng.fill(&mut bytes);
|
||||
hex::encode(bytes)
|
||||
}
|
||||
// User-management request, response, and validation models.
|
||||
mod users;
|
||||
pub(super) use users::*;
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
use super::*;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserLinks {
|
||||
pub(in crate::api) classic: Vec<String>,
|
||||
pub(in crate::api) secure: Vec<String>,
|
||||
pub(in crate::api) tls: Vec<String>,
|
||||
pub(in crate::api) tls_domains: Vec<TlsDomainLink>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct TlsDomainLink {
|
||||
pub(in crate::api) domain: String,
|
||||
pub(in crate::api) link: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserInfo {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) enabled: bool,
|
||||
pub(in crate::api) in_runtime: bool,
|
||||
pub(in crate::api) user_ad_tag: Option<String>,
|
||||
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||
pub(in crate::api) current_connections: u64,
|
||||
pub(in crate::api) active_unique_ips: usize,
|
||||
pub(in crate::api) active_unique_ips_list: Vec<IpAddr>,
|
||||
pub(in crate::api) recent_unique_ips: usize,
|
||||
pub(in crate::api) recent_unique_ips_list: Vec<IpAddr>,
|
||||
pub(in crate::api) total_octets: u64,
|
||||
pub(in crate::api) links: UserLinks,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserActiveIps {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) active_ips: Vec<IpAddr>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct CreateUserResponse {
|
||||
pub(in crate::api) user: UserInfo,
|
||||
pub(in crate::api) secret: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct DeleteUserResponse {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) in_runtime: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct ResetUserQuotaResponse {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) used_bytes: u64,
|
||||
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserQuotaListData {
|
||||
pub(in crate::api) users: Vec<UserQuotaEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(in crate::api) struct UserQuotaEntry {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) data_quota_bytes: u64,
|
||||
pub(in crate::api) used_bytes: u64,
|
||||
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(in crate::api) struct CreateUserRequest {
|
||||
pub(in crate::api) username: String,
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
pub(in crate::api) user_ad_tag: Option<String>,
|
||||
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||
pub(in crate::api) enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(in crate::api) struct PatchUserRequest {
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) user_ad_tag: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) max_tcp_conns: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) expiration_rfc3339: Patch<String>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) data_quota_bytes: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) rate_limit_up_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) rate_limit_down_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) max_unique_ips: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(in crate::api) enabled: Patch<bool>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
pub(in crate::api) struct RotateSecretRequest {
|
||||
pub(in crate::api) secret: Option<String>,
|
||||
}
|
||||
|
||||
pub(in crate::api) fn parse_optional_expiration(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
||||
let Some(raw) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Some(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
|
||||
pub(in crate::api) fn parse_patch_expiration(
|
||||
value: &Patch<String>,
|
||||
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
||||
match value {
|
||||
Patch::Unchanged => Ok(Patch::Unchanged),
|
||||
Patch::Remove => Ok(Patch::Remove),
|
||||
Patch::Set(raw) => {
|
||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_user_secret(secret: &str) -> bool {
|
||||
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(in crate::api) fn is_valid_username(user: &str) -> bool {
|
||||
!user.is_empty()
|
||||
&& user.len() <= MAX_USERNAME_LEN
|
||||
&& user
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
||||
}
|
||||
|
||||
pub(in crate::api) fn random_user_secret() -> String {
|
||||
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
||||
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
||||
let mut bytes = [0u8; 16];
|
||||
rng.fill(&mut bytes);
|
||||
hex::encode(bytes)
|
||||
}
|
||||
+15
-54
@@ -21,8 +21,11 @@ pub(super) struct SecurityWhitelistData {
|
||||
pub(super) struct RuntimeMePoolStateGenerationData {
|
||||
pub(super) active_generation: u64,
|
||||
pub(super) warm_generation: u64,
|
||||
pub(super) warm_generations: Vec<u64>,
|
||||
pub(super) pending_hardswap_generation: u64,
|
||||
pub(super) pending_hardswap_age_secs: Option<u64>,
|
||||
pub(super) reinit_inflight: usize,
|
||||
pub(super) reinit_max_concurrency_effective: usize,
|
||||
pub(super) draining_generations: Vec<u64>,
|
||||
}
|
||||
|
||||
@@ -67,6 +70,8 @@ pub(super) struct RuntimeMePoolStateRefillDcData {
|
||||
pub(super) struct RuntimeMePoolStateRefillData {
|
||||
pub(super) inflight_endpoints_total: usize,
|
||||
pub(super) inflight_dc_total: usize,
|
||||
pub(super) running_dc_total: usize,
|
||||
pub(super) pending_dc_total: usize,
|
||||
pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
|
||||
}
|
||||
|
||||
@@ -291,8 +296,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
||||
};
|
||||
};
|
||||
|
||||
let status = pool.api_status_snapshot().await;
|
||||
let runtime = pool.api_runtime_snapshot().await;
|
||||
let (status, runtime) = pool.api_coherent_snapshots().await;
|
||||
let refill = pool.api_refill_snapshot().await;
|
||||
|
||||
let mut draining_generations = BTreeSet::<u64>::new();
|
||||
@@ -329,8 +333,11 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
||||
generations: RuntimeMePoolStateGenerationData {
|
||||
active_generation: runtime.active_generation,
|
||||
warm_generation: runtime.warm_generation,
|
||||
warm_generations: runtime.warm_generations,
|
||||
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
||||
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
||||
reinit_inflight: runtime.reinit_inflight,
|
||||
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
|
||||
draining_generations: draining_generations.into_iter().collect(),
|
||||
},
|
||||
hardswap: RuntimeMePoolStateHardswapData {
|
||||
@@ -356,6 +363,8 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
||||
refill: RuntimeMePoolStateRefillData {
|
||||
inflight_endpoints_total: refill.inflight_endpoints_total,
|
||||
inflight_dc_total: refill.inflight_dc_total,
|
||||
running_dc_total: refill.running_dc_total,
|
||||
pending_dc_total: refill.pending_dc_total,
|
||||
by_dc: refill
|
||||
.by_dc
|
||||
.into_iter()
|
||||
@@ -532,55 +541,7 @@ pub(super) async fn build_runtime_upstream_quality_data(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||
return RuntimeNatStunData {
|
||||
enabled: false,
|
||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: None,
|
||||
};
|
||||
};
|
||||
|
||||
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||
RuntimeNatStunData {
|
||||
enabled: true,
|
||||
reason: None,
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: Some(RuntimeNatStunPayload {
|
||||
flags: RuntimeNatStunFlagsData {
|
||||
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||
},
|
||||
servers: RuntimeNatStunServersData {
|
||||
configured: snapshot.configured_servers,
|
||||
live: snapshot.live_servers.clone(),
|
||||
live_total: snapshot.live_servers.len(),
|
||||
},
|
||||
reflection: RuntimeNatStunReflectionBlockData {
|
||||
v4: snapshot
|
||||
.reflection_v4
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
v6: snapshot
|
||||
.reflection_v6
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
},
|
||||
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
// NAT/STUN runtime projection and timestamping.
|
||||
mod nat;
|
||||
pub(super) use nat::build_runtime_nat_stun_data;
|
||||
use nat::now_epoch_secs;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use super::*;
|
||||
|
||||
pub(in crate::api) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||
return RuntimeNatStunData {
|
||||
enabled: false,
|
||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: None,
|
||||
};
|
||||
};
|
||||
|
||||
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||
RuntimeNatStunData {
|
||||
enabled: true,
|
||||
reason: None,
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: Some(RuntimeNatStunPayload {
|
||||
flags: RuntimeNatStunFlagsData {
|
||||
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||
},
|
||||
servers: RuntimeNatStunServersData {
|
||||
configured: snapshot.configured_servers,
|
||||
live: snapshot.live_servers.clone(),
|
||||
live_total: snapshot.live_servers.len(),
|
||||
},
|
||||
reflection: RuntimeNatStunReflectionBlockData {
|
||||
v4: snapshot
|
||||
.reflection_v4
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
v6: snapshot
|
||||
.reflection_v6
|
||||
.map(|entry| RuntimeNatStunReflectionData {
|
||||
addr: entry.addr.to_string(),
|
||||
age_secs: entry.age_secs,
|
||||
}),
|
||||
},
|
||||
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
@@ -84,6 +84,7 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
|
||||
reconnect_success_total: stats.get_me_reconnect_success(),
|
||||
handshake_reject_total: stats.get_me_handshake_reject_total(),
|
||||
handshake_error_codes,
|
||||
handshake_error_code_overflow_total: stats.get_me_handshake_error_code_overflow_total(),
|
||||
reader_eof_total: stats.get_me_reader_eof_total(),
|
||||
idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
|
||||
route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
|
||||
@@ -342,8 +343,7 @@ async fn get_minimal_payload_cached(
|
||||
}
|
||||
|
||||
let pool = shared.me_pool.read().await.clone()?;
|
||||
let status = pool.api_status_snapshot().await;
|
||||
let runtime = pool.api_runtime_snapshot().await;
|
||||
let (status, runtime) = pool.api_coherent_snapshots().await;
|
||||
let generated_at_epoch_secs = status.generated_at_epoch_secs;
|
||||
|
||||
let me_writers = MeWritersData {
|
||||
@@ -425,8 +425,11 @@ async fn get_minimal_payload_cached(
|
||||
let me_runtime = MinimalMeRuntimeData {
|
||||
active_generation: runtime.active_generation,
|
||||
warm_generation: runtime.warm_generation,
|
||||
warm_generations: runtime.warm_generations,
|
||||
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
||||
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
||||
reinit_inflight: runtime.reinit_inflight,
|
||||
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
|
||||
hardswap_enabled: runtime.hardswap_enabled,
|
||||
floor_mode: runtime.floor_mode,
|
||||
adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs,
|
||||
@@ -523,57 +526,6 @@ async fn get_minimal_payload_cached(
|
||||
Some((generated_at_epoch_secs, payload))
|
||||
}
|
||||
|
||||
fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
||||
MeWritersData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
summary: MeWritersSummary {
|
||||
configured_dc_groups: 0,
|
||||
configured_endpoints: 0,
|
||||
available_endpoints: 0,
|
||||
available_pct: 0.0,
|
||||
required_writers: 0,
|
||||
alive_writers: 0,
|
||||
coverage_pct: 0.0,
|
||||
fresh_alive_writers: 0,
|
||||
fresh_coverage_pct: 0.0,
|
||||
},
|
||||
writers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
||||
DcStatusData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
dcs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
||||
match value {
|
||||
UpstreamRouteKind::Direct => "direct",
|
||||
UpstreamRouteKind::Socks4 => "socks4",
|
||||
UpstreamRouteKind::Socks5 => "socks5",
|
||||
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
||||
}
|
||||
}
|
||||
|
||||
fn map_ip_preference(value: IpPreference) -> &'static str {
|
||||
match value {
|
||||
IpPreference::Unknown => "unknown",
|
||||
IpPreference::PreferV6 => "prefer_v6",
|
||||
IpPreference::PreferV4 => "prefer_v4",
|
||||
IpPreference::BothWork => "both_work",
|
||||
IpPreference::Unavailable => "unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
// Disabled-state builders and stable upstream enum mappings.
|
||||
mod helpers;
|
||||
use helpers::*;
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
||||
MeWritersData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
summary: MeWritersSummary {
|
||||
configured_dc_groups: 0,
|
||||
configured_endpoints: 0,
|
||||
available_endpoints: 0,
|
||||
available_pct: 0.0,
|
||||
required_writers: 0,
|
||||
alive_writers: 0,
|
||||
coverage_pct: 0.0,
|
||||
fresh_alive_writers: 0,
|
||||
fresh_coverage_pct: 0.0,
|
||||
},
|
||||
writers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
||||
DcStatusData {
|
||||
middle_proxy_enabled: false,
|
||||
reason: Some(reason),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
dcs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
||||
match value {
|
||||
UpstreamRouteKind::Direct => "direct",
|
||||
UpstreamRouteKind::Socks4 => "socks4",
|
||||
UpstreamRouteKind::Socks5 => "socks5",
|
||||
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn map_ip_preference(value: IpPreference) -> &'static str {
|
||||
match value {
|
||||
IpPreference::Unknown => "unknown",
|
||||
IpPreference::PreferV6 => "prefer_v6",
|
||||
IpPreference::PreferV4 => "prefer_v4",
|
||||
IpPreference::BothWork => "both_work",
|
||||
IpPreference::Unavailable => "unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
+34
-15
@@ -4,6 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::maestro::control_plane::ProcessControlPlane;
|
||||
use crate::maestro::generation::RuntimeWatchState;
|
||||
|
||||
use super::ApiRuntimeState;
|
||||
@@ -13,22 +14,29 @@ pub(super) fn spawn_runtime_watchers(
|
||||
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
control_plane: &ProcessControlPlane,
|
||||
) {
|
||||
let _config_watcher = spawn_config_watcher(
|
||||
spawn_config_watcher(
|
||||
runtime_watch_rx.clone(),
|
||||
runtime_state.clone(),
|
||||
runtime_events.clone(),
|
||||
control_plane,
|
||||
);
|
||||
spawn_admission_watcher(
|
||||
runtime_watch_rx,
|
||||
runtime_state,
|
||||
runtime_events,
|
||||
control_plane,
|
||||
);
|
||||
let _admission_watcher =
|
||||
spawn_admission_watcher(runtime_watch_rx, runtime_state, runtime_events);
|
||||
}
|
||||
|
||||
fn spawn_config_watcher(
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
control_plane: &ProcessControlPlane,
|
||||
) {
|
||||
let _ = control_plane.spawn(async move {
|
||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||
return;
|
||||
};
|
||||
@@ -78,15 +86,16 @@ fn spawn_config_watcher(
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn spawn_admission_watcher(
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
control_plane: &ProcessControlPlane,
|
||||
) {
|
||||
let _ = control_plane.spawn(async move {
|
||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||
return;
|
||||
};
|
||||
@@ -124,7 +133,7 @@ fn spawn_admission_watcher(
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn active_generation_id(
|
||||
@@ -246,7 +255,13 @@ mod tests {
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||
let runtime_state = runtime_state();
|
||||
let events = Arc::new(ApiEventStore::new(16));
|
||||
spawn_runtime_watchers(runtime_watch_rx, runtime_state.clone(), events.clone());
|
||||
let control_plane = ProcessControlPlane::new();
|
||||
spawn_runtime_watchers(
|
||||
runtime_watch_rx,
|
||||
runtime_state.clone(),
|
||||
events.clone(),
|
||||
&control_plane,
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
|
||||
@@ -283,6 +298,7 @@ mod tests {
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -291,7 +307,13 @@ mod tests {
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||
let runtime_state = runtime_state();
|
||||
let events = Arc::new(ApiEventStore::new(16));
|
||||
let watcher = spawn_config_watcher(runtime_watch_rx, runtime_state.clone(), events.clone());
|
||||
let control_plane = ProcessControlPlane::new();
|
||||
spawn_config_watcher(
|
||||
runtime_watch_rx,
|
||||
runtime_state.clone(),
|
||||
events.clone(),
|
||||
&control_plane,
|
||||
);
|
||||
drop(initial_config_tx);
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
@@ -302,10 +324,7 @@ mod tests {
|
||||
wait_for_count(&runtime_state, 2).await;
|
||||
|
||||
drop(runtime_watch_tx);
|
||||
tokio::time::timeout(Duration::from_secs(1), watcher)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
|
||||
assert_eq!(
|
||||
events
|
||||
.snapshot(16)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::*;
|
||||
use tracing::warn;
|
||||
|
||||
pub(in crate::api) async fn rotate_secret(
|
||||
user: &str,
|
||||
@@ -108,6 +109,18 @@ pub(in crate::api) async fn delete_user(
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
let revision =
|
||||
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
|
||||
let configured_users = cfg.access.users.keys().cloned().collect();
|
||||
if let Err(error) = shared
|
||||
.quota_state
|
||||
.remove_user(&configured_users, user)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
user,
|
||||
error = %error,
|
||||
"Deleted user quota checkpoint cleanup will be reconciled on restart"
|
||||
);
|
||||
}
|
||||
drop(_guard);
|
||||
shared.ip_tracker.remove_user_limit(user).await;
|
||||
shared.ip_tracker.clear_user_ips(user).await;
|
||||
|
||||
+138
-49
@@ -1,5 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use http_body_util::Full;
|
||||
use hyper::body::{Bytes, Incoming};
|
||||
@@ -13,12 +13,18 @@ use super::model::ApiFailure;
|
||||
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||
use crate::web::manager::{ControlError, SessionDetail, WebProcessRuntime};
|
||||
use crate::web::manager::{ControlError, OperatorLifecycleError, SessionDetail, WebProcessRuntime};
|
||||
|
||||
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
|
||||
mod request;
|
||||
// Ingress, capacity, and decoy telemetry remain separate availability planes.
|
||||
mod observability;
|
||||
use observability::{
|
||||
WebCapacityStatus, WebCarrierNegotiationStatus, WebDecoyFastTrackStatus,
|
||||
WebDecoyUpstreamStatus, WebIngressStatus, WebLifecycleCountersStatus,
|
||||
};
|
||||
use request::{
|
||||
CloseRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
|
||||
CloseRequest, DrainRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
|
||||
valid_runtime_instance,
|
||||
};
|
||||
|
||||
@@ -27,6 +33,9 @@ const SESSIONS_PATH: &str = "/v1/runtime/web/sessions";
|
||||
const CLOSE_PATH: &str = "/v1/runtime/web/sessions/close";
|
||||
const DEBUG_CLEAR_PATH: &str = "/v1/runtime/web/debug/clear";
|
||||
const LEARNING_RESET_PATH: &str = "/v1/runtime/web/carrier-learning/reset";
|
||||
const LIFECYCLE_PAUSE_PATH: &str = "/v1/runtime/web/lifecycle/pause";
|
||||
const LIFECYCLE_DRAIN_PATH: &str = "/v1/runtime/web/lifecycle/drain";
|
||||
const LIFECYCLE_RESUME_PATH: &str = "/v1/runtime/web/lifecycle/resume";
|
||||
const SESSION_DETAIL_PREFIX: &str = "/v1/runtime/web/sessions/";
|
||||
const OPERATION_PREFIX: &str = "/v1/runtime/web/operations/";
|
||||
const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
|
||||
@@ -35,7 +44,12 @@ const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
|
||||
pub(super) fn allowed_methods(path: &str) -> Option<&'static str> {
|
||||
match path {
|
||||
STATUS_PATH | SESSIONS_PATH => Some(ALLOW_GET),
|
||||
CLOSE_PATH | DEBUG_CLEAR_PATH | LEARNING_RESET_PATH => Some(ALLOW_POST),
|
||||
CLOSE_PATH
|
||||
| DEBUG_CLEAR_PATH
|
||||
| LEARNING_RESET_PATH
|
||||
| LIFECYCLE_PAUSE_PATH
|
||||
| LIFECYCLE_DRAIN_PATH
|
||||
| LIFECYCLE_RESUME_PATH => Some(ALLOW_POST),
|
||||
_ if detail_ref(path).is_some() || operation_ref(path).is_some() => Some(ALLOW_GET),
|
||||
_ => None,
|
||||
}
|
||||
@@ -63,7 +77,7 @@ pub(super) async fn handle(
|
||||
reject_query(query)?;
|
||||
let publication = shared.web_runtime_rx.borrow().clone();
|
||||
let runtime = publication.runtime.upgrade();
|
||||
let data = WebStatusData::new(publication, runtime.as_deref(), config.web.enabled);
|
||||
let data = WebStatusData::new(publication, runtime.as_deref(), config);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", SESSIONS_PATH) => {
|
||||
@@ -79,12 +93,20 @@ pub(super) async fn handle(
|
||||
let trace_session_id = parse_session_ref(&runtime, session_ref)?;
|
||||
match runtime.session_detail(trace_session_id) {
|
||||
SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
|
||||
SessionDetail::Gone { attempt } => Ok(success_response(
|
||||
SessionDetail::Gone {
|
||||
attempt,
|
||||
carrier,
|
||||
reason,
|
||||
closed_age_ms,
|
||||
} => Ok(success_response(
|
||||
StatusCode::GONE,
|
||||
GoneSessionData {
|
||||
session_ref: session_ref.to_string(),
|
||||
state: "closed",
|
||||
attempt,
|
||||
carrier,
|
||||
reason,
|
||||
closed_age_ms,
|
||||
},
|
||||
revision,
|
||||
)),
|
||||
@@ -105,6 +127,67 @@ pub(super) async fn handle(
|
||||
.map_err(control_failure)?;
|
||||
Ok(success_response(StatusCode::OK, status, revision))
|
||||
}
|
||||
("POST", LIFECYCLE_PAUSE_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<RuntimeInstanceRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let status = runtime.pause_operator().await.map_err(lifecycle_failure)?;
|
||||
shared.runtime_events.record(
|
||||
"api.web.lifecycle.pause.ok",
|
||||
format!("epoch={}", status.epoch),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, status, revision))
|
||||
}
|
||||
("POST", LIFECYCLE_DRAIN_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<DrainRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let timeout = drain_timeout(request.timeout_secs)?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let status = runtime
|
||||
.drain_operator(timeout)
|
||||
.await
|
||||
.map_err(lifecycle_failure)?;
|
||||
shared.runtime_events.record(
|
||||
"api.web.lifecycle.drain.accepted",
|
||||
format!(
|
||||
"epoch={} timeout_secs={}",
|
||||
status.epoch, request.timeout_secs
|
||||
),
|
||||
);
|
||||
Ok(success_response(StatusCode::ACCEPTED, status, revision))
|
||||
}
|
||||
("POST", LIFECYCLE_RESUME_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<RuntimeInstanceRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let status = runtime.resume_operator().await.map_err(lifecycle_failure)?;
|
||||
shared.runtime_events.record(
|
||||
"api.web.lifecycle.resume.ok",
|
||||
format!("epoch={}", status.epoch),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, status, revision))
|
||||
}
|
||||
("POST", CLOSE_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
@@ -194,6 +277,14 @@ struct WebStatusData {
|
||||
reason: Option<&'static str>,
|
||||
listeners: Vec<String>,
|
||||
effective_config_enabled: bool,
|
||||
ingress: WebIngressStatus,
|
||||
capacity: WebCapacityStatus,
|
||||
decoy_upstream: WebDecoyUpstreamStatus,
|
||||
decoy_fasttrack: WebDecoyFastTrackStatus,
|
||||
carrier_negotiation: WebCarrierNegotiationStatus,
|
||||
lifecycle_counters: WebLifecycleCountersStatus,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
operator_lifecycle: Option<crate::web::manager::OperatorLifecycleStatus>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
runtime: Option<crate::web::manager::WebRuntimeStatus>,
|
||||
}
|
||||
@@ -202,7 +293,7 @@ impl WebStatusData {
|
||||
fn new(
|
||||
publication: WebRuntimePublication,
|
||||
runtime: Option<&WebProcessRuntime>,
|
||||
effective_config_enabled: bool,
|
||||
config: &ProxyConfig,
|
||||
) -> Self {
|
||||
let available = runtime.is_some()
|
||||
&& matches!(
|
||||
@@ -221,6 +312,13 @@ impl WebStatusData {
|
||||
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||
})
|
||||
};
|
||||
let operator_lifecycle = runtime.map(WebProcessRuntime::operator_lifecycle_status);
|
||||
let ingress = WebIngressStatus::new(&publication, runtime.is_some());
|
||||
let capacity = WebCapacityStatus::new(&publication, runtime, config);
|
||||
let decoy_upstream = WebDecoyUpstreamStatus::new(&publication);
|
||||
let decoy_fasttrack = WebDecoyFastTrackStatus::new(&publication, config);
|
||||
let carrier_negotiation = WebCarrierNegotiationStatus::new(&publication);
|
||||
let lifecycle_counters = WebLifecycleCountersStatus::new(&publication, config);
|
||||
Self {
|
||||
lifecycle: publication.lifecycle.as_str(),
|
||||
lifecycle_epoch: publication.epoch,
|
||||
@@ -232,7 +330,14 @@ impl WebStatusData {
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
effective_config_enabled,
|
||||
effective_config_enabled: config.web.enabled,
|
||||
ingress,
|
||||
capacity,
|
||||
decoy_upstream,
|
||||
decoy_fasttrack,
|
||||
carrier_negotiation,
|
||||
lifecycle_counters,
|
||||
operator_lifecycle,
|
||||
runtime: runtime.map(WebProcessRuntime::try_status),
|
||||
}
|
||||
}
|
||||
@@ -243,6 +348,9 @@ struct GoneSessionData {
|
||||
session_ref: String,
|
||||
state: &'static str,
|
||||
attempt: u8,
|
||||
carrier: crate::config::WebCarrier,
|
||||
reason: &'static str,
|
||||
closed_age_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -369,6 +477,17 @@ fn control_failure(error: ControlError) -> ApiFailure {
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_failure(error: OperatorLifecycleError) -> ApiFailure {
|
||||
match error {
|
||||
OperatorLifecycleError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
|
||||
OperatorLifecycleError::OperationInProgress => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_lifecycle_in_progress",
|
||||
"Another WEB drain operation is active",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_busy() -> ApiFailure {
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
@@ -377,6 +496,15 @@ fn snapshot_busy() -> ApiFailure {
|
||||
)
|
||||
}
|
||||
|
||||
fn drain_timeout(timeout_secs: u64) -> Result<Duration, ApiFailure> {
|
||||
if !(1..=3600).contains(&timeout_secs) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"timeout_secs must be within 1..=3600",
|
||||
));
|
||||
}
|
||||
Ok(Duration::from_secs(timeout_secs))
|
||||
}
|
||||
|
||||
fn reject_query(query: Option<&str>) -> Result<(), ApiFailure> {
|
||||
if query.is_some_and(|query| !query.is_empty()) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
@@ -401,44 +529,5 @@ fn millis(duration: std::time::Duration) -> u64 {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hyper::header::HeaderValue;
|
||||
|
||||
#[test]
|
||||
fn route_table_keeps_status_read_only_and_controls_post_only() {
|
||||
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(
|
||||
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
|
||||
Some(ALLOW_GET)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_content_type_is_exact_and_single() {
|
||||
let exact = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(&exact).is_ok());
|
||||
|
||||
let parameterized = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json; charset=utf-8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(¶meterized).is_err());
|
||||
|
||||
let mut duplicated = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
duplicated
|
||||
.headers_mut()
|
||||
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
assert!(require_json_content_type(&duplicated).is_err());
|
||||
}
|
||||
}
|
||||
#[path = "web_runtime/tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::config::{ProxyConfig, WebDecoyFastTrackMode, WebHttpConnectionCapacityAction};
|
||||
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||
use crate::web::manager::{WebCapacityResourceStatus, WebCapacitySnapshot, WebProcessRuntime};
|
||||
use crate::web::telemetry::{
|
||||
WebBridgeRecoveryCounter, WebCarrierFailureCounter, WebCarrierLearningCounter,
|
||||
WebCarrierSelectionCounter, WebDecoyFastTrackCounter, WebSessionCloseCounter,
|
||||
WebSessionLifecycleObservationCounter,
|
||||
};
|
||||
use crate::web::telemetry::{WebOutcomeCounter, WebRejectionCounter};
|
||||
|
||||
/// Private WEB ingress state owned by this Telemt process.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebIngressStatus {
|
||||
configured_listeners: usize,
|
||||
live_acceptors: usize,
|
||||
accepting_connections: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reason: Option<&'static str>,
|
||||
tcp_accept_total: u64,
|
||||
tcp_accept_error_total: u64,
|
||||
}
|
||||
|
||||
impl WebIngressStatus {
|
||||
/// Builds a process-ingress snapshot without probing external TLS termination.
|
||||
pub(super) fn new(publication: &WebRuntimePublication, runtime_available: bool) -> Self {
|
||||
let configured_listeners = publication.listeners.len();
|
||||
let live_acceptors = publication.telemetry.live_acceptors();
|
||||
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
|
||||
&& runtime_available
|
||||
&& configured_listeners != 0
|
||||
&& live_acceptors == configured_listeners;
|
||||
let reason = if accepting_connections {
|
||||
None
|
||||
} else {
|
||||
Some(match publication.lifecycle {
|
||||
WebRuntimeLifecycle::Starting => "starting",
|
||||
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
|
||||
WebRuntimeLifecycle::Draining => "ingress_draining",
|
||||
WebRuntimeLifecycle::Drained => "ingress_drained",
|
||||
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||
WebRuntimeLifecycle::Running if !runtime_available => "runtime_released",
|
||||
WebRuntimeLifecycle::Running if configured_listeners == 0 => "no_web_listener",
|
||||
WebRuntimeLifecycle::Running => "acceptor_unavailable",
|
||||
})
|
||||
};
|
||||
Self {
|
||||
configured_listeners,
|
||||
live_acceptors,
|
||||
accepting_connections,
|
||||
reason,
|
||||
tcp_accept_total: publication.telemetry.accepted(),
|
||||
tcp_accept_error_total: publication.telemetry.accept_errors(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded process-wide WEB capacity and terminal rejection view.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebCapacityStatus {
|
||||
http_connection_capacity_action: WebHttpConnectionCapacityAction,
|
||||
max_http_overload_connections: usize,
|
||||
http_overload_timeout_ms: u64,
|
||||
resources: Vec<WebCapacityResourceStatus>,
|
||||
saturated_resources: Vec<&'static str>,
|
||||
partial: Vec<&'static str>,
|
||||
rejections: Vec<WebRejectionCounter>,
|
||||
http_connection_overload_outcomes: Vec<WebOutcomeCounter>,
|
||||
}
|
||||
|
||||
impl WebCapacityStatus {
|
||||
/// Builds a bounded capacity snapshot from non-blocking runtime observations.
|
||||
pub(super) fn new(
|
||||
publication: &WebRuntimePublication,
|
||||
runtime: Option<&WebProcessRuntime>,
|
||||
config: &ProxyConfig,
|
||||
) -> Self {
|
||||
let snapshot = runtime
|
||||
.map(WebProcessRuntime::capacity_snapshot)
|
||||
.unwrap_or_else(runtime_unavailable_snapshot);
|
||||
Self {
|
||||
http_connection_capacity_action: config.web.http_connection_capacity_action,
|
||||
max_http_overload_connections: config.web.limits.max_http_overload_connections,
|
||||
http_overload_timeout_ms: config.web.timeouts.http_overload_timeout_ms,
|
||||
resources: snapshot.resources,
|
||||
saturated_resources: snapshot.saturated_resources,
|
||||
partial: snapshot.partial,
|
||||
rejections: publication.telemetry.rejection_counters(),
|
||||
http_connection_overload_outcomes: publication.telemetry.overload_counters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime_unavailable_snapshot() -> WebCapacitySnapshot {
|
||||
WebCapacitySnapshot {
|
||||
resources: Vec::new(),
|
||||
saturated_resources: Vec::new(),
|
||||
partial: vec!["runtime"],
|
||||
}
|
||||
}
|
||||
|
||||
/// Passive health of Telemt's internal plain-HTTP decoy origin hop.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebDecoyUpstreamStatus {
|
||||
outcomes: Vec<WebOutcomeCounter>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
last_outcome: Option<&'static str>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
last_outcome_age_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl WebDecoyUpstreamStatus {
|
||||
/// Builds the fixed internal decoy-origin outcome snapshot.
|
||||
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
|
||||
let last = publication.telemetry.last_decoy();
|
||||
Self {
|
||||
outcomes: publication.telemetry.decoy_counters(),
|
||||
last_outcome: last.map(|value| value.0),
|
||||
last_outcome_age_ms: last.map(|value| value.1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-cardinality process-lifetime decoy capability-routing counters.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebDecoyFastTrackStatus {
|
||||
mode: WebDecoyFastTrackMode,
|
||||
requests: Vec<WebDecoyFastTrackCounter>,
|
||||
}
|
||||
|
||||
impl WebDecoyFastTrackStatus {
|
||||
/// Builds effective policy and counters without requiring the runtime manager.
|
||||
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
|
||||
Self {
|
||||
mode: config.web.decoy_fasttrack_mode,
|
||||
requests: publication.telemetry.decoy_fasttrack_counters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-cardinality process-lifetime carrier negotiation counters.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebCarrierNegotiationStatus {
|
||||
selections: Vec<WebCarrierSelectionCounter>,
|
||||
reported_failures: Vec<WebCarrierFailureCounter>,
|
||||
learning_outcomes: Vec<WebCarrierLearningCounter>,
|
||||
}
|
||||
|
||||
impl WebCarrierNegotiationStatus {
|
||||
/// Builds counters from publication ownership even when runtime state is unavailable.
|
||||
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
|
||||
Self {
|
||||
selections: publication.telemetry.carrier_selection_counters(),
|
||||
reported_failures: publication.telemetry.carrier_failure_counters(),
|
||||
learning_outcomes: publication.telemetry.carrier_learning_counters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-cardinality process-lifetime WEB lifecycle counters.
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct WebLifecycleCountersStatus {
|
||||
bridge_recovery_secs: u64,
|
||||
session_closures: Vec<WebSessionCloseCounter>,
|
||||
session_observations: Vec<WebSessionLifecycleObservationCounter>,
|
||||
bridge_recovery_events: Vec<WebBridgeRecoveryCounter>,
|
||||
}
|
||||
|
||||
impl WebLifecycleCountersStatus {
|
||||
/// Builds a complete counter set from process-owned telemetry.
|
||||
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
|
||||
Self {
|
||||
bridge_recovery_secs: config.web.timeouts.bridge_recovery_secs,
|
||||
session_closures: publication.telemetry.session_close_counters(),
|
||||
session_observations: publication.telemetry.session_observation_counters(),
|
||||
bridge_recovery_events: publication.telemetry.bridge_recovery_counters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
|
||||
#[test]
|
||||
fn starting_ingress_does_not_claim_external_availability() {
|
||||
let control = WebRuntimeControl::new();
|
||||
let publication = control.subscribe().borrow().clone();
|
||||
let value =
|
||||
serde_json::to_value(super::WebIngressStatus::new(&publication, false)).unwrap();
|
||||
assert_eq!(value["configured_listeners"], 0);
|
||||
assert_eq!(value["live_acceptors"], 0);
|
||||
assert_eq!(value["accepting_connections"], false);
|
||||
assert_eq!(value["reason"], "starting");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_runtime_keeps_fixed_counter_sets_visible() {
|
||||
let control = WebRuntimeControl::new();
|
||||
let publication = control.subscribe().borrow().clone();
|
||||
let config = ProxyConfig::default();
|
||||
let capacity =
|
||||
serde_json::to_value(super::WebCapacityStatus::new(&publication, None, &config))
|
||||
.unwrap();
|
||||
let decoy = serde_json::to_value(super::WebDecoyUpstreamStatus::new(&publication)).unwrap();
|
||||
let fasttrack =
|
||||
serde_json::to_value(super::WebDecoyFastTrackStatus::new(&publication, &config))
|
||||
.unwrap();
|
||||
let carrier =
|
||||
serde_json::to_value(super::WebCarrierNegotiationStatus::new(&publication)).unwrap();
|
||||
let lifecycle = serde_json::to_value(super::WebLifecycleCountersStatus::new(
|
||||
&publication,
|
||||
&config,
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
capacity["rejections"].as_array().unwrap().len(),
|
||||
crate::web::telemetry::WebRejectionReason::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
capacity["http_connection_overload_outcomes"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
decoy["outcomes"].as_array().unwrap().len(),
|
||||
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
|
||||
);
|
||||
assert_eq!(fasttrack["mode"], "off");
|
||||
assert_eq!(
|
||||
fasttrack["requests"].as_array().unwrap().len(),
|
||||
crate::web::telemetry::WebDecoyFastTrackDisposition::ALL.len()
|
||||
);
|
||||
assert_eq!(capacity["partial"][0], "runtime");
|
||||
assert_eq!(
|
||||
carrier["selections"].as_array().unwrap().len(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierSelectionDisposition::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
carrier["reported_failures"].as_array().unwrap().len(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierFailurePhase::ALL.len()
|
||||
* crate::web::manager::CarrierFailure::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
carrier["learning_outcomes"].as_array().unwrap().len(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierLearningOutcome::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle["session_closures"].as_array().unwrap().len(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::session::SessionCloseReason::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle["session_observations"].as_array().unwrap().len(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle["bridge_recovery_events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,16 @@ pub(super) struct RuntimeInstanceRequest {
|
||||
pub(super) runtime_instance: String,
|
||||
}
|
||||
|
||||
/// Process-fenced graceful drain request with one bounded relative deadline.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct DrainRequest {
|
||||
/// Random process identifier copied from WEB runtime status.
|
||||
pub(super) runtime_instance: String,
|
||||
/// Relative drain deadline frozen into one monotonic server deadline.
|
||||
pub(super) timeout_secs: u64,
|
||||
}
|
||||
|
||||
/// One process-fenced asynchronous close request.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -331,6 +341,14 @@ mod tests {
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<DrainRequest>(serde_json::json!({
|
||||
"runtime_instance": runtime_instance,
|
||||
"timeout_secs": 30,
|
||||
"extra": true,
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
use super::*;
|
||||
|
||||
use hyper::header::HeaderValue;
|
||||
|
||||
#[test]
|
||||
fn route_table_keeps_status_read_only_and_controls_post_only() {
|
||||
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LIFECYCLE_PAUSE_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LIFECYCLE_DRAIN_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LIFECYCLE_RESUME_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(
|
||||
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
|
||||
Some(ALLOW_GET)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_content_type_is_exact_and_single() {
|
||||
let exact = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(&exact).is_ok());
|
||||
|
||||
let parameterized = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json; charset=utf-8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(¶meterized).is_err());
|
||||
|
||||
let mut duplicated = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
duplicated
|
||||
.headers_mut()
|
||||
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
assert!(require_json_content_type(&duplicated).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_timeout_is_bounded_to_the_public_contract() {
|
||||
assert_eq!(drain_timeout(1).unwrap(), Duration::from_secs(1));
|
||||
assert_eq!(drain_timeout(3600).unwrap(), Duration::from_secs(3600));
|
||||
assert!(drain_timeout(0).is_err());
|
||||
assert!(drain_timeout(3601).is_err());
|
||||
}
|
||||
@@ -67,6 +67,11 @@ pub(super) async fn render(
|
||||
push_filter_form(&mut html, &query);
|
||||
html.push_str("<section><h2>Store</h2><table><tbody>");
|
||||
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
|
||||
summary_row(
|
||||
&mut html,
|
||||
"sideband",
|
||||
yes_no(status.policy.bridge_diagnostics_enabled()),
|
||||
);
|
||||
summary_row(&mut html, "body capture", body_mode(&status.policy));
|
||||
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
|
||||
summary_row(
|
||||
|
||||
@@ -97,6 +97,20 @@ pub(super) fn push_lifecycle(html: &mut String, event: &crate::web::trace::Trace
|
||||
);
|
||||
html.push_str("\nreason: ");
|
||||
html.push_str(event.reason.unwrap_or("-"));
|
||||
html.push_str("\npeer gap ms: ");
|
||||
html.push_str(
|
||||
&event
|
||||
.peer_gap_ms
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("\npredecessor session: ");
|
||||
html.push_str(
|
||||
&event
|
||||
.predecessor_session_id
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
if let Some(carrier) = &event.carrier {
|
||||
html.push_str("\nclient class: ");
|
||||
html.push_str(carrier.client_class);
|
||||
|
||||
@@ -22,6 +22,7 @@ fn html_escaping_covers_active_markup_characters() {
|
||||
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
sideband: true,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = crate::config::WebLimitsConfig {
|
||||
@@ -58,6 +59,7 @@ async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let body = std::str::from_utf8(&body).unwrap();
|
||||
assert!(body.contains("session_created"));
|
||||
assert!(body.contains("<th>sideband</th><td>yes</td>"));
|
||||
assert!(body.contains("0123456789abcdef"));
|
||||
assert!(body.contains("192.0.2.40"));
|
||||
}
|
||||
|
||||
+5
-446
@@ -1,6 +1,9 @@
|
||||
use ipnetwork::IpNetwork;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
|
||||
// Extended transport, masking, and ME default values.
|
||||
mod extended;
|
||||
|
||||
pub(crate) use extended::*;
|
||||
|
||||
// Helper defaults kept private to the config module.
|
||||
const DEFAULT_NETWORK_IPV6: Option<bool> = Some(false);
|
||||
@@ -520,447 +523,3 @@ pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize {
|
||||
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
|
||||
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||
DEFAULT_ME_ADMISSION_POLL_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
||||
256 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_max_client_frame() -> usize {
|
||||
16 * 1024 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_desync_all_full() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
||||
25
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
||||
120
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
||||
80
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_minutes() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
||||
15
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_file() -> String {
|
||||
"beobachten.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_serverhello_compact() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
||||
8
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
||||
24
|
||||
}
|
||||
|
||||
pub(crate) fn default_alpn_enforce() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
5 * 1024 * 1024
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
32 * 1024
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
60_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
5_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_stun_servers() -> Vec<String> {
|
||||
vec![
|
||||
"stun.l.google.com:5349".to_string(),
|
||||
"stun1.l.google.com:3478".to_string(),
|
||||
"stun.gmx.net:3478".to_string(),
|
||||
"stun.l.google.com:19302".to_string(),
|
||||
"stun.1und1.de:3478".to_string(),
|
||||
"stun1.l.google.com:19302".to_string(),
|
||||
"stun2.l.google.com:19302".to_string(),
|
||||
"stun3.l.google.com:19302".to_string(),
|
||||
"stun4.l.google.com:19302".to_string(),
|
||||
"stun.services.mozilla.com:3478".to_string(),
|
||||
"stun.stunprotocol.org:3478".to_string(),
|
||||
"stun.nextcloud.com:3478".to_string(),
|
||||
"stun.voip.eutelia.it:3478".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
||||
vec![
|
||||
"https://ifconfig.me/ip".to_string(),
|
||||
"https://api.ipify.org".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_cache_public_ip_path() -> String {
|
||||
"cache/public_ip.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every_secs() -> u64 {
|
||||
5 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every() -> Option<u64> {
|
||||
Some(default_update_every_secs())
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||
15 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||
64
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||
1000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
||||
2000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
||||
500
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||
256
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_instadrain() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||
32
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||
default_me_pool_drain_ttl_secs()
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||
0.8
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_hardswap() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_check() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
||||
vec!["pool.ntp.org".to_string()]
|
||||
}
|
||||
|
||||
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6() -> String {
|
||||
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
||||
Some(default_listen_addr_ipv6())
|
||||
}
|
||||
|
||||
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
||||
HashMap::from([(
|
||||
DEFAULT_ACCESS_USER.to_string(),
|
||||
DEFAULT_ACCESS_SECRET.to_string(),
|
||||
)])
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
// Custom deserializer helpers
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum OneOrMany {
|
||||
One(String),
|
||||
Many(Vec<String>),
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
||||
where
|
||||
D: serde::de::Deserializer<'de>,
|
||||
{
|
||||
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
||||
let mut out = HashMap::new();
|
||||
for (dc, val) in raw {
|
||||
let mut addrs = match val {
|
||||
OneOrMany::One(s) => vec![s],
|
||||
OneOrMany::Many(v) => v,
|
||||
};
|
||||
addrs.retain(|s| !s.trim().is_empty());
|
||||
if !addrs.is_empty() {
|
||||
out.insert(dc, addrs);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||
DEFAULT_ME_ADMISSION_POLL_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
||||
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
||||
256 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_max_client_frame() -> usize {
|
||||
16 * 1024 * 1024
|
||||
}
|
||||
|
||||
pub(crate) fn default_desync_all_full() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
||||
25
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
||||
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
||||
120
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
||||
80
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_minutes() -> u64 {
|
||||
10
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
||||
15
|
||||
}
|
||||
|
||||
pub(crate) fn default_beobachten_file() -> String {
|
||||
"beobachten.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_serverhello_compact() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
||||
8
|
||||
}
|
||||
|
||||
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
||||
24
|
||||
}
|
||||
|
||||
pub(crate) fn default_alpn_enforce() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
||||
512
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
5 * 1024 * 1024
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||
32 * 1024
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
60_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
5_000
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||
100
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_stun_servers() -> Vec<String> {
|
||||
vec![
|
||||
"stun.l.google.com:5349".to_string(),
|
||||
"stun1.l.google.com:3478".to_string(),
|
||||
"stun.gmx.net:3478".to_string(),
|
||||
"stun.l.google.com:19302".to_string(),
|
||||
"stun.1und1.de:3478".to_string(),
|
||||
"stun1.l.google.com:19302".to_string(),
|
||||
"stun2.l.google.com:19302".to_string(),
|
||||
"stun3.l.google.com:19302".to_string(),
|
||||
"stun4.l.google.com:19302".to_string(),
|
||||
"stun.services.mozilla.com:3478".to_string(),
|
||||
"stun.stunprotocol.org:3478".to_string(),
|
||||
"stun.nextcloud.com:3478".to_string(),
|
||||
"stun.voip.eutelia.it:3478".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
||||
vec![
|
||||
"https://ifconfig.me/ip".to_string(),
|
||||
"https://api.ipify.org".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn default_cache_public_ip_path() -> String {
|
||||
"cache/public_ip.txt".to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
||||
60 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every_secs() -> u64 {
|
||||
5 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_update_every() -> Option<u64> {
|
||||
Some(default_update_every_secs())
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||
15 * 60
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_max_concurrency() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||
64
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||
200
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||
1000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
||||
2000
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
||||
3
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
||||
500
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||
256
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||
90
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_instadrain() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||
32
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||
default_me_pool_drain_ttl_secs()
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||
0.8
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_hardswap() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_check() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
||||
vec!["pool.ntp.org".to_string()]
|
||||
}
|
||||
|
||||
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
||||
2
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6() -> String {
|
||||
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
||||
Some(default_listen_addr_ipv6())
|
||||
}
|
||||
|
||||
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
||||
HashMap::from([(
|
||||
DEFAULT_ACCESS_USER.to_string(),
|
||||
DEFAULT_ACCESS_SECRET.to_string(),
|
||||
)])
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
// Custom deserializer helpers
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum OneOrMany {
|
||||
One(String),
|
||||
Many(Vec<String>),
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
||||
where
|
||||
D: serde::de::Deserializer<'de>,
|
||||
{
|
||||
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
||||
let mut out = HashMap::new();
|
||||
for (dc, val) in raw {
|
||||
let mut addrs = match val {
|
||||
OneOrMany::One(s) => vec![s],
|
||||
OneOrMany::Many(v) => v,
|
||||
};
|
||||
addrs.retain(|s| !s.trim().is_empty());
|
||||
if !addrs.is_empty() {
|
||||
out.insert(dc, addrs);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -21,6 +21,8 @@
|
||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
||||
//! reconciled only during privileged startup.
|
||||
//! `web.decoy_fasttrack_mode` is also restart-only so one process never mixes
|
||||
//! capability timing policies or process-lifetime counter semantics.
|
||||
//! Non-hot changes are never mixed into the runtime config snapshot.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
@@ -37,7 +39,7 @@ use super::load::{LoadedConfig, ProxyConfig};
|
||||
#[allow(unused_imports)]
|
||||
use crate::config::{
|
||||
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
||||
MeWriterPickMode, WebDebugConfig, web_debug_fits_limits,
|
||||
MeWriterPickMode, WEB_CARRIER_LEARNING_MIN_ENTRIES, WebDebugConfig, web_debug_fits_limits,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::config::{ListenerConfig, SynLimitMode};
|
||||
|
||||
@@ -85,6 +85,10 @@ pub(super) fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot
|
||||
warned = true;
|
||||
warn!("config reload: server listener settings changed; restart required");
|
||||
}
|
||||
if old.web.decoy_fasttrack_mode != new.web.decoy_fasttrack_mode {
|
||||
warned = true;
|
||||
warn!("config reload: web.decoy_fasttrack_mode changed; restart required");
|
||||
}
|
||||
if old.censorship.tls_domain != new.censorship.tls_domain
|
||||
|| old.censorship.tls_domains != new.censorship.tls_domains
|
||||
|| old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope
|
||||
|
||||
@@ -10,6 +10,7 @@ pub struct HotFields {
|
||||
pub update_every_secs: u64,
|
||||
pub me_reinit_every_secs: u64,
|
||||
pub me_reinit_singleflight: bool,
|
||||
pub me_reinit_max_concurrency: usize,
|
||||
pub me_reinit_coalesce_window_ms: u64,
|
||||
pub hardswap: bool,
|
||||
pub me_pool_drain_ttl_secs: u64,
|
||||
@@ -102,6 +103,7 @@ impl HotFields {
|
||||
update_every_secs: cfg.general.effective_update_every_secs(),
|
||||
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
|
||||
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
|
||||
me_reinit_max_concurrency: cfg.general.me_reinit_max_concurrency,
|
||||
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
|
||||
hardswap: cfg.general.hardswap,
|
||||
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
|
||||
@@ -236,6 +238,7 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
||||
cfg.general.proxy_config_auto_reload_secs = new.general.proxy_config_auto_reload_secs;
|
||||
cfg.general.me_reinit_every_secs = new.general.me_reinit_every_secs;
|
||||
cfg.general.me_reinit_singleflight = new.general.me_reinit_singleflight;
|
||||
cfg.general.me_reinit_max_concurrency = new.general.me_reinit_max_concurrency;
|
||||
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
|
||||
cfg.general.hardswap = new.general.hardswap;
|
||||
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
|
||||
@@ -340,17 +343,26 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
||||
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
||||
cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs;
|
||||
let process_limits = cfg.web.limits.clone();
|
||||
let decoy_fasttrack_mode = cfg.web.decoy_fasttrack_mode;
|
||||
cfg.web = new.web.clone();
|
||||
cfg.web.limits = process_limits;
|
||||
cfg.web.decoy_fasttrack_mode = decoy_fasttrack_mode;
|
||||
if cfg.web.carrier_negotiation_enabled()
|
||||
&& cfg.web.carrier_learning
|
||||
&& cfg.web.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||
{
|
||||
if old.web.carrier_learning != new.web.carrier_learning {
|
||||
cfg.web.carrier_learning = old.web.carrier_learning;
|
||||
} else {
|
||||
cfg.web.carriers = old.web.carriers.clone();
|
||||
}
|
||||
}
|
||||
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
|
||||
cfg.web.debug = old.web.debug.clone();
|
||||
}
|
||||
if cfg.rebuild_runtime_user_auth().is_err() {
|
||||
cfg.runtime_user_auth = None;
|
||||
}
|
||||
if cfg.rebuild_runtime_web().is_err() {
|
||||
cfg.web = old.web.clone();
|
||||
}
|
||||
|
||||
cfg
|
||||
}
|
||||
|
||||
@@ -114,12 +114,14 @@ pub(super) fn log_changes(
|
||||
}
|
||||
if old_hot.me_reinit_every_secs != new_hot.me_reinit_every_secs
|
||||
|| old_hot.me_reinit_singleflight != new_hot.me_reinit_singleflight
|
||||
|| old_hot.me_reinit_max_concurrency != new_hot.me_reinit_max_concurrency
|
||||
|| old_hot.me_reinit_coalesce_window_ms != new_hot.me_reinit_coalesce_window_ms
|
||||
{
|
||||
info!(
|
||||
"config reload: me_reinit: interval={}s singleflight={} coalesce={}ms",
|
||||
"config reload: me_reinit: interval={}s singleflight={} max_concurrency={} coalesce={}ms",
|
||||
new_hot.me_reinit_every_secs,
|
||||
new_hot.me_reinit_singleflight,
|
||||
new_hot.me_reinit_max_concurrency,
|
||||
new_hot.me_reinit_coalesce_window_ms
|
||||
);
|
||||
}
|
||||
|
||||
@@ -47,6 +47,43 @@ fn write_web_reload_config(path: &Path, carriers: &str, carrier_learning: bool)
|
||||
std::fs::write(path, config).unwrap();
|
||||
}
|
||||
|
||||
fn write_web_fasttrack_reload_config(path: &Path, mode: &str, ad_tag: &str) {
|
||||
let config = format!(
|
||||
r#"
|
||||
[general]
|
||||
ad_tag = "{ad_tag}"
|
||||
|
||||
[access.users]
|
||||
alice = "000102030405060708090a0b0c0d0e0f"
|
||||
|
||||
[[server.listeners]]
|
||||
ip = "127.0.0.1"
|
||||
port = 18080
|
||||
transport = "web"
|
||||
proxy_protocol = false
|
||||
web_client_ip_source = "x_forwarded_for"
|
||||
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||
|
||||
[web]
|
||||
enabled = true
|
||||
decoy_fasttrack_mode = "{mode}"
|
||||
|
||||
[[web.vhosts]]
|
||||
host = "proxy.example.com"
|
||||
public_addr = "203.0.113.10:443"
|
||||
|
||||
[web.vhosts.decoy]
|
||||
mode = "http_upstream"
|
||||
upstream = "http://127.0.0.1:18081"
|
||||
|
||||
[[web.vhosts.profiles]]
|
||||
user = "alice"
|
||||
secret_mode = "plain"
|
||||
"#,
|
||||
);
|
||||
std::fs::write(path, config).unwrap();
|
||||
}
|
||||
|
||||
fn temp_config_path(prefix: &str) -> PathBuf {
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -107,11 +144,13 @@ fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
|
||||
let old = sample_config();
|
||||
let mut new = old.clone();
|
||||
new.web.debug.enabled = true;
|
||||
new.web.debug.sideband = true;
|
||||
new.web.debug.default_window_secs = 60;
|
||||
new.web.limits.debug_records_capacity += 1;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
assert!(applied.web.debug.enabled);
|
||||
assert!(applied.web.debug.sideband);
|
||||
assert_eq!(applied.web.debug.default_window_secs, 60);
|
||||
assert_eq!(
|
||||
applied.web.limits.debug_records_capacity,
|
||||
@@ -123,6 +162,65 @@ fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoy_fasttrack_mode_is_deferred_until_restart() {
|
||||
let old = sample_config();
|
||||
let mut new = old.clone();
|
||||
new.web.decoy_fasttrack_mode = crate::config::WebDecoyFastTrackMode::Enforce;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
|
||||
assert_eq!(
|
||||
applied.web.decoy_fasttrack_mode,
|
||||
old.web.decoy_fasttrack_mode
|
||||
);
|
||||
assert_eq!(
|
||||
HotFields::from_config(&old),
|
||||
HotFields::from_config(&applied)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hot_overlay_defers_learning_that_requires_new_process_capacity() {
|
||||
let mut old = sample_config();
|
||||
old.web.limits.max_carrier_learning_entries = 1;
|
||||
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||
old.web.carrier_learning = false;
|
||||
let mut new = old.clone();
|
||||
new.web.limits.max_carrier_learning_entries = 3;
|
||||
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||
crate::config::WebCarrier::Websocket,
|
||||
crate::config::WebCarrier::Https,
|
||||
]);
|
||||
new.web.carrier_learning = true;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
|
||||
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
|
||||
assert!(applied.web.carrier_negotiation_enabled());
|
||||
assert!(!applied.web.carrier_learning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hot_overlay_defers_carriers_for_dormant_learning_with_small_capacity() {
|
||||
let mut old = sample_config();
|
||||
old.web.limits.max_carrier_learning_entries = 1;
|
||||
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||
old.web.carrier_learning = true;
|
||||
let mut new = old.clone();
|
||||
new.web.limits.max_carrier_learning_entries = 3;
|
||||
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||
crate::config::WebCarrier::Websocket,
|
||||
crate::config::WebCarrier::Https,
|
||||
]);
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
|
||||
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
|
||||
assert!(!applied.web.carrier_negotiation_enabled());
|
||||
assert!(applied.web.carrier_learning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
|
||||
let old = sample_config();
|
||||
@@ -272,6 +370,7 @@ async fn candidate_watcher_waits_for_activation_and_reconciles_disk() {
|
||||
None,
|
||||
None,
|
||||
cancellation.clone(),
|
||||
None,
|
||||
Some(activation_rx),
|
||||
);
|
||||
let watcher = tokio::spawn(watcher);
|
||||
@@ -321,6 +420,39 @@ fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_rebuilds_vhosts_with_the_effective_fasttrack_mode() {
|
||||
let initial_tag = "abababababababababababababababab";
|
||||
let final_tag = "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd";
|
||||
let path = temp_config_path("telemt_web_fasttrack_reload");
|
||||
|
||||
write_web_fasttrack_reload_config(&path, "off", initial_tag);
|
||||
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||
.unwrap()
|
||||
.rendered_hash;
|
||||
let (config_tx, _config_rx) = watch::channel(Arc::clone(&initial_cfg));
|
||||
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
|
||||
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||
|
||||
write_web_fasttrack_reload_config(&path, "enforce", final_tag);
|
||||
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||
|
||||
let applied = config_tx.borrow().clone();
|
||||
assert_eq!(applied.general.ad_tag.as_deref(), Some(final_tag));
|
||||
assert_eq!(
|
||||
applied.web.decoy_fasttrack_mode,
|
||||
crate::config::WebDecoyFastTrackMode::Off
|
||||
);
|
||||
let runtime = applied.web.runtime.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
runtime.vhosts["proxy.example.com"].decoy_fasttrack_mode,
|
||||
crate::config::WebDecoyFastTrackMode::Off
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_publishes_web_negotiation_policy_outside_hot_field_reporting() {
|
||||
let path = temp_config_path("telemt_web_negotiation_reload");
|
||||
|
||||
@@ -124,13 +124,14 @@ fn apply_watch_manifest<W1: Watcher, W2: Watcher>(
|
||||
}
|
||||
|
||||
/// Load config, validate, diff against current, and broadcast if changed.
|
||||
pub(super) fn reload_config(
|
||||
fn reload_config_with_resolver(
|
||||
config_path: &PathBuf,
|
||||
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||
log_tx: &watch::Sender<LogLevel>,
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
reload_state: &mut ReloadState,
|
||||
dns_resolver: Option<&crate::network::dns_overrides::GenerationDnsResolver>,
|
||||
) -> Option<WatchManifest> {
|
||||
let loaded = match ProxyConfig::load_with_metadata(config_path) {
|
||||
Ok(loaded) => loaded,
|
||||
@@ -160,7 +161,17 @@ pub(super) fn reload_config(
|
||||
}
|
||||
|
||||
let old_cfg = config_tx.borrow().clone();
|
||||
let applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
|
||||
let mut applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
|
||||
if let Err(error) = applied_cfg
|
||||
.validate_effective_web()
|
||||
.and_then(|_| applied_cfg.rebuild_runtime_web())
|
||||
{
|
||||
error!(
|
||||
"config reload: effective WEB validation failed: {}; keeping old config",
|
||||
error
|
||||
);
|
||||
return Some(next_manifest);
|
||||
}
|
||||
let old_hot = HotFields::from_config(&old_cfg);
|
||||
let applied_hot = HotFields::from_config(&applied_cfg);
|
||||
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
||||
@@ -176,7 +187,8 @@ pub(super) fn reload_config(
|
||||
}
|
||||
|
||||
if old_hot.dns_overrides != applied_hot.dns_overrides
|
||||
&& let Err(e) = crate::network::dns_overrides::install_entries(&applied_hot.dns_overrides)
|
||||
&& let Some(dns_resolver) = dns_resolver
|
||||
&& let Err(e) = dns_resolver.apply_entries(&applied_hot.dns_overrides)
|
||||
{
|
||||
error!(
|
||||
"config reload: invalid network.dns_overrides: {}; keeping old config",
|
||||
@@ -198,6 +210,26 @@ pub(super) fn reload_config(
|
||||
Some(next_manifest)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn reload_config(
|
||||
config_path: &PathBuf,
|
||||
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||
log_tx: &watch::Sender<LogLevel>,
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
reload_state: &mut ReloadState,
|
||||
) -> Option<WatchManifest> {
|
||||
reload_config_with_resolver(
|
||||
config_path,
|
||||
config_tx,
|
||||
log_tx,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
reload_state,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
/// Spawn the hot-reload watcher task.
|
||||
///
|
||||
/// Uses `notify` (inotify on Linux) to detect file changes instantly.
|
||||
@@ -213,6 +245,7 @@ pub fn spawn_config_watcher(
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
cancellation: tokio_util::sync::CancellationToken,
|
||||
dns_resolver: Option<Arc<crate::network::dns_overrides::GenerationDnsResolver>>,
|
||||
mut activation: Option<watch::Receiver<bool>>,
|
||||
) -> (
|
||||
watch::Receiver<Arc<ProxyConfig>>,
|
||||
@@ -364,24 +397,26 @@ pub fn spawn_config_watcher(
|
||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||
while notify_rx.try_recv().is_ok() {}
|
||||
|
||||
let mut next_manifest = reload_config(
|
||||
let mut next_manifest = reload_config_with_resolver(
|
||||
&config_path,
|
||||
&config_tx,
|
||||
&log_tx,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
&mut reload_state,
|
||||
dns_resolver.as_deref(),
|
||||
);
|
||||
if next_manifest.is_none() {
|
||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||
while notify_rx.try_recv().is_ok() {}
|
||||
next_manifest = reload_config(
|
||||
next_manifest = reload_config_with_resolver(
|
||||
&config_path,
|
||||
&config_tx,
|
||||
&log_tx,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
&mut reload_state,
|
||||
dns_resolver.as_deref(),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -216,6 +216,16 @@ impl ProxyConfig {
|
||||
runtime_web::rebuild(self)
|
||||
}
|
||||
|
||||
/// Validates the mixed effective WEB snapshot after restart fields are retained.
|
||||
pub(crate) fn validate_effective_web(&mut self) -> Result<()> {
|
||||
validate_web::validate(self)
|
||||
}
|
||||
|
||||
/// Revalidates decoy separation after restart-only listener fields are resolved.
|
||||
pub(crate) fn validate_web_decoy_listener_separation(&self) -> Result<()> {
|
||||
validate_web::validate_decoy_listener_separation(self)
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
|
||||
self.runtime_user_auth.as_deref()
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
&mut static_bytes,
|
||||
)?;
|
||||
let mut profiles = Vec::with_capacity(vhost.profiles.len());
|
||||
let mut capability_table = Vec::with_capacity(vhost.profiles.len());
|
||||
let mut capabilities = HashSet::with_capacity(vhost.profiles.len());
|
||||
for profile in &vhost.profiles {
|
||||
let user_id = auth.user_id_by_name(&profile.user).ok_or_else(|| {
|
||||
@@ -84,6 +85,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
.max_streams_per_session
|
||||
.unwrap_or(config.web.limits.max_streams_per_session),
|
||||
});
|
||||
capability_table.push(capability);
|
||||
profiles.push(Arc::clone(&runtime_profile));
|
||||
runtime_profiles.push(runtime_profile);
|
||||
}
|
||||
@@ -91,9 +93,11 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
vhost.host.clone(),
|
||||
Arc::new(WebRuntimeVhost {
|
||||
host: vhost.host.clone(),
|
||||
decoy_fasttrack_mode: config.web.decoy_fasttrack_mode,
|
||||
decoy,
|
||||
decoy_header_secs: config.web.timeouts.decoy_header_secs,
|
||||
profiles,
|
||||
capabilities: capability_table.into_boxed_slice(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -161,6 +161,7 @@ const GENERAL_CONFIG_KEYS: &[&str] = &[
|
||||
"proxy_secret_auto_reload_secs",
|
||||
"proxy_config_auto_reload_secs",
|
||||
"me_reinit_singleflight",
|
||||
"me_reinit_max_concurrency",
|
||||
"me_reinit_trigger_channel",
|
||||
"me_reinit_coalesce_window_ms",
|
||||
"me_deterministic_writer_sort",
|
||||
@@ -265,6 +266,8 @@ const WEB_CONFIG_KEYS: &[&str] = &[
|
||||
"carriers",
|
||||
"carrier_learning",
|
||||
"carrier_negotiation_aggressiveness",
|
||||
"decoy_fasttrack_mode",
|
||||
"http_connection_capacity_action",
|
||||
"debug",
|
||||
"limits",
|
||||
"timeouts",
|
||||
@@ -278,6 +281,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"carrier_batch_bytes",
|
||||
"max_frames_per_body",
|
||||
"max_http_connections",
|
||||
"max_http_overload_connections",
|
||||
"max_http_handlers",
|
||||
"max_lane_open_waits_per_session",
|
||||
"pending_bytes_per_lane",
|
||||
@@ -322,6 +326,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
|
||||
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled",
|
||||
"sideband",
|
||||
"capture_lifecycle",
|
||||
"capture_headers",
|
||||
"capture_timings",
|
||||
@@ -341,6 +346,7 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"long_poll_secs",
|
||||
"bridge_request_secs",
|
||||
"bridge_retry_secs",
|
||||
"bridge_recovery_secs",
|
||||
"carrier_probe_coalesce_ms",
|
||||
"lane_open_wait_secs",
|
||||
"carrier_health_secs",
|
||||
@@ -354,6 +360,7 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"bootstrap_lifetime_secs",
|
||||
"reconnect_grace_secs",
|
||||
"http_idle_secs",
|
||||
"http_overload_timeout_ms",
|
||||
"shutdown_secs",
|
||||
"decoy_header_secs",
|
||||
];
|
||||
|
||||
@@ -143,9 +143,15 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
||||
));
|
||||
}
|
||||
|
||||
if config.general.me_reinit_trigger_channel == 0 {
|
||||
if !(1..=8).contains(&config.general.me_reinit_max_concurrency) {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_reinit_trigger_channel must be > 0".to_string(),
|
||||
"general.me_reinit_max_concurrency must be within [1, 8]".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if !(1..=4096).contains(&config.general.me_reinit_trigger_channel) {
|
||||
return Err(ProxyError::Config(
|
||||
"general.me_reinit_trigger_channel must be within [1, 4096]".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
+61
-190
@@ -83,9 +83,59 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
||||
timeouts::validate(&config.web.timeouts)?;
|
||||
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
|
||||
validate_vhosts(config)?;
|
||||
validate_decoy_listener_separation(config)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rejects a direct decoy recursion into an effective WEB listener.
|
||||
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
|
||||
let web_listeners = config
|
||||
.server
|
||||
.listeners
|
||||
.iter()
|
||||
.filter(|listener| listener.transport == ListenerTransport::Web)
|
||||
.filter(|listener| {
|
||||
(listener.ip.is_ipv4() && config.network.ipv4)
|
||||
|| (listener.ip.is_ipv6() && config.network.ipv6 != Some(false))
|
||||
})
|
||||
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
|
||||
.collect::<Vec<_>>();
|
||||
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
|
||||
let WebDecoyConfig::HttpUpstream { upstream } = &vhost.decoy else {
|
||||
continue;
|
||||
};
|
||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||
))
|
||||
})?;
|
||||
let Some(port) = parsed.port_or_known_default() else {
|
||||
continue;
|
||||
};
|
||||
let upstream_ip = match parsed.host() {
|
||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||
_ => continue,
|
||||
};
|
||||
let upstream_addr = SocketAddr::new(upstream_ip, port);
|
||||
if web_listeners
|
||||
.iter()
|
||||
.any(|listener| listener_covers(*listener, upstream_addr))
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn listener_covers(listener: SocketAddr, target: SocketAddr) -> bool {
|
||||
listener.port() == target.port()
|
||||
&& (listener.ip() == target.ip()
|
||||
|| (listener.ip().is_unspecified() && listener.is_ipv4() == target.is_ipv4()))
|
||||
}
|
||||
|
||||
fn validate_web_listener(
|
||||
config: &ProxyConfig,
|
||||
idx: usize,
|
||||
@@ -169,6 +219,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
|
||||
let positive = [
|
||||
("max_http_connections", limits.max_http_connections),
|
||||
(
|
||||
"max_http_overload_connections",
|
||||
limits.max_http_overload_connections,
|
||||
),
|
||||
("max_http_handlers", limits.max_http_handlers),
|
||||
(
|
||||
"max_lane_open_waits_per_session",
|
||||
@@ -222,6 +276,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
}
|
||||
for (field, value) in [
|
||||
("max_http_connections", limits.max_http_connections),
|
||||
(
|
||||
"max_http_overload_connections",
|
||||
limits.max_http_overload_connections,
|
||||
),
|
||||
("max_http_handlers", limits.max_http_handlers),
|
||||
("max_body_readers", limits.max_body_readers),
|
||||
("max_body_bytes_global", limits.max_body_bytes_global),
|
||||
@@ -356,196 +414,9 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
let limits = &config.web.limits;
|
||||
if config.web.vhosts.len() > limits.max_vhosts {
|
||||
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
||||
}
|
||||
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
||||
let mut profile_count = 0usize;
|
||||
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
||||
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
||||
if !hosts.insert(vhost.host.clone()) {
|
||||
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
||||
}
|
||||
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
||||
));
|
||||
}
|
||||
if config.web.enabled && vhost.profiles.is_empty() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
||||
));
|
||||
}
|
||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||
));
|
||||
}
|
||||
if !config.access.users.contains_key(&profile.user) {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||
profile.user
|
||||
));
|
||||
}
|
||||
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
||||
return config_error(&format!(
|
||||
"duplicate WEB profile for user `{}` in vhost `{}`",
|
||||
profile.user, vhost.host
|
||||
));
|
||||
}
|
||||
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
||||
let max_streams_per_session = profile
|
||||
.max_streams_per_session
|
||||
.unwrap_or(limits.max_streams_per_session);
|
||||
if profile.max_sessions == Some(0)
|
||||
|| profile
|
||||
.max_sessions
|
||||
.is_some_and(|value| value > limits.max_sessions_global)
|
||||
|| profile.max_streams == Some(0)
|
||||
|| profile
|
||||
.max_streams
|
||||
.is_some_and(|value| value > limits.max_streams_global)
|
||||
|| profile.max_streams_per_session == Some(0)
|
||||
|| profile
|
||||
.max_streams_per_session
|
||||
.is_some_and(|value| value > limits.max_streams_per_session)
|
||||
|| max_streams_per_session > max_streams
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
||||
));
|
||||
}
|
||||
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
||||
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
if profile_count > limits.max_profiles {
|
||||
return config_error("WEB profiles exceed web.limits.max_profiles");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
||||
let input = value.trim();
|
||||
if input.is_empty()
|
||||
|| input.ends_with('.')
|
||||
|| input
|
||||
.chars()
|
||||
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
||||
));
|
||||
}
|
||||
let host = normalize_domain_to_ascii(input, field)?;
|
||||
if host.len() > 253
|
||||
|| !host.contains('.')
|
||||
|| host.parse::<IpAddr>().is_ok()
|
||||
|| web_host_last_label_is_numeric(&host)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
for label in host.split('.') {
|
||||
if label.is_empty()
|
||||
|| label.len() > 63
|
||||
|| label.starts_with('-')
|
||||
|| label.ends_with('-')
|
||||
|| !label
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} contains a hostname label rejected by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
fn web_host_last_label_is_numeric(host: &str) -> bool {
|
||||
let label = host.rsplit('.').next().unwrap_or_default();
|
||||
let digits = label
|
||||
.strip_prefix("0x")
|
||||
.or_else(|| label.strip_prefix("0X"));
|
||||
if let Some(digits) = digits {
|
||||
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
||||
}
|
||||
label.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||
match decoy {
|
||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||
))
|
||||
})?;
|
||||
if parsed.scheme() != "http"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.path() != "/"
|
||||
|| parsed.port() == Some(0)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||
));
|
||||
}
|
||||
let ip = match parsed.host() {
|
||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||
_ => {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
||||
));
|
||||
}
|
||||
};
|
||||
let private = match ip {
|
||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||
IpAddr::V6(ip) => {
|
||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
||||
}
|
||||
};
|
||||
if !private {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
||||
));
|
||||
}
|
||||
}
|
||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||
if !directory.is_absolute() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
||||
));
|
||||
}
|
||||
if index.is_empty()
|
||||
|| index.contains('\\')
|
||||
|| std::path::Path::new(index).components().count() != 1
|
||||
|| matches!(index.as_str(), "." | "..")
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn config_error<T>(message: &str) -> Result<T> {
|
||||
Err(ProxyError::Config(message.to_string()))
|
||||
}
|
||||
// Virtual-host, hostname, and decoy validation.
|
||||
mod vhosts;
|
||||
use vhosts::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -5,6 +5,8 @@ const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
||||
const WEB_LANE_STATE_BYTES: usize = 512;
|
||||
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
||||
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 32;
|
||||
|
||||
/// Validates process-wide body, header, queue, static, and debug reservations.
|
||||
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
@@ -30,6 +32,12 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
||||
})?;
|
||||
let overload_connection_reservation = limits
|
||||
.max_http_overload_connections
|
||||
.checked_mul(WEB_OVERLOAD_CONNECTION_BYTES)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits HTTP overload reservations overflow usize".to_string())
|
||||
})?;
|
||||
let debug_ring_index = limits
|
||||
.debug_records_capacity
|
||||
.checked_mul(std::mem::size_of::<usize>())
|
||||
@@ -58,6 +66,12 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.carrier learning reservation overflowed usize".to_string())
|
||||
})?;
|
||||
let capability_index_reservation = limits
|
||||
.max_profiles
|
||||
.checked_mul(WEB_CAPABILITY_INDEX_ENTRY_BYTES)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web capability index reservation overflowed usize".to_string())
|
||||
})?;
|
||||
let lane_state_reservation = limits
|
||||
.max_streams_per_session
|
||||
.checked_add(limits.max_tombstones_per_session)
|
||||
@@ -75,8 +89,10 @@ pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
.and_then(|value| value.checked_add(status_pages))
|
||||
.and_then(|value| value.checked_add(debug_reservation))
|
||||
.and_then(|value| value.checked_add(carrier_learning_reservation))
|
||||
.and_then(|value| value.checked_add(capability_index_reservation))
|
||||
.and_then(|value| value.checked_add(lane_state_reservation))
|
||||
.and_then(|value| value.checked_add(http_header_reservation))
|
||||
.and_then(|value| value.checked_add(overload_connection_reservation))
|
||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||
if reserved > limits.memory_envelope_bytes
|
||||
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
||||
@@ -103,4 +119,14 @@ mod tests {
|
||||
};
|
||||
assert!(validate(&previous_envelope).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capability_index_reservation_rejects_size_overflow() {
|
||||
let limits = WebLimitsConfig {
|
||||
max_profiles: usize::MAX,
|
||||
..WebLimitsConfig::default()
|
||||
};
|
||||
let error = validate(&limits).unwrap_err().to_string();
|
||||
assert!(error.contains("web capability index reservation overflowed usize"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ pub(super) fn validate(config: &WebConfig) -> Result<Vec<WebCarrier>> {
|
||||
let candidates = config.carrier_candidates();
|
||||
if config.carrier_negotiation_enabled()
|
||||
&& config.carrier_learning
|
||||
&& config.limits.max_carrier_learning_entries < 3
|
||||
&& config.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||
{
|
||||
return config_error(
|
||||
"web.limits.max_carrier_learning_entries must be >= 3 when carrier learning is enabled",
|
||||
|
||||
@@ -2,6 +2,9 @@ use super::*;
|
||||
|
||||
/// Validates WEB request, learning, and lifecycle timeouts.
|
||||
pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
if !(1..=60_000).contains(&timeouts.http_overload_timeout_ms) {
|
||||
return config_error("web.timeouts.http_overload_timeout_ms must be within [1, 60000]");
|
||||
}
|
||||
let values = [
|
||||
("header_secs", timeouts.header_secs),
|
||||
("body_secs", timeouts.body_secs),
|
||||
@@ -39,6 +42,9 @@ pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
if !(1..=300).contains(&timeouts.bridge_retry_secs) {
|
||||
return config_error("web.timeouts.bridge_retry_secs must be within [1, 300]");
|
||||
}
|
||||
if !(1..=60).contains(&timeouts.bridge_recovery_secs) {
|
||||
return config_error("web.timeouts.bridge_recovery_secs must be within [1, 60]");
|
||||
}
|
||||
if timeouts.bridge_request_secs > timeouts.bridge_retry_secs {
|
||||
return config_error("web.timeouts.bridge_request_secs must not exceed bridge_retry_secs");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
let limits = &config.web.limits;
|
||||
if config.web.vhosts.len() > limits.max_vhosts {
|
||||
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
||||
}
|
||||
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
||||
let mut profile_count = 0usize;
|
||||
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
||||
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
||||
if !hosts.insert(vhost.host.clone()) {
|
||||
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
||||
}
|
||||
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
||||
));
|
||||
}
|
||||
if config.web.enabled && vhost.profiles.is_empty() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
||||
));
|
||||
}
|
||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||
));
|
||||
}
|
||||
if !config.access.users.contains_key(&profile.user) {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||
profile.user
|
||||
));
|
||||
}
|
||||
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
||||
return config_error(&format!(
|
||||
"duplicate WEB profile for user `{}` in vhost `{}`",
|
||||
profile.user, vhost.host
|
||||
));
|
||||
}
|
||||
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
||||
let max_streams_per_session = profile
|
||||
.max_streams_per_session
|
||||
.unwrap_or(limits.max_streams_per_session);
|
||||
if profile.max_sessions == Some(0)
|
||||
|| profile
|
||||
.max_sessions
|
||||
.is_some_and(|value| value > limits.max_sessions_global)
|
||||
|| profile.max_streams == Some(0)
|
||||
|| profile
|
||||
.max_streams
|
||||
.is_some_and(|value| value > limits.max_streams_global)
|
||||
|| profile.max_streams_per_session == Some(0)
|
||||
|| profile
|
||||
.max_streams_per_session
|
||||
.is_some_and(|value| value > limits.max_streams_per_session)
|
||||
|| max_streams_per_session > max_streams
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
||||
));
|
||||
}
|
||||
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
||||
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
if profile_count > limits.max_profiles {
|
||||
return config_error("WEB profiles exceed web.limits.max_profiles");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
||||
let input = value.trim();
|
||||
if input.is_empty()
|
||||
|| input.ends_with('.')
|
||||
|| input
|
||||
.chars()
|
||||
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
||||
));
|
||||
}
|
||||
let host = normalize_domain_to_ascii(input, field)?;
|
||||
if host.len() > 253
|
||||
|| !host.contains('.')
|
||||
|| host.parse::<IpAddr>().is_ok()
|
||||
|| web_host_last_label_is_numeric(&host)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
for label in host.split('.') {
|
||||
if label.is_empty()
|
||||
|| label.len() > 63
|
||||
|| label.starts_with('-')
|
||||
|| label.ends_with('-')
|
||||
|| !label
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
{
|
||||
return config_error(&format!(
|
||||
"{field} contains a hostname label rejected by Telegram Desktop"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
pub(super) fn web_host_last_label_is_numeric(host: &str) -> bool {
|
||||
let label = host.rsplit('.').next().unwrap_or_default();
|
||||
let digits = label
|
||||
.strip_prefix("0x")
|
||||
.or_else(|| label.strip_prefix("0X"));
|
||||
if let Some(digits) = digits {
|
||||
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
||||
}
|
||||
label.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||
match decoy {
|
||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||
))
|
||||
})?;
|
||||
if parsed.scheme() != "http"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
|| parsed.path() != "/"
|
||||
|| parsed.port() == Some(0)
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||
));
|
||||
}
|
||||
let ip = match parsed.host() {
|
||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||
_ => {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
||||
));
|
||||
}
|
||||
};
|
||||
let private = match ip {
|
||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||
IpAddr::V6(ip) => {
|
||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
||||
}
|
||||
};
|
||||
if !private {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
||||
));
|
||||
}
|
||||
}
|
||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||
if !directory.is_absolute() {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
||||
));
|
||||
}
|
||||
if index.is_empty()
|
||||
|| index.contains('\\')
|
||||
|| std::path::Path::new(index).components().count() != 1
|
||||
|| matches!(index.as_str(), "." | "..")
|
||||
{
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn config_error<T>(message: &str) -> Result<T> {
|
||||
Err(ProxyError::Config(message.to_string()))
|
||||
}
|
||||
@@ -41,6 +41,9 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
||||
.get("proxy.example.com")
|
||||
.expect("canonical WEB vhost");
|
||||
assert_eq!(vhost.profiles.len(), 1);
|
||||
assert_eq!(vhost.capabilities.len(), vhost.profiles.len());
|
||||
assert_eq!(vhost.capabilities[0], vhost.profiles[0].capability);
|
||||
assert_eq!(vhost.decoy_fasttrack_mode, WebDecoyFastTrackMode::Off);
|
||||
assert_eq!(vhost.profiles[0].user, "alice");
|
||||
assert_eq!(vhost.profiles[0].secret_mode, WebSecretMode::Dd);
|
||||
assert_eq!(vhost.profiles[0].carrier, WebCarrier::HttpsLanes);
|
||||
@@ -56,6 +59,109 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_decoy_fasttrack_mode_is_typed_and_defaults_off() {
|
||||
let defaults = ProxyConfig::default();
|
||||
assert_eq!(
|
||||
defaults.web.decoy_fasttrack_mode,
|
||||
WebDecoyFastTrackMode::Off
|
||||
);
|
||||
|
||||
for (token, expected) in [
|
||||
("shadow", WebDecoyFastTrackMode::Shadow),
|
||||
("enforce", WebDecoyFastTrackMode::Enforce),
|
||||
] {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
&format!("carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"{token}\""),
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert_eq!(config.web.decoy_fasttrack_mode, expected);
|
||||
assert_eq!(
|
||||
config.web.runtime.as_ref().unwrap().vhosts["proxy.example.com"].decoy_fasttrack_mode,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"automatic\"",
|
||||
);
|
||||
assert!(load_config_error_from_temp_toml(&invalid).contains("decoy_fasttrack_mode"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_http_connection_capacity_policy_is_bounded_and_configurable() {
|
||||
let configured = WEB_CONFIG
|
||||
.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"wait\"",
|
||||
)
|
||||
.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.limits]\nmax_http_overload_connections = 23\n\n[web.timeouts]\nhttp_overload_timeout_ms = 731\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
|
||||
assert_eq!(
|
||||
config.web.http_connection_capacity_action,
|
||||
WebHttpConnectionCapacityAction::Wait
|
||||
);
|
||||
assert_eq!(config.web.limits.max_http_overload_connections, 23);
|
||||
assert_eq!(config.web.timeouts.http_overload_timeout_ms, 731);
|
||||
|
||||
let defaults = ProxyConfig::default();
|
||||
assert_eq!(
|
||||
defaults.web.http_connection_capacity_action,
|
||||
WebHttpConnectionCapacityAction::Drop
|
||||
);
|
||||
assert_eq!(defaults.web.limits.max_http_overload_connections, 64);
|
||||
assert_eq!(defaults.web.timeouts.http_overload_timeout_ms, 250);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_http_connection_capacity_policy_rejects_unknown_or_unbounded_values() {
|
||||
let unknown = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"queue\"",
|
||||
);
|
||||
assert!(load_config_error_from_temp_toml(&unknown).contains("http_connection_capacity_action"));
|
||||
|
||||
for timeout in [0, 60_001] {
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
&format!("[web.timeouts]\nhttp_overload_timeout_ms = {timeout}\n\n[[web.vhosts]]"),
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&invalid)
|
||||
.contains("web.timeouts.http_overload_timeout_ms")
|
||||
);
|
||||
}
|
||||
|
||||
let no_overload_slots = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.limits]\nmax_http_overload_connections = 0\n\n[[web.vhosts]]",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&no_overload_slots)
|
||||
.contains("web.limits.max_http_overload_connections")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_decoy_rejects_direct_and_wildcard_listener_loops() {
|
||||
let direct = WEB_CONFIG.replace("http://127.0.0.1:18081", "http://127.0.0.1:18080");
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&direct).contains("decoy upstream overlaps WEB listener")
|
||||
);
|
||||
|
||||
let wildcard = direct.replace("ip = \"127.0.0.1\"", "ip = \"0.0.0.0\"");
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&wildcard)
|
||||
.contains("decoy upstream overlaps WEB listener")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_profile_user_labels_are_bounded_for_runtime_status() {
|
||||
let user = "a".repeat(65);
|
||||
@@ -122,7 +228,7 @@ fn web_carriers_reject_true_empty_and_duplicates() {
|
||||
fn web_carrier_and_bridge_deadlines_are_configurable() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert_eq!(
|
||||
@@ -132,6 +238,7 @@ fn web_carrier_and_bridge_deadlines_are_configurable() {
|
||||
assert_eq!(config.web.timeouts.carrier_learning_secs, 30);
|
||||
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
|
||||
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||
}
|
||||
|
||||
@@ -139,13 +246,14 @@ fn web_carrier_and_bridge_deadlines_are_configurable() {
|
||||
fn web_bridge_deadlines_are_known_in_strict_mode() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
);
|
||||
let configured = format!("[general]\nconfig_strict = true\n{configured}");
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
|
||||
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
|
||||
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||
}
|
||||
|
||||
@@ -156,6 +264,8 @@ fn web_bridge_deadlines_are_bounded_and_ordered() {
|
||||
("bridge_request_secs", "61"),
|
||||
("bridge_retry_secs", "0"),
|
||||
("bridge_retry_secs", "301"),
|
||||
("bridge_recovery_secs", "0"),
|
||||
("bridge_recovery_secs", "61"),
|
||||
("carrier_probe_coalesce_ms", "11"),
|
||||
] {
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
@@ -191,17 +301,31 @@ fn web_carrier_learning_capacity_must_remain_nonzero() {
|
||||
|
||||
#[test]
|
||||
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
|
||||
assert!(!crate::config::WebDebugConfig::default().sideband);
|
||||
let mut ineffective = crate::config::WebDebugConfig {
|
||||
enabled: true,
|
||||
sideband: true,
|
||||
..Default::default()
|
||||
};
|
||||
ineffective.capture_lifecycle = false;
|
||||
assert!(!ineffective.bridge_diagnostics_enabled());
|
||||
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
|
||||
"[web.debug]\nenabled = true\nsideband = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert!(config.web.debug.enabled);
|
||||
assert!(config.web.debug.sideband);
|
||||
assert!(config.web.debug.bridge_diagnostics_enabled());
|
||||
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
|
||||
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
|
||||
assert_eq!(config.web.debug.default_window_secs, 180);
|
||||
assert_eq!(config.web.debug.max_window_secs, 900);
|
||||
|
||||
let strict = format!("[general]\nconfig_strict = true\n{configured}");
|
||||
assert!(load_config_from_temp_toml(&strict).web.debug.sideband);
|
||||
|
||||
let old_name = format!(
|
||||
"[general]\nconfig_strict = true\n{}",
|
||||
WEB_CONFIG.replace(
|
||||
@@ -211,6 +335,16 @@ fn web_debug_table_uses_debug_name_and_bounded_defaults() {
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&old_name);
|
||||
assert!(error.contains("web.trace"));
|
||||
|
||||
let old_parameter = format!(
|
||||
"[general]\nconfig_strict = true\n{}",
|
||||
WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.debug]\nbridge_diagnostics = true\n\n[[web.vhosts]]",
|
||||
)
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&old_parameter);
|
||||
assert!(error.contains("web.debug.bridge_diagnostics"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+4
-2
@@ -53,13 +53,15 @@ pub use server::{
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use web::{
|
||||
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebLimitsConfig,
|
||||
WebProfileConfig, WebSecretMode, WebTimeoutsConfig, WebVhostConfig,
|
||||
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebDecoyFastTrackMode,
|
||||
WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
||||
WebTimeoutsConfig, WebVhostConfig,
|
||||
};
|
||||
pub(crate) use web::{
|
||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||
WebStaticSite,
|
||||
};
|
||||
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
||||
#[allow(unused_imports)]
|
||||
pub use web_carrier::{WebCarrier, WebCarriers};
|
||||
pub(crate) use web_debug::web_debug_fits_limits;
|
||||
|
||||
@@ -441,6 +441,9 @@ pub struct GeneralConfig {
|
||||
/// Serialize ME reinit cycles across all trigger sources.
|
||||
#[serde(default = "default_me_reinit_singleflight")]
|
||||
pub me_reinit_singleflight: bool,
|
||||
/// Maximum concurrent ME reinit warmups when single-flight mode is disabled.
|
||||
#[serde(default = "default_me_reinit_max_concurrency")]
|
||||
pub me_reinit_max_concurrency: usize,
|
||||
/// Trigger queue capacity for reinit scheduler.
|
||||
#[serde(default = "default_me_reinit_trigger_channel")]
|
||||
pub me_reinit_trigger_channel: usize,
|
||||
|
||||
@@ -159,6 +159,7 @@ impl Default for GeneralConfig {
|
||||
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
|
||||
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
|
||||
me_reinit_singleflight: default_me_reinit_singleflight(),
|
||||
me_reinit_max_concurrency: default_me_reinit_max_concurrency(),
|
||||
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
||||
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
||||
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
||||
|
||||
+34
-83
@@ -12,6 +12,12 @@ use super::web_debug::WebDebugConfig;
|
||||
// Serialized WEB defaults remain separate from the runtime data model.
|
||||
mod defaults;
|
||||
use defaults::*;
|
||||
// Decoy fast-track policy remains isolated from the bulky WEB data model.
|
||||
mod fasttrack;
|
||||
pub use fasttrack::WebDecoyFastTrackMode;
|
||||
// Accepted-socket overload policy remains separate from the bulky WEB data model.
|
||||
mod overload;
|
||||
pub use overload::WebHttpConnectionCapacityAction;
|
||||
|
||||
/// Client-facing secret representation used to derive a WEB capability.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -95,6 +101,9 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide accepted WEB HTTP connection ceiling.
|
||||
#[serde(default = "default_web_max_http_connections")]
|
||||
pub max_http_connections: usize,
|
||||
/// Accepted overload sockets allowed to wait or emit a retryable response.
|
||||
#[serde(default = "default_web_max_http_overload_connections")]
|
||||
pub max_http_overload_connections: usize,
|
||||
/// Process-wide concurrently executing HTTP handler ceiling.
|
||||
#[serde(default = "default_web_max_http_handlers")]
|
||||
pub max_http_handlers: usize,
|
||||
@@ -194,7 +203,7 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide retained and in-flight WEB debug byte ceiling.
|
||||
#[serde(default = "default_web_debug_bytes_global")]
|
||||
pub debug_bytes_global: usize,
|
||||
/// Declared process envelope for HTTP, queues, lane state, learning, and static snapshots.
|
||||
/// Declared process envelope for HTTP, queues, capabilities, learning, and static snapshots.
|
||||
#[serde(default = "default_web_memory_envelope_bytes")]
|
||||
pub memory_envelope_bytes: usize,
|
||||
/// Sustained process-wide bootstrap issuance rate.
|
||||
@@ -226,6 +235,7 @@ impl Default for WebLimitsConfig {
|
||||
carrier_batch_bytes: default_web_carrier_batch_bytes(),
|
||||
max_frames_per_body: default_web_max_frames_per_body(),
|
||||
max_http_connections: default_web_max_http_connections(),
|
||||
max_http_overload_connections: default_web_max_http_overload_connections(),
|
||||
max_http_handlers: default_web_max_http_handlers(),
|
||||
max_lane_open_waits_per_session: default_web_max_lane_open_waits_per_session(),
|
||||
pending_bytes_per_lane: default_web_pending_bytes_per_lane(),
|
||||
@@ -294,6 +304,9 @@ pub struct WebTimeoutsConfig {
|
||||
/// Absolute generated-bridge budget for one retryable HTTP operation.
|
||||
#[serde(default = "default_web_bridge_retry_secs")]
|
||||
pub bridge_retry_secs: u64,
|
||||
/// Absolute post-commit budget for one surviving bridge recovery epoch.
|
||||
#[serde(default = "default_web_bridge_recovery_secs")]
|
||||
pub bridge_recovery_secs: u64,
|
||||
/// Optional delay for coalescing the first OPEN with immediate DATA.
|
||||
#[serde(default = "default_web_carrier_probe_coalesce_ms")]
|
||||
pub carrier_probe_coalesce_ms: u64,
|
||||
@@ -327,12 +340,15 @@ pub struct WebTimeoutsConfig {
|
||||
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
|
||||
#[serde(default = "default_web_bootstrap_lifetime_secs")]
|
||||
pub bootstrap_lifetime_secs: u64,
|
||||
/// Maximum carrier inactivity before a session is closed.
|
||||
/// Maximum validated peer inactivity before a session is closed.
|
||||
#[serde(default = "default_web_reconnect_grace_secs")]
|
||||
pub reconnect_grace_secs: u64,
|
||||
/// Maximum idle lifetime of a WEB HTTP keep-alive connection.
|
||||
#[serde(default = "default_web_http_idle_secs")]
|
||||
pub http_idle_secs: u64,
|
||||
/// Per-phase wait or response deadline for accepted HTTP overload sockets.
|
||||
#[serde(default = "default_web_http_overload_timeout_ms")]
|
||||
pub http_overload_timeout_ms: u64,
|
||||
/// Maximum graceful wait for WEB connections and process-owned tasks.
|
||||
#[serde(default = "default_web_shutdown_secs")]
|
||||
pub shutdown_secs: u64,
|
||||
@@ -351,6 +367,7 @@ impl Default for WebTimeoutsConfig {
|
||||
long_poll_secs: default_web_long_poll_timeout_secs(),
|
||||
bridge_request_secs: default_web_bridge_request_secs(),
|
||||
bridge_retry_secs: default_web_bridge_retry_secs(),
|
||||
bridge_recovery_secs: default_web_bridge_recovery_secs(),
|
||||
carrier_probe_coalesce_ms: default_web_carrier_probe_coalesce_ms(),
|
||||
lane_open_wait_secs: default_web_lane_open_wait_secs(),
|
||||
carrier_health_secs: default_web_carrier_health_secs(),
|
||||
@@ -364,6 +381,7 @@ impl Default for WebTimeoutsConfig {
|
||||
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
|
||||
reconnect_grace_secs: default_web_reconnect_grace_secs(),
|
||||
http_idle_secs: default_web_http_idle_secs(),
|
||||
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
|
||||
shutdown_secs: default_web_shutdown_secs(),
|
||||
decoy_header_secs: default_web_decoy_header_timeout_secs(),
|
||||
}
|
||||
@@ -401,6 +419,12 @@ pub struct WebConfig {
|
||||
/// Controls the evidence thresholds used by automatic carrier ranking.
|
||||
#[serde(default)]
|
||||
pub carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
/// Restart-only capability-scan policy for structurally impossible bridge requests.
|
||||
#[serde(default)]
|
||||
pub decoy_fasttrack_mode: WebDecoyFastTrackMode,
|
||||
/// Action applied when accepted HTTP connection capacity is exhausted.
|
||||
#[serde(default)]
|
||||
pub http_connection_capacity_action: WebHttpConnectionCapacityAction,
|
||||
/// Hard process and protocol limits.
|
||||
#[serde(default)]
|
||||
pub limits: WebLimitsConfig,
|
||||
@@ -447,6 +471,8 @@ impl Default for WebConfig {
|
||||
carriers: WebCarriers::default(),
|
||||
carrier_learning: default_web_carrier_learning(),
|
||||
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||
decoy_fasttrack_mode: WebDecoyFastTrackMode::default(),
|
||||
http_connection_capacity_action: WebHttpConnectionCapacityAction::default(),
|
||||
limits: WebLimitsConfig::default(),
|
||||
debug: WebDebugConfig::default(),
|
||||
timeouts: WebTimeoutsConfig::default(),
|
||||
@@ -456,84 +482,9 @@ impl Default for WebConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeConfig {
|
||||
/// Canonical host lookup used by HTTP request routing.
|
||||
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
||||
/// Flat profile inventory used by startup link emission.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed immutable virtual-host data.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeVhost {
|
||||
/// Canonical lowercase ACE hostname.
|
||||
pub(crate) host: String,
|
||||
/// Immutable ordinary-site fallback snapshot.
|
||||
pub(crate) decoy: WebRuntimeDecoy,
|
||||
/// Upstream connect and response-head deadline.
|
||||
pub(crate) decoy_header_secs: u64,
|
||||
/// Exact capability profiles accepted by this host.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed exact-user capability entry.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeProfile {
|
||||
/// Canonical host that owns this profile.
|
||||
pub(crate) host: String,
|
||||
/// Stable public destination tuple supplied to relay routing.
|
||||
pub(crate) public_addr: SocketAddr,
|
||||
/// Exact access user authenticated by logical streams.
|
||||
pub(crate) user: String,
|
||||
/// Client secret representation and inner protocol policy.
|
||||
pub(crate) secret_mode: WebSecretMode,
|
||||
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||
pub(crate) carrier_negotiation_enabled: bool,
|
||||
/// Whether automatic outcomes consult and update process-local evidence.
|
||||
pub(crate) carrier_learning: bool,
|
||||
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
pub(crate) key_fingerprint: String,
|
||||
/// Per-profile live session ceiling.
|
||||
pub(crate) max_sessions: usize,
|
||||
/// Per-profile live logical-stream ceiling.
|
||||
pub(crate) max_streams: usize,
|
||||
/// Per-session live relay-task ceiling.
|
||||
pub(crate) max_streams_per_session: usize,
|
||||
}
|
||||
|
||||
/// Runtime-ready ordinary-site fallback.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum WebRuntimeDecoy {
|
||||
HttpUpstream { addr: SocketAddr, authority: String },
|
||||
StaticDirectory(Arc<WebStaticSite>),
|
||||
}
|
||||
|
||||
/// Immutable bounded static-site snapshot.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticSite {
|
||||
/// Canonical URL-path to immutable response asset mapping.
|
||||
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
||||
/// Configured root index file name.
|
||||
pub(crate) index: String,
|
||||
}
|
||||
|
||||
/// One immutable static response body and metadata.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticAsset {
|
||||
/// Immutable response body retained by the runtime snapshot.
|
||||
pub(crate) body: Bytes,
|
||||
/// Extension-derived static content type.
|
||||
pub(crate) content_type: &'static str,
|
||||
/// Strong SHA-256 entity tag.
|
||||
pub(crate) etag: String,
|
||||
}
|
||||
// Immutable runtime WEB configuration consumed by hot paths.
|
||||
mod runtime;
|
||||
pub(crate) use runtime::{
|
||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||
WebStaticSite,
|
||||
};
|
||||
|
||||
@@ -40,6 +40,7 @@ usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
|
||||
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frames_per_body, 4096);
|
||||
usize_default!(default_web_max_http_connections, 1024);
|
||||
usize_default!(default_web_max_http_overload_connections, 64);
|
||||
usize_default!(default_web_max_http_handlers, 512);
|
||||
usize_default!(default_web_max_lane_open_waits_per_session, 16);
|
||||
usize_default!(default_web_pending_bytes_per_lane, 8 * 1024 * 1024);
|
||||
@@ -87,6 +88,7 @@ u64_default!(default_web_stream_first_byte_secs, 30);
|
||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||
u64_default!(default_web_bridge_request_secs, 10);
|
||||
u64_default!(default_web_bridge_retry_secs, 90);
|
||||
u64_default!(default_web_bridge_recovery_secs, 15);
|
||||
u64_default!(default_web_carrier_probe_coalesce_ms, 0);
|
||||
u64_default!(default_web_lane_open_wait_secs, 2);
|
||||
u64_default!(default_web_carrier_health_secs, 30);
|
||||
@@ -105,5 +107,6 @@ pub(super) fn default_web_carrier_learning() -> bool {
|
||||
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
||||
u64_default!(default_web_reconnect_grace_secs, 120);
|
||||
u64_default!(default_web_http_idle_secs, 75);
|
||||
u64_default!(default_web_http_overload_timeout_ms, 250);
|
||||
u64_default!(default_web_shutdown_secs, 15);
|
||||
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Capability-scan policy for structurally impossible WEB bridge requests.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum WebDecoyFastTrackMode {
|
||||
/// Preserve the legacy full scan without collecting fast-track decisions.
|
||||
#[default]
|
||||
Off,
|
||||
/// Record eligible requests while preserving the legacy full scan.
|
||||
Shadow,
|
||||
/// Skip the scan only when the public request shape cannot open a bridge.
|
||||
Enforce,
|
||||
}
|
||||
|
||||
impl WebDecoyFastTrackMode {
|
||||
/// Complete fixed mode set in stable API and metric order.
|
||||
pub const ALL: [Self; 3] = [Self::Off, Self::Shadow, Self::Enforce];
|
||||
|
||||
/// Returns the stable serialized mode token.
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Off => "off",
|
||||
Self::Shadow => "shadow",
|
||||
Self::Enforce => "enforce",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Action applied after an accepted WEB socket finds HTTP connection capacity exhausted.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum WebHttpConnectionCapacityAction {
|
||||
/// Close the accepted socket without emitting an HTTP response.
|
||||
#[default]
|
||||
Drop,
|
||||
/// Wait for ordinary HTTP connection capacity under the overload deadline.
|
||||
Wait,
|
||||
/// Emit a bounded retryable HTTP response without parsing the request.
|
||||
Respond,
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
use super::*;
|
||||
|
||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeConfig {
|
||||
/// Canonical host lookup used by HTTP request routing.
|
||||
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
||||
/// Flat profile inventory used by startup link emission.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
}
|
||||
|
||||
/// Precomputed immutable virtual-host data.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeVhost {
|
||||
/// Canonical lowercase ACE hostname.
|
||||
pub(crate) host: String,
|
||||
/// Restart-frozen decoy capability-scan policy.
|
||||
pub(crate) decoy_fasttrack_mode: WebDecoyFastTrackMode,
|
||||
/// Immutable ordinary-site fallback snapshot.
|
||||
pub(crate) decoy: WebRuntimeDecoy,
|
||||
/// Upstream connect and response-head deadline.
|
||||
pub(crate) decoy_header_secs: u64,
|
||||
/// Exact capability profiles accepted by this host.
|
||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||
/// Contiguous capability table aligned one-to-one with `profiles`.
|
||||
pub(crate) capabilities: Box<[[u8; 32]]>,
|
||||
}
|
||||
|
||||
/// Precomputed exact-user capability entry.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebRuntimeProfile {
|
||||
/// Canonical host that owns this profile.
|
||||
pub(crate) host: String,
|
||||
/// Stable public destination tuple supplied to relay routing.
|
||||
pub(crate) public_addr: SocketAddr,
|
||||
/// Exact access user authenticated by logical streams.
|
||||
pub(crate) user: String,
|
||||
/// Client secret representation and inner protocol policy.
|
||||
pub(crate) secret_mode: WebSecretMode,
|
||||
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||
pub(crate) carrier_negotiation_enabled: bool,
|
||||
/// Whether automatic outcomes consult and update process-local evidence.
|
||||
pub(crate) carrier_learning: bool,
|
||||
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
pub(crate) key_fingerprint: String,
|
||||
/// Per-profile live session ceiling.
|
||||
pub(crate) max_sessions: usize,
|
||||
/// Per-profile live logical-stream ceiling.
|
||||
pub(crate) max_streams: usize,
|
||||
/// Per-session live relay-task ceiling.
|
||||
pub(crate) max_streams_per_session: usize,
|
||||
}
|
||||
|
||||
/// Runtime-ready ordinary-site fallback.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum WebRuntimeDecoy {
|
||||
HttpUpstream { addr: SocketAddr, authority: String },
|
||||
StaticDirectory(Arc<WebStaticSite>),
|
||||
}
|
||||
|
||||
/// Immutable bounded static-site snapshot.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticSite {
|
||||
/// Canonical URL-path to immutable response asset mapping.
|
||||
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
||||
/// Configured root index file name.
|
||||
pub(crate) index: String,
|
||||
}
|
||||
|
||||
/// One immutable static response body and metadata.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct WebStaticAsset {
|
||||
/// Immutable response body retained by the runtime snapshot.
|
||||
pub(crate) body: Bytes,
|
||||
/// Extension-derived static content type.
|
||||
pub(crate) content_type: &'static str,
|
||||
/// Strong SHA-256 entity tag.
|
||||
pub(crate) etag: String,
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Minimum restart-owned entries required for one complete learning sample.
|
||||
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
|
||||
|
||||
/// Carrier selected for one newly issued WEB relay session.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
|
||||
@@ -23,6 +23,9 @@ pub struct WebDebugConfig {
|
||||
/// Enables process-owned WEB debug collection.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Enables generated-bridge diagnostic reports over the HTTPS sideband.
|
||||
#[serde(default)]
|
||||
pub sideband: bool,
|
||||
/// Records typed bridge, session, stream, handshake, and relay events.
|
||||
#[serde(default = "default_true")]
|
||||
pub capture_lifecycle: bool,
|
||||
@@ -56,6 +59,7 @@ impl Default for WebDebugConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
sideband: false,
|
||||
capture_lifecycle: true,
|
||||
capture_headers: true,
|
||||
capture_timings: true,
|
||||
@@ -69,6 +73,13 @@ impl Default for WebDebugConfig {
|
||||
}
|
||||
}
|
||||
|
||||
impl WebDebugConfig {
|
||||
/// Returns whether newly issued bridges may report diagnostic lifecycle events.
|
||||
pub(crate) const fn bridge_diagnostics_enabled(&self) -> bool {
|
||||
self.enabled && self.sideband && self.capture_lifecycle
|
||||
}
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
+26
-5
@@ -6,6 +6,8 @@ use serde_json::Value;
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
const HEALTHCHECK_RESPONSE_MAX_BYTES: u64 = 64 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HealthcheckMode {
|
||||
Liveness,
|
||||
@@ -73,10 +75,7 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
||||
.flush()
|
||||
.map_err(|error| format!("request flush failed: {error}"))?;
|
||||
|
||||
let mut raw_response = Vec::new();
|
||||
stream
|
||||
.read_to_end(&mut raw_response)
|
||||
.map_err(|error| format!("response read failed: {error}"))?;
|
||||
let raw_response = read_response_bounded(&mut stream)?;
|
||||
let response =
|
||||
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
|
||||
|
||||
@@ -89,6 +88,18 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_response_bounded(reader: &mut impl Read) -> Result<Vec<u8>, String> {
|
||||
let mut raw_response = Vec::new();
|
||||
reader
|
||||
.take(HEALTHCHECK_RESPONSE_MAX_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut raw_response)
|
||||
.map_err(|error| format!("response read failed: {error}"))?;
|
||||
if raw_response.len() as u64 > HEALTHCHECK_RESPONSE_MAX_BYTES {
|
||||
return Err("response exceeds the 64 KiB healthcheck limit".to_string());
|
||||
}
|
||||
Ok(raw_response)
|
||||
}
|
||||
|
||||
fn probe_target(listen: SocketAddr) -> SocketAddr {
|
||||
match listen {
|
||||
SocketAddr::V4(addr) => {
|
||||
@@ -180,7 +191,10 @@ fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
|
||||
use super::{
|
||||
HEALTHCHECK_RESPONSE_MAX_BYTES, HealthcheckMode, parse_status_code, read_response_bounded,
|
||||
split_response, validate_payload,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn parse_status_code_reads_http_200() {
|
||||
@@ -208,4 +222,11 @@ mod tests {
|
||||
let result = validate_payload(HealthcheckMode::Ready, body);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_reader_rejects_oversized_health_response() {
|
||||
let payload = vec![b'x'; HEALTHCHECK_RESPONSE_MAX_BYTES as usize + 1];
|
||||
|
||||
assert!(read_response_bounded(&mut payload.as_slice()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
+30
-37
@@ -8,10 +8,10 @@ use std::hash::{Hash, Hasher};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU8, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use dashmap::DashMap;
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::{Mutex as AsyncMutex, RwLock};
|
||||
|
||||
use crate::config::UserMaxUniqueIpsMode;
|
||||
@@ -39,6 +39,25 @@ struct CleanupShard {
|
||||
queue: Mutex<HashMap<String, HashMap<IpAddr, usize>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct UserIpLimitPolicy {
|
||||
max_ips: Arc<HashMap<String, usize>>,
|
||||
default_max_ips: usize,
|
||||
mode: UserMaxUniqueIpsMode,
|
||||
window_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for UserIpLimitPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_ips: Arc::new(HashMap::new()),
|
||||
default_max_ips: 0,
|
||||
mode: UserMaxUniqueIpsMode::ActiveWindow,
|
||||
window_secs: 30,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks active and recent client IPs for per-user admission control.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UserIpTracker {
|
||||
@@ -48,10 +67,7 @@ pub struct UserIpTracker {
|
||||
active_cap_rejects: Arc<AtomicU64>,
|
||||
recent_cap_rejects: Arc<AtomicU64>,
|
||||
cleanup_deferred_releases: Arc<AtomicU64>,
|
||||
max_ips: Arc<DashMap<String, usize>>,
|
||||
default_max_ips: Arc<AtomicUsize>,
|
||||
limit_mode: Arc<AtomicU8>,
|
||||
limit_window_secs: Arc<AtomicU64>,
|
||||
limit_policy: Arc<ArcSwap<UserIpLimitPolicy>>,
|
||||
last_compact_epoch_secs: Arc<AtomicU64>,
|
||||
cleanup_queue_len: Arc<AtomicU64>,
|
||||
cleanup_shards: Arc<Box<[CleanupShard]>>,
|
||||
@@ -102,12 +118,7 @@ impl UserIpTracker {
|
||||
active_cap_rejects: Arc::new(AtomicU64::new(0)),
|
||||
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
|
||||
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
|
||||
max_ips: Arc::new(DashMap::new()),
|
||||
default_max_ips: Arc::new(AtomicUsize::new(0)),
|
||||
limit_mode: Arc::new(AtomicU8::new(Self::mode_to_u8(
|
||||
UserMaxUniqueIpsMode::ActiveWindow,
|
||||
))),
|
||||
limit_window_secs: Arc::new(AtomicU64::new(30)),
|
||||
limit_policy: Arc::new(ArcSwap::from_pointee(UserIpLimitPolicy::default())),
|
||||
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
|
||||
cleanup_shards: Arc::new(cleanup_shards),
|
||||
@@ -117,41 +128,23 @@ impl UserIpTracker {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mode_to_u8(mode: UserMaxUniqueIpsMode) -> u8 {
|
||||
match mode {
|
||||
UserMaxUniqueIpsMode::ActiveWindow => 0,
|
||||
UserMaxUniqueIpsMode::TimeWindow => 1,
|
||||
UserMaxUniqueIpsMode::Combined => 2,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn mode_from_u8(raw: u8) -> UserMaxUniqueIpsMode {
|
||||
match raw {
|
||||
1 => UserMaxUniqueIpsMode::TimeWindow,
|
||||
2 => UserMaxUniqueIpsMode::Combined,
|
||||
_ => UserMaxUniqueIpsMode::ActiveWindow,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn shard_idx(username: &str) -> usize {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
username.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & USER_IP_TRACKER_SHARD_MASK
|
||||
}
|
||||
|
||||
pub(super) fn limit_window(&self) -> Duration {
|
||||
Duration::from_secs(self.limit_window_secs.load(Ordering::Relaxed).max(1))
|
||||
fn limit_window(policy: &UserIpLimitPolicy) -> Duration {
|
||||
Duration::from_secs(policy.window_secs)
|
||||
}
|
||||
|
||||
pub(super) fn user_limit(&self, username: &str) -> Option<usize> {
|
||||
self.max_ips
|
||||
fn user_limit(policy: &UserIpLimitPolicy, username: &str) -> Option<usize> {
|
||||
policy
|
||||
.max_ips
|
||||
.get(username)
|
||||
.map(|limit| *limit)
|
||||
.copied()
|
||||
.filter(|limit| *limit > 0)
|
||||
.or_else(|| {
|
||||
let default_limit = self.default_max_ips.load(Ordering::Relaxed);
|
||||
(default_limit > 0).then_some(default_limit)
|
||||
})
|
||||
.or_else(|| (policy.default_max_ips > 0).then_some(policy.default_max_ips))
|
||||
}
|
||||
|
||||
pub(super) fn decrement_counter(counter: &AtomicU64, amount: usize) {
|
||||
|
||||
+36
-14
@@ -2,26 +2,47 @@ use super::*;
|
||||
|
||||
impl UserIpTracker {
|
||||
pub async fn set_limit_policy(&self, mode: UserMaxUniqueIpsMode, window_secs: u64) {
|
||||
self.limit_mode
|
||||
.store(Self::mode_to_u8(mode), Ordering::Relaxed);
|
||||
self.limit_window_secs
|
||||
.store(window_secs.max(1), Ordering::Relaxed);
|
||||
self.limit_policy.rcu(|current| {
|
||||
Arc::new(UserIpLimitPolicy {
|
||||
mode,
|
||||
window_secs: window_secs.max(1),
|
||||
..(**current).clone()
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn set_user_limit(&self, username: &str, max_ips: usize) {
|
||||
self.max_ips.insert(username.to_string(), max_ips);
|
||||
let username = username.to_string();
|
||||
self.limit_policy.rcu(|current| {
|
||||
let mut limits = current.max_ips.as_ref().clone();
|
||||
limits.insert(username.clone(), max_ips);
|
||||
Arc::new(UserIpLimitPolicy {
|
||||
max_ips: Arc::new(limits),
|
||||
..(**current).clone()
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn remove_user_limit(&self, username: &str) {
|
||||
self.max_ips.remove(username);
|
||||
self.limit_policy.rcu(|current| {
|
||||
let mut limits = current.max_ips.as_ref().clone();
|
||||
limits.remove(username);
|
||||
Arc::new(UserIpLimitPolicy {
|
||||
max_ips: Arc::new(limits),
|
||||
..(**current).clone()
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn load_limits(&self, default_limit: usize, limits: &HashMap<String, usize>) {
|
||||
self.default_max_ips.store(default_limit, Ordering::Relaxed);
|
||||
self.max_ips.clear();
|
||||
for (username, limit) in limits {
|
||||
self.max_ips.insert(username.clone(), *limit);
|
||||
}
|
||||
let limits = Arc::new(limits.clone());
|
||||
self.limit_policy.rcu(|current| {
|
||||
Arc::new(UserIpLimitPolicy {
|
||||
max_ips: Arc::clone(&limits),
|
||||
default_max_ips: default_limit,
|
||||
..(**current).clone()
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn prune_recent(
|
||||
@@ -40,9 +61,10 @@ impl UserIpTracker {
|
||||
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
|
||||
self.drain_cleanup_for_user(username).await;
|
||||
self.maybe_compact_empty_users().await;
|
||||
let limit = self.user_limit(username);
|
||||
let mode = Self::mode_from_u8(self.limit_mode.load(Ordering::Relaxed));
|
||||
let window = self.limit_window();
|
||||
let policy = self.limit_policy.load();
|
||||
let limit = Self::user_limit(&policy, username);
|
||||
let mode = policy.mode;
|
||||
let window = Self::limit_window(&policy);
|
||||
let now = Instant::now();
|
||||
|
||||
let shard_idx = Self::shard_idx(username);
|
||||
|
||||
@@ -21,7 +21,8 @@ impl UserIpTracker {
|
||||
return;
|
||||
}
|
||||
|
||||
let window = self.limit_window();
|
||||
let policy = self.limit_policy.load();
|
||||
let window = Self::limit_window(&policy);
|
||||
let now = Instant::now();
|
||||
for shard_lock in self.shards.iter() {
|
||||
let mut shard = shard_lock.write().await;
|
||||
@@ -113,7 +114,8 @@ impl UserIpTracker {
|
||||
&self,
|
||||
users: &[String],
|
||||
) -> HashMap<String, usize> {
|
||||
let window = self.limit_window();
|
||||
let policy = self.limit_policy.load();
|
||||
let window = Self::limit_window(&policy);
|
||||
let now = Instant::now();
|
||||
|
||||
let mut counts = HashMap::with_capacity(users.len());
|
||||
@@ -152,7 +154,8 @@ impl UserIpTracker {
|
||||
|
||||
pub async fn get_recent_ips_for_users(&self, users: &[String]) -> HashMap<String, Vec<IpAddr>> {
|
||||
self.drain_cleanup_queue().await;
|
||||
let window = self.limit_window();
|
||||
let policy = self.limit_policy.load();
|
||||
let window = Self::limit_window(&policy);
|
||||
let now = Instant::now();
|
||||
|
||||
let mut out = HashMap::with_capacity(users.len());
|
||||
@@ -202,6 +205,7 @@ impl UserIpTracker {
|
||||
}
|
||||
|
||||
pub(crate) async fn get_stats_snapshot(&self) -> Vec<(String, usize, usize)> {
|
||||
let policy = self.limit_policy.load();
|
||||
let mut active_counts = Vec::new();
|
||||
for shard_lock in self.shards.iter() {
|
||||
let shard = shard_lock.read().await;
|
||||
@@ -215,7 +219,7 @@ impl UserIpTracker {
|
||||
|
||||
let mut stats = Vec::with_capacity(active_counts.len());
|
||||
for (username, active_count) in active_counts {
|
||||
let limit = self.user_limit(&username).unwrap_or(0);
|
||||
let limit = Self::user_limit(&policy, &username).unwrap_or(0);
|
||||
stats.push((username, active_count, limit));
|
||||
}
|
||||
|
||||
@@ -273,7 +277,8 @@ impl UserIpTracker {
|
||||
}
|
||||
|
||||
pub async fn get_user_limit(&self, username: &str) -> Option<usize> {
|
||||
self.user_limit(username)
|
||||
let policy = self.limit_policy.load();
|
||||
Self::user_limit(&policy, username)
|
||||
}
|
||||
|
||||
pub async fn format_stats(&self) -> String {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use super::*;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn test_ipv4(oct1: u8, oct2: u8, oct3: u8, oct4: u8) -> IpAddr {
|
||||
@@ -232,6 +233,55 @@ async fn test_load_limits_replaces_previous_map() {
|
||||
assert_eq!(tracker.get_user_limit("user2").await, Some(5));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrent_policy_replacement_never_exposes_partial_limit_map() {
|
||||
const USER_COUNT: usize = 4_096;
|
||||
const REPLACEMENTS: usize = 32;
|
||||
|
||||
let tracker = Arc::new(UserIpTracker::new());
|
||||
let first = (0..USER_COUNT)
|
||||
.map(|index| (format!("user-{index}"), 3usize))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let second = (0..USER_COUNT)
|
||||
.map(|index| (format!("user-{index}"), 5usize))
|
||||
.collect::<HashMap<_, _>>();
|
||||
tracker.load_limits(7, &first).await;
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let writer_tracker = Arc::clone(&tracker);
|
||||
let writer_running = Arc::clone(&running);
|
||||
let writer = tokio::spawn(async move {
|
||||
for _ in 0..REPLACEMENTS {
|
||||
writer_tracker.load_limits(7, &second).await;
|
||||
tokio::task::yield_now().await;
|
||||
writer_tracker.load_limits(7, &first).await;
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
writer_running.store(false, Ordering::Release);
|
||||
});
|
||||
|
||||
let mut readers = Vec::new();
|
||||
for reader in 0..3usize {
|
||||
let reader_tracker = Arc::clone(&tracker);
|
||||
let reader_running = Arc::clone(&running);
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut index = reader;
|
||||
while reader_running.load(Ordering::Acquire) {
|
||||
let username = format!("user-{}", index % USER_COUNT);
|
||||
let limit = reader_tracker.get_user_limit(&username).await;
|
||||
assert!(matches!(limit, Some(3 | 5)), "partial policy: {limit:?}");
|
||||
index = index.wrapping_add(17);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
writer.await.unwrap();
|
||||
for reader in readers {
|
||||
reader.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_global_each_limit_applies_without_user_override() {
|
||||
let tracker = UserIpTracker::new();
|
||||
|
||||
@@ -253,10 +253,6 @@ pub(super) async fn bootstrap(
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) {
|
||||
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
set_maestro_colors_enabled(!config.general.disable_colors);
|
||||
startup_tracker
|
||||
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
const CONTROL_TASK_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
|
||||
const CONTROL_TASK_REGISTRATION_COUNT: usize = CONTROL_TASK_ADMISSION_CLOSED - 1;
|
||||
|
||||
struct ControlTaskAdmission {
|
||||
state: AtomicUsize,
|
||||
registrations_drained: Notify,
|
||||
}
|
||||
|
||||
struct ControlTaskRegistration<'a> {
|
||||
admission: &'a ControlTaskAdmission,
|
||||
}
|
||||
|
||||
impl ControlTaskAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
registrations_drained: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn try_register(&self) -> Option<ControlTaskRegistration<'_>> {
|
||||
let mut state = self.state.load(Ordering::Acquire);
|
||||
loop {
|
||||
if state & CONTROL_TASK_ADMISSION_CLOSED != 0
|
||||
|| state & CONTROL_TASK_REGISTRATION_COUNT == CONTROL_TASK_REGISTRATION_COUNT
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match self.state.compare_exchange_weak(
|
||||
state,
|
||||
state + 1,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return Some(ControlTaskRegistration { admission: self }),
|
||||
Err(observed) => state = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.state
|
||||
.fetch_or(CONTROL_TASK_ADMISSION_CLOSED, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
async fn wait_for_registrations(&self) {
|
||||
loop {
|
||||
let notified = self.registrations_drained.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.state.load(Ordering::Acquire) & CONTROL_TASK_REGISTRATION_COUNT == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ControlTaskRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||
if previous & CONTROL_TASK_REGISTRATION_COUNT == 1 {
|
||||
self.admission.registrations_drained.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ProcessControlPlaneInner {
|
||||
admission: ControlTaskAdmission,
|
||||
cancellation: CancellationToken,
|
||||
tasks: TaskTracker,
|
||||
shutdown_completed: AtomicBool,
|
||||
}
|
||||
|
||||
/// Process-owned cancellation and join scope for API, metrics, and signal tasks.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ProcessControlPlane {
|
||||
inner: Arc<ProcessControlPlaneInner>,
|
||||
}
|
||||
|
||||
impl ProcessControlPlane {
|
||||
/// Creates an open process control-plane scope.
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(ProcessControlPlaneInner {
|
||||
admission: ControlTaskAdmission::new(),
|
||||
cancellation: CancellationToken::new(),
|
||||
tasks: TaskTracker::new(),
|
||||
shutdown_completed: AtomicBool::new(false),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers a cancellable process control-plane task before it can be unpolled.
|
||||
pub(crate) fn spawn<F>(&self, future: F) -> Result<(), F>
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(registration) = self.inner.admission.try_register() else {
|
||||
return Err(future);
|
||||
};
|
||||
let cancellation = self.inner.cancellation.clone();
|
||||
self.inner.tasks.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
drop(registration);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Closes task admission, cancels all owned work, and joins it within the deadline.
|
||||
pub(crate) async fn shutdown(&self, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
self.inner.admission.close();
|
||||
self.inner.cancellation.cancel();
|
||||
self.inner.tasks.close();
|
||||
if self.inner.shutdown_completed.load(Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
let registrations_stopped =
|
||||
tokio::time::timeout_at(deadline, self.inner.admission.wait_for_registrations())
|
||||
.await
|
||||
.is_ok();
|
||||
let tasks_stopped = tokio::time::timeout_at(deadline, self.inner.tasks.wait())
|
||||
.await
|
||||
.is_ok();
|
||||
let outcome = registrations_stopped && tasks_stopped;
|
||||
if outcome {
|
||||
self.inner.shutdown_completed.store(true, Ordering::Release);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_cancels_owned_tasks_and_rejects_late_registration() {
|
||||
struct DropSignal(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
let scope = ProcessControlPlane::new();
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let drop_signal = DropSignal(dropped.clone());
|
||||
assert!(
|
||||
scope
|
||||
.spawn(async move {
|
||||
let _drop_signal = drop_signal;
|
||||
std::future::pending::<()>().await;
|
||||
})
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
assert!(scope.spawn(async {}).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_shutdown_callers_wait_for_completion() {
|
||||
let scope = ProcessControlPlane::new();
|
||||
let registration = scope.inner.admission.try_register().unwrap();
|
||||
let first_scope = scope.clone();
|
||||
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(1)).await });
|
||||
tokio::task::yield_now().await;
|
||||
let second_scope = scope.clone();
|
||||
let second =
|
||||
tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!first.is_finished());
|
||||
assert!(!second.is_finished());
|
||||
drop(registration);
|
||||
|
||||
assert!(first.await.unwrap());
|
||||
assert!(second.await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_shutdown_caller_cannot_orphan_the_control_plane() {
|
||||
let scope = ProcessControlPlane::new();
|
||||
let registration = scope.inner.admission.try_register().unwrap();
|
||||
let first_scope = scope.clone();
|
||||
let first =
|
||||
tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
assert!(scope.spawn(async {}).is_err());
|
||||
drop(registration);
|
||||
|
||||
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||
}
|
||||
}
|
||||
+66
-10
@@ -3,7 +3,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tokio::sync::{Notify, RwLock, Semaphore, watch};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
@@ -29,6 +29,7 @@ const SESSION_REGISTRATION_COUNT: usize = SESSION_ADMISSION_CLOSED - 1;
|
||||
|
||||
struct SessionAdmission {
|
||||
state: AtomicUsize,
|
||||
registrations_drained: Notify,
|
||||
}
|
||||
|
||||
struct SessionRegistration<'a> {
|
||||
@@ -39,6 +40,7 @@ impl SessionAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
registrations_drained: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,15 +75,24 @@ impl SessionAdmission {
|
||||
}
|
||||
|
||||
async fn wait_for_registrations(&self) {
|
||||
while self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
loop {
|
||||
let notified = self.registrations_drained.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT == 0 {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SessionRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.admission.state.fetch_sub(1, Ordering::Release);
|
||||
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||
if previous & SESSION_REGISTRATION_COUNT == 1 {
|
||||
self.admission.registrations_drained.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +109,7 @@ pub(crate) struct RuntimeWatchState {
|
||||
pub(crate) struct RuntimeTaskScope {
|
||||
tracker: TaskTracker,
|
||||
cancel: CancellationToken,
|
||||
admission: Arc<SessionAdmission>,
|
||||
}
|
||||
|
||||
impl RuntimeTaskScope {
|
||||
@@ -106,6 +118,7 @@ impl RuntimeTaskScope {
|
||||
Self {
|
||||
tracker: TaskTracker::new(),
|
||||
cancel: CancellationToken::new(),
|
||||
admission: Arc::new(SessionAdmission::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,9 +127,13 @@ impl RuntimeTaskScope {
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(_registration) = self.admission.try_register() else {
|
||||
return;
|
||||
};
|
||||
let cancel = self.cancel.clone();
|
||||
self.tracker.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
@@ -130,6 +147,8 @@ impl RuntimeTaskScope {
|
||||
|
||||
/// Cancels the scope and waits within the bounded background-task budget.
|
||||
pub(crate) async fn stop(&self) {
|
||||
self.admission.close();
|
||||
self.admission.wait_for_registrations().await;
|
||||
self.cancel.cancel();
|
||||
self.tracker.close();
|
||||
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
|
||||
@@ -263,6 +282,7 @@ impl RuntimeGeneration {
|
||||
let cancel = self.session_cancel.clone();
|
||||
self.sessions.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
@@ -275,11 +295,6 @@ impl RuntimeGeneration {
|
||||
self.session_admission.close();
|
||||
}
|
||||
|
||||
/// Reopens admission after a candidate activation rolls back.
|
||||
pub(crate) fn resume_accepting_sessions(&self) {
|
||||
self.session_admission.reopen();
|
||||
}
|
||||
|
||||
/// Waits for registered sessions and cancels them when the deadline expires.
|
||||
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
|
||||
self.stop_accepting_sessions();
|
||||
@@ -308,6 +323,27 @@ impl RuntimeGeneration {
|
||||
pub(crate) async fn stop_background_tasks(&self) {
|
||||
self.background_tasks.stop().await;
|
||||
}
|
||||
|
||||
/// Terminally stops the generation's Middle-End task and writer scope.
|
||||
pub(crate) async fn stop_middle_end(&self, timeout: Duration) -> bool {
|
||||
let Some(pool) = self.current_me_pool().await else {
|
||||
return true;
|
||||
};
|
||||
pool.shutdown_until(timeout).await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RuntimeGeneration {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pool) = self.me_pool.as_ref() {
|
||||
pool.begin_shutdown();
|
||||
}
|
||||
if let Ok(pool) = self.me_pool_runtime.try_read()
|
||||
&& let Some(pool) = pool.as_ref()
|
||||
{
|
||||
pool.begin_shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -393,11 +429,31 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_task_scope_joins_cancelled_background_task() {
|
||||
struct DropSignal(Arc<AtomicUsize>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
let scope = RuntimeTaskScope::new();
|
||||
scope.spawn(std::future::pending());
|
||||
let dropped = Arc::new(AtomicUsize::new(0));
|
||||
let drop_signal = DropSignal(dropped.clone());
|
||||
scope.spawn(async move {
|
||||
let _drop_signal = drop_signal;
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(dropped.load(Ordering::Acquire), 1);
|
||||
|
||||
let late_drop_signal = DropSignal(dropped.clone());
|
||||
scope.spawn(async move {
|
||||
let _late_drop_signal = late_drop_signal;
|
||||
});
|
||||
assert_eq!(dropped.load(Ordering::Acquire), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! - `bind` prepares and activates sockets without partial startup binding.
|
||||
//! - `accept` runs cancellation-aware TCP accept loops.
|
||||
//! - `control` coordinates reversible listener transitions and shutdown.
|
||||
//! - `web_overload` handles accepted WEB sockets outside ordinary capacity.
|
||||
|
||||
mod accept;
|
||||
mod bind;
|
||||
@@ -13,6 +14,7 @@ mod control;
|
||||
mod plan;
|
||||
#[cfg(unix)]
|
||||
mod unix;
|
||||
mod web_overload;
|
||||
|
||||
pub(crate) use bind::bind_listeners;
|
||||
pub(crate) use control::{ListenerManager, PreparedListenerTransition};
|
||||
|
||||
@@ -12,10 +12,12 @@ use tracing::{debug, error, info, warn};
|
||||
use crate::config::{ListenerTransport, RstOnCloseMode};
|
||||
use crate::proxy::ClientHandler;
|
||||
use crate::transport::socket::set_linger_zero;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||
use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};
|
||||
|
||||
use super::bind::BoundTcpListener;
|
||||
use super::plan::ListenerBindSpec;
|
||||
use super::web_overload;
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::maestro::helpers::{
|
||||
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
|
||||
@@ -190,6 +192,7 @@ async fn run_accept_loop(
|
||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||
connections: TaskTracker,
|
||||
cancellation: CancellationToken,
|
||||
_web_acceptor_guard: Option<WebAcceptorGuard>,
|
||||
) {
|
||||
loop {
|
||||
let accepted = tokio::select! {
|
||||
@@ -204,9 +207,79 @@ async fn run_accept_loop(
|
||||
error!(addr = %spec.addr, "WEB listener has no process runtime");
|
||||
return;
|
||||
};
|
||||
let Some(connection_permit) = web_runtime.try_http_connection() else {
|
||||
web_runtime.telemetry().record_accept();
|
||||
if cancellation.is_cancelled() {
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
if web_runtime.is_shutdown() {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let connection_permit = match web_runtime.try_http_connection() {
|
||||
Ok(permit) => permit,
|
||||
Err(HttpConnectionAdmissionError::Closed) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||
let config = web_runtime.active_generation().config();
|
||||
let action = config.web.http_connection_capacity_action;
|
||||
let phase_timeout =
|
||||
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
|
||||
drop(config);
|
||||
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
);
|
||||
web_runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let overload_permit = match web_runtime.try_http_overload_connection() {
|
||||
Ok(permit) => permit,
|
||||
Err(HttpConnectionAdmissionError::Closed) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||
);
|
||||
web_runtime.telemetry().record_overload(
|
||||
WebHttpConnectionOverloadOutcome::ShutdownDrop,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||
web_runtime.telemetry().record_rejection(
|
||||
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||
);
|
||||
web_runtime.telemetry().record_overload(
|
||||
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
connections.spawn(web_overload::serve(
|
||||
stream,
|
||||
peer_addr,
|
||||
spec.web_client_ip_source,
|
||||
Arc::clone(&spec.web_trusted_proxy_cidrs),
|
||||
Arc::clone(web_runtime),
|
||||
cancellation.clone(),
|
||||
overload_permit,
|
||||
action,
|
||||
phase_timeout,
|
||||
));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
connections.spawn(crate::web::http::serve_connection(
|
||||
stream,
|
||||
@@ -245,6 +318,9 @@ async fn run_accept_loop(
|
||||
}
|
||||
}
|
||||
Err(error_value) => {
|
||||
if let Some(web_runtime) = &web_runtime {
|
||||
web_runtime.telemetry().record_accept_error();
|
||||
}
|
||||
error!(addr = %spec.addr, error = %error_value, "TCP accept error");
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -262,8 +338,16 @@ impl ListenerSlot {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||
) -> Self {
|
||||
let web_runtime = if bound.spec.transport == ListenerTransport::Web {
|
||||
web_runtime
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let cancellation = CancellationToken::new();
|
||||
let connections = TaskTracker::new();
|
||||
let web_acceptor_guard = web_runtime
|
||||
.as_ref()
|
||||
.map(|runtime| runtime.telemetry().acceptor_guard());
|
||||
let task = tokio::spawn(run_accept_loop(
|
||||
bound.listener.clone(),
|
||||
bound.spec.clone(),
|
||||
@@ -271,6 +355,7 @@ impl ListenerSlot {
|
||||
web_runtime.clone(),
|
||||
connections.clone(),
|
||||
cancellation.clone(),
|
||||
web_acceptor_guard,
|
||||
));
|
||||
Self {
|
||||
spec: bound.spec,
|
||||
@@ -364,6 +449,10 @@ impl ListenerSlot {
|
||||
self.active_runtime = active_runtime.clone();
|
||||
self.cancellation = CancellationToken::new();
|
||||
self.connections = TaskTracker::new();
|
||||
let web_acceptor_guard = self
|
||||
.web_runtime
|
||||
.as_ref()
|
||||
.map(|runtime| runtime.telemetry().acceptor_guard());
|
||||
self.task = Some(tokio::spawn(run_accept_loop(
|
||||
self.listener.clone(),
|
||||
self.spec.clone(),
|
||||
@@ -371,6 +460,7 @@ impl ListenerSlot {
|
||||
self.web_runtime.clone(),
|
||||
self.connections.clone(),
|
||||
self.cancellation.clone(),
|
||||
web_acceptor_guard,
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,8 +58,13 @@ impl ListenerManager {
|
||||
.map(|listener| listener.spec.addr)
|
||||
.collect();
|
||||
let has_web = !web_listeners.is_empty();
|
||||
let web_runtime =
|
||||
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
||||
let web_runtime = has_web.then(|| {
|
||||
WebProcessRuntime::start_with_trace(
|
||||
active_runtime.clone(),
|
||||
trace,
|
||||
web_control.telemetry(),
|
||||
)
|
||||
});
|
||||
let mut slots = BTreeMap::new();
|
||||
for listener in bound.listeners {
|
||||
let addr = listener.spec.addr;
|
||||
@@ -243,6 +248,18 @@ impl ListenerManager {
|
||||
);
|
||||
}
|
||||
|
||||
/// Publishes one generation through the process-owned WEB policy fence.
|
||||
pub(crate) fn activate_runtime_generation(
|
||||
&self,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
) -> Arc<RuntimeGeneration> {
|
||||
if let Some(runtime) = &self.web_runtime {
|
||||
runtime.activate_generation(generation)
|
||||
} else {
|
||||
self.active_runtime.swap(generation)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops every accept task and applies one deadline to the complete WEB ingress.
|
||||
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
||||
self.web_control.publish(
|
||||
@@ -461,4 +478,30 @@ mod tests {
|
||||
manager.shutdown().await.unwrap();
|
||||
runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acceptor_liveness_counts_only_web_listeners() {
|
||||
let runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||
let active_runtime = Arc::new(ArcSwap::from(runtime.clone()));
|
||||
let (native_listener, _native_addr) = bound_listener().await;
|
||||
let (mut web_listener, _web_addr) = bound_listener().await;
|
||||
web_listener.spec.transport = ListenerTransport::Web;
|
||||
let bound = BoundListeners {
|
||||
listeners: vec![native_listener, web_listener],
|
||||
#[cfg(unix)]
|
||||
unix_listener: None,
|
||||
};
|
||||
let trace = WebTraceStore::new(
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
);
|
||||
let control = WebRuntimeControl::new();
|
||||
let receiver = control.subscribe();
|
||||
let mut manager = ListenerManager::start(bound, active_runtime, trace, control);
|
||||
|
||||
assert_eq!(receiver.borrow().telemetry.live_acceptors(), 1);
|
||||
|
||||
manager.shutdown().await.unwrap();
|
||||
runtime.stop_sessions().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use ipnetwork::IpNetwork;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
|
||||
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
|
||||
|
||||
/// Exact bounded retryable response emitted before HTTP request parsing.
|
||||
pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
|
||||
|
||||
/// Handles one accepted WEB socket outside ordinary connection capacity.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn serve(
|
||||
stream: TcpStream,
|
||||
peer: SocketAddr,
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: Arc<[IpNetwork]>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
cancellation: CancellationToken,
|
||||
overload_permit: OwnedSemaphorePermit,
|
||||
action: WebHttpConnectionCapacityAction,
|
||||
phase_timeout: Duration,
|
||||
) {
|
||||
match action {
|
||||
WebHttpConnectionCapacityAction::Drop => unreachable!("drop is handled before spawn"),
|
||||
WebHttpConnectionCapacityAction::Respond => {
|
||||
let outcome = respond(stream, &cancellation, phase_timeout).await;
|
||||
record_final_capacity_rejection(&runtime, outcome);
|
||||
runtime.telemetry().record_overload(outcome);
|
||||
}
|
||||
WebHttpConnectionCapacityAction::Wait => {
|
||||
let connection_permit = tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => {
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
|
||||
return;
|
||||
}
|
||||
permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
|
||||
permit
|
||||
}
|
||||
};
|
||||
let connection_permit = match connection_permit {
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(HttpConnectionAdmissionError::Closed)) => {
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_rejection(WebRejectionReason::RuntimeClosed);
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
|
||||
return;
|
||||
}
|
||||
Ok(Err(HttpConnectionAdmissionError::AtCapacity)) | Err(_) => {
|
||||
let outcome = match respond(stream, &cancellation, phase_timeout).await {
|
||||
WebHttpConnectionOverloadOutcome::Responded503 => {
|
||||
WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
record_final_capacity_rejection(&runtime, outcome);
|
||||
runtime.telemetry().record_overload(outcome);
|
||||
return;
|
||||
}
|
||||
};
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_overload(WebHttpConnectionOverloadOutcome::WaitAdmitted);
|
||||
drop(overload_permit);
|
||||
crate::web::http::serve_connection(
|
||||
stream,
|
||||
peer,
|
||||
client_ip_source,
|
||||
trusted_proxy_cidrs,
|
||||
runtime,
|
||||
cancellation,
|
||||
connection_permit,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_final_capacity_rejection(
|
||||
runtime: &WebProcessRuntime,
|
||||
outcome: WebHttpConnectionOverloadOutcome,
|
||||
) {
|
||||
if matches!(
|
||||
outcome,
|
||||
WebHttpConnectionOverloadOutcome::Responded503
|
||||
| WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||
| WebHttpConnectionOverloadOutcome::ResponseErrorDrop
|
||||
) {
|
||||
runtime
|
||||
.telemetry()
|
||||
.record_rejection(WebRejectionReason::HttpConnectionCapacity);
|
||||
}
|
||||
}
|
||||
|
||||
async fn respond(
|
||||
stream: TcpStream,
|
||||
cancellation: &CancellationToken,
|
||||
phase_timeout: Duration,
|
||||
) -> WebHttpConnectionOverloadOutcome {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => WebHttpConnectionOverloadOutcome::ShutdownDrop,
|
||||
written = write_service_unavailable(stream, phase_timeout) => {
|
||||
if written {
|
||||
WebHttpConnectionOverloadOutcome::Responded503
|
||||
} else {
|
||||
WebHttpConnectionOverloadOutcome::ResponseErrorDrop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_service_unavailable(mut stream: TcpStream, deadline: Duration) -> bool {
|
||||
tokio::time::timeout(deadline, async {
|
||||
stream.write_all(SERVICE_UNAVAILABLE_RESPONSE).await?;
|
||||
stream.shutdown().await
|
||||
})
|
||||
.await
|
||||
.is_ok_and(|result| result.is_ok())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{ProxyConfig, WebClientIpSource, WebHttpConnectionCapacityAction};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
|
||||
|
||||
async fn tcp_pair() -> (TcpStream, TcpStream) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let client = TcpStream::connect(addr);
|
||||
let server = listener.accept();
|
||||
let (client, server) = tokio::join!(client, server);
|
||||
(server.unwrap().0, client.unwrap())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overload_response_is_exact_retryable_http() {
|
||||
let (server, mut client) = tcp_pair().await;
|
||||
assert!(super::write_service_unavailable(server, Duration::from_secs(1)).await);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
client.read_to_end(&mut bytes).await.unwrap();
|
||||
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_timeout_is_one_rejection_and_one_retryable_response() {
|
||||
let (runtime, generation) = runtime();
|
||||
let held = runtime.try_http_connection().unwrap();
|
||||
let overload = runtime.try_http_overload_connection().unwrap();
|
||||
let (server, mut client) = tcp_pair().await;
|
||||
let peer = server.peer_addr().unwrap();
|
||||
|
||||
super::serve(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
trusted_loopback(),
|
||||
Arc::clone(&runtime),
|
||||
CancellationToken::new(),
|
||||
overload,
|
||||
WebHttpConnectionCapacityAction::Wait,
|
||||
Duration::from_millis(10),
|
||||
)
|
||||
.await;
|
||||
drop(held);
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
client.read_to_end(&mut bytes).await.unwrap();
|
||||
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.telemetry()
|
||||
.overload_total(WebHttpConnectionOverloadOutcome::WaitTimeout503,),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.telemetry()
|
||||
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
|
||||
1
|
||||
);
|
||||
stop(runtime, generation).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admitted_wait_is_not_counted_as_a_rejection() {
|
||||
let (runtime, generation) = runtime();
|
||||
let held = runtime.try_http_connection().unwrap();
|
||||
let overload = runtime.try_http_overload_connection().unwrap();
|
||||
let (server, _client) = tcp_pair().await;
|
||||
let peer = server.peer_addr().unwrap();
|
||||
let cancellation = CancellationToken::new();
|
||||
let task = tokio::spawn(super::serve(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
trusted_loopback(),
|
||||
Arc::clone(&runtime),
|
||||
cancellation.clone(),
|
||||
overload,
|
||||
WebHttpConnectionCapacityAction::Wait,
|
||||
Duration::from_secs(1),
|
||||
));
|
||||
tokio::task::yield_now().await;
|
||||
drop(held);
|
||||
for _ in 0..100 {
|
||||
if runtime
|
||||
.telemetry()
|
||||
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted)
|
||||
== 1
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
cancellation.cancel();
|
||||
tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
runtime
|
||||
.telemetry()
|
||||
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.telemetry()
|
||||
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
|
||||
0
|
||||
);
|
||||
stop(runtime, generation).await;
|
||||
}
|
||||
|
||||
fn runtime() -> (
|
||||
Arc<WebProcessRuntime>,
|
||||
Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||
) {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.web.limits.max_http_connections = 1;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
(runtime, generation)
|
||||
}
|
||||
|
||||
fn trusted_loopback() -> Arc<[ipnetwork::IpNetwork]> {
|
||||
Arc::from(["127.0.0.1/32".parse().unwrap()])
|
||||
}
|
||||
|
||||
async fn stop(
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
generation: Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||
) {
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
+7
-115
@@ -22,57 +22,11 @@ use crate::transport::middle_proxy::MePool;
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::load_startup_proxy_config_snapshot;
|
||||
|
||||
async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
loop {
|
||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||
match result {
|
||||
Ok(()) => warn!(
|
||||
task = task_name,
|
||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||
),
|
||||
Err(error) => {
|
||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_me_supervisors(
|
||||
task_scope: RuntimeTaskScope,
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
min_connections: usize,
|
||||
) {
|
||||
let health_pool = pool.clone();
|
||||
let health_rng = rng;
|
||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||
let pool = health_pool.clone();
|
||||
let rng = health_rng.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||
}
|
||||
}));
|
||||
|
||||
let drain_pool = pool.clone();
|
||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||
let pool = drain_pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||
}
|
||||
}));
|
||||
|
||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||
let pool = pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Restarting supervisors for long-lived ME maintenance tasks.
|
||||
mod supervisor;
|
||||
use supervisor::spawn_me_supervisors;
|
||||
#[cfg(test)]
|
||||
use supervisor::supervise_me_task;
|
||||
|
||||
pub(crate) async fn initialize_me_pool(
|
||||
use_middle_proxy: bool,
|
||||
@@ -345,6 +299,7 @@ pub(crate) async fn initialize_me_pool(
|
||||
config.general.me_route_blocking_send_timeout_ms,
|
||||
config.general.me_route_inline_recovery_attempts,
|
||||
config.general.me_route_inline_recovery_wait_ms,
|
||||
(config.server.max_connections as usize).saturating_add(128),
|
||||
);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
@@ -586,67 +541,4 @@ pub(crate) async fn initialize_me_pool(
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_supervisor_aborts_its_current_child() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let dropped_for_task = dropped.clone();
|
||||
scope.spawn(supervise_me_task("test", move || {
|
||||
let dropped = dropped_for_task.clone();
|
||||
async move {
|
||||
let _signal = DropSignal(dropped);
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
scope.stop().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let starts = Arc::new(AtomicUsize::new(0));
|
||||
let restarted = Arc::new(Notify::new());
|
||||
let starts_task = starts.clone();
|
||||
let restarted_task = restarted.clone();
|
||||
scope.spawn(supervise_me_task("restart_test", move || {
|
||||
let starts = starts_task.clone();
|
||||
let restarted = restarted_task.clone();
|
||||
async move {
|
||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||
restarted.notify_one();
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
scope.stop().await;
|
||||
let stopped_at = starts.load(Ordering::Acquire);
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(stopped_at >= 3);
|
||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||
}
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
use super::*;
|
||||
|
||||
pub(super) async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
loop {
|
||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||
match result {
|
||||
Ok(()) => warn!(
|
||||
task = task_name,
|
||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||
),
|
||||
Err(error) => {
|
||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn_me_supervisors(
|
||||
task_scope: RuntimeTaskScope,
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
min_connections: usize,
|
||||
) {
|
||||
let health_pool = pool.clone();
|
||||
let health_rng = rng;
|
||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||
let pool = health_pool.clone();
|
||||
let rng = health_rng.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||
}
|
||||
}));
|
||||
|
||||
let drain_pool = pool.clone();
|
||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||
let pool = drain_pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||
}
|
||||
}));
|
||||
|
||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||
let pool = pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_supervisor_aborts_its_current_child() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let dropped_for_task = dropped.clone();
|
||||
scope.spawn(supervise_me_task("test", move || {
|
||||
let dropped = dropped_for_task.clone();
|
||||
async move {
|
||||
let _signal = DropSignal(dropped);
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
scope.stop().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let starts = Arc::new(AtomicUsize::new(0));
|
||||
let restarted = Arc::new(Notify::new());
|
||||
let starts_task = starts.clone();
|
||||
let restarted_task = restarted.clone();
|
||||
scope.spawn(supervise_me_task("restart_test", move || {
|
||||
let starts = starts_task.clone();
|
||||
let restarted = restarted_task.clone();
|
||||
async move {
|
||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||
restarted.notify_one();
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
scope.stop().await;
|
||||
let stopped_at = starts.load(Ordering::Acquire);
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(stopped_at >= 3);
|
||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||
}
|
||||
@@ -6,6 +6,7 @@
|
||||
// - admission: conditional-cast gate and route mode switching.
|
||||
// - bootstrap: configuration and tracing initialization.
|
||||
// - connectivity: startup ME/DC connectivity diagnostics.
|
||||
// - control_plane: process-owned API, metrics, and signal task lifecycle.
|
||||
// - generation: runtime generation state and task ownership.
|
||||
// - helpers: CLI and shared startup/runtime helper routines.
|
||||
// - listeners: TCP/Unix listener planning, binding, and lifecycle control.
|
||||
@@ -21,6 +22,7 @@
|
||||
mod admission;
|
||||
mod bootstrap;
|
||||
mod connectivity;
|
||||
pub(crate) mod control_plane;
|
||||
pub(crate) mod generation;
|
||||
mod helpers;
|
||||
mod listeners;
|
||||
|
||||
+59
-18
@@ -1,9 +1,10 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::{RwLock, watch};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::api;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
@@ -15,14 +16,15 @@ use crate::startup::{COMPONENT_API_BOOTSTRAP, COMPONENT_NETWORK_PROBE};
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{QuotaStore, Stats};
|
||||
use crate::synlimit_control;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::{
|
||||
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
||||
shutdown, tls_bootstrap,
|
||||
bootstrap, control_plane, generation, listeners, reload, reload_supervisor, runtime_startup,
|
||||
runtime_tasks, shutdown, tls_bootstrap,
|
||||
};
|
||||
|
||||
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
||||
@@ -45,10 +47,15 @@ pub(super) async fn run_telemt_core(
|
||||
|
||||
let quota_store = Arc::new(QuotaStore::default());
|
||||
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
|
||||
let tls_full_cert_budget = Arc::new(TlsFullCertBudget::new());
|
||||
let process_control_plane = control_plane::ProcessControlPlane::new();
|
||||
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||
let quota_state_path = config.general.quota_state_path.clone();
|
||||
crate::quota_state::load_quota_state("a_state_path, stats.as_ref()).await;
|
||||
let quota_state =
|
||||
crate::quota_state::QuotaStateOwner::new(quota_state_path, quota_store.clone());
|
||||
let configured_quota_users = config.access.users.keys().cloned().collect::<BTreeSet<_>>();
|
||||
quota_state.load(&configured_quota_users).await;
|
||||
|
||||
let upstream_manager = Arc::new(
|
||||
UpstreamManager::new(
|
||||
@@ -136,15 +143,29 @@ pub(super) async fn run_telemt_core(
|
||||
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
||||
Ok(listen) => listen,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
listen = %config.server.api.listen,
|
||||
"Invalid server.api.listen; API is disabled"
|
||||
let message = format!(
|
||||
"invalid server.api.listen \"{}\": {}",
|
||||
config.server.api.listen, error
|
||||
);
|
||||
SocketAddr::from(([127, 0, 0, 1], 0))
|
||||
startup_tracker
|
||||
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.clone()))
|
||||
.await;
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, message).into());
|
||||
}
|
||||
};
|
||||
if listen.port() != 0 {
|
||||
let api_listener = match tokio::net::TcpListener::bind(listen).await {
|
||||
Ok(listener) => listener,
|
||||
Err(error) => {
|
||||
startup_tracker
|
||||
.fail_component(
|
||||
COMPONENT_API_BOOTSTRAP,
|
||||
Some(format!("API listener bind failed on {listen}: {error}")),
|
||||
)
|
||||
.await;
|
||||
return Err(error.into());
|
||||
}
|
||||
};
|
||||
let stats_api = stats.clone();
|
||||
let ip_tracker_api = ip_tracker.clone();
|
||||
let me_pool_api = api_me_pool.clone();
|
||||
@@ -152,7 +173,7 @@ pub(super) async fn run_telemt_core(
|
||||
let route_runtime_api = route_runtime.clone();
|
||||
let proxy_shared_api = shared_state.clone();
|
||||
let config_path_api = config_path.clone();
|
||||
let quota_state_path_api = quota_state_path.clone();
|
||||
let quota_state_api = quota_state.clone();
|
||||
let startup_tracker_api = startup_tracker.clone();
|
||||
let detected_ips_rx_api = detected_ips_rx.clone();
|
||||
let reload_control_api = reload_control.clone();
|
||||
@@ -160,9 +181,11 @@ pub(super) async fn run_telemt_core(
|
||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||
let web_trace_api = web_trace.clone();
|
||||
let web_runtime_rx_api = web_runtime_control.subscribe();
|
||||
tokio::spawn(async move {
|
||||
let api_control_plane = process_control_plane.clone();
|
||||
let api_task_control_plane = process_control_plane.clone();
|
||||
let api_task = async move {
|
||||
api::serve(
|
||||
listen,
|
||||
api_listener,
|
||||
stats_api,
|
||||
ip_tracker_api,
|
||||
me_pool_api,
|
||||
@@ -170,7 +193,7 @@ pub(super) async fn run_telemt_core(
|
||||
proxy_shared_api,
|
||||
upstream_manager_api,
|
||||
config_path_api,
|
||||
quota_state_path_api,
|
||||
quota_state_api,
|
||||
detected_ips_rx_api,
|
||||
process_started_at_epoch_secs,
|
||||
startup_tracker_api,
|
||||
@@ -179,13 +202,21 @@ pub(super) async fn run_telemt_core(
|
||||
runtime_watch_rx_api,
|
||||
web_trace_api,
|
||||
web_runtime_rx_api,
|
||||
api_task_control_plane,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
};
|
||||
if api_control_plane.spawn(api_task).is_err() {
|
||||
let message = "process control-plane task admission closed during API startup";
|
||||
startup_tracker
|
||||
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.to_string()))
|
||||
.await;
|
||||
return Err(std::io::Error::other(message).into());
|
||||
}
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_API_BOOTSTRAP,
|
||||
Some(format!("api task spawned on {}", listen)),
|
||||
Some(format!("API listener bound and supervised on {}", listen)),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
@@ -219,6 +250,7 @@ pub(super) async fn run_telemt_core(
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
runtime_task_scope.clone(),
|
||||
tls_full_cert_budget.clone(),
|
||||
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await?;
|
||||
@@ -315,8 +347,11 @@ pub(super) async fn run_telemt_core(
|
||||
&runtime.config,
|
||||
&startup_tracker,
|
||||
active_runtime.clone(),
|
||||
web_runtime_control.subscribe(),
|
||||
tls_full_cert_budget.clone(),
|
||||
process_control_plane.clone(),
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
|
||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||
@@ -334,6 +369,7 @@ pub(super) async fn run_telemt_core(
|
||||
reload_commands,
|
||||
config_path,
|
||||
quota_store,
|
||||
tls_full_cert_budget,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
@@ -341,12 +377,17 @@ pub(super) async fn run_telemt_core(
|
||||
web_trace,
|
||||
);
|
||||
|
||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
||||
shutdown::spawn_signal_handlers(
|
||||
active_runtime.clone(),
|
||||
process_started_at,
|
||||
process_control_plane.clone(),
|
||||
);
|
||||
shutdown::wait_for_shutdown(
|
||||
process_started_at,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
quota_state,
|
||||
reload_supervisor,
|
||||
process_control_plane,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::stats::QuotaStore;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
@@ -26,6 +27,7 @@ pub(crate) struct ReloadSupervisor {
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
@@ -81,12 +83,7 @@ fn revision_gate_action(
|
||||
|
||||
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
|
||||
generation.stop_background_tasks().await;
|
||||
let Some(pool) = generation.current_me_pool().await else {
|
||||
return false;
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
.is_err()
|
||||
!generation.stop_middle_end(Duration::from_secs(5)).await
|
||||
}
|
||||
|
||||
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
|
||||
@@ -103,6 +100,7 @@ impl ReloadSupervisor {
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
@@ -116,6 +114,7 @@ impl ReloadSupervisor {
|
||||
commands,
|
||||
config_path,
|
||||
quota_store,
|
||||
tls_full_cert_budget,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
@@ -160,7 +159,13 @@ impl ReloadSupervisor {
|
||||
.mark_phase(command.reload_id, ReloadPhase::Preparing)
|
||||
.await;
|
||||
let old_runtime = self.active_runtime.load_full();
|
||||
let resolved = resolve_reload_config(&old_runtime.config(), &command.config);
|
||||
let resolved = match resolve_reload_config(&old_runtime.config(), &command.config) {
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => {
|
||||
self.control.fail(command.reload_id, error).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.control
|
||||
.set_deferred_fields(command.reload_id, resolved.deferred_process_fields.clone())
|
||||
.await;
|
||||
@@ -171,6 +176,7 @@ impl ReloadSupervisor {
|
||||
&self.config_path,
|
||||
self.quota_store.clone(),
|
||||
self.runtime_log_filter.clone(),
|
||||
self.tls_full_cert_budget.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -207,25 +213,18 @@ impl ReloadSupervisor {
|
||||
prepared,
|
||||
listener_transition,
|
||||
revision_action,
|
||||
|entries| {
|
||||
crate::network::dns_overrides::install_entries(entries)
|
||||
.map_err(|error| error.to_string())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn activate_prepared<InstallDns>(
|
||||
async fn activate_prepared(
|
||||
&self,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
prepared: PreparedRuntime,
|
||||
revision_action: RevisionGateAction,
|
||||
install_dns: InstallDns,
|
||||
) where
|
||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
||||
{
|
||||
) {
|
||||
let listener_transition = match self
|
||||
.listener_manager
|
||||
.lock()
|
||||
@@ -245,22 +244,18 @@ impl ReloadSupervisor {
|
||||
prepared,
|
||||
listener_transition,
|
||||
revision_action,
|
||||
install_dns,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn activate_prepared_with_transition<InstallDns>(
|
||||
async fn activate_prepared_with_transition(
|
||||
&self,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
prepared: PreparedRuntime,
|
||||
listener_transition: Option<PreparedListenerTransition>,
|
||||
revision_action: RevisionGateAction,
|
||||
install_dns: InstallDns,
|
||||
) where
|
||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
||||
{
|
||||
) {
|
||||
match revision_action {
|
||||
RevisionGateAction::Proceed => {}
|
||||
RevisionGateAction::Warn(warning) => {
|
||||
@@ -283,18 +278,6 @@ impl ReloadSupervisor {
|
||||
detected_ips,
|
||||
config_watcher_activation,
|
||||
} = prepared;
|
||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
||||
let message = format!("runtime DNS activation failed: {}", error);
|
||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
||||
old_runtime.resume_accepting_sessions();
|
||||
let _ = cleanup_candidate(&new_runtime).await;
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&old_runtime.config().general.log_level);
|
||||
self.control.rolled_back(command.reload_id, message).await;
|
||||
return;
|
||||
}
|
||||
self.control.add_warning(command.reload_id, message).await;
|
||||
}
|
||||
let pending_listener_transition = if let Some(listener_transition) = listener_transition {
|
||||
match self
|
||||
.listener_manager
|
||||
@@ -315,8 +298,11 @@ impl ReloadSupervisor {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
old_runtime.stop_accepting_sessions();
|
||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||
let replaced = {
|
||||
let listener_manager = self.listener_manager.lock().await;
|
||||
old_runtime.stop_accepting_sessions();
|
||||
listener_manager.activate_runtime_generation(new_runtime.clone())
|
||||
};
|
||||
self.web_trace
|
||||
.apply_policy(new_runtime.id, &new_runtime.config().web.debug);
|
||||
config_watcher_activation.send_replace(true);
|
||||
@@ -367,7 +353,7 @@ impl ReloadSupervisor {
|
||||
|
||||
if stop_background_and_middle_end(&replaced).await {
|
||||
let warning = format!(
|
||||
"generation {} Middle-End close broadcast timed out",
|
||||
"generation {} Middle-End lifecycle shutdown timed out",
|
||||
replaced.id
|
||||
);
|
||||
warn!(reload_id = command.reload_id, warning = %warning);
|
||||
|
||||
@@ -53,6 +53,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
commands,
|
||||
config_path: PathBuf::new(),
|
||||
quota_store: Arc::new(QuotaStore::default()),
|
||||
tls_full_cert_budget: Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter: runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
@@ -131,7 +132,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
fixture.old_runtime.clone(),
|
||||
prepared_runtime(fixture.new_runtime),
|
||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -154,44 +154,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
||||
for policy in [ReloadFailurePolicy::Rollback, ReloadFailurePolicy::KeepNew] {
|
||||
let fixture = fixture(ReloadRequest {
|
||||
failure_policy: policy,
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
fixture
|
||||
.supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
prepared_runtime(fixture.new_runtime.clone()),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Err("invalid DNS entry".to_string()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
match policy {
|
||||
ReloadFailurePolicy::Rollback => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
|
||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
ReloadFailurePolicy::KeepNew => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 2);
|
||||
assert_eq!(status.state, ReloadPhase::Succeeded);
|
||||
assert_eq!(status.warnings.len(), 1);
|
||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
let mut fixture = fixture(ReloadRequest {
|
||||
@@ -220,7 +182,6 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
old_runtime,
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -273,7 +234,6 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
||||
old_runtime,
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -304,6 +264,7 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
||||
commands,
|
||||
PathBuf::new(),
|
||||
Arc::new(QuotaStore::default()),
|
||||
Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
|
||||
@@ -5,7 +5,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
|
||||
use crate::config::{ProxyConfig, ServerConfig, web_debug_fits_limits};
|
||||
use crate::config::{
|
||||
ProxyConfig, ServerConfig, WEB_CARRIER_LEARNING_MIN_ENTRIES, web_debug_fits_limits,
|
||||
};
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::network::probe::{decide_network_capabilities, run_probe};
|
||||
@@ -19,6 +21,7 @@ use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
@@ -44,7 +47,11 @@ pub(crate) async fn prepare_runtime(
|
||||
config_path: &Path,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
) -> Result<PreparedRuntime, String> {
|
||||
config
|
||||
.validate_web_decoy_listener_separation()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let started_at_epoch_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
@@ -118,6 +125,7 @@ pub(crate) async fn prepare_runtime(
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
task_scope.clone(),
|
||||
tls_full_cert_budget,
|
||||
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
@@ -331,7 +339,7 @@ pub(crate) struct ResolvedReloadConfig {
|
||||
pub(crate) fn resolve_reload_config(
|
||||
old: &ProxyConfig,
|
||||
desired: &ProxyConfig,
|
||||
) -> ResolvedReloadConfig {
|
||||
) -> Result<ResolvedReloadConfig, String> {
|
||||
let mut effective = desired.clone();
|
||||
let mut fields = Vec::new();
|
||||
let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server);
|
||||
@@ -418,21 +426,42 @@ pub(crate) fn resolve_reload_config(
|
||||
{
|
||||
fields.push("web.limits".to_string());
|
||||
effective.web.limits = old.web.limits.clone();
|
||||
if effective.rebuild_runtime_web().is_err() {
|
||||
fields.push("web".to_string());
|
||||
effective.web = old.web.clone();
|
||||
}
|
||||
if old.web.decoy_fasttrack_mode != desired.web.decoy_fasttrack_mode {
|
||||
fields.push("web.decoy_fasttrack_mode".to_string());
|
||||
effective.web.decoy_fasttrack_mode = old.web.decoy_fasttrack_mode;
|
||||
}
|
||||
if effective.web.carrier_negotiation_enabled()
|
||||
&& effective.web.carrier_learning
|
||||
&& effective.web.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||
{
|
||||
if old.web.carrier_learning != desired.web.carrier_learning {
|
||||
fields.push("web.carrier_learning".to_string());
|
||||
effective.web.carrier_learning = old.web.carrier_learning;
|
||||
} else {
|
||||
fields.push("web.carriers".to_string());
|
||||
effective.web.carriers = old.web.carriers.clone();
|
||||
}
|
||||
}
|
||||
if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
|
||||
fields.push("web.debug".to_string());
|
||||
effective.web.debug = old.web.debug.clone();
|
||||
}
|
||||
effective
|
||||
.validate_effective_web()
|
||||
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
|
||||
effective
|
||||
.rebuild_runtime_user_auth()
|
||||
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
|
||||
effective
|
||||
.rebuild_runtime_web()
|
||||
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
|
||||
let runtime_changed = !configs_equal(old, &effective);
|
||||
ResolvedReloadConfig {
|
||||
Ok(ResolvedReloadConfig {
|
||||
effective,
|
||||
deferred_process_fields: fields,
|
||||
runtime_changed,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool {
|
||||
@@ -463,8 +492,11 @@ fn listener_process_fields_equal(old: &ServerConfig, desired: &ServerConfig) ->
|
||||
}
|
||||
|
||||
/// Returns process-owned fields that cannot change in the current generation.
|
||||
pub(crate) fn deferred_process_fields(old: &ProxyConfig, new: &ProxyConfig) -> Vec<String> {
|
||||
resolve_reload_config(old, new).deferred_process_fields
|
||||
pub(crate) fn deferred_process_fields(
|
||||
old: &ProxyConfig,
|
||||
new: &ProxyConfig,
|
||||
) -> Result<Vec<String>, String> {
|
||||
resolve_reload_config(old, new).map(|resolved| resolved.deferred_process_fields)
|
||||
}
|
||||
|
||||
fn configs_equal(old: &ProxyConfig, new: &ProxyConfig) -> bool {
|
||||
|
||||
@@ -24,6 +24,43 @@ fn test_listener(port: u16) -> crate::config::ListenerConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn web_config_with_fasttrack(mode: &str) -> ProxyConfig {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.toml");
|
||||
let config = format!(
|
||||
r#"
|
||||
[access.users]
|
||||
alice = "000102030405060708090a0b0c0d0e0f"
|
||||
|
||||
[[server.listeners]]
|
||||
ip = "127.0.0.1"
|
||||
port = 18080
|
||||
transport = "web"
|
||||
proxy_protocol = false
|
||||
web_client_ip_source = "x_forwarded_for"
|
||||
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||
|
||||
[web]
|
||||
enabled = true
|
||||
decoy_fasttrack_mode = "{mode}"
|
||||
|
||||
[[web.vhosts]]
|
||||
host = "proxy.example.com"
|
||||
public_addr = "203.0.113.10:443"
|
||||
|
||||
[web.vhosts.decoy]
|
||||
mode = "http_upstream"
|
||||
upstream = "http://127.0.0.1:18081"
|
||||
|
||||
[[web.vhosts.profiles]]
|
||||
user = "alice"
|
||||
secret_mode = "plain"
|
||||
"#,
|
||||
);
|
||||
std::fs::write(&path, config).unwrap();
|
||||
ProxyConfig::load(path).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_socket_and_logging_changes_are_deferred() {
|
||||
let old = ProxyConfig::default();
|
||||
@@ -31,7 +68,7 @@ fn process_socket_and_logging_changes_are_deferred() {
|
||||
new.server.listen_backlog = new.server.listen_backlog.saturating_add(1);
|
||||
new.general.disable_colors = !new.general.disable_colors;
|
||||
|
||||
let fields = deferred_process_fields(&old, &new);
|
||||
let fields = deferred_process_fields(&old, &new).unwrap();
|
||||
assert!(fields.contains(&"server.listeners".to_string()));
|
||||
assert!(fields.contains(&"general.disable_colors".to_string()));
|
||||
}
|
||||
@@ -43,7 +80,7 @@ fn global_mss_profiles_are_deferred_with_the_listener_socket_group() {
|
||||
desired.server.client_mss = Some("92".to_string());
|
||||
desired.server.client_mss_bulk = Some("1400".to_string());
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
@@ -64,7 +101,7 @@ fn mixed_reload_retains_process_state_and_applies_runtime_state() {
|
||||
desired.server.client_mss = Some("92".to_string());
|
||||
desired.censorship.tls_domain = "reload.example".to_string();
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(resolved.effective.server.client_mss, old.server.client_mss);
|
||||
assert_eq!(
|
||||
@@ -105,7 +142,7 @@ fn listener_announcement_is_runtime_owned_when_bind_identity_is_stable() {
|
||||
let mut desired = old.clone();
|
||||
desired.server.listeners[0].announce = Some("proxy.example".to_string());
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert!(resolved.deferred_process_fields.is_empty());
|
||||
assert_eq!(
|
||||
@@ -128,7 +165,7 @@ fn process_field_labels_are_stable_ordered_and_unique() {
|
||||
.saturating_add(1);
|
||||
desired.general.disable_colors = !desired.general.disable_colors;
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
@@ -146,7 +183,7 @@ fn runtime_only_change_does_not_require_process_rebind() {
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = old.clone();
|
||||
new.censorship.tls_domain = "reload.example".to_string();
|
||||
assert!(deferred_process_fields(&old, &new).is_empty());
|
||||
assert!(deferred_process_fields(&old, &new).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -157,7 +194,7 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
||||
let mut desired = old.clone();
|
||||
desired.web.limits.max_sessions_global += 1;
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
@@ -170,6 +207,89 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
||||
assert!(!resolved.runtime_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_decoy_fasttrack_mode_is_deferred_without_runtime_publication() {
|
||||
let old = web_config_with_fasttrack("off");
|
||||
let desired = web_config_with_fasttrack("enforce");
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
vec!["web.decoy_fasttrack_mode".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.effective.web.decoy_fasttrack_mode,
|
||||
old.web.decoy_fasttrack_mode
|
||||
);
|
||||
let effective_runtime = resolved.effective.web.runtime.as_ref().unwrap();
|
||||
let effective_vhost = &effective_runtime.vhosts["proxy.example.com"];
|
||||
assert_eq!(
|
||||
effective_vhost.decoy_fasttrack_mode,
|
||||
old.web.decoy_fasttrack_mode
|
||||
);
|
||||
assert!(!resolved.runtime_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabling_learning_is_deferred_when_retained_capacity_is_too_small() {
|
||||
let mut old = ProxyConfig::default();
|
||||
old.web.limits.max_carrier_learning_entries = 1;
|
||||
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||
old.web.carrier_learning = false;
|
||||
old.rebuild_runtime_user_auth().unwrap();
|
||||
old.rebuild_runtime_web().unwrap();
|
||||
let mut desired = old.clone();
|
||||
desired.web.limits.max_carrier_learning_entries = 3;
|
||||
desired.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||
crate::config::WebCarrier::Websocket,
|
||||
crate::config::WebCarrier::Https,
|
||||
]);
|
||||
desired.web.carrier_learning = true;
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
vec!["web.limits".to_string(), "web.carrier_learning".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.effective.web.limits.max_carrier_learning_entries,
|
||||
1
|
||||
);
|
||||
assert!(resolved.effective.web.carrier_negotiation_enabled());
|
||||
assert!(!resolved.effective.web.carrier_learning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabling_carriers_is_deferred_for_dormant_learning_with_small_capacity() {
|
||||
let mut old = ProxyConfig::default();
|
||||
old.web.limits.max_carrier_learning_entries = 1;
|
||||
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||
old.web.carrier_learning = true;
|
||||
old.rebuild_runtime_user_auth().unwrap();
|
||||
old.rebuild_runtime_web().unwrap();
|
||||
let mut desired = old.clone();
|
||||
desired.web.limits.max_carrier_learning_entries = 3;
|
||||
desired.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||
crate::config::WebCarrier::Websocket,
|
||||
crate::config::WebCarrier::Https,
|
||||
]);
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
vec!["web.limits".to_string(), "web.carriers".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.effective.web.limits.max_carrier_learning_entries,
|
||||
1
|
||||
);
|
||||
assert!(!resolved.effective.web.carrier_negotiation_enabled());
|
||||
assert!(resolved.effective.web.carrier_learning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
|
||||
let mut old = ProxyConfig::default();
|
||||
@@ -179,7 +299,7 @@ fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
|
||||
desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||
desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
@@ -206,7 +326,7 @@ fn endpoint_only_listener_move_is_runtime_rebindable() {
|
||||
let mut desired = old.clone();
|
||||
desired.server.listeners[0].port = Some(8443);
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert!(resolved.deferred_process_fields.is_empty());
|
||||
assert_eq!(resolved.effective.server.listeners[0].port, Some(8443));
|
||||
@@ -221,7 +341,7 @@ fn synlimited_endpoint_move_remains_restart_only() {
|
||||
let mut desired = old.clone();
|
||||
desired.server.listeners[0].port = Some(8443);
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
@@ -230,3 +350,28 @@ fn synlimited_endpoint_move_remains_restart_only() {
|
||||
assert_eq!(resolved.effective.server.listeners[0].port, Some(443));
|
||||
assert!(!resolved.runtime_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_listener_identity_cannot_create_an_effective_decoy_loop() {
|
||||
let mut old = ProxyConfig::default();
|
||||
old.server.listeners = vec![test_listener(18080)];
|
||||
old.server.listeners[0].transport = crate::config::ListenerTransport::Web;
|
||||
let mut desired = old.clone();
|
||||
desired.server.listeners[0].port = Some(18081);
|
||||
desired.server.listen_backlog = desired.server.listen_backlog.saturating_add(1);
|
||||
desired.web.vhosts = vec![
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"host": "proxy.example",
|
||||
"public_addr": "203.0.113.10:443",
|
||||
"decoy": {
|
||||
"mode": "http_upstream",
|
||||
"upstream": "http://127.0.0.1:18080"
|
||||
},
|
||||
"profiles": []
|
||||
}))
|
||||
.unwrap(),
|
||||
];
|
||||
|
||||
assert!(desired.validate_web_decoy_listener_separation().is_ok());
|
||||
assert!(resolve_reload_config(&old, &desired).is_err());
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
||||
|
||||
use super::control_plane::ProcessControlPlane;
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::write_beobachten_snapshot;
|
||||
@@ -158,6 +159,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
task_scope.cancellation_token(),
|
||||
Some(upstream_manager.dns_resolver()),
|
||||
config_watcher_activation,
|
||||
);
|
||||
task_scope.spawn(config_watcher_task);
|
||||
@@ -168,7 +170,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
)
|
||||
.await;
|
||||
let stats_policy = stats.clone();
|
||||
let upstream_policy = upstream_manager.clone();
|
||||
let mut config_rx_policy = config_rx.clone();
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
@@ -178,9 +179,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
let cfg = config_rx_policy.borrow_and_update().clone();
|
||||
stats_policy
|
||||
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
|
||||
if let Err(error) = upstream_policy.update_dns_overrides(&cfg.network.dns_overrides) {
|
||||
warn!(error = %error, "Failed to update generation DNS overrides");
|
||||
}
|
||||
if let Some(pool) = &me_pool_for_policy {
|
||||
pool.update_runtime_transport_policy(
|
||||
cfg.general.me_socks_kdf_policy,
|
||||
@@ -405,7 +403,10 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
config: &Arc<ProxyConfig>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
) {
|
||||
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
|
||||
tls_full_cert_budget: Arc<crate::tls_front::cache::TlsFullCertBudget>,
|
||||
control_plane: ProcessControlPlane,
|
||||
) -> std::io::Result<()> {
|
||||
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
||||
let metrics_target: Option<(u16, Option<String>)> =
|
||||
if let Some(ref listen) = config.server.metrics_listen {
|
||||
@@ -413,12 +414,15 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
Ok(addr) => Some((addr.port(), Some(listen.clone()))),
|
||||
Err(e) => {
|
||||
startup_tracker
|
||||
.skip_component(
|
||||
.fail_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some(format!("invalid metrics_listen \"{}\": {}", listen, e)),
|
||||
)
|
||||
.await;
|
||||
None
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid metrics_listen \"{}\": {}", listen, e),
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -434,15 +438,30 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
Some(format!("spawn metrics endpoint on {}", label)),
|
||||
)
|
||||
.await;
|
||||
let active_runtime = active_runtime.clone();
|
||||
let listen_backlog = config.server.listen_backlog;
|
||||
tokio::spawn(async move {
|
||||
metrics::serve(port, listen, listen_backlog, active_runtime).await;
|
||||
});
|
||||
let bound = match metrics::bind(port, listen, listen_backlog) {
|
||||
Ok(bound) => bound,
|
||||
Err(error) => {
|
||||
startup_tracker
|
||||
.fail_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some(format!("metrics listener bind failed: {error}")),
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
metrics::serve(
|
||||
bound,
|
||||
active_runtime,
|
||||
web_runtime_rx,
|
||||
tls_full_cert_budget,
|
||||
control_plane,
|
||||
);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_METRICS_START,
|
||||
Some("metrics task spawned".to_string()),
|
||||
Some("metrics listeners bound and supervised".to_string()),
|
||||
)
|
||||
.await;
|
||||
} else if config.server.metrics_listen.is_none() {
|
||||
@@ -453,6 +472,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc<StartupTracker>) {
|
||||
|
||||
+30
-22
@@ -8,7 +8,7 @@
|
||||
//!
|
||||
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -19,9 +19,11 @@ use tokio::signal;
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::control_plane::ProcessControlPlane;
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::helpers::{format_uptime, unit_label};
|
||||
use super::reload_supervisor::ReloadSupervisorHandle;
|
||||
use crate::quota_state::QuotaStateOwner;
|
||||
use crate::stats::Stats;
|
||||
use crate::synlimit_control;
|
||||
|
||||
@@ -50,16 +52,18 @@ impl std::fmt::Display for ShutdownSignal {
|
||||
pub(crate) async fn wait_for_shutdown(
|
||||
process_started_at: Instant,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
quota_state: Arc<QuotaStateOwner>,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
let signal = wait_for_shutdown_signal().await;
|
||||
perform_shutdown(
|
||||
signal,
|
||||
process_started_at,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
quota_state,
|
||||
reload_supervisor,
|
||||
process_control_plane,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -89,8 +93,9 @@ async fn perform_shutdown(
|
||||
signal: ShutdownSignal,
|
||||
process_started_at: Instant,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
quota_state: Arc<QuotaStateOwner>,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
let shutdown_started_at = Instant::now();
|
||||
info!(signal = %signal, "Received shutdown signal");
|
||||
@@ -115,37 +120,38 @@ async fn perform_shutdown(
|
||||
// Graceful ME pool shutdown
|
||||
runtime.stop_sessions().await;
|
||||
runtime.stop_background_tasks().await;
|
||||
if let Some(pool) = runtime.current_me_pool().await {
|
||||
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
{
|
||||
Ok(total) => {
|
||||
info!(
|
||||
close_conn_sent = total,
|
||||
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
|
||||
}
|
||||
}
|
||||
if runtime.stop_middle_end(Duration::from_secs(5)).await {
|
||||
info!("ME shutdown: pool lifecycle completed");
|
||||
} else {
|
||||
warn!("ME shutdown: pool lifecycle deadline expired");
|
||||
}
|
||||
|
||||
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
|
||||
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
||||
}
|
||||
|
||||
match crate::quota_state::save_quota_state("a_state_path, stats).await {
|
||||
if !process_control_plane.shutdown(Duration::from_secs(5)).await {
|
||||
warn!("Process control-plane task shutdown deadline expired");
|
||||
}
|
||||
|
||||
let configured_quota_users = runtime
|
||||
.config()
|
||||
.access
|
||||
.users
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
match quota_state.save(&configured_quota_users).await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
path = %quota_state_path.display(),
|
||||
path = %quota_state.path().display(),
|
||||
"Persisted per-user quota state"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
error = %error,
|
||||
path = %quota_state_path.display(),
|
||||
path = %quota_state.path().display(),
|
||||
"Failed to persist per-user quota state"
|
||||
);
|
||||
}
|
||||
@@ -205,8 +211,9 @@ fn dump_stats(stats: &Stats, process_started_at: Instant) {
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
process_started_at: Instant,
|
||||
process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let _ = process_control_plane.spawn(async move {
|
||||
let mut sigusr1 =
|
||||
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
||||
let mut sigusr2 =
|
||||
@@ -231,6 +238,7 @@ pub(crate) fn spawn_signal_handlers(
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
_process_started_at: Instant,
|
||||
_process_control_plane: ProcessControlPlane,
|
||||
) {
|
||||
// No SIGUSR1/SIGUSR2 on non-Unix
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::config::ProxyConfig;
|
||||
use crate::error::{ProxyError, Result};
|
||||
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::tls_front::cache::TlsFullCertBudget;
|
||||
use crate::tls_front::fetcher::TlsFetchStrategy;
|
||||
use crate::transport::UpstreamManager;
|
||||
|
||||
@@ -109,6 +110,7 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
full_cert_budget: Arc<TlsFullCertBudget>,
|
||||
policy: TlsBootstrapPolicy,
|
||||
) -> Result<Option<Arc<TlsFrontCache>>> {
|
||||
startup_tracker
|
||||
@@ -128,10 +130,11 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let cache = Arc::new(TlsFrontCache::new(
|
||||
let cache = Arc::new(TlsFrontCache::new_with_full_cert_budget(
|
||||
tls_domains,
|
||||
config.censorship.fake_cert_len,
|
||||
&config.censorship.tls_front_dir,
|
||||
full_cert_budget,
|
||||
));
|
||||
cache.load_from_disk().await;
|
||||
|
||||
@@ -301,6 +304,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await;
|
||||
@@ -336,6 +340,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
@@ -364,6 +369,7 @@ mod tests {
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
Arc::new(TlsFullCertBudget::new()),
|
||||
TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await
|
||||
|
||||
+104
-3938
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
use super::*;
|
||||
|
||||
// Process, buffer, and TLS cache metrics.
|
||||
mod process;
|
||||
// Connection, quota, and conntrack metrics.
|
||||
mod connections;
|
||||
// Rate limiter, upstream, and initial ME metrics.
|
||||
mod traffic;
|
||||
// ME lifecycle and relay event metrics.
|
||||
mod me_lifecycle;
|
||||
// ME batching and resident-memory metrics.
|
||||
mod me_buffers;
|
||||
// ME writer selection, KDF, and hardswap metrics.
|
||||
mod me_policy;
|
||||
// Adaptive-floor and writer-cap metrics.
|
||||
mod me_floor;
|
||||
// Desync, pool recovery, and refill metrics.
|
||||
mod me_recovery;
|
||||
// Bounded per-user and IP-tracker metrics.
|
||||
mod users;
|
||||
|
||||
pub(super) async fn render_metrics(
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
config: &ProxyConfig,
|
||||
ip_tracker: &UserIpTracker,
|
||||
tls_cache: Option<&TlsFrontCache>,
|
||||
tls_full_cert_budget: &TlsFullCertBudget,
|
||||
web_publication: &crate::web::control::WebRuntimePublication,
|
||||
) -> String {
|
||||
let mut out = String::with_capacity(4096);
|
||||
let telemetry = stats.telemetry_policy();
|
||||
let core_enabled = telemetry.core_enabled;
|
||||
let user_enabled = telemetry.user_enabled;
|
||||
let me_allows_normal = telemetry.me_level.allows_normal();
|
||||
let me_allows_debug = telemetry.me_level.allows_debug();
|
||||
|
||||
process::render(
|
||||
&mut out,
|
||||
stats,
|
||||
shared_state,
|
||||
telemetry,
|
||||
tls_full_cert_budget,
|
||||
);
|
||||
super::render_tls_front_profile_health(&mut out, config, tls_cache).await;
|
||||
connections::render(&mut out, stats, shared_state, core_enabled);
|
||||
traffic::render(
|
||||
&mut out,
|
||||
stats,
|
||||
shared_state,
|
||||
config,
|
||||
core_enabled,
|
||||
me_allows_normal,
|
||||
me_allows_debug,
|
||||
);
|
||||
me_lifecycle::render(&mut out, stats, me_allows_normal);
|
||||
me_buffers::render(
|
||||
&mut out,
|
||||
stats,
|
||||
core_enabled,
|
||||
me_allows_normal,
|
||||
me_allows_debug,
|
||||
);
|
||||
me_policy::render(&mut out, stats, me_allows_normal, me_allows_debug);
|
||||
me_floor::render(&mut out, stats, config, me_allows_normal);
|
||||
me_recovery::render(&mut out, stats, me_allows_normal, me_allows_debug);
|
||||
users::render(
|
||||
&mut out,
|
||||
stats,
|
||||
config,
|
||||
ip_tracker,
|
||||
core_enabled,
|
||||
user_enabled,
|
||||
)
|
||||
.await;
|
||||
super::web::render(&mut out, web_publication, config);
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
core_enabled: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_total Total accepted connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_total {}",
|
||||
if core_enabled {
|
||||
stats.get_connects_all()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_bad_total Bad/rejected connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_bad_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_bad_total {}",
|
||||
if core_enabled {
|
||||
stats.get_connects_bad()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter");
|
||||
if core_enabled {
|
||||
for (class, total) in stats.get_connects_bad_class_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_connections_bad_by_class_total{{class=\"{}\"}} {}",
|
||||
class, total
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_handshake_timeouts_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_handshake_timeouts_total {}",
|
||||
if core_enabled {
|
||||
stats.get_handshake_timeouts()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_handshake_failures_by_class_total Handshake failures by class"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_handshake_failures_by_class_total counter"
|
||||
);
|
||||
if core_enabled {
|
||||
for (class, total) in stats.get_handshake_failure_class_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}",
|
||||
class, total
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_auth_expensive_checks_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_auth_expensive_checks_total {}",
|
||||
if core_enabled {
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_expensive_checks_total
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_auth_budget_exhausted_total Handshake validations that hit authentication candidate budget limits"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_auth_budget_exhausted_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_auth_budget_exhausted_total {}",
|
||||
if core_enabled {
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_budget_exhausted_total
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_accept_permit_timeout_total Accepted connections dropped due to permit wait timeout"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_accept_permit_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_accept_permit_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_accept_permit_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_route_cutover_parked_current Sessions currently parked in route cutover stagger delay"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_route_cutover_parked_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_current{{route=\"direct\"}} {}",
|
||||
stats.get_route_cutover_parked_direct_current()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_current{{route=\"middle\"}} {}",
|
||||
stats.get_route_cutover_parked_middle_current()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_route_cutover_parked_total Sessions parked in route cutover stagger delay"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_route_cutover_parked_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_total{{route=\"direct\"}} {}",
|
||||
stats.get_route_cutover_parked_direct_total()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_route_cutover_parked_total{{route=\"middle\"}} {}",
|
||||
stats.get_route_cutover_parked_middle_total()
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_refund_bytes_total Reserved quota bytes returned before commit"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_refund_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_refund_bytes_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_refund_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_contention_total Quota reservation CAS contention events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_contention_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_contention_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_contention_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_contention_timeout_total Quota reservations that hit the bounded contention budget"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_contention_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_contention_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_contention_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_quota_acquire_cancelled_total Quota acquisitions cancelled before reservation completed"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_quota_acquire_cancelled_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_quota_acquire_cancelled_total {}",
|
||||
if core_enabled {
|
||||
stats.get_quota_acquire_cancelled_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_control_state gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"enabled\"}} {}",
|
||||
if stats.get_conntrack_control_enabled() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"available\"}} {}",
|
||||
if stats.get_conntrack_control_available() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"pressure_active\"}} {}",
|
||||
if stats.get_conntrack_pressure_active() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_control_state{{flag=\"rule_apply_ok\"}} {}",
|
||||
if stats.get_conntrack_rule_apply_ok() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_event_queue_depth Pending close events in conntrack control queue"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_event_queue_depth gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_event_queue_depth {}",
|
||||
stats.get_conntrack_event_queue_depth()
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_delete_total Conntrack delete attempts by outcome"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_conntrack_delete_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"attempt\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"success\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"not_found\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_not_found_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_delete_total{{result=\"error\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_delete_error_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_conntrack_close_event_drop_total Dropped conntrack close events due to queue pressure or unavailable sender"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_conntrack_close_event_drop_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_conntrack_close_event_drop_total {}",
|
||||
if core_enabled {
|
||||
stats.get_conntrack_close_event_drop_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,489 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
core_enabled: bool,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"batch_bytes\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_batch_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"max_delay\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_max_delay_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"ack_immediate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_ack_immediate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"close\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_data_frames_total DC->Client data frames"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_data_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_data_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_data_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_ack_frames_total DC->Client quick-ack frames"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_ack_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_ack_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_ack_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_payload_bytes_total DC->Client payload bytes before transport framing"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_payload_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_payload_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_payload_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_write_mode_total DC->Client writer mode selection"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_write_mode_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_write_mode_total{{mode=\"coalesced\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_write_mode_coalesced_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_write_mode_total{{mode=\"split\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_write_mode_split_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_quota_reject_total DC->Client quota rejects"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_quota_reject_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_quota_reject_total{{stage=\"pre_write\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_quota_reject_pre_write_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_quota_reject_total{{stage=\"post_write\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_quota_reject_post_write_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_child_join_timeout_total Middle relay child tasks that did not join before cleanup deadline"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_child_join_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_child_join_timeout_total {}",
|
||||
if core_enabled {
|
||||
stats.get_me_child_join_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_child_abort_total Middle relay child tasks aborted after bounded cleanup timeout"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_child_abort_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_child_abort_total {}",
|
||||
if core_enabled {
|
||||
stats.get_me_child_abort_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_flow_wait_events_total Flow wait events by reason, direction, and outcome"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_flow_wait_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"waited\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"cancelled\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_cancelled_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_flow_wait_ms_total Flow wait time in milliseconds by reason and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_flow_wait_ms_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_flow_wait_ms_total{{reason=\"middle_rate_limit\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_flow_wait_middle_rate_limit_ms_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_session_drop_fallback_total Session reservations cleaned by Drop instead of explicit async release"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_session_drop_fallback_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_session_drop_fallback_total {}",
|
||||
if core_enabled {
|
||||
stats.get_session_drop_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_frame_buf_shrink_total DC->Client reusable frame buffer shrink events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_frame_buf_shrink_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_frame_buf_shrink_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_frame_buf_shrink_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_frame_buf_shrink_bytes_total DC->Client reusable frame buffer bytes released"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_frame_buf_shrink_bytes_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_frame_buf_shrink_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_frame_buf_shrink_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_frames_bucket_total DC->Client batch frame count buckets"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_frames_bucket_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"1\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_1()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"2_4\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_2_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"5_8\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_5_8()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"9_16\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_9_16()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"17_32\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_17_32()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_bucket_total{{bucket=\"gt_32\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_frames_bucket_gt_32()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_bytes_bucket_total DC->Client batch byte size buckets"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_bytes_bucket_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"0_1k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_0_1k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"1k_4k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_1k_4k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"4k_16k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_4k_16k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"16k_64k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_16k_64k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"64k_128k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_64k_128k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_bucket_total{{bucket=\"gt_128k\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_bytes_bucket_gt_128k()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_flush_duration_us_bucket_total DC->Client flush duration buckets"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"0_50\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_0_50()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"51_200\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_51_200()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"201_1000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_201_1000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"1001_5000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_1001_5000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"5001_20000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_5001_20000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_duration_us_bucket_total{{bucket=\"gt_20000\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_flush_duration_us_bucket_gt_20000()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_timeout_armed_total DC->Client max-delay timer armed events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_timeout_armed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_timeout_armed_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_timeout_armed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_timeout_fired_total DC->Client max-delay timer fired events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_d2c_batch_timeout_fired_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_timeout_fired_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_d2c_batch_timeout_fired_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_limit_bytes Configured resident-memory budget per ME writer"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_byte_budget_limit_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_limit_bytes {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_limit_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
me_allows_normal: bool,
|
||||
) {
|
||||
let floor_mode = config.general.me_floor_mode;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"static\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Static) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode{{mode=\"adaptive\"}} {}",
|
||||
if matches!(floor_mode, crate::config::MeFloorMode::Adaptive) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode_switch_all_total Runtime ME floor mode switches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode_switch_all_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_all_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"static\",to=\"adaptive\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_static_to_adaptive_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_mode_switch_total{{from=\"adaptive\",to=\"static\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_mode_switch_adaptive_to_static_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_cpu_cores_detected Runtime detected logical CPU cores for adaptive floor"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_cpu_cores_detected gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_cpu_cores_detected {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cpu_cores_detected_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_cpu_cores_effective Runtime effective logical CPU cores for adaptive floor"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_cpu_cores_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_cpu_cores_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cpu_cores_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_global_cap_raw Runtime raw global adaptive floor cap"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_adaptive_floor_global_cap_raw gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_global_cap_raw {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_global_cap_raw_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_global_cap_effective Runtime effective global adaptive floor cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_global_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_global_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_global_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_target_writers_total Runtime adaptive floor target writers total"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_target_writers_total gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_target_writers_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_target_writers_total_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_active_cap_configured Runtime configured active writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_active_cap_configured gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_active_cap_configured {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_active_cap_configured_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_active_cap_effective Runtime effective active writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_active_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_active_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_active_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_warm_cap_configured Runtime configured warm writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_warm_cap_configured gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_warm_cap_configured {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_warm_cap_configured_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_adaptive_floor_warm_cap_effective Runtime effective warm writer cap"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_adaptive_floor_warm_cap_effective gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_adaptive_floor_warm_cap_effective {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_warm_cap_effective_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writers_active_current Current non-draining active ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writers_active_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writers_active_current {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writers_active_current_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writers_warm_current Current non-draining warm ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writers_warm_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writers_warm_current {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writers_warm_current_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_cap_block_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_cap_block_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_cap_block_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_swap_idle_total Adaptive floor cap recovery via idle writer swap"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_swap_idle_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_swap_idle_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_swap_idle_failed_total Failed idle swap attempts under adaptive floor caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_floor_swap_idle_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_floor_swap_idle_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(out: &mut String, stats: &Stats, me_allows_normal: bool) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reconnect_attempts_total ME reconnect attempts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reconnect_attempts_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reconnect_attempts_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reconnect_attempts()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reconnect_success_total ME reconnect successes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reconnect_success_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reconnect_success_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reconnect_success()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_handshake_reject_total ME handshake rejects from upstream"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_reject_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_reject_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_handshake_reject_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_handshake_error_code_total ME handshake reject errors by code"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_handshake_error_code_total counter");
|
||||
if me_allows_normal {
|
||||
for (error_code, count) in stats.get_me_handshake_error_code_counts() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_error_code_total{{error_code=\"{}\"}} {}",
|
||||
error_code, count
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_handshake_error_code_total{{error_code=\"overflow\"}} {}",
|
||||
stats.get_me_handshake_error_code_overflow_total()
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_reader_eof_total ME reader EOF terminations"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_reader_eof_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_reader_eof_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_reader_eof_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_idle_close_by_peer_total ME idle writers closed by peer"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_idle_close_by_peer_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_idle_close_by_peer_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_idle_close_by_peer_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_idle_soft_mark_total Middle-relay sessions marked as soft-idle candidates"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_idle_soft_mark_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_idle_soft_mark_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_idle_soft_mark_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_idle_hard_close_total Middle-relay sessions closed by hard-idle policy"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_idle_hard_close_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_idle_hard_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_idle_hard_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_pressure_evict_total Middle-relay sessions evicted under resource pressure"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_relay_pressure_evict_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_pressure_evict_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_pressure_evict_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_relay_protocol_desync_close_total Middle-relay sessions closed due to protocol desync"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_relay_protocol_desync_close_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_relay_protocol_desync_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_relay_protocol_desync_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches");
|
||||
let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_crc_mismatch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_crc_mismatch()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_seq_mismatch_total ME sequence mismatches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_seq_mismatch_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_seq_mismatch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_seq_mismatch()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_no_conn_total ME route drops: no conn"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_route_drop_no_conn_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_no_conn_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_no_conn()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_channel_closed_total ME route drops: channel closed"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_route_drop_channel_closed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_channel_closed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_channel_closed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_queue_full_total ME route drops: queue full"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_route_drop_queue_full_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_route_drop_queue_full_profile_total ME route drops: queue full by adaptive profile"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_route_drop_queue_full_profile_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_profile_total{{profile=\"base\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full_base()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_route_drop_queue_full_profile_total{{profile=\"high\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_route_drop_queue_full_high()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_pressure_state Worker-local fairness pressure state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_pressure_state gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_pressure_state {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_pressure_state_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_active_flows Fair-scheduler active flow count"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_active_flows gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_active_flows {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_active_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_queued_bytes Fair-scheduler queued bytes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_queued_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_queued_bytes {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_queued_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_flow_state_gauge Fair-scheduler flow health classes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_flow_state_gauge gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_flow_state_gauge{{class=\"standing\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_standing_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_flow_state_gauge{{class=\"backpressured\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_backpressured_flows_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_fair_events_total Fair-scheduler event counters"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_fair_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"scheduler_round\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_scheduler_rounds_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"deficit_grant\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_deficit_grants_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"deficit_skip\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_deficit_skips_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"enqueue_reject\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_enqueue_rejects_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"shed_drop\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_shed_drops_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"penalty\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_penalties_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_fair_events_total{{event=\"downstream_stall\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_fair_downstream_stalls_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_c2me_enqueue_events_total ME client->ME enqueue outcomes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_c2me_enqueue_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"high_water\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_high_water_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_c2me_enqueue_events_total{{event=\"timeout\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_c2me_send_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batches_total Total DC->Client flush batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batches_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batches_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batches_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_frames_total Total DC->Client frames flushed in batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_frames_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_frames_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batch_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_batch_bytes_total Total DC->Client bytes flushed in batches"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_batch_bytes_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_batch_bytes_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_batch_bytes_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_d2c_flush_reason_total DC->Client flush reasons"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_d2c_flush_reason_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"queue_drain\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_queue_drain_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_d2c_flush_reason_total{{reason=\"batch_frames\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_d2c_flush_reason_batch_frames_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_reserved_bytes Aggregate ME writer memory reservations by lifecycle state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_reserved_bytes gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"queued\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_queued_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"inflight\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_inflight_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_events_total ME writer byte-budget outcomes"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_events_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"wait\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_wait_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"timeout\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"oversize\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_oversize_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_try\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_success_try_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_fallback\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_success_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"closed\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_closed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"no_candidate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_sorted_rr_no_candidate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_try\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_success_try_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_fallback\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_success_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"full\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_full_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"closed\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_closed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"no_candidate\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_p2c_no_candidate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_blocking_fallback_total ME writer-pick blocking fallback attempts"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_pick_blocking_fallback_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_blocking_fallback_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_blocking_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_mode_switch_total Writer-pick mode switches via runtime updates"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_pick_mode_switch_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_pick_mode_switch_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_pick_mode_switch_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_socks_kdf_policy_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_socks_kdf_policy_total{{policy=\"strict\",outcome=\"reject\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_socks_kdf_strict_reject()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_socks_kdf_policy_total{{policy=\"compat\",outcome=\"fallback\"}} {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_socks_kdf_compat_fallback()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_total ME endpoint quarantines due to rapid flaps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_endpoint_quarantine_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_unexpected_total ME endpoint quarantines caused by unexpected writer removals"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_unexpected_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_unexpected_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_endpoint_quarantine_draining_suppressed_total Draining writer removals that skipped endpoint quarantine"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_endpoint_quarantine_draining_suppressed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_endpoint_quarantine_draining_suppressed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_kdf_drift_total ME KDF input drift detections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_drift_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_kdf_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_kdf_port_only_drift_total ME KDF client-port changes with stable non-port material"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_kdf_port_only_drift_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_kdf_port_only_drift_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_kdf_port_only_drift_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_reuse_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_reuse_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_hardswap_pending_reuse_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hardswap_pending_ttl_expired_total Pending hardswap generations reset by TTL expiration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_hardswap_pending_ttl_expired_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hardswap_pending_ttl_expired_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_hardswap_pending_ttl_expired_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_enter_total Single-endpoint DC outage transitions to active state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_enter_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_enter_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_enter_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_exit_total Single-endpoint DC outage recovery transitions"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_exit_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_exit_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_exit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_attempt_total Reconnect attempts performed during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_attempt_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_attempt_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_outage_reconnect_success_total Successful reconnect attempts during single-endpoint outages"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_outage_reconnect_success_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_outage_reconnect_success_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_outage_reconnect_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_quarantine_bypass_total Outage reconnect attempts that bypassed quarantine"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_quarantine_bypass_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_quarantine_bypass_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_quarantine_bypass_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_total Successful periodic shadow rotations for single-endpoint DC groups"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total Shadow rotations skipped because endpoint is quarantined"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_single_endpoint_shadow_rotate_skipped_quarantine_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_floor_mode Runtime ME writer floor policy mode"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_floor_mode gauge");
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_secure_padding_invalid_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_secure_padding_invalid()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_total Total crypto-desync detections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_full_logged_total Full forensic desync logs emitted"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_full_logged_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_full_logged_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_full_logged()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_suppressed_total Suppressed desync forensic events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_suppressed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_suppressed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_suppressed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_desync_frames_bucket_total Desync count by frames_ok bucket"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_desync_frames_bucket_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"0\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_0()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"1_2\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_1_2()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"3_10\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_3_10()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_desync_frames_bucket_total{{bucket=\"gt_10\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_desync_frames_bucket_gt_10()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_swap_total Successful ME pool swaps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_swap_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_swap_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_swap_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_drain_active Active draining ME writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_drain_active gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_drain_active {}",
|
||||
if me_allows_debug {
|
||||
stats.get_pool_drain_active()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_force_close_total Forced close events for draining writers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_force_close_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_force_close_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_force_close_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_pool_stale_pick_total Stale writer fallback picks for new binds"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_pool_stale_pick_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_pool_stale_pick_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_pool_stale_pick_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_total Total ME writer removals"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_removed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_writer_removed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_unexpected_total Unexpected ME writer removals that triggered refill"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_removed_unexpected_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_unexpected_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_removed_unexpected_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_triggered_total Immediate ME refill runs started"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_refill_triggered_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_triggered_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_refill_triggered_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_skipped_inflight_total Immediate ME refill skips due to inflight dedup"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_refill_skipped_inflight_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_skipped_inflight_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_refill_skipped_inflight_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_refill_failed_total Immediate ME refill failures"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_refill_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_refill_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_refill_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_restored_same_endpoint_total Refilled ME writer restored on the same endpoint"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_restored_same_endpoint_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_restored_same_endpoint_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_restored_same_endpoint_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_restored_fallback_total Refilled ME writer restored via fallback endpoint"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_restored_fallback_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_restored_fallback_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_restored_fallback_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_no_writer_failfast_total ME route failfast errors due to missing writer in bounded wait window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_no_writer_failfast_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_no_writer_failfast_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_no_writer_failfast_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_hybrid_timeout_total ME hybrid route timeouts after bounded retry window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_hybrid_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_hybrid_timeout_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_hybrid_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_async_recovery_trigger_total Async ME recovery trigger attempts from route path"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_async_recovery_trigger_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_async_recovery_trigger_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_async_recovery_trigger_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_inline_recovery_total Legacy inline ME recovery attempts from route path"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_inline_recovery_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_inline_recovery_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_inline_recovery_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let unresolved_writer_losses = if me_allows_normal {
|
||||
stats
|
||||
.get_me_writer_removed_unexpected_total()
|
||||
.saturating_sub(
|
||||
stats
|
||||
.get_me_writer_restored_same_endpoint_total()
|
||||
.saturating_add(stats.get_me_writer_restored_fallback_total()),
|
||||
)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_removed_unexpected_minus_restored_total Unexpected writer removals not yet compensated by restore"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_removed_unexpected_minus_restored_total {}",
|
||||
unresolved_writer_losses
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
telemetry: crate::stats::telemetry::TelemetryPolicy,
|
||||
tls_full_cert_budget: &TlsFullCertBudget,
|
||||
) {
|
||||
let core_enabled = telemetry.core_enabled;
|
||||
let user_enabled = telemetry.user_enabled;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_build_info Build information for the running telemt binary"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_build_info gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
|
||||
let _ = writeln!(out, "# HELP telemt_uptime_seconds Proxy uptime");
|
||||
let _ = writeln!(out, "# TYPE telemt_uptime_seconds gauge");
|
||||
let _ = writeln!(out, "telemt_uptime_seconds {:.1}", stats.uptime_secs());
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_core_enabled Runtime core telemetry switch"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_core_enabled gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_core_enabled {}",
|
||||
if core_enabled { 1 } else { 0 }
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_enabled Runtime per-user telemetry switch"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_enabled gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_enabled {}",
|
||||
if user_enabled { 1 } else { 0 }
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_stats_user_entries Retained per-user stats entries"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_stats_user_entries gauge");
|
||||
let _ = writeln!(out, "telemt_stats_user_entries {}", stats.user_stats_len());
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_me_level Runtime ME telemetry level flag"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_me_level gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"silent\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Silent) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"normal\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Normal) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_me_level{{level=\"debug\"}} {}",
|
||||
if matches!(telemetry.me_level, crate::config::MeTelemetryLevel::Debug) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_buffer_pool_buffers_total Snapshot of pooled and allocated buffers"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_buffer_pool_buffers_total gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"pooled\"}} {}",
|
||||
stats.get_buffer_pool_pooled_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"allocated\"}} {}",
|
||||
stats.get_buffer_pool_allocated_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"in_use\"}} {}",
|
||||
stats.get_buffer_pool_in_use_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_buffer_pool_events_total Buffer-pool allocation lifecycle events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_buffer_pool_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_events_total{{event=\"replaced_nonstandard\"}} {}",
|
||||
stats.get_buffer_pool_replaced_nonstandard_total()
|
||||
);
|
||||
|
||||
let direct_budget = shared_state.direct_buffer_budget.snapshot();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_bytes Direct relay copy-buffer budget and memory inputs"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_budget_bytes gauge");
|
||||
for (kind, value) in [
|
||||
("hard_limit", direct_budget.hard_limit_bytes),
|
||||
("target", direct_budget.target_bytes),
|
||||
("reserved", direct_budget.reserved_bytes),
|
||||
("memory_total", direct_budget.memory_total_bytes),
|
||||
("memory_available", direct_budget.memory_available_bytes),
|
||||
("process_rss", direct_budget.process_rss_bytes),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_bytes{{kind=\"{}\"}} {}",
|
||||
kind, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_events_total Direct relay buffer-budget lifecycle events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_direct_relay_buffer_budget_events_total counter"
|
||||
);
|
||||
for (result, value) in [
|
||||
("promotion", direct_budget.promotion_total),
|
||||
("promotion_denied", direct_budget.promotion_denied_total),
|
||||
("minimum_fallback", direct_budget.minimum_fallback_total),
|
||||
("admission_rejected", direct_budget.admission_rejected_total),
|
||||
("quiet_demotion", direct_budget.quiet_demotion_total),
|
||||
(
|
||||
"write_pressure_demotion",
|
||||
direct_budget.write_pressure_demotion_total,
|
||||
),
|
||||
(
|
||||
"global_pressure_demotion",
|
||||
direct_budget.global_pressure_demotion_total,
|
||||
),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_events_total{{result=\"{}\"}} {}",
|
||||
result, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_sessions Current Direct relay sessions by adaptive tier"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_sessions gauge");
|
||||
for (tier, value) in ["base", "tier1", "tier2", "tier3"]
|
||||
.into_iter()
|
||||
.zip(direct_budget.tier_sessions)
|
||||
{
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_sessions{{tier=\"{}\"}} {}",
|
||||
tier, value
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_fetch_profile_cache_entries Current adaptive TLS fetch profile-cache entries"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_tls_fetch_profile_cache_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_fetch_profile_cache_entries {}",
|
||||
fetcher::profile_cache_entries_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_fetch_profile_cache_cap_drops_total Profile-cache winner inserts skipped because the cache cap was reached"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_fetch_profile_cache_cap_drops_total {}",
|
||||
fetcher::profile_cache_cap_drops_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_full_cert_budget_entries Current domain and IP entries tracked by the process-owned TLS full-cert budget"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_full_cert_budget_entries gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_full_cert_budget_entries {}",
|
||||
tls_full_cert_budget.entries_for_metrics()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_full_cert_budget_cap_drops_total New domain and IP entries denied full-cert budget tracking because a bound was reached"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
|
||||
tls_full_cert_budget.cap_drops_for_metrics()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,516 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
shared_state: &ProxySharedState,
|
||||
config: &ProxyConfig,
|
||||
core_enabled: bool,
|
||||
me_allows_normal: bool,
|
||||
me_allows_debug: bool,
|
||||
) {
|
||||
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_burst_bound_bytes Configured upper bound for one direct relay rate-limit burst"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_burst_bound_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_burst_bound_bytes{{direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
config.general.direct_relay_copy_buf_c2s_bytes
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_burst_bound_bytes{{direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
config.general.direct_relay_copy_buf_s2c_bytes
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_throttle_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_throttle_up_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_throttle_down_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_throttle_up_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_throttle_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_throttle_down_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_wait_ms_total Traffic limiter accumulated wait time in milliseconds by scope and direction"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_wait_ms_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_wait_up_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"user\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_wait_down_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"up\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_wait_up_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_wait_ms_total{{scope=\"cidr\",direction=\"down\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_wait_down_ms_total
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_active_leases Active relay leases under rate limiting by scope"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_active_leases gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_active_leases{{scope=\"user\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_active_leases
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_active_leases{{scope=\"cidr\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_active_leases
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_rate_limiter_policy_entries Active rate-limit policy entries by scope"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_rate_limiter_policy_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_policy_entries{{scope=\"user\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.user_policy_entries
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_rate_limiter_policy_entries{{scope=\"cidr\"}} {}",
|
||||
if core_enabled {
|
||||
limiter_metrics.cidr_policy_entries
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_attempt_total Upstream connect attempts across all requests"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_attempt_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempt_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempt_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_success_total Successful upstream connect request cycles"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_success_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_success_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_success_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_fail_total Failed upstream connect request cycles"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_upstream_connect_fail_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_fail_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_fail_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_failfast_hard_error_total Hard errors that triggered upstream connect failfast"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_failfast_hard_error_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_failfast_hard_error_total {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_failfast_hard_error_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_attempts_per_request Histogram-like buckets for attempts per upstream connect request cycle"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_attempts_per_request counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"1\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_1()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"2\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_2()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"3_4\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_3_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_attempts_per_request{{bucket=\"gt_4\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_attempts_bucket_gt_4()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_duration_success_total Histogram-like buckets of successful upstream connect cycle duration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_duration_success_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"le_100ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_le_100ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"101_500ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_101_500ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"501_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_501_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_success_total{{bucket=\"gt_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_success_bucket_gt_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_upstream_connect_duration_fail_total Histogram-like buckets of failed upstream connect cycle duration"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_upstream_connect_duration_fail_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"le_100ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_le_100ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"101_500ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_101_500ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"501_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_501_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_upstream_connect_duration_fail_total{{bucket=\"gt_1000ms\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_upstream_connect_duration_fail_bucket_gt_1000ms()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_sent_total ME keepalive frames sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_sent_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_sent_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_keepalive_sent()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_failed_total ME keepalive send failures"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_failed_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_keepalive_failed()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_pong_total ME keepalive pong replies"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_pong_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_pong_total {}",
|
||||
if me_allows_debug {
|
||||
stats.get_me_keepalive_pong()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_keepalive_timeout_total ME keepalive ping timeouts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_keepalive_timeout_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_keepalive_timeout_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_keepalive_timeout()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_sent_total Service RPC_PROXY_REQ activity signals sent"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_sent_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_sent_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_failed_total Service RPC_PROXY_REQ activity signal failures"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_failed_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_failed_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_failed_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_skipped_no_meta_total Service RPC_PROXY_REQ skipped due to missing writer metadata"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_skipped_no_meta_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_skipped_no_meta_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_skipped_no_meta_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_response_total Service RPC_PROXY_REQ responses observed"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_response_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_response_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_response_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_rpc_proxy_req_signal_close_sent_total Service RPC_CLOSE_EXT sent after activity signals"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_rpc_proxy_req_signal_close_sent_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_rpc_proxy_req_signal_close_sent_total {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_rpc_proxy_req_signal_close_sent_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
use super::*;
|
||||
use std::fmt::Write;
|
||||
|
||||
pub(super) async fn render(
|
||||
out: &mut String,
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
ip_tracker: &UserIpTracker,
|
||||
core_enabled: bool,
|
||||
user_enabled: bool,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_connections_total Per-user total connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_connections_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_connections_current Per-user active connections"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_connections_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_octets_from_client_total Per-user total bytes received"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_octets_from_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_octets_to_client_total Per-user total bytes sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_octets_to_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_msgs_from_client_total Per-user total messages received"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_msgs_from_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_msgs_to_client_total Per-user total messages sent"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_msgs_to_client_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_reservation_rollback_total IP reservation rollbacks caused by later limit checks"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_reservation_rollback_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_reservation_rollback_total{{reason=\"tcp_limit\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_ip_reservation_rollback_tcp_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_reservation_rollback_total{{reason=\"quota_limit\"}} {}",
|
||||
if core_enabled {
|
||||
stats.get_ip_reservation_rollback_quota_limit_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let ip_memory = ip_tracker.memory_stats().await;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_users Number of users tracked by IP limiter state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_users gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_users{{scope=\"active\"}} {}",
|
||||
ip_memory.active_users
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_users{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_users
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_entries Number of IP entries tracked by limiter state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_entries gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_entries{{scope=\"active\"}} {}",
|
||||
ip_memory.active_entries
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_entries{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_entries
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cleanup_queue_len Deferred disconnect cleanup queue length"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_queue_len gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cleanup_queue_len {}",
|
||||
ip_memory.cleanup_queue_len
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cleanup_total Release cleanups deferred through the cleanup queue"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cleanup_total{{path=\"deferred\"}} {}",
|
||||
ip_memory.cleanup_deferred_releases
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_ip_tracker_cap_rejects_total New connection rejects caused by global IP tracker caps"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cap_rejects_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cap_rejects_total{{scope=\"active\"}} {}",
|
||||
ip_memory.active_cap_rejects
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_ip_tracker_cap_rejects_total{{scope=\"recent\"}} {}",
|
||||
ip_memory.recent_cap_rejects
|
||||
);
|
||||
|
||||
let mut user_stats_emitted = 0usize;
|
||||
let mut user_stats_suppressed = 0usize;
|
||||
let mut unique_ip_emitted = 0usize;
|
||||
let mut unique_ip_suppressed = 0usize;
|
||||
|
||||
if user_enabled {
|
||||
for entry in stats.iter_user_stats() {
|
||||
if user_stats_emitted >= USER_LABELED_METRICS_MAX_USERS {
|
||||
user_stats_suppressed = user_stats_suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
let user = entry.key();
|
||||
let s = entry.value();
|
||||
user_stats_emitted = user_stats_emitted.saturating_add(1);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_connections_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.connects.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_connections_current{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.curr_connects.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_octets_from_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.octets_from_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_octets_to_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.octets_to_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_msgs_from_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.msgs_from_client
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_msgs_to_client_total{{user=\"{}\"}} {}",
|
||||
user,
|
||||
s.msgs_to_client.load(std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
}
|
||||
|
||||
let ip_stats = ip_tracker.get_stats_snapshot().await;
|
||||
let ip_counts: HashMap<String, usize> = ip_stats
|
||||
.into_iter()
|
||||
.map(|(user, count, _)| (user, count))
|
||||
.collect();
|
||||
|
||||
let mut unique_users = BTreeSet::new();
|
||||
unique_users.extend(config.access.users.keys().cloned());
|
||||
unique_users.extend(config.access.user_max_unique_ips.keys().cloned());
|
||||
unique_users.extend(ip_counts.keys().cloned());
|
||||
let unique_users_vec: Vec<String> = unique_users.iter().cloned().collect();
|
||||
let recent_counts = ip_tracker
|
||||
.get_recent_counts_for_users_snapshot(&unique_users_vec)
|
||||
.await;
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_current Per-user current number of unique active IPs"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_current gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_recent_window Per-user unique IPs seen in configured observation window"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_recent_window gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_limit Effective per-user unique IP limit (0 means unlimited)"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_limit gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_user_unique_ips_utilization Per-user unique IP usage ratio (0 for unlimited)"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge");
|
||||
|
||||
for user in unique_users {
|
||||
if unique_ip_emitted >= USER_LABELED_METRICS_MAX_USERS {
|
||||
unique_ip_suppressed = unique_ip_suppressed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
unique_ip_emitted = unique_ip_emitted.saturating_add(1);
|
||||
let current = ip_counts.get(&user).copied().unwrap_or(0);
|
||||
let limit = config
|
||||
.access
|
||||
.user_max_unique_ips
|
||||
.get(&user)
|
||||
.copied()
|
||||
.filter(|limit| *limit > 0)
|
||||
.or((config.access.user_max_unique_ips_global_each > 0)
|
||||
.then_some(config.access.user_max_unique_ips_global_each))
|
||||
.unwrap_or(0);
|
||||
let utilization = if limit > 0 {
|
||||
current as f64 / limit as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_current{{user=\"{}\"}} {}",
|
||||
user, current
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_recent_window{{user=\"{}\"}} {}",
|
||||
user,
|
||||
recent_counts.get(&user).copied().unwrap_or(0)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_limit{{user=\"{}\"}} {}",
|
||||
user, limit
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_user_unique_ips_utilization{{user=\"{}\"}} {:.6}",
|
||||
user, utilization
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_suppressed {}",
|
||||
if user_enabled && user_stats_suppressed == 0 && unique_ip_suppressed == 0 {
|
||||
0
|
||||
} else {
|
||||
1
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_telemetry_user_series_users User-labeled metric users by export status"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_users gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"emitted\"}} {}",
|
||||
user_stats_emitted
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"suppressed\"}} {}",
|
||||
user_stats_suppressed
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"emitted\"}} {}",
|
||||
unique_ip_emitted
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"suppressed\"}} {}",
|
||||
unique_ip_suppressed
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use std::net::IpAddr;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use crate::tls_front::types::{
|
||||
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
|
||||
};
|
||||
|
||||
fn test_web_publication() -> crate::web::control::WebRuntimePublication {
|
||||
let control = crate::web::control::WebRuntimeControl::new();
|
||||
control.subscribe().borrow().clone()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_metrics_format() {
|
||||
let stats = Arc::new(Stats::new());
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let mut config = ProxyConfig::default();
|
||||
config
|
||||
.access
|
||||
.user_max_unique_ips
|
||||
.insert("alice".to_string(), 4);
|
||||
|
||||
stats.increment_connects_all();
|
||||
stats.increment_connects_all();
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
stats.increment_handshake_timeouts();
|
||||
stats.increment_handshake_failure_class("timeout");
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_expensive_checks_total
|
||||
.fetch_add(9, std::sync::atomic::Ordering::Relaxed);
|
||||
shared_state
|
||||
.handshake
|
||||
.auth_budget_exhausted_total
|
||||
.fetch_add(2, std::sync::atomic::Ordering::Relaxed);
|
||||
stats.increment_upstream_connect_attempt_total();
|
||||
stats.increment_upstream_connect_attempt_total();
|
||||
stats.increment_upstream_connect_success_total();
|
||||
stats.increment_upstream_connect_fail_total();
|
||||
stats.increment_upstream_connect_failfast_hard_error_total();
|
||||
stats.observe_upstream_connect_attempts_per_request(2);
|
||||
stats.observe_upstream_connect_duration_ms(220, true);
|
||||
stats.observe_upstream_connect_duration_ms(1500, false);
|
||||
stats.increment_me_rpc_proxy_req_signal_sent_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_failed_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_response_total();
|
||||
stats.increment_me_rpc_proxy_req_signal_close_sent_total();
|
||||
stats.increment_me_idle_close_by_peer_total();
|
||||
stats.increment_relay_idle_soft_mark_total();
|
||||
stats.increment_relay_idle_hard_close_total();
|
||||
stats.increment_relay_pressure_evict_total();
|
||||
stats.increment_relay_protocol_desync_close_total();
|
||||
stats.increment_me_d2c_batches_total();
|
||||
stats.add_me_d2c_batch_frames_total(3);
|
||||
stats.add_me_d2c_batch_bytes_total(2048);
|
||||
stats.increment_me_d2c_flush_reason(crate::stats::MeD2cFlushReason::AckImmediate);
|
||||
stats.increment_me_d2c_data_frames_total();
|
||||
stats.increment_me_d2c_ack_frames_total();
|
||||
stats.add_me_d2c_payload_bytes_total(1800);
|
||||
stats.increment_me_d2c_write_mode(crate::stats::MeD2cWriteMode::Coalesced);
|
||||
stats.increment_me_d2c_quota_reject_total(crate::stats::MeD2cQuotaRejectStage::PostWrite);
|
||||
stats.observe_me_d2c_frame_buf_shrink(4096);
|
||||
stats.increment_me_endpoint_quarantine_total();
|
||||
stats.increment_me_endpoint_quarantine_unexpected_total();
|
||||
stats.increment_me_endpoint_quarantine_draining_suppressed_total();
|
||||
stats.increment_user_connects("alice");
|
||||
stats.increment_user_curr_connects("alice");
|
||||
stats.add_user_octets_from("alice", 1024);
|
||||
stats.add_user_octets_to("alice", 2048);
|
||||
stats.increment_user_msgs_from("alice");
|
||||
stats.increment_user_msgs_to("alice");
|
||||
stats.increment_user_msgs_to("alice");
|
||||
tracker
|
||||
.check_and_add("alice", "203.0.113.10".parse().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
shared_state.as_ref(),
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains(&format!(
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
)));
|
||||
assert!(output.contains("telemt_connections_total 2"));
|
||||
assert!(output.contains("telemt_connections_bad_total 1"));
|
||||
assert!(
|
||||
output.contains(
|
||||
"telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1"
|
||||
)
|
||||
);
|
||||
assert!(output.contains("telemt_handshake_timeouts_total 1"));
|
||||
assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1"));
|
||||
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
|
||||
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
|
||||
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
|
||||
assert!(output.contains("telemt_upstream_connect_success_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_fail_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_failfast_hard_error_total 1"));
|
||||
assert!(output.contains("telemt_upstream_connect_attempts_per_request{bucket=\"2\"} 1"));
|
||||
assert!(
|
||||
output.contains("telemt_upstream_connect_duration_success_total{bucket=\"101_500ms\"} 1")
|
||||
);
|
||||
assert!(output.contains("telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\"} 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_sent_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_failed_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_skipped_no_meta_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_response_total 1"));
|
||||
assert!(output.contains("telemt_me_rpc_proxy_req_signal_close_sent_total 1"));
|
||||
assert!(output.contains("telemt_me_idle_close_by_peer_total 1"));
|
||||
assert!(output.contains("telemt_relay_idle_soft_mark_total 1"));
|
||||
assert!(output.contains("telemt_relay_idle_hard_close_total 1"));
|
||||
assert!(output.contains("telemt_relay_pressure_evict_total 1"));
|
||||
assert!(output.contains("telemt_relay_protocol_desync_close_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_batches_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_batch_frames_total 3"));
|
||||
assert!(output.contains("telemt_me_d2c_batch_bytes_total 2048"));
|
||||
assert!(output.contains("telemt_me_d2c_flush_reason_total{reason=\"ack_immediate\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_data_frames_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_ack_frames_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_payload_bytes_total 1800"));
|
||||
assert!(output.contains("telemt_me_d2c_write_mode_total{mode=\"coalesced\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_quota_reject_total{stage=\"post_write\"} 1"));
|
||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_total 1"));
|
||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_bytes_total 4096"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_total 1"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_unexpected_total 1"));
|
||||
assert!(output.contains("telemt_me_endpoint_quarantine_draining_suppressed_total 1"));
|
||||
assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_octets_from_client_total{user=\"alice\"} 1024"));
|
||||
assert!(output.contains("telemt_user_octets_to_client_total{user=\"alice\"} 2048"));
|
||||
assert!(output.contains("telemt_user_msgs_from_client_total{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_msgs_to_client_total{user=\"alice\"} 2"));
|
||||
assert!(output.contains("telemt_user_unique_ips_current{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_unique_ips_recent_window{user=\"alice\"} 1"));
|
||||
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 4"));
|
||||
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.250000"));
|
||||
assert!(output.contains("telemt_ip_tracker_users{scope=\"active\"} 1"));
|
||||
assert!(output.contains("telemt_ip_tracker_entries{scope=\"active\"} 1"));
|
||||
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_tls_front_profile_health() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_domain = "primary.example".to_string();
|
||||
config.censorship.tls_domains = vec!["fallback.example".to_string()];
|
||||
|
||||
let cache = TlsFrontCache::new(
|
||||
&[
|
||||
"primary.example".to_string(),
|
||||
"fallback.example".to_string(),
|
||||
],
|
||||
1024,
|
||||
"tlsfront-profile-health-test",
|
||||
);
|
||||
cache
|
||||
.set(
|
||||
"primary.example",
|
||||
CachedTlsData {
|
||||
server_hello_template: ParsedServerHello {
|
||||
version: [0x03, 0x03],
|
||||
random: [0u8; 32],
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: {
|
||||
let mut key_share = vec![0x00, 0x1d, 0x00, 0x20];
|
||||
key_share.resize(36, 0x42);
|
||||
vec![
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x002b,
|
||||
data: vec![0x03, 0x04],
|
||||
},
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x0033,
|
||||
data: key_share,
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
cert_info: None,
|
||||
cert_payload: Some(TlsCertPayload {
|
||||
cert_chain_der: vec![vec![0x30, 0x01]],
|
||||
certificate_message: vec![0x0b, 0x00, 0x00, 0x00],
|
||||
}),
|
||||
app_data_records_sizes: vec![1024, 512],
|
||||
total_app_data_len: 1536,
|
||||
behavior_profile: TlsBehaviorProfile {
|
||||
change_cipher_spec_count: 1,
|
||||
app_data_record_sizes: vec![1024, 512],
|
||||
ticket_record_sizes: vec![69],
|
||||
source: TlsProfileSource::Merged,
|
||||
..TlsBehaviorProfile::default()
|
||||
},
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "primary.example".to_string(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
Some(&cache),
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
|
||||
assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0"));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"primary.example\",quality=\"raw_strict\",key_share_group=\"x25519\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"fallback.example\",quality=\"fallback\",key_share_group=\"none\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output
|
||||
.contains("telemt_tls_front_profile_server_hello_bytes{domain=\"primary.example\"} 90")
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_server_hello_extensions{domain=\"primary.example\"} 2"
|
||||
));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1")
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1"
|
||||
));
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn process_tls_budget_metrics_survive_a_generation_without_tls_cache() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let budget = Arc::new(TlsFullCertBudget::new());
|
||||
let cache = TlsFrontCache::new_with_full_cert_budget(
|
||||
&["example.com".to_string()],
|
||||
1024,
|
||||
"tlsfront-test-cache",
|
||||
Arc::clone(&budget),
|
||||
);
|
||||
assert!(
|
||||
cache
|
||||
.take_full_cert_budget_for_ip(
|
||||
"example.com",
|
||||
"127.0.0.1".parse().unwrap(),
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
.await
|
||||
);
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
budget.as_ref(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_tls_front_full_cert_budget_entries 1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_empty_stats() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
assert!(output.contains("telemt_connections_total 0"));
|
||||
assert!(output.contains("telemt_connections_bad_total 0"));
|
||||
assert!(output.contains("telemt_handshake_timeouts_total 0"));
|
||||
assert!(output.contains("telemt_auth_expensive_checks_total 0"));
|
||||
assert!(output.contains("telemt_auth_budget_exhausted_total 0"));
|
||||
assert!(output.contains("telemt_user_unique_ips_current{user="));
|
||||
assert!(output.contains("telemt_user_unique_ips_recent_window{user="));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_uses_global_each_unique_ip_limit() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
stats.increment_user_connects("alice");
|
||||
stats.increment_user_curr_connects("alice");
|
||||
let tracker = UserIpTracker::new();
|
||||
tracker
|
||||
.check_and_add("alice", "203.0.113.10".parse().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut config = ProxyConfig::default();
|
||||
config.access.user_max_unique_ips_global_each = 2;
|
||||
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
|
||||
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_render_has_type_annotations() {
|
||||
let stats = Stats::new();
|
||||
let shared_state = ProxySharedState::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let config = ProxyConfig::default();
|
||||
let output = render_metrics(
|
||||
&stats,
|
||||
&shared_state,
|
||||
&config,
|
||||
&tracker,
|
||||
None,
|
||||
&TlsFullCertBudget::new(),
|
||||
&test_web_publication(),
|
||||
)
|
||||
.await;
|
||||
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
|
||||
assert!(output.contains("# TYPE telemt_connections_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_idle_soft_mark_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_idle_hard_close_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_pressure_evict_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_relay_protocol_desync_close_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_batches_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_flush_reason_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_write_mode_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_batch_frames_bucket_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"));
|
||||
assert!(
|
||||
output.contains("# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter")
|
||||
);
|
||||
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
||||
assert!(
|
||||
output.contains("# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge")
|
||||
);
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_current gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_recent_window gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge"));
|
||||
assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge"));
|
||||
assert!(output.contains("# TYPE telemt_stats_user_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_telemetry_user_series_users gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_users gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_queue_len gauge"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_ip_tracker_cap_rejects_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_entries gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_quality_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_bytes gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_extensions gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_endpoint_integration() {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.general.beobachten = true;
|
||||
config.general.beobachten_minutes = 10;
|
||||
let runtime = crate::maestro::generation::test_runtime_generation(1, config);
|
||||
let web_publication = test_web_publication();
|
||||
let tls_full_cert_budget = TlsFullCertBudget::new();
|
||||
runtime.stats.increment_connects_all();
|
||||
runtime.stats.increment_connects_all();
|
||||
runtime.stats.increment_connects_all();
|
||||
|
||||
let req = Request::builder().uri("/metrics").body(()).unwrap();
|
||||
let resp = handle(req, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
let body = resp.into_body().collect().await.unwrap().to_bytes();
|
||||
assert!(
|
||||
std::str::from_utf8(body.as_ref())
|
||||
.unwrap()
|
||||
.contains("telemt_connections_total 3")
|
||||
);
|
||||
assert!(
|
||||
std::str::from_utf8(body.as_ref())
|
||||
.unwrap()
|
||||
.contains(&format!(
|
||||
"telemt_build_info{{version=\"{}\"}} 1",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
))
|
||||
);
|
||||
|
||||
runtime.beobachten.record(
|
||||
"TLS-scanner",
|
||||
"203.0.113.10".parse::<IpAddr>().unwrap(),
|
||||
Duration::from_secs(600),
|
||||
);
|
||||
let req_beob = Request::builder().uri("/beobachten").body(()).unwrap();
|
||||
let resp_beob = handle(req_beob, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp_beob.status(), StatusCode::OK);
|
||||
let body_beob = resp_beob.into_body().collect().await.unwrap().to_bytes();
|
||||
let beob_text = std::str::from_utf8(body_beob.as_ref()).unwrap();
|
||||
assert!(beob_text.contains("[TLS-scanner]"));
|
||||
assert!(beob_text.contains("203.0.113.10-1"));
|
||||
|
||||
let req404 = Request::builder().uri("/other").body(()).unwrap();
|
||||
let resp404 = handle(req404, &runtime, &web_publication, &tls_full_cert_budget)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp404.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::config::{
|
||||
ProxyConfig, WebCarrier, WebCarrierNegotiationAggressiveness, WebHttpConnectionCapacityAction,
|
||||
};
|
||||
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||
use crate::web::manager::{CarrierFailure, OperatorLifecycleState};
|
||||
use crate::web::telemetry::{
|
||||
WebCarrierFailurePhase, WebCarrierLearningOutcome, WebCarrierSelectionDisposition,
|
||||
WebDecoyUpstreamOutcome, WebHttpConnectionOverloadOutcome, WebRejectionReason,
|
||||
};
|
||||
|
||||
// Decoy fast-track metrics stay isolated from the main WEB renderer.
|
||||
mod fasttrack;
|
||||
// Session lifecycle and aggregate families stay isolated from capacity rendering.
|
||||
mod lifecycle;
|
||||
|
||||
/// Renders fixed-cardinality process-owned WEB observability families.
|
||||
pub(super) fn render(out: &mut String, publication: &WebRuntimePublication, config: &ProxyConfig) {
|
||||
let runtime = publication.runtime.upgrade();
|
||||
let configured_listeners = publication.listeners.len();
|
||||
let live_acceptors = publication.telemetry.live_acceptors();
|
||||
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
|
||||
&& runtime.is_some()
|
||||
&& configured_listeners != 0
|
||||
&& live_acceptors == configured_listeners;
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_ingress_lifecycle_state Current process-owned WEB ingress lifecycle"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_ingress_lifecycle_state gauge");
|
||||
for state in WebRuntimeLifecycle::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_ingress_lifecycle_state{{state=\"{}\"}} {}",
|
||||
state.as_str(),
|
||||
flag(publication.lifecycle == state)
|
||||
);
|
||||
}
|
||||
|
||||
let operator_status = runtime
|
||||
.as_deref()
|
||||
.map(crate::web::manager::WebProcessRuntime::operator_lifecycle_status);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_operator_lifecycle_state Current reversible WEB operator lifecycle"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_operator_lifecycle_state gauge");
|
||||
for state in OPERATOR_STATES {
|
||||
let active = match operator_status.as_ref() {
|
||||
Some(status) => operator_state_token(status.state) == state,
|
||||
None => state == "unavailable",
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_operator_lifecycle_state{{state=\"{state}\"}} {}",
|
||||
flag(active)
|
||||
);
|
||||
}
|
||||
|
||||
let operator_admission_open = operator_status
|
||||
.as_ref()
|
||||
.is_some_and(|status| status.admission_open);
|
||||
let effective_new_work_admission = operator_status
|
||||
.as_ref()
|
||||
.is_some_and(|status| status.effective_new_work_admission);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_ingress_state Independent WEB ingress and admission flags"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_ingress_state gauge");
|
||||
for (name, value) in [
|
||||
("runtime_available", runtime.is_some()),
|
||||
("accepting_connections", accepting_connections),
|
||||
("config_enabled", config.web.enabled),
|
||||
("operator_admission_open", operator_admission_open),
|
||||
("effective_new_work_admission", effective_new_work_admission),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_ingress_state{{flag=\"{name}\"}} {}",
|
||||
flag(value)
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_listeners Process-owned WEB listener and acceptor counts"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_listeners gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_listeners{{status=\"configured\"}} {configured_listeners}"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_listeners{{status=\"acceptors_live\"}} {live_acceptors}"
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_tcp_accept_total Accepted sockets and accept syscall errors"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_tcp_accept_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_tcp_accept_total{{result=\"accepted\"}} {}",
|
||||
publication.telemetry.accepted()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_tcp_accept_total{{result=\"error\"}} {}",
|
||||
publication.telemetry.accept_errors()
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_rejections_total WEB operational rejection decisions by fixed reason"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_rejections_total counter");
|
||||
for reason in WebRejectionReason::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_rejections_total{{reason=\"{}\"}} {}",
|
||||
reason.as_str(),
|
||||
publication.telemetry.rejection_total(reason)
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_http_connection_overload_total Accepted saturated sockets by terminal outcome"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_http_connection_overload_total counter"
|
||||
);
|
||||
for outcome in WebHttpConnectionOverloadOutcome::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_http_connection_overload_total{{outcome=\"{}\"}} {}",
|
||||
outcome.as_str(),
|
||||
publication.telemetry.overload_total(outcome)
|
||||
);
|
||||
}
|
||||
|
||||
let action = config.web.http_connection_capacity_action;
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_http_connection_capacity_action Effective overload action"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_http_connection_capacity_action gauge"
|
||||
);
|
||||
for (token, variant) in [
|
||||
("drop", WebHttpConnectionCapacityAction::Drop),
|
||||
("wait", WebHttpConnectionCapacityAction::Wait),
|
||||
("respond", WebHttpConnectionCapacityAction::Respond),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_http_connection_capacity_action{{action=\"{token}\"}} {}",
|
||||
flag(action == variant)
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_http_overload_timeout_milliseconds Effective overload phase timeout"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_http_overload_timeout_milliseconds gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_http_overload_timeout_milliseconds {}",
|
||||
config.web.timeouts.http_overload_timeout_ms
|
||||
);
|
||||
|
||||
if let Some(runtime) = runtime.as_deref() {
|
||||
render_capacity(out, &runtime.capacity_snapshot());
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_decoy_upstream_requests_total Internal plain-HTTP decoy origin outcomes"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_decoy_upstream_requests_total counter"
|
||||
);
|
||||
for outcome in WebDecoyUpstreamOutcome::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_decoy_upstream_requests_total{{outcome=\"{}\"}} {}",
|
||||
outcome.as_str(),
|
||||
publication.telemetry.decoy_total(outcome)
|
||||
);
|
||||
}
|
||||
|
||||
fasttrack::render(out, publication, config);
|
||||
render_carrier_negotiation(out, publication, runtime.as_deref(), config);
|
||||
lifecycle::render(out, publication, config);
|
||||
}
|
||||
|
||||
fn render_carrier_negotiation(
|
||||
out: &mut String,
|
||||
publication: &WebRuntimePublication,
|
||||
runtime: Option<&crate::web::manager::WebProcessRuntime>,
|
||||
config: &ProxyConfig,
|
||||
) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_selections_total Successful carrier selections by learning disposition"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_carrier_selections_total counter");
|
||||
for carrier in WebCarrier::ALL {
|
||||
for disposition in WebCarrierSelectionDisposition::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_selections_total{{carrier=\"{}\",disposition=\"{}\"}} {}",
|
||||
carrier.as_str(),
|
||||
disposition.as_str(),
|
||||
publication
|
||||
.telemetry
|
||||
.carrier_selection_total(carrier, disposition)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_reported_failures_total Canonical client-reported failures for authenticated carrier chains"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_carrier_reported_failures_total counter"
|
||||
);
|
||||
for carrier in WebCarrier::ALL {
|
||||
for phase in WebCarrierFailurePhase::ALL {
|
||||
for reason in CarrierFailure::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_reported_failures_total{{carrier=\"{}\",phase=\"{}\",reason=\"{}\"}} {}",
|
||||
carrier.as_str(),
|
||||
phase.as_str(),
|
||||
reason.as_str(),
|
||||
publication
|
||||
.telemetry
|
||||
.carrier_failure_total(carrier, phase, reason)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_learning_outcomes_total Terminal carrier health and evidence publication outcomes"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_carrier_learning_outcomes_total counter"
|
||||
);
|
||||
for carrier in WebCarrier::ALL {
|
||||
for outcome in WebCarrierLearningOutcome::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_learning_outcomes_total{{carrier=\"{}\",outcome=\"{}\"}} {}",
|
||||
carrier.as_str(),
|
||||
outcome.as_str(),
|
||||
publication
|
||||
.telemetry
|
||||
.carrier_learning_total(carrier, outcome)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let learning = runtime.and_then(|runtime| runtime.try_carrier_learning_status());
|
||||
let policy_matches = learning.as_ref().is_some_and(|status| {
|
||||
status.policy_generation == runtime.map(|runtime| runtime.active_generation().id)
|
||||
&& status.enabled
|
||||
== (config.web.carrier_negotiation_enabled() && config.web.carrier_learning)
|
||||
&& status.aggressiveness == config.web.carrier_negotiation_aggressiveness
|
||||
&& status.lifetime_secs == config.web.timeouts.carrier_learning_secs
|
||||
&& status.health_secs == config.web.timeouts.carrier_health_secs
|
||||
});
|
||||
let active_state = if runtime.is_none() {
|
||||
"unavailable"
|
||||
} else if learning.is_none() {
|
||||
"partial"
|
||||
} else if !policy_matches {
|
||||
"pending"
|
||||
} else if learning
|
||||
.as_ref()
|
||||
.is_some_and(|status| status.epoch.is_none())
|
||||
{
|
||||
"exhausted"
|
||||
} else if learning.as_ref().is_some_and(|status| status.enabled) {
|
||||
"enabled"
|
||||
} else {
|
||||
"disabled"
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_learning_state Current bounded learning policy state"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_carrier_learning_state gauge");
|
||||
for state in LEARNING_STATES {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_learning_state{{state=\"{state}\"}} {}",
|
||||
flag(active_state == state)
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_learning_entries Current bounded evidence entries"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_carrier_learning_entries gauge");
|
||||
for (kind, value) in [
|
||||
("used", learning.as_ref().map_or(0, |status| status.entries)),
|
||||
(
|
||||
"limit",
|
||||
learning.as_ref().map_or(0, |status| status.capacity),
|
||||
),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_learning_entries{{kind=\"{kind}\"}} {value}"
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_carrier_learning_policy Effective learning aggressiveness"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_carrier_learning_policy gauge");
|
||||
for aggressiveness in [
|
||||
WebCarrierNegotiationAggressiveness::Conservative,
|
||||
WebCarrierNegotiationAggressiveness::Balanced,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
] {
|
||||
let token = match aggressiveness {
|
||||
WebCarrierNegotiationAggressiveness::Conservative => "conservative",
|
||||
WebCarrierNegotiationAggressiveness::Balanced => "balanced",
|
||||
WebCarrierNegotiationAggressiveness::Aggressive => "aggressive",
|
||||
};
|
||||
let active = learning
|
||||
.as_ref()
|
||||
.is_some_and(|status| status.aggressiveness == aggressiveness);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_carrier_learning_policy{{aggressiveness=\"{token}\"}} {}",
|
||||
flag(active)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn render_capacity(out: &mut String, snapshot: &crate::web::manager::WebCapacitySnapshot) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_capacity_snapshot_partial Whether a non-blocking capacity plane was omitted"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_capacity_snapshot_partial gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_capacity_snapshot_partial{{plane=\"budget\"}} {}",
|
||||
flag(snapshot.partial.contains(&"budget"))
|
||||
);
|
||||
for (unit, family) in [
|
||||
("slots", "telemt_web_capacity_slots"),
|
||||
("bytes", "telemt_web_capacity_bytes"),
|
||||
("items", "telemt_web_capacity_items"),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP {family} Current process-wide WEB capacity in {unit}"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE {family} gauge");
|
||||
for status in snapshot
|
||||
.resources
|
||||
.iter()
|
||||
.filter(|status| status.unit == unit)
|
||||
{
|
||||
for (kind, value) in [
|
||||
("used", status.used),
|
||||
("available", status.available),
|
||||
("limit", status.limit),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"{family}{{resource=\"{}\",kind=\"{kind}\"}} {value}",
|
||||
status.resource
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_capacity_closed Whether terminal shutdown closed a WEB capacity authority"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_capacity_closed gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_capacity_saturated Whether a WEB resource has no immediately available capacity"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_capacity_saturated gauge");
|
||||
for status in &snapshot.resources {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_capacity_closed{{resource=\"{}\"}} {}",
|
||||
status.resource,
|
||||
flag(status.closed)
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_capacity_saturated{{resource=\"{}\"}} {}",
|
||||
status.resource,
|
||||
flag(status.available == 0 && !status.closed)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn operator_state_token(state: OperatorLifecycleState) -> &'static str {
|
||||
match state {
|
||||
OperatorLifecycleState::Running => "running",
|
||||
OperatorLifecycleState::Paused => "paused",
|
||||
OperatorLifecycleState::Draining => "draining",
|
||||
OperatorLifecycleState::ForceClosing => "force_closing",
|
||||
OperatorLifecycleState::Drained => "drained",
|
||||
}
|
||||
}
|
||||
|
||||
const OPERATOR_STATES: [&str; 6] = [
|
||||
"unavailable",
|
||||
"running",
|
||||
"paused",
|
||||
"draining",
|
||||
"force_closing",
|
||||
"drained",
|
||||
];
|
||||
|
||||
const LEARNING_STATES: [&str; 6] = [
|
||||
"unavailable",
|
||||
"partial",
|
||||
"pending",
|
||||
"exhausted",
|
||||
"disabled",
|
||||
"enabled",
|
||||
];
|
||||
|
||||
const fn flag(value: bool) -> u8 {
|
||||
if value { 1 } else { 0 }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
|
||||
#[test]
|
||||
fn renderer_emits_complete_zeroed_fixed_counter_sets() {
|
||||
let control = WebRuntimeControl::new();
|
||||
let publication = control.subscribe().borrow().clone();
|
||||
let mut output = String::new();
|
||||
super::render(&mut output, &publication, &ProxyConfig::default());
|
||||
|
||||
assert_eq!(
|
||||
output.matches("telemt_web_rejections_total{").count(),
|
||||
crate::web::telemetry::WebRejectionReason::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_http_connection_overload_total{")
|
||||
.count(),
|
||||
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_decoy_upstream_requests_total{")
|
||||
.count(),
|
||||
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
|
||||
);
|
||||
assert!(output.contains("telemt_web_ingress_lifecycle_state{state=\"starting\"} 1"));
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_carrier_selections_total{")
|
||||
.count(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierSelectionDisposition::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_carrier_reported_failures_total{")
|
||||
.count(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierFailurePhase::ALL.len()
|
||||
* crate::web::manager::CarrierFailure::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_carrier_learning_outcomes_total{")
|
||||
.count(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebCarrierLearningOutcome::ALL.len()
|
||||
);
|
||||
assert!(output.contains("telemt_web_carrier_learning_state{state=\"unavailable\"} 1"));
|
||||
assert_eq!(
|
||||
output.matches("telemt_web_session_closures_total{").count(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::session::SessionCloseReason::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_session_lifecycle_observations_total{")
|
||||
.count(),
|
||||
crate::config::WebCarrier::ALL.len()
|
||||
* crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_bridge_recovery_events_total{")
|
||||
.count(),
|
||||
crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
|
||||
);
|
||||
assert!(output.contains("telemt_web_bridge_recovery_seconds 15"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::config::{ProxyConfig, WebDecoyFastTrackMode};
|
||||
use crate::web::control::WebRuntimePublication;
|
||||
use crate::web::telemetry::WebDecoyFastTrackDisposition;
|
||||
|
||||
/// Renders fixed-cardinality decoy capability-routing metrics.
|
||||
pub(super) fn render(out: &mut String, publication: &WebRuntimePublication, config: &ProxyConfig) {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_decoy_fasttrack_mode Effective restart-frozen decoy fast-track mode"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_web_decoy_fasttrack_mode gauge");
|
||||
for mode in WebDecoyFastTrackMode::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_decoy_fasttrack_mode{{mode=\"{}\"}} {}",
|
||||
mode.as_str(),
|
||||
u8::from(config.web.decoy_fasttrack_mode == mode)
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_web_decoy_fasttrack_requests_total WEB root requests classified by decoy capability-routing work"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_web_decoy_fasttrack_requests_total counter"
|
||||
);
|
||||
for disposition in WebDecoyFastTrackDisposition::ALL {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_web_decoy_fasttrack_requests_total{{disposition=\"{}\"}} {}",
|
||||
disposition.as_str(),
|
||||
publication.telemetry.decoy_fasttrack_total(disposition)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
|
||||
#[test]
|
||||
fn renderer_emits_one_hot_mode_and_complete_counters() {
|
||||
let control = WebRuntimeControl::new();
|
||||
control
|
||||
.telemetry()
|
||||
.record_decoy_fasttrack(WebDecoyFastTrackDisposition::EnforceFastTrack);
|
||||
let publication = control.subscribe().borrow().clone();
|
||||
let mut config = ProxyConfig::default();
|
||||
config.web.decoy_fasttrack_mode = WebDecoyFastTrackMode::Enforce;
|
||||
let mut output = String::new();
|
||||
|
||||
render(&mut output, &publication, &config);
|
||||
|
||||
assert!(output.contains("telemt_web_decoy_fasttrack_mode{mode=\"off\"} 0"));
|
||||
assert!(output.contains("telemt_web_decoy_fasttrack_mode{mode=\"enforce\"} 1"));
|
||||
assert_eq!(
|
||||
output
|
||||
.matches("telemt_web_decoy_fasttrack_requests_total{")
|
||||
.count(),
|
||||
WebDecoyFastTrackDisposition::ALL.len()
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_web_decoy_fasttrack_requests_total{disposition=\"enforce_fasttrack\"} 1"
|
||||
));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user