mirror of
https://github.com/telemt/telemt.git
synced 2026-10-31 21:02:01 +03:00
Bridge Request/Buffers/Recovery
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
@@ -97,6 +97,20 @@ pub(super) fn push_lifecycle(html: &mut String, event: &crate::web::trace::Trace
|
|||||||
);
|
);
|
||||||
html.push_str("\nreason: ");
|
html.push_str("\nreason: ");
|
||||||
html.push_str(event.reason.unwrap_or("-"));
|
html.push_str(event.reason.unwrap_or("-"));
|
||||||
|
html.push_str("\npeer gap ms: ");
|
||||||
|
html.push_str(
|
||||||
|
&event
|
||||||
|
.peer_gap_ms
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("\npredecessor session: ");
|
||||||
|
html.push_str(
|
||||||
|
&event
|
||||||
|
.predecessor_session_id
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
if let Some(carrier) = &event.carrier {
|
if let Some(carrier) = &event.carrier {
|
||||||
html.push_str("\nclient class: ");
|
html.push_str("\nclient class: ");
|
||||||
html.push_str(carrier.client_class);
|
html.push_str(carrier.client_class);
|
||||||
|
|||||||
@@ -500,5 +500,24 @@ mod tests {
|
|||||||
assert!(output.contains(
|
assert!(output.contains(
|
||||||
"telemt_web_carrier_learning_state{state=\"unavailable\"} 1"
|
"telemt_web_carrier_learning_state{state=\"unavailable\"} 1"
|
||||||
));
|
));
|
||||||
|
assert_eq!(
|
||||||
|
output.matches("telemt_web_session_closures_total{").count(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::session::SessionCloseReason::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
output
|
||||||
|
.matches("telemt_web_session_lifecycle_observations_total{")
|
||||||
|
.count(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
output
|
||||||
|
.matches("telemt_web_bridge_recovery_events_total{")
|
||||||
|
.count(),
|
||||||
|
crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
|
||||||
|
);
|
||||||
|
assert!(output.contains("telemt_web_bridge_recovery_seconds 15"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ function create(settings){
|
|||||||
while(attempt<maxAttempts){
|
while(attempt<maxAttempts){
|
||||||
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
||||||
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
||||||
const controller=new AbortController(),abort=()=>controller.abort();
|
const controller=new AbortController(),abort=()=>controller.abort();let timedOut=false;
|
||||||
if(external)external.addEventListener('abort',abort,{once:true});
|
if(external)external.addEventListener('abort',abort,{once:true});
|
||||||
const requestOptions=Object.assign({},frozenOptions,{signal:controller.signal});
|
const requestOptions=Object.assign({},frozenOptions,{signal:controller.signal});
|
||||||
const timer=setTimeout(abort,Math.max(1,Math.min(attemptLimit,remaining)));
|
const timer=setTimeout(()=>{timedOut=true;controller.abort()},Math.max(1,Math.min(attemptLimit,remaining)));
|
||||||
let response=null,wait=0;
|
let response=null,wait=0;
|
||||||
try{
|
try{
|
||||||
const fetched=await fetch(settings.origin()+path,requestOptions);
|
const fetched=await fetch(settings.origin()+path,requestOptions);
|
||||||
@@ -45,12 +45,18 @@ function create(settings){
|
|||||||
}else{
|
}else{
|
||||||
const policy=responsePolicy(path,fetched.status);let body;
|
const policy=responsePolicy(path,fetched.status);let body;
|
||||||
try{body=await settings.read(fetched,policy.limit,policy.exact,controller.signal)}
|
try{body=await settings.read(fetched,policy.limit,policy.exact,controller.signal)}
|
||||||
catch(error){controller.abort();throw settings.failure('protocol',error&&error.message)}
|
catch(error){
|
||||||
|
controller.abort();
|
||||||
|
if(external&&external.aborted)throw error;
|
||||||
|
if(timedOut)throw settings.failure('timeout','response deadline exceeded');
|
||||||
|
throw settings.failure('protocol',error&&error.message);
|
||||||
|
}
|
||||||
response={status:fetched.status,headers:fetched.headers,body};return response;
|
response={status:fetched.status,headers:fetched.headers,body};return response;
|
||||||
}
|
}
|
||||||
}catch(error){
|
}catch(error){
|
||||||
controller.abort();
|
controller.abort();
|
||||||
if(settings.closed()||(external&&external.aborted))throw error;
|
if(settings.closed()||(external&&external.aborted))throw error;
|
||||||
|
if(timedOut)lastReason='timeout';
|
||||||
if(settings.reason(error,'')==='protocol')throw error;
|
if(settings.reason(error,'')==='protocol')throw error;
|
||||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||||
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=maxAttempts||after<=0)break;
|
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=maxAttempts||after<=0)break;
|
||||||
|
|||||||
@@ -268,6 +268,7 @@ async function runUp(){
|
|||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink replay acknowledgement rejected');
|
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink replay acknowledgement rejected');
|
||||||
});
|
});
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token)return;
|
if(!recovered||closed||lease.cancelled||sessionToken!==token)return;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});upSequence++;lease=null;
|
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});upSequence++;lease=null;
|
||||||
@@ -421,6 +422,7 @@ async function runLaneUp(lane){
|
|||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink replay acknowledgement rejected');
|
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink replay acknowledgement rejected');
|
||||||
});
|
});
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
if(!recovered||closed||lease.cancelled||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lane.sequence++;lease=null;
|
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lane.sequence++;lease=null;
|
||||||
|
|||||||
@@ -36,6 +36,9 @@ fn rendered_page_contains_bounded_negotiation_contract() {
|
|||||||
assert!(page.body.contains("tproxy-auto-v1."));
|
assert!(page.body.contains("tproxy-auto-v1."));
|
||||||
assert!(page.body.contains("tproxy-auto-lane-v1."));
|
assert!(page.body.contains("tproxy-auto-lane-v1."));
|
||||||
assert!(page.body.contains("globalThis.TelemtBridgeResponse"));
|
assert!(page.body.contains("globalThis.TelemtBridgeResponse"));
|
||||||
|
assert!(page.body.contains("globalThis.TelemtBridgeRequest"));
|
||||||
|
assert!(page.body.contains("globalThis.TelemtBridgeBuffers"));
|
||||||
|
assert!(page.body.contains("globalThis.TelemtBridgeRecovery"));
|
||||||
assert!(page.body.contains("responseBody.read"));
|
assert!(page.body.contains("responseBody.read"));
|
||||||
assert!(!page.body.contains("arrayBuffer()"));
|
assert!(!page.body.contains("arrayBuffer()"));
|
||||||
assert!(page.body.contains("maxChunks=4096"));
|
assert!(page.body.contains("maxChunks=4096"));
|
||||||
@@ -77,13 +80,13 @@ fn rendered_page_embeds_the_configured_bridge_timing_policy() {
|
|||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(page.body.contains("const longPollMs=17*1000"));
|
assert!(page.body.contains("let longPollMs=17*1000"));
|
||||||
assert!(page.body.contains("bridgeRequestMs=7*1000"));
|
assert!(page.body.contains("bridgeRequestMs=7*1000"));
|
||||||
assert!(page.body.contains("bridgeRetryMs=41*1000"));
|
assert!(page.body.contains("bridgeRetryMs=41*1000"));
|
||||||
assert!(page.body.contains("bridgeRecoveryMs=13*1000"));
|
assert!(page.body.contains("bridgeRecoveryMs=13*1000"));
|
||||||
assert!(page.body.contains("websocketOpenMs=11*1000"));
|
assert!(page.body.contains("websocketOpenMs=11*1000"));
|
||||||
assert!(page.body.contains("reconnectGraceMs=119*1000"));
|
assert!(page.body.contains("reconnectGraceMs=119*1000"));
|
||||||
assert!(page.body.contains("const probeCoalesceMs=4"));
|
assert!(page.body.contains("let probeCoalesceMs=4"));
|
||||||
assert!(page.body.contains(
|
assert!(page.body.contains(
|
||||||
"helloTimer=setTimeout(()=>fail('timeout'),bridgeRequestMs)"
|
"helloTimer=setTimeout(()=>fail('timeout'),bridgeRequestMs)"
|
||||||
));
|
));
|
||||||
@@ -137,12 +140,12 @@ fn retry_and_attempt_state_are_frozen_before_fetch() {
|
|||||||
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
||||||
assert!(
|
assert!(
|
||||||
page.body
|
page.body
|
||||||
.contains("async function request(path,frozenOptions,remainingBudget,maxAttempts)")
|
.contains("async function send(path,frozenOptions,remainingBudget,maxAttempts)")
|
||||||
);
|
);
|
||||||
assert!(!page.body.contains("makeOptions"));
|
assert!(!page.body.contains("makeOptions"));
|
||||||
assert!(
|
assert!(
|
||||||
page.body
|
page.body
|
||||||
.contains("if(closed||(external&&external.aborted))throw new Error('request aborted')")
|
.contains("if(settings.closed()||(external&&external.aborted))throw new Error('request aborted')")
|
||||||
);
|
);
|
||||||
assert!(page.body.contains(
|
assert!(page.body.contains(
|
||||||
"const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal)"
|
"const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal)"
|
||||||
|
|||||||
+11
-4
@@ -243,6 +243,12 @@ async fn handle_root(
|
|||||||
.headers()
|
.headers()
|
||||||
.get(header::USER_AGENT)
|
.get(header::USER_AGENT)
|
||||||
.and_then(|value| value.to_str().ok());
|
.and_then(|value| value.to_str().ok());
|
||||||
|
let recovery_session = match representation {
|
||||||
|
recovery::RootRepresentation::Recovery(Some(hash)) => {
|
||||||
|
runtime.bridge_recovery_session(hash, &vhost.host, &profile)
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
let bootstrap = match match representation {
|
let bootstrap = match match representation {
|
||||||
recovery::RootRepresentation::Bridge => runtime.issue_bootstrap_for_request(
|
recovery::RootRepresentation::Bridge => runtime.issue_bootstrap_for_request(
|
||||||
&generation,
|
&generation,
|
||||||
@@ -256,6 +262,9 @@ async fn handle_root(
|
|||||||
Arc::clone(&profile),
|
Arc::clone(&profile),
|
||||||
client_ip,
|
client_ip,
|
||||||
user_agent,
|
user_agent,
|
||||||
|
recovery_session
|
||||||
|
.as_ref()
|
||||||
|
.map(|session| session.trace_session_id()),
|
||||||
),
|
),
|
||||||
recovery::RootRepresentation::Invalid => {
|
recovery::RootRepresentation::Invalid => {
|
||||||
strip_query(&mut request);
|
strip_query(&mut request);
|
||||||
@@ -281,10 +290,8 @@ async fn handle_root(
|
|||||||
trace.register_redaction(bootstrap.token.as_bytes());
|
trace.register_redaction(bootstrap.token.as_bytes());
|
||||||
}
|
}
|
||||||
let config = generation.config();
|
let config = generation.config();
|
||||||
if let recovery::RootRepresentation::Recovery(previous_hash) = representation {
|
if let recovery::RootRepresentation::Recovery(_) = representation {
|
||||||
if let Some(session) = previous_hash.and_then(|hash| {
|
if let Some(session) = recovery_session {
|
||||||
runtime.bridge_recovery_session(hash, &vhost.host, &profile)
|
|
||||||
}) {
|
|
||||||
let outcome = session.close(crate::web::session::SessionCloseReason::BridgeRecovery);
|
let outcome = session.close(crate::web::session::SessionCloseReason::BridgeRecovery);
|
||||||
if outcome != crate::web::session::SessionCloseOutcome::Closed
|
if outcome != crate::web::session::SessionCloseOutcome::Closed
|
||||||
&& tokio::time::timeout(
|
&& tokio::time::timeout(
|
||||||
|
|||||||
@@ -0,0 +1,244 @@
|
|||||||
|
use super::*;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
use crate::web::session::{SessionCloseOutcome, SessionCloseReason};
|
||||||
|
use crate::web::telemetry::{WebBridgeRecoveryEvent, WebSessionLifecycleObservation};
|
||||||
|
|
||||||
|
const RECOVERY_TYPE: &str = "application/vnd.telemt.web-recovery+json";
|
||||||
|
|
||||||
|
async fn bridge_bootstrap(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
encoded_capability: &str,
|
||||||
|
) -> String {
|
||||||
|
let root = format!(
|
||||||
|
"GET /?bridge={encoded_capability} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.40\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let response = request(listener, runtime, root).await;
|
||||||
|
let (_, body) = split_response(&response);
|
||||||
|
std::str::from_utf8(body)
|
||||||
|
.unwrap()
|
||||||
|
.split_once("bootstrap=\"")
|
||||||
|
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||||
|
.map(|(token, _)| token.to_string())
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_session(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
bootstrap: &str,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||||
|
let mut create = format!(
|
||||||
|
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.40\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
hello.len()
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
create.extend_from_slice(&hello);
|
||||||
|
request(listener, runtime, create).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recover(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
encoded_capability: &str,
|
||||||
|
authorization: &str,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
request(
|
||||||
|
listener,
|
||||||
|
runtime,
|
||||||
|
format!(
|
||||||
|
"GET /?bridge={encoded_capability} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.40\r\nAccept: {RECOVERY_TYPE}\r\nAuthorization: {authorization}\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn recovery_retires_current_bearer_before_single_slot_recreation() {
|
||||||
|
let capability = [40u8; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.limits.max_sessions_global = 1;
|
||||||
|
config.web.limits.max_sessions_per_ip = 1;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(active_runtime);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let bootstrap = bridge_bootstrap(&listener, &runtime, &encoded).await;
|
||||||
|
let create_response = create_session(&listener, &runtime, &bootstrap).await;
|
||||||
|
let (create_headers, _) = split_response(&create_response);
|
||||||
|
assert!(create_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
let old_session = response_header(create_headers, "x-session-token").to_string();
|
||||||
|
|
||||||
|
let recovery_response = recover(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
&encoded,
|
||||||
|
&format!("Bearer {old_session}"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (recovery_headers, recovery_body) = split_response(&recovery_response);
|
||||||
|
assert!(recovery_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(response_header(recovery_headers, "content-type"), RECOVERY_TYPE);
|
||||||
|
assert_eq!(response_header(recovery_headers, "cache-control"), "no-store");
|
||||||
|
assert!(recovery_body.len() <= 1024);
|
||||||
|
let document: serde_json::Value = serde_json::from_slice(recovery_body).unwrap();
|
||||||
|
assert_eq!(document["v"], 1);
|
||||||
|
assert_eq!(document["timeouts"]["bridge_recovery_secs"], 15);
|
||||||
|
let recovery_bootstrap = document["bootstrap"].as_str().unwrap();
|
||||||
|
|
||||||
|
let recreated = create_session(&listener, &runtime, recovery_bootstrap).await;
|
||||||
|
assert!(recreated.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(
|
||||||
|
runtime.telemetry().session_close_total(
|
||||||
|
WebCarrier::Https,
|
||||||
|
SessionCloseReason::BridgeRecovery,
|
||||||
|
),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.bridge_recovery_total(WebBridgeRecoveryEvent::BootstrapIssued),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.bridge_recovery_total(WebBridgeRecoveryEvent::SessionCreated),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
|
||||||
|
let late = format!(
|
||||||
|
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.40\r\nAuthorization: Bearer {old_session}\r\nX-Down-Cursor: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let late_response = request(&listener, &runtime, late).await;
|
||||||
|
assert!(late_response.starts_with(b"HTTP/1.1 404"));
|
||||||
|
assert_eq!(
|
||||||
|
runtime.telemetry().session_observation_total(
|
||||||
|
WebCarrier::Https,
|
||||||
|
WebSessionLifecycleObservation::RequestAfterClose,
|
||||||
|
),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
|
||||||
|
let retired_recovery = recover(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
&encoded,
|
||||||
|
&format!("Bearer {old_session}"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (retired_headers, retired_body) = split_response(&retired_recovery);
|
||||||
|
assert!(retired_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(response_header(retired_headers, "content-type"), RECOVERY_TYPE);
|
||||||
|
assert!(serde_json::from_slice::<serde_json::Value>(retired_body).is_ok());
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.list_sessions(SessionListRequest {
|
||||||
|
limit: 10,
|
||||||
|
cursor: None,
|
||||||
|
filter: SessionFilter::default(),
|
||||||
|
})
|
||||||
|
.sessions
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn malformed_or_over_capacity_recovery_is_indistinguishable_from_decoy() {
|
||||||
|
let capability = [41u8; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(active_runtime);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
|
||||||
|
let malformed = recover(&listener, &runtime, &encoded, "Bearer malformed").await;
|
||||||
|
let (malformed_headers, malformed_body) = split_response(&malformed);
|
||||||
|
assert!(malformed_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(response_header(malformed_headers, "cache-control"), "no-store");
|
||||||
|
assert_eq!(malformed_body, b"<!doctype html><title>decoy</title>");
|
||||||
|
|
||||||
|
let _held = bridge_bootstrap(&listener, &runtime, &encoded).await;
|
||||||
|
let over_capacity = recover(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
&encoded,
|
||||||
|
&format!("Bearer {}", "U".repeat(43)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (capacity_headers, capacity_body) = split_response(&over_capacity);
|
||||||
|
assert!(capacity_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(response_header(capacity_headers, "cache-control"), "no-store");
|
||||||
|
assert_eq!(capacity_body, b"<!doctype html><title>decoy</title>");
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn deferred_close_preserves_the_first_reason_across_replacement_cancel() {
|
||||||
|
let capability = [42u8; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(active_runtime);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let bootstrap = bridge_bootstrap(&listener, &runtime, &encoded).await;
|
||||||
|
let created = create_session(&listener, &runtime, &bootstrap).await;
|
||||||
|
let (created_headers, _) = split_response(&created);
|
||||||
|
let bearer = response_header(created_headers, "x-session-token");
|
||||||
|
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||||
|
.decode(bearer)
|
||||||
|
.unwrap();
|
||||||
|
let hash: crate::web::manager::TokenHash = Sha256::digest(raw).into();
|
||||||
|
let session = runtime
|
||||||
|
.get_session(hash, "proxy.example.com")
|
||||||
|
.unwrap();
|
||||||
|
let trace_session_id = session.trace_session_id();
|
||||||
|
|
||||||
|
assert!(session.begin_carrier_supersede());
|
||||||
|
assert_eq!(
|
||||||
|
session.close(SessionCloseReason::ApiClose),
|
||||||
|
SessionCloseOutcome::Deferred
|
||||||
|
);
|
||||||
|
session.cancel_carrier_supersede();
|
||||||
|
session.wait_close_complete().await;
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.session_detail(trace_session_id),
|
||||||
|
SessionDetail::Gone {
|
||||||
|
reason: "api_close",
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.session_close_total(WebCarrier::Https, SessionCloseReason::ApiClose),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.telemetry().session_close_total(
|
||||||
|
WebCarrier::Https,
|
||||||
|
SessionCloseReason::CarrierSuperseded,
|
||||||
|
),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
@@ -36,6 +36,9 @@ mod control_tests;
|
|||||||
// Reversible operator lifecycle coverage stays separate from terminal shutdown tests.
|
// Reversible operator lifecycle coverage stays separate from terminal shutdown tests.
|
||||||
#[path = "operator_lifecycle_tests.rs"]
|
#[path = "operator_lifecycle_tests.rs"]
|
||||||
mod operator_lifecycle_tests;
|
mod operator_lifecycle_tests;
|
||||||
|
// Positive-only recovery representation coverage remains isolated from ordinary root routing.
|
||||||
|
#[path = "recovery_tests.rs"]
|
||||||
|
mod recovery_tests;
|
||||||
|
|
||||||
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
||||||
|
|
||||||
@@ -276,8 +279,8 @@ async fn https_carrier_bootstraps_and_closes_one_session() {
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
next_root_body
|
next_root_body
|
||||||
.windows(b"const negotiationEnabled=false".len())
|
.windows(b"let negotiationEnabled=false".len())
|
||||||
.any(|value| value == b"const negotiationEnabled=false")
|
.any(|value| value == b"let negotiationEnabled=false")
|
||||||
);
|
);
|
||||||
|
|
||||||
let close = format!(
|
let close = format!(
|
||||||
@@ -482,7 +485,7 @@ async fn https_lanes_is_advertised_and_requires_canonical_lane_headers() {
|
|||||||
let root_response = request(&listener, &runtime, root).await;
|
let root_response = request(&listener, &runtime, root).await;
|
||||||
let (_, root_body) = split_response(&root_response);
|
let (_, root_body) = split_response(&root_response);
|
||||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||||
assert!(root_body.contains("const negotiationEnabled=false"));
|
assert!(root_body.contains("let negotiationEnabled=false"));
|
||||||
let bootstrap = root_body
|
let bootstrap = root_body
|
||||||
.split_once("bootstrap=\"")
|
.split_once("bootstrap=\"")
|
||||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ async fn enabled_debug_records_bridge_request_response_without_credentials() {
|
|||||||
let capability = [18u8; 32];
|
let capability = [18u8; 32];
|
||||||
let mut config = runtime_config(capability, WebCarrier::Https);
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
config.web.debug.enabled = true;
|
config.web.debug.enabled = true;
|
||||||
config.web.debug.body_capture = WebDebugBodyCapture::Prefix;
|
config.web.debug.body_capture = WebDebugBodyCapture::Full;
|
||||||
config.web.debug.body_prefix_bytes = 4096;
|
config.web.debug.body_prefix_bytes = 4096;
|
||||||
let generation = test_runtime_generation(1, config);
|
let generation = test_runtime_generation(1, config);
|
||||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
|||||||
@@ -225,6 +225,9 @@ async fn run_multiplex(
|
|||||||
&payload,
|
&payload,
|
||||||
Instant::now(),
|
Instant::now(),
|
||||||
);
|
);
|
||||||
|
if !session.record_websocket_peer_activity() {
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
connection.mark_peer_activity();
|
connection.mark_peer_activity();
|
||||||
next_ping = Instant::now() + liveness_interval;
|
next_ping = Instant::now() + liveness_interval;
|
||||||
}
|
}
|
||||||
@@ -247,6 +250,9 @@ async fn run_multiplex(
|
|||||||
&payload,
|
&payload,
|
||||||
started,
|
started,
|
||||||
);
|
);
|
||||||
|
if !session.record_websocket_peer_activity() {
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
connection.mark_peer_activity();
|
connection.mark_peer_activity();
|
||||||
next_ping = Instant::now() + liveness_interval;
|
next_ping = Instant::now() + liveness_interval;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -133,6 +133,9 @@ pub(super) async fn run_lane(
|
|||||||
&payload,
|
&payload,
|
||||||
Instant::now(),
|
Instant::now(),
|
||||||
);
|
);
|
||||||
|
if !session.record_websocket_peer_activity() {
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
connection.mark_peer_activity();
|
connection.mark_peer_activity();
|
||||||
next_ping = Instant::now() + liveness_interval;
|
next_ping = Instant::now() + liveness_interval;
|
||||||
}
|
}
|
||||||
@@ -155,6 +158,9 @@ pub(super) async fn run_lane(
|
|||||||
&payload,
|
&payload,
|
||||||
started,
|
started,
|
||||||
);
|
);
|
||||||
|
if !session.record_websocket_peer_activity() {
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
connection.mark_peer_activity();
|
connection.mark_peer_activity();
|
||||||
next_ping = Instant::now() + liveness_interval;
|
next_ping = Instant::now() + liveness_interval;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ impl WebProcessRuntime {
|
|||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
let generation = self.active_generation();
|
let generation = self.active_generation();
|
||||||
self.issue_bootstrap_inner(&generation, profile, client_ip, None, false)
|
self.issue_bootstrap_inner(&generation, profile, client_ip, None, false, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Issues one bootstrap against the generation that selected the bridge profile.
|
/// Issues one bootstrap against the generation that selected the bridge profile.
|
||||||
@@ -35,7 +35,7 @@ impl WebProcessRuntime {
|
|||||||
profile: Arc<WebRuntimeProfile>,
|
profile: Arc<WebRuntimeProfile>,
|
||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
self.issue_bootstrap_inner(generation, profile, client_ip, None, false)
|
self.issue_bootstrap_inner(generation, profile, client_ip, None, false, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Issues one bridge bootstrap with bounded non-secret request metadata.
|
/// Issues one bridge bootstrap with bounded non-secret request metadata.
|
||||||
@@ -46,7 +46,7 @@ impl WebProcessRuntime {
|
|||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
user_agent: Option<&str>,
|
user_agent: Option<&str>,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent, false)
|
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent, false, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Issues one recovery bootstrap with the same positive-only admission boundary.
|
/// Issues one recovery bootstrap with the same positive-only admission boundary.
|
||||||
@@ -56,8 +56,16 @@ impl WebProcessRuntime {
|
|||||||
profile: Arc<WebRuntimeProfile>,
|
profile: Arc<WebRuntimeProfile>,
|
||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
user_agent: Option<&str>,
|
user_agent: Option<&str>,
|
||||||
|
predecessor_session_id: Option<u64>,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent, true)
|
self.issue_bootstrap_inner(
|
||||||
|
generation,
|
||||||
|
profile,
|
||||||
|
client_ip,
|
||||||
|
user_agent,
|
||||||
|
true,
|
||||||
|
predecessor_session_id,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn issue_bootstrap_inner(
|
fn issue_bootstrap_inner(
|
||||||
@@ -67,6 +75,7 @@ impl WebProcessRuntime {
|
|||||||
client_ip: IpAddr,
|
client_ip: IpAddr,
|
||||||
user_agent: Option<&str>,
|
user_agent: Option<&str>,
|
||||||
recovery: bool,
|
recovery: bool,
|
||||||
|
predecessor_session_id: Option<u64>,
|
||||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||||
let config = generation.config();
|
let config = generation.config();
|
||||||
let profile = config
|
let profile = config
|
||||||
@@ -170,6 +179,7 @@ impl WebProcessRuntime {
|
|||||||
session_ip_learning_eligible: false,
|
session_ip_learning_eligible: false,
|
||||||
used: false,
|
used: false,
|
||||||
recovery,
|
recovery,
|
||||||
|
predecessor_session_id,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
@@ -183,14 +193,32 @@ impl WebProcessRuntime {
|
|||||||
self.telemetry
|
self.telemetry
|
||||||
.record_bridge_recovery(WebBridgeRecoveryEvent::BootstrapIssued);
|
.record_bridge_recovery(WebBridgeRecoveryEvent::BootstrapIssued);
|
||||||
}
|
}
|
||||||
self.trace.record_profile_lifecycle(
|
if recovery {
|
||||||
client_ip,
|
self.trace.record_lifecycle_with_context(
|
||||||
Some(trace_session_id),
|
None,
|
||||||
&profile,
|
Some(client_ip),
|
||||||
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
|
crate::web::trace::TraceIdentity::from_optional_profile(
|
||||||
None,
|
Some(trace_session_id),
|
||||||
None,
|
&profile,
|
||||||
);
|
),
|
||||||
|
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
crate::web::trace::TraceLifecycleContext {
|
||||||
|
peer_gap_ms: None,
|
||||||
|
predecessor_session_id,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
self.trace.record_profile_lifecycle(
|
||||||
|
client_ip,
|
||||||
|
Some(trace_session_id),
|
||||||
|
&profile,
|
||||||
|
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
Ok(BootstrapResult {
|
Ok(BootstrapResult {
|
||||||
token,
|
token,
|
||||||
trace_session_id,
|
trace_session_id,
|
||||||
@@ -227,13 +255,28 @@ impl WebProcessRuntime {
|
|||||||
hash: TokenHash,
|
hash: TokenHash,
|
||||||
host: &str,
|
host: &str,
|
||||||
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
||||||
self.state
|
let state = self.state.lock();
|
||||||
.lock()
|
if let Some(session) = state
|
||||||
.sessions
|
.sessions
|
||||||
.get(&hash)
|
.get(&hash)
|
||||||
.cloned()
|
.cloned()
|
||||||
.filter(|session| session.matches_host(host))
|
.filter(|session| session.matches_host(host))
|
||||||
.ok_or(ManagerError::Authentication)
|
{
|
||||||
|
return Ok(session);
|
||||||
|
}
|
||||||
|
let retired_carrier = state
|
||||||
|
.closed_tokens
|
||||||
|
.get(&hash)
|
||||||
|
.filter(|closed| closed.host == host)
|
||||||
|
.map(|closed| closed.carrier);
|
||||||
|
drop(state);
|
||||||
|
if let Some(carrier) = retired_carrier {
|
||||||
|
self.telemetry.record_session_observation(
|
||||||
|
carrier,
|
||||||
|
crate::web::telemetry::WebSessionLifecycleObservation::RequestAfterClose,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(ManagerError::Authentication)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolves a current bearer only when it belongs to the recovering profile.
|
/// Resolves a current bearer only when it belongs to the recovering profile.
|
||||||
|
|||||||
@@ -82,9 +82,7 @@ impl WebProcessRuntime {
|
|||||||
&mut state,
|
&mut state,
|
||||||
hash,
|
hash,
|
||||||
profile_host,
|
profile_host,
|
||||||
session.trace_session_id(),
|
|
||||||
session.carrier(),
|
session.carrier(),
|
||||||
reason,
|
|
||||||
closed_token_lifetime,
|
closed_token_lifetime,
|
||||||
self.limits.max_sessions_global.saturating_mul(16),
|
self.limits.max_sessions_global.saturating_mul(16),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -339,7 +339,14 @@ impl WebProcessRuntime {
|
|||||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||||
let (issuance_ip, candidate_count, user_agent, user_agent_id, recovery) = {
|
let (
|
||||||
|
issuance_ip,
|
||||||
|
candidate_count,
|
||||||
|
user_agent,
|
||||||
|
user_agent_id,
|
||||||
|
recovery,
|
||||||
|
predecessor_session_id,
|
||||||
|
) = {
|
||||||
let entry = state
|
let entry = state
|
||||||
.bootstraps
|
.bootstraps
|
||||||
.get_mut(&bootstrap_hash)
|
.get_mut(&bootstrap_hash)
|
||||||
@@ -367,6 +374,7 @@ impl WebProcessRuntime {
|
|||||||
entry.user_agent.clone(),
|
entry.user_agent.clone(),
|
||||||
entry.user_agent_id,
|
entry.user_agent_id,
|
||||||
entry.recovery,
|
entry.recovery,
|
||||||
|
entry.predecessor_session_id,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||||
@@ -422,13 +430,17 @@ impl WebProcessRuntime {
|
|||||||
scores,
|
scores,
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
self.trace.record_lifecycle(
|
self.trace.record_lifecycle_with_context(
|
||||||
None,
|
None,
|
||||||
Some(client_ip),
|
Some(client_ip),
|
||||||
identity,
|
identity,
|
||||||
TraceLifecycleEvent::SessionCreated,
|
TraceLifecycleEvent::SessionCreated,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
|
crate::web::trace::TraceLifecycleContext {
|
||||||
|
peer_gap_ms: None,
|
||||||
|
predecessor_session_id,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,9 +98,7 @@ impl WebProcessRuntime {
|
|||||||
&mut state,
|
&mut state,
|
||||||
old_hash,
|
old_hash,
|
||||||
&replacement.profile.host,
|
&replacement.profile.host,
|
||||||
replacement.old_session.trace_session_id(),
|
|
||||||
replacement.old_session.carrier(),
|
replacement.old_session.carrier(),
|
||||||
crate::web::session::SessionCloseReason::CarrierSuperseded,
|
|
||||||
Duration::from_secs(replacement.old_session.timeouts().bootstrap_lifetime_secs),
|
Duration::from_secs(replacement.old_session.timeouts().bootstrap_lifetime_secs),
|
||||||
self.limits.max_sessions_global.saturating_mul(16),
|
self.limits.max_sessions_global.saturating_mul(16),
|
||||||
);
|
);
|
||||||
@@ -126,6 +124,9 @@ impl WebProcessRuntime {
|
|||||||
self.telemetry.record_bridge_recovery(
|
self.telemetry.record_bridge_recovery(
|
||||||
crate::web::telemetry::WebBridgeRecoveryEvent::SessionCreated,
|
crate::web::telemetry::WebBridgeRecoveryEvent::SessionCreated,
|
||||||
);
|
);
|
||||||
|
self.telemetry.record_bridge_recovery(
|
||||||
|
crate::web::telemetry::WebBridgeRecoveryEvent::ClosedBeforeCommit,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
self.telemetry.record_session_closed(
|
self.telemetry.record_session_closed(
|
||||||
replacement.old_session.carrier(),
|
replacement.old_session.carrier(),
|
||||||
|
|||||||
@@ -88,6 +88,8 @@ pub(super) struct Bootstrap {
|
|||||||
pub(super) used: bool,
|
pub(super) used: bool,
|
||||||
/// Whether this credential was issued by the post-commit recovery representation.
|
/// Whether this credential was issued by the post-commit recovery representation.
|
||||||
pub(super) recovery: bool,
|
pub(super) recovery: bool,
|
||||||
|
/// Previous logical session authenticated by the recovery request, if current.
|
||||||
|
pub(super) predecessor_session_id: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bounded replay marker for one explicitly or naturally closed session token.
|
/// Bounded replay marker for one explicitly or naturally closed session token.
|
||||||
@@ -96,12 +98,8 @@ pub(super) struct ClosedToken {
|
|||||||
pub(super) expires_at: Instant,
|
pub(super) expires_at: Instant,
|
||||||
/// Canonical host that owned the session.
|
/// Canonical host that owned the session.
|
||||||
pub(super) host: String,
|
pub(super) host: String,
|
||||||
/// Non-secret logical trace owner retained for exact late-request diagnostics.
|
|
||||||
pub(super) trace_session_id: u64,
|
|
||||||
/// Carrier that owned the retired bearer.
|
/// Carrier that owned the retired bearer.
|
||||||
pub(super) carrier: WebCarrier,
|
pub(super) carrier: WebCarrier,
|
||||||
/// First-writer terminal cause for the retired bearer.
|
|
||||||
pub(super) reason: crate::web::session::SessionCloseReason,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Current logical-session owner stored without exposing bearer credentials.
|
/// Current logical-session owner stored without exposing bearer credentials.
|
||||||
@@ -363,9 +361,7 @@ pub(super) fn remember_closed_token_locked(
|
|||||||
state: &mut ManagerState,
|
state: &mut ManagerState,
|
||||||
hash: TokenHash,
|
hash: TokenHash,
|
||||||
host: &str,
|
host: &str,
|
||||||
trace_session_id: u64,
|
|
||||||
carrier: WebCarrier,
|
carrier: WebCarrier,
|
||||||
reason: crate::web::session::SessionCloseReason,
|
|
||||||
lifetime: Duration,
|
lifetime: Duration,
|
||||||
capacity: usize,
|
capacity: usize,
|
||||||
) {
|
) {
|
||||||
@@ -374,9 +370,7 @@ pub(super) fn remember_closed_token_locked(
|
|||||||
ClosedToken {
|
ClosedToken {
|
||||||
expires_at: Instant::now() + lifetime,
|
expires_at: Instant::now() + lifetime,
|
||||||
host: host.to_string(),
|
host: host.to_string(),
|
||||||
trace_session_id,
|
|
||||||
carrier,
|
carrier,
|
||||||
reason,
|
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
while state.closed_tokens.len() > capacity {
|
while state.closed_tokens.len() > capacity {
|
||||||
|
|||||||
@@ -1,11 +1,76 @@
|
|||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use super::{SessionState, WebSession};
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
/// Session-local activity clocks with distinct lease and diagnostic authority.
|
/// Session-local activity clocks with distinct lease and diagnostic authority.
|
||||||
pub(super) struct SessionActivity {
|
pub(super) struct SessionActivity {
|
||||||
last_peer: Instant,
|
last_peer: Instant,
|
||||||
last_progress: Instant,
|
last_progress: Instant,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl WebSession {
|
||||||
|
/// Refreshes the authenticated peer lease and records only threshold-crossing gaps.
|
||||||
|
pub(super) fn touch_peer_locked(
|
||||||
|
&self,
|
||||||
|
state: &mut SessionState,
|
||||||
|
now: Instant,
|
||||||
|
observation: WebSessionLifecycleObservation,
|
||||||
|
) {
|
||||||
|
let gap = state.activity.touch_peer(now);
|
||||||
|
if gap >= Duration::from_secs(self.timeouts.reconnect_grace_secs)
|
||||||
|
&& let Some(manager) = self.manager.upgrade()
|
||||||
|
{
|
||||||
|
manager
|
||||||
|
.telemetry()
|
||||||
|
.record_session_observation(self.carrier(), observation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one valid WebSocket control message as authenticated peer activity.
|
||||||
|
pub(crate) fn record_websocket_peer_activity(&self) -> bool {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
if state.closed {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::WebSocketActivityAfterGap,
|
||||||
|
);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn progress_does_not_extend_the_authenticated_peer_lease() {
|
||||||
|
let started = Instant::now();
|
||||||
|
let mut activity = SessionActivity::new(started);
|
||||||
|
activity.touch_progress(started + Duration::from_secs(4));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
activity.peer_idle(started + Duration::from_secs(9)),
|
||||||
|
Duration::from_secs(9)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
activity.progress_idle(started + Duration::from_secs(9)),
|
||||||
|
Duration::from_secs(5)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
activity.touch_peer(started + Duration::from_secs(9)),
|
||||||
|
Duration::from_secs(9)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
activity.peer_idle(started + Duration::from_secs(10)),
|
||||||
|
Duration::from_secs(1)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl SessionActivity {
|
impl SessionActivity {
|
||||||
/// Starts both activity clocks at the same session creation instant.
|
/// Starts both activity clocks at the same session creation instant.
|
||||||
pub(super) fn new(now: Instant) -> Self {
|
pub(super) fn new(now: Instant) -> Self {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use super::{
|
|||||||
};
|
};
|
||||||
use crate::web::frame::{self, FrameType};
|
use crate::web::frame::{self, FrameType};
|
||||||
use crate::web::manager::ManagerError;
|
use crate::web::manager::ManagerError;
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
impl WebSession {
|
impl WebSession {
|
||||||
/// Polls pending downlink frames with cursor replay and newest-poll-wins semantics.
|
/// Polls pending downlink frames with cursor replay and newest-poll-wins semantics.
|
||||||
@@ -22,14 +23,19 @@ impl WebSession {
|
|||||||
if state.closed {
|
if state.closed {
|
||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
state.activity.touch_peer(Instant::now());
|
|
||||||
if let Some(unacked) = &state.unacked {
|
if let Some(unacked) = &state.unacked {
|
||||||
if cursor == unacked.base_cursor {
|
if cursor == unacked.base_cursor {
|
||||||
return Ok(PollResult {
|
let result = PollResult {
|
||||||
body: unacked.body.clone(),
|
body: unacked.body.clone(),
|
||||||
next_cursor: unacked.next_cursor,
|
next_cursor: unacked.next_cursor,
|
||||||
lane_closed: false,
|
lane_closed: false,
|
||||||
});
|
};
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
|
return Ok(result);
|
||||||
}
|
}
|
||||||
if cursor != unacked.next_cursor {
|
if cursor != unacked.next_cursor {
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -53,6 +59,11 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
};
|
};
|
||||||
state.down_epoch = epoch;
|
state.down_epoch = epoch;
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||||
(state.down_epoch, healthy)
|
(state.down_epoch, healthy)
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use super::{PendingClass, SessionCloseReason, WebSession, insert_carrier_lane};
|
|||||||
use crate::config::WebCarrier;
|
use crate::config::WebCarrier;
|
||||||
use crate::web::frame::{self, Frame, FrameType};
|
use crate::web::frame::{self, Frame, FrameType};
|
||||||
use crate::web::manager::{ManagerError, TokenHash};
|
use crate::web::manager::{ManagerError, TokenHash};
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
impl WebSession {
|
impl WebSession {
|
||||||
/// Applies one exactly-once uplink batch to an independent HTTPS lane.
|
/// Applies one exactly-once uplink batch to an independent HTTPS lane.
|
||||||
@@ -46,7 +47,6 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
self.ensure_carrier_active_locked(&state)?;
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
state.activity.touch_peer(Instant::now());
|
|
||||||
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
||||||
if new_lane {
|
if new_lane {
|
||||||
if lane_id != 0
|
if lane_id != 0
|
||||||
@@ -55,6 +55,11 @@ impl WebSession {
|
|||||||
.is_some_and(|value| value.frame_type != FrameType::Open)
|
.is_some_and(|value| value.frame_type != FrameType::Open)
|
||||||
&& only_late_frames(&frames)
|
&& only_late_frames(&frames)
|
||||||
{
|
{
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
return if self.automatic_carrier
|
return if self.automatic_carrier
|
||||||
&& state.negotiation_phase != super::SessionNegotiationPhase::Committed
|
&& state.negotiation_phase != super::SessionNegotiationPhase::Committed
|
||||||
{
|
{
|
||||||
@@ -89,6 +94,11 @@ impl WebSession {
|
|||||||
});
|
});
|
||||||
if sequence == last_sequence && sequence != 0 {
|
if sequence == last_sequence && sequence != 0 {
|
||||||
return if bool::from(last_digest.ct_eq(&digest)) {
|
return if bool::from(last_digest.ct_eq(&digest)) {
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
Ok(sequence)
|
Ok(sequence)
|
||||||
} else {
|
} else {
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -109,6 +119,11 @@ impl WebSession {
|
|||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||||
if !self.reserve_locked(
|
if !self.reserve_locked(
|
||||||
&mut state,
|
&mut state,
|
||||||
|
|||||||
+31
-11
@@ -11,6 +11,7 @@ use super::{
|
|||||||
};
|
};
|
||||||
use crate::web::frame::{self, FrameType};
|
use crate::web::frame::{self, FrameType};
|
||||||
use crate::web::manager::ManagerError;
|
use crate::web::manager::ManagerError;
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
impl WebSession {
|
impl WebSession {
|
||||||
/// Polls one lane with independent cursor replay and newest-poll-wins semantics.
|
/// Polls one lane with independent cursor replay and newest-poll-wins semantics.
|
||||||
@@ -65,8 +66,7 @@ impl WebSession {
|
|||||||
if state.closed {
|
if state.closed {
|
||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
state.activity.touch_peer(Instant::now());
|
let (acknowledged, replay) = {
|
||||||
let acknowledged = {
|
|
||||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||||
return Ok(PollResult {
|
return Ok(PollResult {
|
||||||
body: Bytes::new(),
|
body: Bytes::new(),
|
||||||
@@ -83,27 +83,40 @@ impl WebSession {
|
|||||||
}
|
}
|
||||||
if let Some(unacked) = &lane.unacked {
|
if let Some(unacked) = &lane.unacked {
|
||||||
if cursor == unacked.base_cursor {
|
if cursor == unacked.base_cursor {
|
||||||
return Ok(PollResult {
|
(
|
||||||
body: unacked.body.clone(),
|
None,
|
||||||
next_cursor: unacked.next_cursor,
|
Some(PollResult {
|
||||||
lane_closed: false,
|
body: unacked.body.clone(),
|
||||||
});
|
next_cursor: unacked.next_cursor,
|
||||||
}
|
lane_closed: false,
|
||||||
if cursor != unacked.next_cursor {
|
}),
|
||||||
|
)
|
||||||
|
} else if cursor != unacked.next_cursor {
|
||||||
drop(state);
|
drop(state);
|
||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
|
} else {
|
||||||
|
(lane.unacked.take(), None)
|
||||||
}
|
}
|
||||||
lane.unacked.take()
|
|
||||||
} else {
|
} else {
|
||||||
if cursor != lane.down_cursor {
|
if cursor != lane.down_cursor {
|
||||||
drop(state);
|
drop(state);
|
||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
None
|
(None, None)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if let Some(result) = replay {
|
||||||
|
if expected_instance.is_none() {
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
if let Some(batch) = acknowledged {
|
if let Some(batch) = acknowledged {
|
||||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||||
lane.pending_bytes = lane.pending_bytes.saturating_sub(batch.data_bytes);
|
lane.pending_bytes = lane.pending_bytes.saturating_sub(batch.data_bytes);
|
||||||
@@ -139,6 +152,13 @@ impl WebSession {
|
|||||||
lane.down_epoch = epoch;
|
lane.down_epoch = epoch;
|
||||||
let instance = lane.instance;
|
let instance = lane.instance;
|
||||||
let notify = Arc::clone(&lane.notify);
|
let notify = Arc::clone(&lane.notify);
|
||||||
|
if expected_instance.is_none() {
|
||||||
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
|
}
|
||||||
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
let healthy = self.carrier_health_ready_locked(&mut state, Instant::now());
|
||||||
(instance, epoch, notify, healthy)
|
(instance, epoch, notify, healthy)
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ struct ReleasedQueues {
|
|||||||
control_items: usize,
|
control_items: usize,
|
||||||
closed_before_health: bool,
|
closed_before_health: bool,
|
||||||
reason: SessionCloseReason,
|
reason: SessionCloseReason,
|
||||||
|
peer_gap: Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Deferred queue release after manager publication linearizes a supersede.
|
/// Deferred queue release after manager publication linearizes a supersede.
|
||||||
@@ -248,6 +249,7 @@ impl WebSession {
|
|||||||
state: &mut super::SessionState,
|
state: &mut super::SessionState,
|
||||||
reason: SessionCloseReason,
|
reason: SessionCloseReason,
|
||||||
) -> ReleasedQueues {
|
) -> ReleasedQueues {
|
||||||
|
let peer_gap = state.activity.peer_idle(Instant::now());
|
||||||
let closed_before_health = self.automatic_carrier
|
let closed_before_health = self.automatic_carrier
|
||||||
&& state.negotiation_phase == SessionNegotiationPhase::Committed
|
&& state.negotiation_phase == SessionNegotiationPhase::Committed
|
||||||
&& self.reject_carrier_health_on_close();
|
&& self.reject_carrier_health_on_close();
|
||||||
@@ -304,6 +306,7 @@ impl WebSession {
|
|||||||
control_items,
|
control_items,
|
||||||
closed_before_health,
|
closed_before_health,
|
||||||
reason,
|
reason,
|
||||||
|
peer_gap,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,11 +340,25 @@ impl WebSession {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||||
self.trace_lifecycle(
|
if let Some(manager) = &manager {
|
||||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
manager.trace().record_lifecycle_with_context(
|
||||||
None,
|
None,
|
||||||
Some(released.reason.as_str()),
|
Some(self.client_ip),
|
||||||
);
|
self.trace_identity(),
|
||||||
|
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||||
|
None,
|
||||||
|
Some(released.reason.as_str()),
|
||||||
|
crate::web::trace::TraceLifecycleContext {
|
||||||
|
peer_gap_ms: Some(
|
||||||
|
released
|
||||||
|
.peer_gap
|
||||||
|
.as_millis()
|
||||||
|
.min(u128::from(u64::MAX)) as u64,
|
||||||
|
),
|
||||||
|
predecessor_session_id: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
if released.reason != SessionCloseReason::CarrierSuperseded
|
if released.reason != SessionCloseReason::CarrierSuperseded
|
||||||
&& let Some(manager) = manager
|
&& let Some(manager) = manager
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -298,6 +298,7 @@ mod tests {
|
|||||||
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||||
};
|
};
|
||||||
use crate::web::manager::{CarrierClientClass, WebProcessRuntime};
|
use crate::web::manager::{CarrierClientClass, WebProcessRuntime};
|
||||||
|
use crate::web::session::{SessionCloseOutcome, SessionCloseReason};
|
||||||
|
|
||||||
fn session(carrier: WebCarrier, deadline: Instant) -> Arc<WebSession> {
|
fn session(carrier: WebCarrier, deadline: Instant) -> Arc<WebSession> {
|
||||||
let profile = Arc::new(WebRuntimeProfile {
|
let profile = Arc::new(WebRuntimeProfile {
|
||||||
@@ -460,7 +461,7 @@ mod tests {
|
|||||||
let close = std::thread::spawn(move || {
|
let close = std::thread::spawn(move || {
|
||||||
close_barrier.wait();
|
close_barrier.wait();
|
||||||
std::thread::yield_now();
|
std::thread::yield_now();
|
||||||
close_session.close(super::SessionCloseReason::ApiClose);
|
close_session.close(SessionCloseReason::ApiClose);
|
||||||
});
|
});
|
||||||
barrier.wait();
|
barrier.wait();
|
||||||
health.join().unwrap();
|
health.join().unwrap();
|
||||||
@@ -473,8 +474,8 @@ mod tests {
|
|||||||
));
|
));
|
||||||
assert!(!session.publish_carrier_health());
|
assert!(!session.publish_carrier_health());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
session.close(super::SessionCloseReason::ApiClose),
|
session.close(SessionCloseReason::ApiClose),
|
||||||
super::SessionCloseOutcome::AlreadyClosing
|
SessionCloseOutcome::AlreadyClosing
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use super::{
|
|||||||
};
|
};
|
||||||
use crate::web::frame::{self, Frame, FrameType};
|
use crate::web::frame::{self, Frame, FrameType};
|
||||||
use crate::web::manager::{ManagerError, TokenHash};
|
use crate::web::manager::{ManagerError, TokenHash};
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Default)]
|
#[derive(Clone, Copy, Default)]
|
||||||
pub(super) struct AppliedProgress {
|
pub(super) struct AppliedProgress {
|
||||||
@@ -34,7 +35,11 @@ impl WebSession {
|
|||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
) -> Result<u64, ManagerError> {
|
) -> Result<u64, ManagerError> {
|
||||||
let (acknowledged, progressed) = self.process_up_inner(sequence, body)?;
|
let (acknowledged, progressed) = self.process_up_inner(
|
||||||
|
sequence,
|
||||||
|
body,
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
)?;
|
||||||
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
||||||
return Err(ManagerError::Backpressure);
|
return Err(ManagerError::Backpressure);
|
||||||
}
|
}
|
||||||
@@ -47,14 +52,19 @@ impl WebSession {
|
|||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
) -> Result<bool, ManagerError> {
|
) -> Result<bool, ManagerError> {
|
||||||
self.process_up_inner(sequence, body)
|
self.process_up_inner(
|
||||||
.map(|(_, progress)| progress)
|
sequence,
|
||||||
|
body,
|
||||||
|
WebSessionLifecycleObservation::WebSocketActivityAfterGap,
|
||||||
|
)
|
||||||
|
.map(|(_, progress)| progress)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_up_inner(
|
fn process_up_inner(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
|
observation: WebSessionLifecycleObservation,
|
||||||
) -> Result<(u64, bool), ManagerError> {
|
) -> Result<(u64, bool), ManagerError> {
|
||||||
if !self.carrier().is_multiplexed() {
|
if !self.carrier().is_multiplexed() {
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
@@ -92,9 +102,9 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
self.ensure_carrier_active_locked(&state)?;
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
state.activity.touch_peer(Instant::now());
|
|
||||||
if sequence == state.last_up_sequence && sequence != 0 {
|
if sequence == state.last_up_sequence && sequence != 0 {
|
||||||
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
||||||
|
self.touch_peer_locked(&mut state, Instant::now(), observation);
|
||||||
Ok((sequence, false))
|
Ok((sequence, false))
|
||||||
} else {
|
} else {
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -112,6 +122,7 @@ impl WebSession {
|
|||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
|
self.touch_peer_locked(&mut state, Instant::now(), observation);
|
||||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||||
if !self.reserve_locked(
|
if !self.reserve_locked(
|
||||||
&mut state,
|
&mut state,
|
||||||
|
|||||||
@@ -391,7 +391,11 @@ impl WebSession {
|
|||||||
lane.last_up_digest = digest;
|
lane.last_up_digest = digest;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
state.activity.touch_peer(Instant::now());
|
self.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
crate::web::telemetry::WebSessionLifecycleObservation::WebSocketActivityAfterGap,
|
||||||
|
);
|
||||||
if applied {
|
if applied {
|
||||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,6 @@ pub(crate) use store::{
|
|||||||
};
|
};
|
||||||
pub(crate) use types::{
|
pub(crate) use types::{
|
||||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
||||||
TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind, TraceRoute,
|
TraceLifecycleContext, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind,
|
||||||
TraceWebSocketContext,
|
TraceRoute, TraceWebSocketContext,
|
||||||
};
|
};
|
||||||
|
|||||||
+31
-2
@@ -9,8 +9,8 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
|||||||
|
|
||||||
use super::exchange::HttpTraceExchange;
|
use super::exchange::HttpTraceExchange;
|
||||||
use super::types::{
|
use super::types::{
|
||||||
TraceCarrierDetail, TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord,
|
TraceCarrierDetail, TraceIdentity, TraceLifecycleContext, TraceLifecycleEvent,
|
||||||
TraceRecordKind,
|
TraceLifecycleRecord, TraceRecord, TraceRecordKind,
|
||||||
};
|
};
|
||||||
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
||||||
|
|
||||||
@@ -227,6 +227,31 @@ impl WebTraceStore {
|
|||||||
stream_id,
|
stream_id,
|
||||||
reason,
|
reason,
|
||||||
None,
|
None,
|
||||||
|
TraceLifecycleContext::default(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records one lifecycle event with bounded non-secret recovery context.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn record_lifecycle_with_context(
|
||||||
|
&self,
|
||||||
|
peer_ip: Option<IpAddr>,
|
||||||
|
effective_ip: Option<IpAddr>,
|
||||||
|
identity: TraceIdentity,
|
||||||
|
event: TraceLifecycleEvent,
|
||||||
|
stream_id: Option<u32>,
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
context: TraceLifecycleContext,
|
||||||
|
) {
|
||||||
|
self.record_lifecycle_detail(
|
||||||
|
peer_ip,
|
||||||
|
effective_ip,
|
||||||
|
identity,
|
||||||
|
event,
|
||||||
|
stream_id,
|
||||||
|
reason,
|
||||||
|
None,
|
||||||
|
context,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,6 +281,7 @@ impl WebTraceStore {
|
|||||||
attempt,
|
attempt,
|
||||||
scores,
|
scores,
|
||||||
}),
|
}),
|
||||||
|
TraceLifecycleContext::default(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,6 +295,7 @@ impl WebTraceStore {
|
|||||||
stream_id: Option<u32>,
|
stream_id: Option<u32>,
|
||||||
reason: Option<&'static str>,
|
reason: Option<&'static str>,
|
||||||
carrier: Option<TraceCarrierDetail>,
|
carrier: Option<TraceCarrierDetail>,
|
||||||
|
context: TraceLifecycleContext,
|
||||||
) {
|
) {
|
||||||
if !self.enabled.load(Ordering::Acquire) {
|
if !self.enabled.load(Ordering::Acquire) {
|
||||||
return;
|
return;
|
||||||
@@ -301,6 +328,8 @@ impl WebTraceStore {
|
|||||||
stream_id,
|
stream_id,
|
||||||
reason,
|
reason,
|
||||||
carrier,
|
carrier,
|
||||||
|
peer_gap_ms: context.peer_gap_ms,
|
||||||
|
predecessor_session_id: context.predecessor_session_id,
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
if !self.try_commit(record, reservation, epoch) {
|
if !self.try_commit(record, reservation, epoch) {
|
||||||
|
|||||||
@@ -109,3 +109,28 @@ fn explicit_clear_fences_inflight_commits_and_preserves_snapshot_leases() {
|
|||||||
drop(snapshot);
|
drop(snapshot);
|
||||||
assert_eq!(store.status().used_bytes, 0);
|
assert_eq!(store.status().used_bytes, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lifecycle_context_retains_only_bounded_non_secret_correlations() {
|
||||||
|
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||||
|
store.record_lifecycle_with_context(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.40".parse().unwrap()),
|
||||||
|
TraceIdentity::default(),
|
||||||
|
TraceLifecycleEvent::SessionClosed,
|
||||||
|
None,
|
||||||
|
Some("bridge_recovery"),
|
||||||
|
TraceLifecycleContext {
|
||||||
|
peer_gap_ms: Some(125_000),
|
||||||
|
predecessor_session_id: Some(41),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let snapshot = store.snapshot_matching(|_| true);
|
||||||
|
let TraceRecordKind::Lifecycle(event) = &snapshot[0].record.kind else {
|
||||||
|
panic!("expected lifecycle record");
|
||||||
|
};
|
||||||
|
assert_eq!(event.reason, Some("bridge_recovery"));
|
||||||
|
assert_eq!(event.peer_gap_ms, Some(125_000));
|
||||||
|
assert_eq!(event.predecessor_session_id, Some(41));
|
||||||
|
}
|
||||||
|
|||||||
@@ -313,6 +313,15 @@ pub(crate) struct TraceCarrierDetail {
|
|||||||
pub(crate) scores: [i16; 4],
|
pub(crate) scores: [i16; 4],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Optional non-secret context for one lifecycle transition.
|
||||||
|
#[derive(Clone, Copy, Debug, Default)]
|
||||||
|
pub(crate) struct TraceLifecycleContext {
|
||||||
|
/// Monotonic peer inactivity preceding the transition.
|
||||||
|
pub(crate) peer_gap_ms: Option<u64>,
|
||||||
|
/// Previous logical session replaced by this transition.
|
||||||
|
pub(crate) predecessor_session_id: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
/// One typed WEB lifecycle observation.
|
/// One typed WEB lifecycle observation.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct TraceLifecycleRecord {
|
pub(crate) struct TraceLifecycleRecord {
|
||||||
@@ -324,6 +333,10 @@ pub(crate) struct TraceLifecycleRecord {
|
|||||||
pub(crate) reason: Option<&'static str>,
|
pub(crate) reason: Option<&'static str>,
|
||||||
/// Carrier negotiation detail when this is a carrier lifecycle event.
|
/// Carrier negotiation detail when this is a carrier lifecycle event.
|
||||||
pub(crate) carrier: Option<TraceCarrierDetail>,
|
pub(crate) carrier: Option<TraceCarrierDetail>,
|
||||||
|
/// Optional monotonic peer inactivity preceding the transition.
|
||||||
|
pub(crate) peer_gap_ms: Option<u64>,
|
||||||
|
/// Optional logical predecessor for a recovered session incarnation.
|
||||||
|
pub(crate) predecessor_session_id: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trace record payload variant.
|
/// Trace record payload variant.
|
||||||
|
|||||||
Reference in New Issue
Block a user