astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-10-31 04:50:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

WEB Bounded Bridge Recovery added

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-09-04 19:01:56 +03:00
parent 0d044c7372
commit 9f023ff9c7
35 changed files with 407 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/manager/carrier_learning.rs
+2 -5
View File
@@ -265,10 +265,8 @@ impl CarrierLearning {
{
return CarrierLearningEpoch::Pending;
}
self.epoch.map_or(
CarrierLearningEpoch::Exhausted,
CarrierLearningEpoch::Ready,
)
self.epoch
.map_or(CarrierLearningEpoch::Exhausted, CarrierLearningEpoch::Ready)
}
/// Ranks supported configured candidates without scanning the evidence store.
@@ -482,7 +480,6 @@ impl CarrierLearning {
entry.update(slot, deltas, cohort);
}
}
}
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
src/web/manager/carrier_learning/tests.rs
+5 -3
View File
@@ -352,7 +352,10 @@ fn health_window_change_starts_a_new_empty_epoch() {
assert_ne!(changed.epoch, Some(epoch));
assert_eq!(
changed.detached.as_ref().map(|detached| detached.entries.len()),
changed
.detached
.as_ref()
.map(|detached| detached.entries.len()),
Some(3)
);
assert!(learning.entries.is_empty());
@@ -404,8 +407,7 @@ async fn runtime_activation_publishes_matching_policy_and_generation() {
let mut config = crate::config::ProxyConfig::default();
config.web.carriers = crate::config::WebCarriers::Enabled(vec![WebCarrier::Websocket]);
config.web.carrier_learning = true;
config.web.carrier_negotiation_aggressiveness =
WebCarrierNegotiationAggressiveness::Aggressive;
config.web.carrier_negotiation_aggressiveness = WebCarrierNegotiationAggressiveness::Aggressive;
config.web.timeouts.carrier_learning_secs = 10;
config.web.timeouts.carrier_health_secs = 3;
let first = crate::maestro::generation::test_runtime_generation(1, config.clone());
src/web/manager/carrier_outcome.rs
+7 -17
View File
@@ -108,20 +108,15 @@ impl WebProcessRuntime {
&& entry.carrier_phase == CarrierChainPhase::Provisional
{
entry.carrier_phase = CarrierChainPhase::CommittedPendingHealth;
Some((entry.carrier_scores, entry.recovery))
Some(entry.carrier_scores)
} else {
None
}
});
drop(state);
let Some((scores, recovery)) = outcome else {
let Some(scores) = outcome else {
return false;
};
if recovery {
self.telemetry.record_bridge_recovery(
crate::web::telemetry::WebBridgeRecoveryEvent::Committed,
);
}
self.trace.record_carrier_lifecycle(
client_ip,
identity.clone(),
@@ -150,9 +145,7 @@ impl WebProcessRuntime {
websocket_owner: Option<u64>,
) -> CarrierHealthPublicationOutcome {
if carrier.uses_websocket()
&& websocket_owner.is_none_or(|owner| {
!self.claim_websocket_health(owner, session_hash)
})
&& websocket_owner.is_none_or(|owner| !self.claim_websocket_health(owner, session_hash))
{
let outcome = WebCarrierLearningOutcome::OwnerNotLive;
self.telemetry.record_carrier_learning(carrier, outcome);
@@ -187,13 +180,10 @@ impl WebProcessRuntime {
};
let learning_outcome = if let Some(context) = learning_context {
let now = Instant::now();
let outcome = self.learning.lock().record_chain(
now,
context.epoch,
context,
&[],
carrier,
);
let outcome =
self.learning
.lock()
.record_chain(now, context.epoch, context, &[], carrier);
match outcome {
super::learning::CarrierLearningRecordOutcome::Recorded => {
WebCarrierLearningOutcome::Recorded
src/web/manager/credentials.rs
+11 -9
View File
@@ -319,15 +319,17 @@ impl WebProcessRuntime {
.is_some_and(|current| current.token_hash() == hash)
{
bootstrap.close_requested = true;
failure_phase = Some(if matches!(
bootstrap.carrier_phase,
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
) || session.is_carrier_committed()
{
crate::web::telemetry::WebCarrierFailurePhase::Committed
} else {
crate::web::telemetry::WebCarrierFailurePhase::Provisional
});
failure_phase = Some(
if matches!(
bootstrap.carrier_phase,
CarrierChainPhase::CommittedPendingHealth | CarrierChainPhase::Healthy
) || session.is_carrier_committed()
{
crate::web::telemetry::WebCarrierFailurePhase::Committed
} else {
crate::web::telemetry::WebCarrierFailurePhase::Provisional
},
);
break;
}
}
src/web/manager/lifecycle.rs
+4 -14
View File
@@ -46,7 +46,10 @@ impl WebProcessRuntime {
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
Duration::from_secs(config.web.timeouts.carrier_health_secs),
);
debug_assert!(outcome.applied, "runtime generations must increase monotonically");
debug_assert!(
outcome.applied,
"runtime generations must increase monotonically"
);
let replaced = self.active_runtime.swap(generation);
(replaced, outcome.detached)
};
@@ -68,14 +71,6 @@ impl WebProcessRuntime {
let Some(session) = state.sessions.remove(&hash) else {
return;
};
let recovery_closed_before_commit = state.bootstraps.values().any(|bootstrap| {
bootstrap.recovery
&& bootstrap
.session
.as_ref()
.is_some_and(|current| current.token_hash() == hash)
&& !session.is_carrier_committed()
});
decrement_map(&mut state.sessions_per_ip, &client_ip);
decrement_map(&mut state.sessions_per_profile, &profile_key);
remember_closed_token_locked(
@@ -119,11 +114,6 @@ impl WebProcessRuntime {
}
self.telemetry
.record_session_closed(session.carrier(), reason);
if recovery_closed_before_commit {
self.telemetry.record_bridge_recovery(
crate::web::telemetry::WebBridgeRecoveryEvent::ClosedBeforeCommit,
);
}
drop(state);
self.notify_operator_work_changed();
}
src/web/manager/session_creation.rs
+56 -64
View File
@@ -35,6 +35,7 @@ struct Replacement {
learning_disposition: WebCarrierSelectionDisposition,
ip_learning_eligible: bool,
carrier_deadline_at: Instant,
recovery: bool,
}
impl WebProcessRuntime {
@@ -194,6 +195,7 @@ impl WebProcessRuntime {
learning_disposition: entry.carrier_learning_disposition,
ip_learning_eligible,
carrier_deadline_at: entry.carrier_deadline_at.ok_or(ManagerError::Protocol)?,
recovery: entry.recovery,
};
let _operator_admission = self.try_operator_admission()?;
state
@@ -213,6 +215,7 @@ impl WebProcessRuntime {
let trace_session_id = entry.trace_session_id;
let issued_profile = Arc::clone(&entry.profile);
let issued_timeouts = entry.timeouts.clone();
let recovery = entry.recovery;
let profile = config
.web
.runtime
@@ -232,69 +235,68 @@ impl WebProcessRuntime {
Duration::from_secs(config.web.timeouts.carrier_learning_secs),
Duration::from_secs(config.web.timeouts.carrier_health_secs),
);
let (candidates, scores, learning_epoch, learning_disposition) = if capability_selection
&& profile.carrier_learning
&& learning_policy.0
{
let learning = self.learning.lock();
match learning.epoch_for_policy(
let (candidates, scores, learning_epoch, learning_disposition) =
if capability_selection && profile.carrier_learning && learning_policy.0 {
let learning = self.learning.lock();
match learning.epoch_for_policy(
generation.id,
learning_policy.0,
learning_policy.1,
learning_policy.2,
learning_policy.3,
) {
super::learning::CarrierLearningEpoch::Ready(epoch) => {
let (candidates, scores) = learning.rank(
now,
&profile.carriers,
carrier_request,
profile_key,
client_ip,
ip_learning_eligible,
);
let disposition = if scores.iter().any(|score| *score != 0) {
WebCarrierSelectionDisposition::Applied
} else {
WebCarrierSelectionDisposition::Cold
};
(candidates, scores, Some(epoch), disposition)
super::learning::CarrierLearningEpoch::Ready(epoch) => {
let (candidates, scores) = learning.rank(
now,
&profile.carriers,
carrier_request,
profile_key,
client_ip,
ip_learning_eligible,
);
let disposition = if scores.iter().any(|score| *score != 0) {
WebCarrierSelectionDisposition::Applied
} else {
WebCarrierSelectionDisposition::Cold
};
(candidates, scores, Some(epoch), disposition)
}
super::learning::CarrierLearningEpoch::Pending => (
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
WebCarrierSelectionDisposition::PolicyPending,
),
super::learning::CarrierLearningEpoch::Exhausted => (
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
WebCarrierSelectionDisposition::EpochExhausted,
),
}
super::learning::CarrierLearningEpoch::Pending => (
} else if capability_selection {
(
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
WebCarrierSelectionDisposition::PolicyPending,
),
super::learning::CarrierLearningEpoch::Exhausted => (
supported_candidates(&profile.carriers, carrier_request),
if learning_policy.0 {
WebCarrierSelectionDisposition::ProfileDisabled
} else {
WebCarrierSelectionDisposition::PolicyDisabled
},
)
} else if carrier_request.uses_capabilities()
&& !carrier_request.supports(profile.carrier)
{
return Err(ManagerError::Protocol);
} else {
(
vec![profile.carrier],
[0; 4],
None,
WebCarrierSelectionDisposition::EpochExhausted,
),
}
} else if capability_selection {
(
supported_candidates(&profile.carriers, carrier_request),
[0; 4],
None,
if learning_policy.0 {
WebCarrierSelectionDisposition::ProfileDisabled
} else {
WebCarrierSelectionDisposition::PolicyDisabled
},
)
} else if carrier_request.uses_capabilities() && !carrier_request.supports(profile.carrier)
{
return Err(ManagerError::Protocol);
} else {
(
vec![profile.carrier],
[0; 4],
None,
WebCarrierSelectionDisposition::ProfileDisabled,
)
};
WebCarrierSelectionDisposition::ProfileDisabled,
)
};
let Some(carrier) = candidates.first().copied() else {
return Err(ManagerError::Protocol);
};
@@ -333,20 +335,14 @@ impl WebProcessRuntime {
carrier_request.class(),
learning_context,
carrier_request.is_automatic(),
recovery,
self.limits.clone(),
issued_timeouts.clone(),
);
state.sessions.insert(session_hash, Arc::clone(&session));
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
let (
issuance_ip,
candidate_count,
user_agent,
user_agent_id,
recovery,
predecessor_session_id,
) = {
let (issuance_ip, candidate_count, user_agent, user_agent_id, predecessor_session_id) = {
let entry = state
.bootstraps
.get_mut(&bootstrap_hash)
@@ -373,7 +369,6 @@ impl WebProcessRuntime {
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
entry.user_agent.clone(),
entry.user_agent_id,
entry.recovery,
entry.predecessor_session_id,
)
};
@@ -446,10 +441,7 @@ impl WebProcessRuntime {
}
}
fn supported_candidates(
configured: &[WebCarrier],
request: CarrierRequest,
) -> Vec<WebCarrier> {
fn supported_candidates(configured: &[WebCarrier], request: CarrierRequest) -> Vec<WebCarrier> {
configured
.iter()
.copied()
src/web/manager/session_creation/replacement.rs
+3 -7
View File
@@ -83,6 +83,7 @@ impl WebProcessRuntime {
replacement.request.class(),
learning_context,
true,
replacement.recovery,
self.limits.clone(),
replacement.old_session.timeouts().clone(),
);
@@ -124,9 +125,6 @@ impl WebProcessRuntime {
self.telemetry.record_bridge_recovery(
crate::web::telemetry::WebBridgeRecoveryEvent::SessionCreated,
);
self.telemetry.record_bridge_recovery(
crate::web::telemetry::WebBridgeRecoveryEvent::ClosedBeforeCommit,
);
}
self.telemetry.record_session_closed(
replacement.old_session.carrier(),
@@ -152,10 +150,8 @@ impl WebProcessRuntime {
let old_identity = replacement.old_session.trace_identity();
drop(state);
supersede.finish();
self.telemetry.record_carrier_selection(
replacement.carrier,
replacement.learning_disposition,
);
self.telemetry
.record_carrier_selection(replacement.carrier, replacement.learning_disposition);
if let Some(failure) = replacement.request.failure() {
self.telemetry.record_carrier_failure(
replacement.old_session.carrier(),
src/web/manager/status.rs
+5 -5
View File
@@ -510,15 +510,15 @@ impl WebProcessRuntime {
.unwrap_or(SessionDetail::Busy);
}
let now = Instant::now();
state.closed_sessions.get(&trace_session_id).map_or(
SessionDetail::NotFound,
|closed| SessionDetail::Gone {
state
.closed_sessions
.get(&trace_session_id)
.map_or(SessionDetail::NotFound, |closed| SessionDetail::Gone {
attempt: closed.attempt,
carrier: closed.carrier,
reason: closed.reason.as_str(),
closed_age_ms: millis(now.saturating_duration_since(closed.closed_at)),
},
)
})
}
fn row(&self, candidate: Candidate, status: WebSessionStatus) -> SessionRow {