Bounded Debugging + Websocket Carriers + Carriers Negotiation

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/web/http/websocket/driver.rs

@@ -2,7 +2,6 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use bytes::Bytes;
-use futures_util::{SinkExt, StreamExt};
 use hyper_util::rt::TokioIo;
 use tokio_tungstenite::WebSocketStream;
 use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
@@ -10,7 +9,7 @@ use tokio_util::sync::CancellationToken;
 
 use super::ConnectionIo;
 use crate::web::manager::{
-    ManagerError, WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
+    WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
 };
 use crate::web::session::{WebSession, WebSocketLaneReservation};
 use crate::web::trace::{TraceDirection, TraceWebSocketContext};
@@ -18,6 +17,10 @@ use crate::web::trace::{TraceDirection, TraceWebSocketContext};
 const READ_BUFFER_BYTES: usize = 64 * 1024;
 const WRITE_BUFFER_BYTES: usize = 64 * 1024;
 
+// Cancellation-safe message I/O and budget retries remain separate from carrier loops.
+mod io;
+use io::{flush, process_lane, process_multiplex, read_message, record_message, reserve_data, send};
+
 pub(super) async fn run_upgraded(
     on_upgrade: hyper::upgrade::OnUpgrade,
     runtime: Arc<WebProcessRuntime>,
@@ -25,6 +28,7 @@ pub(super) async fn run_upgraded(
     connection: WebSocketConnection,
     mut lane_reservation: Option<WebSocketLaneReservation>,
     trace: Option<TraceWebSocketContext>,
+    acknowledge_commit: bool,
 ) {
     let Ok(upgraded) = on_upgrade.await else {
         return;
@@ -57,6 +61,7 @@ pub(super) async fn run_upgraded(
             reservation,
             cancellation.clone(),
             trace.as_ref(),
+            acknowledge_commit,
         )
         .await;
     } else {
@@ -67,6 +72,7 @@ pub(super) async fn run_upgraded(
             &connection,
             cancellation.clone(),
             trace.as_ref(),
+            acknowledge_commit,
         )
         .await;
     }
@@ -82,7 +88,7 @@ pub(super) async fn run_upgraded(
     if let Some(reservation) = lane_reservation {
         session.close_websocket_lane(reservation.lane_id());
         drop(reservation);
-    } else {
+    } else if !acknowledge_commit || session.is_carrier_committed() {
         session.close();
     }
 }
@@ -96,6 +102,7 @@ async fn run_multiplex(
     connection: &WebSocketConnection,
     cancellation: CancellationToken,
     trace: Option<&TraceWebSocketContext>,
+    acknowledge_commit: bool,
 ) -> Result<(), ()> {
     let mut sequence = 1u64;
     let mut cursor = 0u64;
@@ -136,6 +143,18 @@ async fn run_multiplex(
                         started,
                     );
                     result?;
+                    if acknowledge_commit && sequence == 1 && session.is_carrier_committed() {
+                        let started = Instant::now();
+                        send(socket, runtime, Message::Binary(Bytes::new())).await?;
+                        record_message(
+                            runtime,
+                            trace,
+                            TraceDirection::Response,
+                            "carrier-ack",
+                            &[],
+                            started,
+                        );
+                    }
                     sequence = sequence.checked_add(1).ok_or(())?;
                     connection.mark_peer_activity();
                     next_ping = Instant::now() + liveness_interval;
@@ -255,6 +274,7 @@ async fn run_multiplex(
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn run_lane(
     socket: &mut CarrierSocket,
     runtime: &Arc<WebProcessRuntime>,
@@ -263,6 +283,7 @@ async fn run_lane(
     reservation: &mut WebSocketLaneReservation,
     cancellation: CancellationToken,
     trace: Option<&TraceWebSocketContext>,
+    acknowledge_commit: bool,
 ) -> Result<(), ()> {
     let mut sequence = 1u64;
     let mut cursor = 0u64;
@@ -309,6 +330,18 @@ async fn run_lane(
                         started,
                     );
                     result?;
+                    if acknowledge_commit && sequence == 1 && session.is_carrier_committed() {
+                        let started = Instant::now();
+                        send(socket, runtime, Message::Binary(Bytes::new())).await?;
+                        record_message(
+                            runtime,
+                            trace,
+                            TraceDirection::Response,
+                            "carrier-ack",
+                            &[],
+                            started,
+                        );
+                    }
                     sequence = sequence.checked_add(1).ok_or(())?;
                     connection.mark_peer_activity();
                     next_ping = Instant::now() + liveness_interval;
@@ -436,203 +469,3 @@ enum DriverEvent {
     Down(crate::web::session::PollResult),
     Liveness,
 }
-
-async fn read_message(
-    socket: &mut CarrierSocket,
-    runtime: &Arc<WebProcessRuntime>,
-    owner: crate::web::manager::ProfileKey,
-    cancellation: &CancellationToken,
-    retained_budget: &mut Option<WebSocketBudgetLease>,
-) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
-    tokio::select! {
-        _ = cancellation.cancelled() => return Err(()),
-        ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
-    }
-    if retained_budget.is_none() {
-        let maximum = runtime
-            .active_generation()
-            .config()
-            .web
-            .limits
-            .carrier_batch_bytes;
-        *retained_budget = Some(reserve_data(runtime, owner, maximum, cancellation).await?);
-    }
-    let message = tokio::select! {
-        _ = cancellation.cancelled() => return Err(()),
-        message = socket.next() => message.ok_or(())?.map_err(|_| ())?,
-    };
-    if socket.get_ref().websocket_fragmented_message() {
-        return Ok((message, None));
-    }
-    let mut budget = retained_budget.take().ok_or(())?;
-    budget.shrink_to(message.len());
-    Ok((message, Some(budget)))
-}
-
-async fn reserve_data(
-    runtime: &Arc<WebProcessRuntime>,
-    owner: crate::web::manager::ProfileKey,
-    bytes: usize,
-    cancellation: &CancellationToken,
-) -> Result<WebSocketBudgetLease, ()> {
-    let timeout = Duration::from_secs(
-        runtime
-            .active_generation()
-            .config()
-            .web
-            .timeouts
-            .websocket_backpressure_secs,
-    );
-    tokio::time::timeout(timeout, async {
-        loop {
-            let notify = runtime.budget_notify();
-            let notified = notify.notified();
-            if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
-                return Ok(budget);
-            }
-            tokio::select! {
-                _ = cancellation.cancelled() => return Err(()),
-                _ = notified => {}
-            }
-        }
-    })
-    .await
-    .map_err(|_| ())?
-}
-
-async fn process_multiplex(
-    runtime: &Arc<WebProcessRuntime>,
-    session: &Arc<WebSession>,
-    sequence: u64,
-    body: &[u8],
-    cancellation: &CancellationToken,
-) -> Result<(), ()> {
-    retry_backpressure(runtime, cancellation, || {
-        session.process_up(sequence, body).map(|_| ())
-    })
-    .await
-}
-
-async fn process_lane(
-    runtime: &Arc<WebProcessRuntime>,
-    session: &Arc<WebSession>,
-    reservation: &mut WebSocketLaneReservation,
-    sequence: u64,
-    body: &[u8],
-    cancellation: &CancellationToken,
-) -> Result<(), ()> {
-    let timeout = Duration::from_secs(
-        runtime
-            .active_generation()
-            .config()
-            .web
-            .timeouts
-            .websocket_backpressure_secs,
-    );
-    tokio::time::timeout(timeout, async {
-        loop {
-            let notify = runtime.budget_notify();
-            let notified = notify.notified();
-            match session.process_websocket_lane(reservation, sequence, body) {
-                Ok(()) => return Ok(()),
-                Err(ManagerError::Backpressure) => {}
-                Err(_) => return Err(()),
-            }
-            tokio::select! {
-                _ = cancellation.cancelled() => return Err(()),
-                _ = notified => {}
-            }
-        }
-    })
-    .await
-    .map_err(|_| ())?
-}
-
-async fn retry_backpressure<F>(
-    runtime: &Arc<WebProcessRuntime>,
-    cancellation: &CancellationToken,
-    mut operation: F,
-) -> Result<(), ()>
-where
-    F: FnMut() -> Result<(), ManagerError>,
-{
-    let timeout = Duration::from_secs(
-        runtime
-            .active_generation()
-            .config()
-            .web
-            .timeouts
-            .websocket_backpressure_secs,
-    );
-    tokio::time::timeout(timeout, async {
-        loop {
-            let notify = runtime.budget_notify();
-            let notified = notify.notified();
-            match operation() {
-                Ok(()) => return Ok(()),
-                Err(ManagerError::Backpressure) => {}
-                Err(_) => return Err(()),
-            }
-            tokio::select! {
-                _ = cancellation.cancelled() => return Err(()),
-                _ = notified => {}
-            }
-        }
-    })
-    .await
-    .map_err(|_| ())?
-}
-
-async fn send(
-    socket: &mut CarrierSocket,
-    runtime: &WebProcessRuntime,
-    message: Message,
-) -> Result<(), ()> {
-    let timeout = Duration::from_secs(
-        runtime
-            .active_generation()
-            .config()
-            .web
-            .timeouts
-            .websocket_write_secs,
-    );
-    tokio::time::timeout(timeout, socket.send(message))
-        .await
-        .map_err(|_| ())?
-        .map_err(|_| ())
-}
-
-async fn flush(socket: &mut CarrierSocket, runtime: &WebProcessRuntime) -> Result<(), ()> {
-    let timeout = Duration::from_secs(
-        runtime
-            .active_generation()
-            .config()
-            .web
-            .timeouts
-            .websocket_write_secs,
-    );
-    tokio::time::timeout(timeout, socket.flush())
-        .await
-        .map_err(|_| ())?
-        .map_err(|_| ())
-}
-
-fn record_message(
-    runtime: &WebProcessRuntime,
-    trace: Option<&TraceWebSocketContext>,
-    direction: TraceDirection,
-    message_type: &'static str,
-    payload: &[u8],
-    started: Instant,
-) {
-    let Some(trace) = trace else {
-        return;
-    };
-    runtime.trace().record_websocket_message(
-        trace,
-        direction,
-        message_type,
-        payload,
-        started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64,
-    );
-}

src/web/http/websocket/driver/io.rs

@@ -0,0 +1,214 @@
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use futures_util::{SinkExt, StreamExt};
+use tokio_tungstenite::tungstenite::protocol::Message;
+use tokio_util::sync::CancellationToken;
+
+use super::CarrierSocket;
+use crate::web::manager::{ManagerError, WebProcessRuntime, WebSocketBudgetLease};
+use crate::web::session::{WebSession, WebSocketLaneReservation};
+use crate::web::trace::{TraceDirection, TraceWebSocketContext};
+
+pub(super) async fn read_message(
+    socket: &mut CarrierSocket,
+    runtime: &Arc<WebProcessRuntime>,
+    owner: crate::web::manager::ProfileKey,
+    cancellation: &CancellationToken,
+    retained_budget: &mut Option<WebSocketBudgetLease>,
+) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
+    tokio::select! {
+        _ = cancellation.cancelled() => return Err(()),
+        ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
+    }
+    if retained_budget.is_none() {
+        let maximum = runtime
+            .active_generation()
+            .config()
+            .web
+            .limits
+            .carrier_batch_bytes;
+        *retained_budget = Some(reserve_data(runtime, owner, maximum, cancellation).await?);
+    }
+    let message = tokio::select! {
+        _ = cancellation.cancelled() => return Err(()),
+        message = socket.next() => message.ok_or(())?.map_err(|_| ())?,
+    };
+    if socket.get_ref().websocket_fragmented_message() {
+        return Ok((message, None));
+    }
+    let mut budget = retained_budget.take().ok_or(())?;
+    budget.shrink_to(message.len());
+    Ok((message, Some(budget)))
+}
+
+pub(super) async fn reserve_data(
+    runtime: &Arc<WebProcessRuntime>,
+    owner: crate::web::manager::ProfileKey,
+    bytes: usize,
+    cancellation: &CancellationToken,
+) -> Result<WebSocketBudgetLease, ()> {
+    let timeout = Duration::from_secs(
+        runtime
+            .active_generation()
+            .config()
+            .web
+            .timeouts
+            .websocket_backpressure_secs,
+    );
+    tokio::time::timeout(timeout, async {
+        loop {
+            let notify = runtime.budget_notify();
+            let notified = notify.notified();
+            if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
+                return Ok(budget);
+            }
+            tokio::select! {
+                _ = cancellation.cancelled() => return Err(()),
+                _ = notified => {}
+            }
+        }
+    })
+    .await
+    .map_err(|_| ())?
+}
+
+pub(super) async fn process_multiplex(
+    runtime: &Arc<WebProcessRuntime>,
+    session: &Arc<WebSession>,
+    sequence: u64,
+    body: &[u8],
+    cancellation: &CancellationToken,
+) -> Result<(), ()> {
+    retry_backpressure(runtime, cancellation, || {
+        session.process_up(sequence, body).map(|_| ())
+    })
+    .await
+}
+
+pub(super) async fn process_lane(
+    runtime: &Arc<WebProcessRuntime>,
+    session: &Arc<WebSession>,
+    reservation: &mut WebSocketLaneReservation,
+    sequence: u64,
+    body: &[u8],
+    cancellation: &CancellationToken,
+) -> Result<(), ()> {
+    let timeout = Duration::from_secs(
+        runtime
+            .active_generation()
+            .config()
+            .web
+            .timeouts
+            .websocket_backpressure_secs,
+    );
+    tokio::time::timeout(timeout, async {
+        loop {
+            let notify = runtime.budget_notify();
+            let notified = notify.notified();
+            match session.process_websocket_lane(reservation, sequence, body) {
+                Ok(()) => return Ok(()),
+                Err(ManagerError::Backpressure) => {}
+                Err(_) => return Err(()),
+            }
+            tokio::select! {
+                _ = cancellation.cancelled() => return Err(()),
+                _ = notified => {}
+            }
+        }
+    })
+    .await
+    .map_err(|_| ())?
+}
+
+async fn retry_backpressure<F>(
+    runtime: &Arc<WebProcessRuntime>,
+    cancellation: &CancellationToken,
+    mut operation: F,
+) -> Result<(), ()>
+where
+    F: FnMut() -> Result<(), ManagerError>,
+{
+    let timeout = Duration::from_secs(
+        runtime
+            .active_generation()
+            .config()
+            .web
+            .timeouts
+            .websocket_backpressure_secs,
+    );
+    tokio::time::timeout(timeout, async {
+        loop {
+            let notify = runtime.budget_notify();
+            let notified = notify.notified();
+            match operation() {
+                Ok(()) => return Ok(()),
+                Err(ManagerError::Backpressure) => {}
+                Err(_) => return Err(()),
+            }
+            tokio::select! {
+                _ = cancellation.cancelled() => return Err(()),
+                _ = notified => {}
+            }
+        }
+    })
+    .await
+    .map_err(|_| ())?
+}
+
+pub(super) async fn send(
+    socket: &mut CarrierSocket,
+    runtime: &WebProcessRuntime,
+    message: Message,
+) -> Result<(), ()> {
+    let timeout = Duration::from_secs(
+        runtime
+            .active_generation()
+            .config()
+            .web
+            .timeouts
+            .websocket_write_secs,
+    );
+    tokio::time::timeout(timeout, socket.send(message))
+        .await
+        .map_err(|_| ())?
+        .map_err(|_| ())
+}
+
+pub(super) async fn flush(
+    socket: &mut CarrierSocket,
+    runtime: &WebProcessRuntime,
+) -> Result<(), ()> {
+    let timeout = Duration::from_secs(
+        runtime
+            .active_generation()
+            .config()
+            .web
+            .timeouts
+            .websocket_write_secs,
+    );
+    tokio::time::timeout(timeout, socket.flush())
+        .await
+        .map_err(|_| ())?
+        .map_err(|_| ())
+}
+
+pub(super) fn record_message(
+    runtime: &WebProcessRuntime,
+    trace: Option<&TraceWebSocketContext>,
+    direction: TraceDirection,
+    message_type: &'static str,
+    payload: &[u8],
+    started: Instant,
+) {
+    let Some(trace) = trace else {
+        return;
+    };
+    runtime.trace().record_websocket_message(
+        trace,
+        direction,
+        message_type,
+        payload,
+        started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64,
+    );
+}

src/web/http/websocket/tests.rs

@@ -14,8 +14,10 @@ use tokio_util::sync::CancellationToken;
 
 use crate::maestro::generation::{RuntimeGeneration, test_runtime_generation};
 use crate::web::frame::{self, FrameType};
-use crate::web::http::tests::runtime_config;
-use crate::web::manager::WebProcessRuntime;
+use crate::web::http::tests::{negotiation_runtime_config, runtime_config};
+use crate::web::manager::{
+    CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierRequest, WebProcessRuntime,
+};
 
 fn request(protocol: &str) -> Request<()> {
     Request::builder()
@@ -39,6 +41,14 @@ fn canonical_multiplex_and_lane_protocols_are_accepted() {
 
     let lane = parse_upgrade(&request(&format!("tproxy-lane-v1.{token}.16777215"))).unwrap();
     assert!(matches!(lane.carrier, ParsedCarrier::Lane(16_777_215)));
+
+    let automatic = parse_upgrade(&request(&format!("tproxy-auto-v1.{token}"))).unwrap();
+    assert!(automatic.acknowledge_commit);
+    let automatic_lane =
+        parse_upgrade(&request(&format!("tproxy-auto-lane-v1.{token}.7"))).unwrap();
+    assert!(matches!(automatic_lane.carrier, ParsedCarrier::Lane(7)));
+    assert!(automatic_lane.acknowledge_commit);
+    assert!(!multiplex.acknowledge_commit);
 }
 
 #[test]
@@ -92,7 +102,20 @@ fn live_runtime(carrier: WebCarrier) -> LiveRuntime {
 }
 
 fn live_runtime_with_long_poll(carrier: WebCarrier, long_poll_secs: u64) -> LiveRuntime {
-    let mut config = runtime_config([31; 32], carrier);
+    live_runtime_from_config(runtime_config([31; 32], carrier), long_poll_secs)
+}
+
+fn live_negotiation_runtime(carrier: WebCarrier, carriers: Arc<[WebCarrier]>) -> LiveRuntime {
+    live_runtime_from_config(
+        negotiation_runtime_config([31; 32], carrier, false, carriers),
+        1,
+    )
+}
+
+fn live_runtime_from_config(
+    mut config: crate::config::ProxyConfig,
+    long_poll_secs: u64,
+) -> LiveRuntime {
     config.web.timeouts.long_poll_secs = long_poll_secs;
     config.web.timeouts.websocket_write_secs = 2;
     config.web.timeouts.websocket_backpressure_secs = 2;
@@ -105,6 +128,48 @@ fn live_runtime_with_long_poll(carrier: WebCarrier, long_poll_secs: u64) -> Live
     }
 }
 
+fn create_automatic_session(
+    runtime: &Arc<WebProcessRuntime>,
+) -> (TokenHash, Bytes, String, TokenHash) {
+    let profile = runtime
+        .active_generation()
+        .config()
+        .web
+        .runtime
+        .as_ref()
+        .unwrap()
+        .profiles[0]
+        .clone();
+    let client_ip = "192.0.2.10".parse().unwrap();
+    let bootstrap = runtime.issue_bootstrap(profile, client_ip).unwrap().token;
+    let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
+        .decode(&bootstrap)
+        .unwrap();
+    let bootstrap_hash = Sha256::digest(raw).into();
+    let hello = frame::encode(FrameType::Hello, 0, &[1]);
+    let session = runtime
+        .create_session(
+            bootstrap_hash,
+            "proxy.example.com",
+            client_ip,
+            &hello,
+            CarrierRequest::automatic(
+                CarrierClientClass::Bridge,
+                CarrierCapabilities::all(),
+                1,
+                None,
+                [9; 32],
+            ),
+        )
+        .unwrap()
+        .token;
+    let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
+        .decode(&session)
+        .unwrap();
+    let session_hash = Sha256::digest(raw).into();
+    (bootstrap_hash, hello, session, session_hash)
+}
+
 fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
     let profile = runtime
         .active_generation()
@@ -123,7 +188,13 @@ fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
     let bootstrap_hash = Sha256::digest(raw).into();
     let hello = frame::encode(FrameType::Hello, 0, &[1]);
     let session = runtime
-        .create_session(bootstrap_hash, "proxy.example.com", client_ip, &hello)
+        .create_session(
+            bootstrap_hash,
+            "proxy.example.com",
+            client_ip,
+            &hello,
+            CarrierRequest::legacy([0; 32]),
+        )
         .unwrap()
         .token;
     let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
@@ -309,3 +380,78 @@ async fn malformed_websocket_lane_closes_only_that_lane() {
     let _ = second.close(None).await;
     live.shutdown().await;
 }
+
+#[tokio::test]
+async fn automatic_websocket_carriers_ack_the_first_committing_message() {
+    for carrier in [WebCarrier::Websocket, WebCarrier::WebsocketLanes] {
+        let live = live_negotiation_runtime(carrier, Arc::from([carrier]));
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let (_, _, session, session_hash) = create_automatic_session(&live.runtime);
+        let protocol = match carrier {
+            WebCarrier::Websocket => format!("tproxy-auto-v1.{session}"),
+            WebCarrier::WebsocketLanes => format!("tproxy-auto-lane-v1.{session}.7"),
+            _ => unreachable!(),
+        };
+        let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
+        socket
+            .send(Message::Binary(frame::encode(FrameType::Open, 7, &[])))
+            .await
+            .unwrap();
+        let acknowledgement = tokio::time::timeout(Duration::from_secs(2), socket.next())
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap();
+        assert_eq!(acknowledgement, Message::Binary(Bytes::new()));
+        assert!(
+            live.runtime
+                .get_session(session_hash, "proxy.example.com")
+                .unwrap()
+                .is_carrier_committed()
+        );
+
+        let _ = socket.close(None).await;
+        live.shutdown().await;
+    }
+}
+
+#[tokio::test]
+async fn failed_automatic_multiplex_socket_remains_supersedable() {
+    let live = live_negotiation_runtime(
+        WebCarrier::Https,
+        Arc::from([WebCarrier::Websocket, WebCarrier::Https]),
+    );
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let (bootstrap_hash, hello, session, session_hash) =
+        create_automatic_session(&live.runtime);
+    let protocol = format!("tproxy-auto-v1.{session}");
+    let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
+    socket.close(None).await.unwrap();
+    tokio::time::sleep(Duration::from_millis(200)).await;
+    assert!(
+        live.runtime
+            .get_session(session_hash, "proxy.example.com")
+            .is_ok()
+    );
+
+    let replacement = live
+        .runtime
+        .create_session(
+            bootstrap_hash,
+            "proxy.example.com",
+            "192.0.2.10".parse().unwrap(),
+            &hello,
+            CarrierRequest::automatic(
+                CarrierClientClass::Bridge,
+                CarrierCapabilities::all(),
+                2,
+                Some(CarrierFailure::Upgrade),
+                [9; 32],
+            ),
+        )
+        .unwrap();
+    assert_eq!(replacement.carrier, WebCarrier::Https);
+    assert_eq!(replacement.attempt, Some(2));
+
+    live.shutdown().await;
+}