Retry Jitter

src/web/bridge/behavior_tests.js

@@ -11,7 +11,7 @@ function renderedPage(){
  page=page.replace('__DIAGNOSTIC_RUNTIME__','');
  for(const [key,file] of Object.entries(modules))if(page.includes('__'+key+'_RUNTIME__'))page=page.replace('__'+key+'_RUNTIME__',()=>fs.readFileSync(path.join(__dirname,file+'.js'),'utf8'));
  page=page.replace('__RUNTIME__',()=>fs.readFileSync(path.join(__dirname,'runtime.js'),'utf8'));
- const values={BOOTSTRAP:'A'.repeat(43),HOST:'proxy.example.com',BASE_PREFIX:'',CARRIER_METHOD:'POST',NEGOTIATION_ENABLED:'true',CANDIDATE_COUNT:4,CARRIER_DEADLINES:'3,5,8,12',LONG_POLL_SECS:25,BRIDGE_REQUEST_SECS:10,BRIDGE_RETRY_SECS:90,BRIDGE_RECOVERY_SECS:15,WEBSOCKET_OPEN_SECS:15,RECONNECT_GRACE_SECS:120,CARRIER_PROBE_COALESCE_MS:0,BATCH_LIMIT:2097152,QUEUE_LIMIT:33554432,QUEUE_ITEMS:16384,MAX_STREAMS:1024,STATUS_FUNCTION:"state=>{if(port&&!closed)port.postMessage({t:'status',state})}",HELLO_TIMEOUT_CALLBACK:"()=>fail('timeout')",PAGEHIDE_CALLBACK:'()=>close(true)'};
+ const values={RETRY_JITTER:'{enabled:false,min:0,max:250,peak:125,triangular:false}',BOOTSTRAP:'A'.repeat(43),HOST:'proxy.example.com',BASE_PREFIX:'',CARRIER_METHOD:'POST',NEGOTIATION_ENABLED:'true',CANDIDATE_COUNT:4,CARRIER_DEADLINES:'3,5,8,12',LONG_POLL_SECS:25,BRIDGE_REQUEST_SECS:10,BRIDGE_RETRY_SECS:90,BRIDGE_RECOVERY_SECS:15,WEBSOCKET_OPEN_SECS:15,RECONNECT_GRACE_SECS:120,CARRIER_PROBE_COALESCE_MS:0,BATCH_LIMIT:2097152,QUEUE_LIMIT:33554432,QUEUE_ITEMS:16384,MAX_STREAMS:1024,STATUS_FUNCTION:"state=>{if(port&&!closed)port.postMessage({t:'status',state})}",HELLO_TIMEOUT_CALLBACK:"()=>fail('timeout')",PAGEHIDE_CALLBACK:'()=>close(true)'};
  return page.replace(/__([A-Z_]+)__/g,(all,key)=>key.startsWith('DIAGNOSTIC_')?'':String(values[key]??''));
 }
 function frame(type,id=0,payload=[]){
@@ -87,6 +87,15 @@ test('closing the bridge cancels the 250 ms overload retry',async page=>{
  await env.tick(249);env.close();await flush();await env.tick(1000);
  assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/up')).length,1);
 });
+test('retry jitter distributions, fallback, deadlines and page ownership',require('./retry_jitter_tests'));
+test('configured jitter is added once to the Telemt delay',async page=>{
+ page=page.replace('{enabled:false,min:0,max:250,peak:125,triangular:false}','{enabled:true,min:100,max:100,peak:100,triangular:false}');
+ const env=await session(page);env.send(frame(1,1));await flush();
+ env.answer(env.pending('/api/v1/up')[0],503,null,{'Retry-After':'1','X-Telemt-Retry-After-Ms':'250'});await flush();
+ await env.tick(349);assert.equal(env.pending('/api/v1/up').length,0);
+ await env.tick(1);assert.equal(env.pending('/api/v1/up').length,1);
+ env.close();await flush();
+});
 test('already queued OPEN and DATA share the zero-wait startup request',async page=>{
  const env=await session(page);const body=join(frame(1,1),frame(2,1,[7,8]));env.send(body);await flush();
  assert.equal(env.pending('/api/v1/up')[0].options.body.byteLength,body.byteLength);env.close();await flush();

src/web/bridge/behavior_tests.rs

@@ -4,6 +4,7 @@ use std::process::{Command, Stdio};
 /// Exercises the actual generated page with deterministic network and native-boundary doubles.
 pub(super) fn run(page: &super::BridgePage) {
     let mut child = Command::new("node")
+        .current_dir(concat!(env!("CARGO_MANIFEST_DIR"), "/src/web/bridge"))
         .args(["-e", include_str!("behavior_tests.js"), "--", "--stdin"])
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())

src/web/bridge/render_reference.rs

@@ -23,6 +23,7 @@ pub(super) fn render(
     carrier_probe_coalesce_ms: u64,
     bridge_diagnostics_enabled: bool,
     carrier_method: WebCarrierMethod,
+    retry_jitter: &crate::config::WebRetryJitterConfig,
     nonce: &str,
 ) -> BridgePage {
     let diagnostic_script = if bridge_diagnostics_enabled {
@@ -110,6 +111,7 @@ pub(super) fn render(
         )
         .replace("__NONCE__", &nonce)
         .replace("__HOST__", host)
+        .replace("__RETRY_JITTER__", &retry_jitter.bridge_policy())
         .replace("__BASE_PREFIX__", base_prefix)
         .replace("__BOOTSTRAP__", bootstrap)
         .replace("__CARRIER_METHOD__", carrier_method.as_str())

src/web/bridge/render_tests.rs

@@ -55,11 +55,13 @@ struct Case {
     negotiation: bool,
     candidates: usize,
     value: usize,
+    jitter: crate::config::WebRetryJitterConfig,
 }
 
 impl Case {
     fn new(index: usize) -> Self {
         Self {
+            jitter: crate::config::WebRetryJitterConfig::default(),
             host: format!("proxy-{index}.example.com"),
             base: "/telegram/web/".to_string(),
             bootstrap: format!("{index:043}"),
@@ -96,6 +98,7 @@ impl Case {
             self.value as u64 - 1,
             self.diagnostics,
             self.method,
+            &self.jitter,
             rng,
         )
     }
@@ -121,6 +124,7 @@ impl Case {
             self.value as u64 - 1,
             self.diagnostics,
             self.method,
+            &self.jitter,
             nonce,
         )
     }
@@ -223,3 +227,25 @@ fn concurrent_responses_keep_nonces_and_configuration_response_owned() {
         assert!(template(diagnostics).contains("__BASE_PREFIX__"));
     }
 }
+
+#[test]
+fn retry_jitter_is_rendered_per_page_without_changing_existing_pages() {
+    let rng = SecureRandom::new();
+    let mut case = Case::new(1);
+    let old = case.render(&rng);
+    case.jitter.enabled = true;
+    case.jitter.min_ms = 10;
+    case.jitter.max_ms = 301;
+    case.jitter.distribution = crate::config::WebRetryJitterDistribution::Triangular;
+    let new = case.render(&rng);
+    assert!(
+        old.body
+            .contains("{enabled:false,min:0,max:250,peak:125,triangular:false}")
+    );
+    assert!(
+        new.body
+            .contains("{enabled:true,min:10,max:301,peak:155.5,triangular:true}")
+    );
+    let nonce = case.assert_page(&new);
+    assert_eq!(new.body, case.reference(&nonce).body);
+}

src/web/bridge/request.js

@@ -1,5 +1,21 @@
 (()=>{'use strict';
 function create(settings){
+ // Snapshot policy per page, including during session recovery.
+ const jitter=Object.freeze(Object.assign({},settings.jitter||{enabled:false}));
+ const randomWord=jitter.enabled?new Uint32Array(1):null;
+ function randomUnit(){
+  try{if(globalThis.crypto&&typeof globalThis.crypto.getRandomValues==='function'){
+   globalThis.crypto.getRandomValues(randomWord);return randomWord[0]/4294967296;
+  }}catch(error){}
+  return Math.random();
+ }
+ function jitterMs(){
+  const a=jitter.min,b=jitter.max;if(a===b)return a;
+  const u=randomUnit();
+  if(!jitter.triangular)return a+Math.floor(u*(b-a+1));
+  const c=jitter.peak,span=b-a;
+  return Math.round(u<(c-a)/span?a+Math.sqrt(u*span*(c-a)):b-Math.sqrt((1-u)*span*(b-c)));
+ }
  const pause=(milliseconds,signal)=>new Promise((resolve,reject)=>{
   if(signal&&signal.aborted){reject(new Error('request aborted'));return}
   const timer=setTimeout(done,milliseconds);function done(){if(signal)signal.removeEventListener('abort',abort);resolve()}
@@ -15,10 +31,12 @@ function create(settings){
   // Only the exact Telemt overload hint overrides its legacy whole-second fallback.
   if(response.status===503&&header==='1'&&response.headers.get('X-Telemt-Retry-After-Ms')==='250')return 250;
   if(!header)return 0;
+  // Enabled policies must never shorten a valid server minimum to the legacy 30 s cap.
+  if(jitter.enabled&&/^\d+$/.test(header))return Number(header)*1000;
   const seconds=Number(header);
-  if(Number.isFinite(seconds)&&seconds>=0)return Math.min(seconds*1000,30000);
+  if(Number.isFinite(seconds)&&seconds>=0)return jitter.enabled?seconds*1000:Math.min(seconds*1000,30000);
   const when=Date.parse(header);
-  if(Number.isFinite(when)){const delta=when-Date.now();return delta>0?Math.min(delta,30000):0}
+  if(Number.isFinite(when)){const delta=when-Date.now();return delta>0?(jitter.enabled?delta:Math.min(delta,30000)):0}
   return 0;
  }
  function retryableStatus(status){return status===408||status===429||status===502||status===503||status===504}
@@ -65,7 +83,7 @@ function create(settings){
    }finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
    const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=attempts()||after<=0)break;
    settings.retrying();
-   const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
+   const backoff=jitter.enabled?(wait||delay)+jitterMs():wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
    await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,2000);
   }
   throw settings.failure(lastReason,'carrier retry limit reached');

src/web/bridge/retry_jitter_tests.js

@@ -0,0 +1,72 @@
+'use strict';
+const assert=require('node:assert/strict');
+const fs=require('node:fs');
+const vm=require('node:vm');
+const path=require('node:path');
+const source=fs.readFileSync(path.join(__dirname,'request.js'),'utf8');
+const flush=async()=>{for(let i=0;i<20;i++)await Promise.resolve()};
+function harness(policy,unit=0.5,cryptoMode='ok',budget=90000){
+ let now=0,next=1,calls=0,draws=0;const timers=new Map();
+ const context={Uint32Array,AbortController,Date:class extends Date{static now(){return now}},
+  Math:Object.assign(Object.create(Math),{random:()=>unit}),
+  setTimeout:(fn,ms)=>{const id=next++;timers.set(id,{fn,ms});return id},clearTimeout:id=>timers.delete(id),
+  fetch:async()=>{calls++;return {status:503,headers:new Headers({'Retry-After':'1','X-Telemt-Retry-After-Ms':'250'})}}
+ };
+ if(cryptoMode!=='absent')context.crypto={getRandomValues(word){draws++;if(cryptoMode==='throw')throw Error('unavailable');word[0]=Math.floor(unit*4294967296);return word}};
+ vm.createContext(context);vm.runInContext(source,context);
+ const settings={jitter:policy,base:()=>'',closed:()=>false,retryMs:()=>budget,longPollMs:()=>0,requestMs:()=>10000,
+  batchLimit:()=>8,cancel(){},retrying(){},failure:(_,message)=>new Error(message),reason:()=>'',read:async()=>new ArrayBuffer(0)};
+ const client=context.TelemtBridgeRequest.create(settings);
+ return {context,timers,client,get calls(){return calls},get draws(){return draws},
+  async sample(status=503,headers={'Retry-After':'1','X-Telemt-Retry-After-Ms':'250'}){
+   context.fetch=async()=>{calls++;if(status===0)throw Error('network');return {status,headers:new Headers(headers)}};
+   const controller=new AbortController();const promise=client.send('/api/v1/up',{signal:controller.signal},null).catch(()=>{});
+   await flush();const timer=[...timers.values()][0];assert.ok(timer);const delay=timer.ms;
+   controller.abort();await promise;assert.equal(timers.size,0);return delay;
+  },async expire(){const item=[...timers.entries()][0];assert.ok(item);timers.delete(item[0]);now+=item[1].ms;item[1].fn();await flush()}
+ };
+}
+module.exports=async()=>{
+ const config={enabled:true,min:0,max:250,peak:125,triangular:false};
+ for(const mode of ['ok','absent','throw']){
+  const h=harness(config,0.5,mode);assert.equal(await h.sample(),375);
+ }
+ for(const triangular of [false,true]){
+  for(const peak of [0,25,100]){
+   let sum=0;
+   for(let i=0;i<256;i++){
+    const h=harness({...config,min:0,max:100,peak,triangular},(i+0.5)/256);
+    const value=await h.sample()-250;assert.ok(value>=0&&value<=100);sum+=value;
+   }
+   assert.ok(Math.abs(sum/256-(triangular?(100+peak)/3:50))<1);
+  }
+ }
+ for(const u of [0,0.25,0.5,0.75,1-1/4294967296]){
+  const h=harness({...config,min:0,max:100,peak:50,triangular:true},u);
+  const expected=[0,35,50,65,100][[0,0.25,0.5,0.75,1-1/4294967296].indexOf(u)];
+  assert.equal(await h.sample()-250,expected);
+ }
+ const frozen={...config,min:100,max:100};const h=harness(frozen);frozen.min=0;frozen.max=0;
+ assert.equal(await h.sample(),350);assert.equal(h.draws,0);
+ for(const status of [0,408,429,502,503,504])assert.equal(await h.sample(status,{}),350);
+ assert.equal(await h.sample(429,{'Retry-After':'2'}),2100);
+ assert.equal(await h.sample(503,{'Retry-After':'40'}),40100);
+ assert.equal(await h.sample(503,{'Retry-After':new Date(40000).toUTCString()}),40100);
+ assert.equal(await h.sample(503,{'Retry-After':'bad'}),350);
+ // Budget exhaustion must not turn a long minimum into an early retry.
+ const bounded=harness(config,0.5,'ok',1000);
+ bounded.context.fetch=async()=>({status:503,headers:new Headers({'Retry-After':'99999999999999999999999999999999999999999'})});
+ const pending=bounded.client.send('/api/v1/up',{},null).then(()=>assert.fail('unexpected success'),()=>{});
+ await flush();await bounded.expire();await pending;assert.equal(bounded.timers.size,0);
+ // A new draw is made per retry while the exponential base and attempt cap remain.
+ const retries=harness(config,0.5);let attempts=0;
+ retries.context.fetch=async()=>{attempts++;throw Error('network')};
+ const failed=retries.client.send('/api/v1/up',{},null).catch(()=>{});await flush();
+ for(const delay of [375,625,1125,2125,2125,2125,2125,2125]){
+  assert.equal([...retries.timers.values()][0].ms,delay);await retries.expire();
+ }
+ await failed;assert.equal(attempts,9);assert.equal(retries.draws,8);assert.equal(retries.timers.size,0);
+ // Successful responses do not draw entropy or wait for jitter.
+ const success=harness(config);success.context.fetch=async()=>({status:204,headers:new Headers()});
+ await success.client.send('/api/v1/up',{},null);assert.equal(success.draws,0);assert.equal(success.timers.size,0);
+};

src/web/bridge/runtime.js

@@ -30,6 +30,7 @@ const failureReason=(error,fallback)=>error&&canonicalFailures.includes(error.te
 const status=__STATUS_FUNCTION__;
 const socketURL=()=>relayBase.replace(/^https:/,'wss:')+'/api/v1/ws';
 const requestClient=requestSupport.create({
+ jitter:Object.freeze(__RETRY_JITTER__),
  base:()=>relayBase,closed:()=>closed,retryMs:()=>bridgeRetryMs,longPollMs:()=>longPollMs,requestMs:()=>bridgeRequestMs,
  batchLimit:()=>batchLimit,read:(response,limit,exact,signal)=>responseBody.read(response,limit,exact,signal),cancel:responseBody.cancel,
  failure,reason:failureReason,retrying:()=>status('reconnecting')

src/web/bridge/tests.rs

@@ -23,6 +23,7 @@ fn carrier_method_is_page_owned_and_used_by_every_https_request() {
             0,
             true,
             method,
+            &crate::config::WebRetryJitterConfig::default(),
             &SecureRandom::new(),
         );
         assert!(!page.body.contains("__"));
@@ -80,6 +81,7 @@ fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
         0,
         false,
         WebCarrierMethod::Post,
+        &crate::config::WebRetryJitterConfig::default(),
         &SecureRandom::new(),
     )
 }
@@ -105,6 +107,7 @@ fn render_diagnostic_page(bootstrap: &str) -> BridgePage {
         0,
         true,
         WebCarrierMethod::Post,
+        &crate::config::WebRetryJitterConfig::default(),
         &SecureRandom::new(),
     )
 }
@@ -157,6 +160,7 @@ fn rendered_page_resolves_carriers_against_the_exact_base_path() {
         0,
         true,
         WebCarrierMethod::Post,
+        &crate::config::WebRetryJitterConfig::default(),
         &SecureRandom::new(),
     );
 
@@ -211,6 +215,7 @@ fn rendered_page_embeds_the_configured_bridge_timing_policy() {
         4,
         false,
         WebCarrierMethod::Post,
+        &crate::config::WebRetryJitterConfig::default(),
         &SecureRandom::new(),
     );
 
@@ -265,6 +270,7 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
         0,
         false,
         WebCarrierMethod::Post,
+        &crate::config::WebRetryJitterConfig::default(),
         &SecureRandom::new(),
     );
     assert!(page.body.contains(