UpstreamManager: Backoff Retries

src/config/defaults.rs

@@ -8,6 +8,8 @@ const DEFAULT_STUN_TCP_FALLBACK: bool = true;
 const DEFAULT_MIDDLE_PROXY_WARM_STANDBY: usize = 16;
 const DEFAULT_ME_RECONNECT_MAX_CONCURRENT_PER_DC: u32 = 8;
 const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16;
+const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3;
+const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4;
 const DEFAULT_LISTEN_ADDR_IPV6: &str = "::";
 const DEFAULT_ACCESS_USER: &str = "default";
 const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000";
@@ -158,6 +160,18 @@ pub(crate) fn default_me_reconnect_fast_retry_count() -> u32 {
     DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT
 }
 
+pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
+    DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
+}
+
+pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
+    250
+}
+
+pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
+    DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
+}
+
 pub(crate) fn default_crypto_pending_buffer() -> usize {
     256 * 1024
 }

src/config/hot_reload.rs

@@ -117,6 +117,14 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) {
     if old.general.stun_nat_probe_concurrency != new.general.stun_nat_probe_concurrency {
         warn!("config reload: general.stun_nat_probe_concurrency changed; restart required");
     }
+    if old.general.upstream_connect_retry_attempts != new.general.upstream_connect_retry_attempts
+        || old.general.upstream_connect_retry_backoff_ms
+            != new.general.upstream_connect_retry_backoff_ms
+        || old.general.upstream_unhealthy_fail_threshold
+            != new.general.upstream_unhealthy_fail_threshold
+    {
+        warn!("config reload: general.upstream_* changed; restart required");
+    }
 }
 
 /// Resolve the public host for link generation — mirrors the logic in main.rs.

src/config/load.rs

@@ -237,6 +237,18 @@ impl ProxyConfig {
             ));
         }
 
+        if config.general.upstream_connect_retry_attempts == 0 {
+            return Err(ProxyError::Config(
+                "general.upstream_connect_retry_attempts must be > 0".to_string(),
+            ));
+        }
+
+        if config.general.upstream_unhealthy_fail_threshold == 0 {
+            return Err(ProxyError::Config(
+                "general.upstream_unhealthy_fail_threshold must be > 0".to_string(),
+            ));
+        }
+
         if config.general.me_reinit_every_secs == 0 {
             return Err(ProxyError::Config(
                 "general.me_reinit_every_secs must be > 0".to_string(),
@@ -567,6 +579,18 @@ mod tests {
             cfg.general.me_reconnect_fast_retry_count,
             default_me_reconnect_fast_retry_count()
         );
+        assert_eq!(
+            cfg.general.upstream_connect_retry_attempts,
+            default_upstream_connect_retry_attempts()
+        );
+        assert_eq!(
+            cfg.general.upstream_connect_retry_backoff_ms,
+            default_upstream_connect_retry_backoff_ms()
+        );
+        assert_eq!(
+            cfg.general.upstream_unhealthy_fail_threshold,
+            default_upstream_unhealthy_fail_threshold()
+        );
         assert_eq!(cfg.general.update_every, default_update_every());
         assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4());
         assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt());
@@ -593,6 +617,18 @@ mod tests {
             general.me_reconnect_fast_retry_count,
             default_me_reconnect_fast_retry_count()
         );
+        assert_eq!(
+            general.upstream_connect_retry_attempts,
+            default_upstream_connect_retry_attempts()
+        );
+        assert_eq!(
+            general.upstream_connect_retry_backoff_ms,
+            default_upstream_connect_retry_backoff_ms()
+        );
+        assert_eq!(
+            general.upstream_unhealthy_fail_threshold,
+            default_upstream_unhealthy_fail_threshold()
+        );
         assert_eq!(general.update_every, default_update_every());
 
         let server = ServerConfig::default();
@@ -765,6 +801,46 @@ mod tests {
         let _ = std::fs::remove_file(path);
     }
 
+    #[test]
+    fn upstream_connect_retry_attempts_zero_is_rejected() {
+        let toml = r#"
+            [general]
+            upstream_connect_retry_attempts = 0
+
+            [censorship]
+            tls_domain = "example.com"
+
+            [access.users]
+            user = "00000000000000000000000000000000"
+        "#;
+        let dir = std::env::temp_dir();
+        let path = dir.join("telemt_upstream_connect_retry_attempts_zero_test.toml");
+        std::fs::write(&path, toml).unwrap();
+        let err = ProxyConfig::load(&path).unwrap_err().to_string();
+        assert!(err.contains("general.upstream_connect_retry_attempts must be > 0"));
+        let _ = std::fs::remove_file(path);
+    }
+
+    #[test]
+    fn upstream_unhealthy_fail_threshold_zero_is_rejected() {
+        let toml = r#"
+            [general]
+            upstream_unhealthy_fail_threshold = 0
+
+            [censorship]
+            tls_domain = "example.com"
+
+            [access.users]
+            user = "00000000000000000000000000000000"
+        "#;
+        let dir = std::env::temp_dir();
+        let path = dir.join("telemt_upstream_unhealthy_fail_threshold_zero_test.toml");
+        std::fs::write(&path, toml).unwrap();
+        let err = ProxyConfig::load(&path).unwrap_err().to_string();
+        assert!(err.contains("general.upstream_unhealthy_fail_threshold must be > 0"));
+        let _ = std::fs::remove_file(path);
+    }
+
     #[test]
     fn me_hardswap_warmup_defaults_are_set() {
         let toml = r#"

src/config/types.rs

@@ -365,6 +365,18 @@ pub struct GeneralConfig {
     #[serde(default = "default_me_reconnect_fast_retry_count")]
     pub me_reconnect_fast_retry_count: u32,
 
+    /// Connect attempts for the selected upstream before returning error/fallback.
+    #[serde(default = "default_upstream_connect_retry_attempts")]
+    pub upstream_connect_retry_attempts: u32,
+
+    /// Delay in milliseconds between upstream connect attempts.
+    #[serde(default = "default_upstream_connect_retry_backoff_ms")]
+    pub upstream_connect_retry_backoff_ms: u64,
+
+    /// Consecutive failed requests before upstream is marked unhealthy.
+    #[serde(default = "default_upstream_unhealthy_fail_threshold")]
+    pub upstream_unhealthy_fail_threshold: u32,
+
     /// Ignore STUN/interface IP mismatch (keep using Middle Proxy even if NAT detected).
     #[serde(default)]
     pub stun_iface_mismatch_ignore: bool,
@@ -522,6 +534,9 @@ impl Default for GeneralConfig {
             me_reconnect_backoff_base_ms: default_reconnect_backoff_base_ms(),
             me_reconnect_backoff_cap_ms: default_reconnect_backoff_cap_ms(),
             me_reconnect_fast_retry_count: default_me_reconnect_fast_retry_count(),
+            upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
+            upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
+            upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
             stun_iface_mismatch_ignore: false,
             unknown_dc_log_path: default_unknown_dc_log_path(),
             log_level: LogLevel::Normal,