use TCP RTT timeout-style calculation for max_rtt

this was one of the annoying limitations of the old system, that the max_rtt had to be set manually and I couldn't figure out how to effectively auto-detect it. This appears to work alright.
2025-12-07 20:38:34 -08:00
parent a656adf98f
commit bf7ad0a43a
3 changed files with 120 additions and 5 deletions
@@ -5,14 +5,69 @@ log_level_conf = 5
 metrics_addr = '127.0.0.1'
 metrics_port = 6969

+# interval at which to calculate and update gateway biases and states
+update_interval = 350
+
+
+# =================================================================
+#
+#   ping configuration
+#
+# =================================================================
+
 # minimum ping interval
 ping_interval = 100

 # ping timeout before assuming the destination is unreachable
 ping_timeout = 1000

-# maximum "acceptable" ping rtt before a gateway is considered down
-max_rtt = 200
+
+# =================================================================
+#
+#   RTT estimation configuration
+#
+# =================================================================
+
+# part of the new bias calculations
+# based on TCP timeout calculation
+# the est_rtt is calculated for all nodes averaged together
+# any nodes with RTT higher than the calculated max_rtt will be considered down
+
+# alpha value used for est_rtt running average calculation
+# - est_rtt = est_rtt_alpha * sample_rtt + (1 - est_rtt_alpha) * est_rtt
+# - higher values increase sensitivity to RTT samples
+# - lower values will be slower to react
+# - valid range: 0.0 - 1.0
+est_rtt_alpha = 0.25
+
+
+# beta value used for dev_rtt running average calculation
+# - dev_rtt = dev_rtt_beta * abs(sample_rtt - est_rtt) + (1 - dev_rtt_beta) * dev_rtt
+# - higher values increase sensitivity to deviations in est_rtt vs sample_rtt
+# - lower values will be slower to react
+# - valid range: 0.0 - 1.0
+dev_rtt_beta = 0.5
+
+
+# magnitude of dev_rtt when adding the safety margin to get the max allowed rtt 
+# before marking a tunnel node down. it's ok to set this a little higher than normal.
+# setting it too high will reduce the effectiveness of gateway biasing
+# - max_rtt = est_rtt + max_rtt_safezone * dev_rtt
+# - higher values increase the margin, increasing tolerance of higher deviation in RTT times across nodes
+# - lower values will be more likely to exclude nodes with high relative RTT
+# - valid range: 0.0+
+max_rtt_safezone = 4.20
+
+
+# =================================================================
+#
+#   gateway bias configuration
+#
+# =================================================================
+
+# here is where you can configure how gateway biases are calculated
+# biases make packets statistically more likely to go down tunnel nodes with lower RTT
+

 # max number of rtt samples to keep per gateway
 # lower values will increase responsiveness and sensitivity to rtt fluctuations
@@ -44,13 +99,22 @@ bias_score_samples = 200
 bias_score_decimals = 3

 # bias will be finalized as an integer on a scale of 0 to this number
-bias_granularity = 10
+bias_granularity = 100

 # function to modify slope of bias score
 # input is on scale of 0 - 1
 # output can be >=0
 bias_score_slope = (lambda x: x**3)

+
+# =================================================================
+#
+#   helper scripts
+#
+# =================================================================
+
+# you will likely want to change these to something more useful
+
 gw_setup_script = ['env']
 gw_update_script = ['env']

@@ -3,7 +3,7 @@ import argparse
 from helpers import *
 from log import Log, ansi_logging, concurrent_logging, shutdown_printer, log_level
 from ping import blast_ping, ping
-from stats import SampleData, nan, lerp
+from stats import SampleData, nan, lerp, avg_ignore_nan
 import math

 from threading import Thread
@@ -23,6 +23,11 @@ for metric in ['min', 'q1', 'mean', 'q3', 'max', 'stdev', 'count']:
    rtt_gauges[metric] = Gauge(NAMESPACE + 'rtt_' + metric, '%s of samples of rtt from ping test results in milliseconds' % metric, [GATEWAY])


+est_rtt_gauge = Gauge(NAMESPACE + 'est_rtt', 'estimated RTT of an exactly average tunnel node. used for max_rtt calculation')
+dev_rtt_gauge = Gauge(NAMESPACE + 'dev_rtt', 'dev RTT of an exactly average tunnel node. used for max_rtt calculation')
+max_rtt_gauge = Gauge(NAMESPACE + 'max_rtt', 'maximum RTT allowed before considering a node down. used to exclude nodes that deviate too much from the rest')
+
+
 class GatewayAddr(object):
    def __init__(self, addr):
        self.addr = addr
@@ -253,6 +258,11 @@ def update_metrics():
        summary = gw.mon_addr.rtt_samples.summarize()
        for stat in rtt_gauges.keys():
            rtt_gauges[stat].labels(gw.label).set(summary[stat])
+     
+    est_rtt_gauge.set(est_rtt)
+    dev_rtt_gauge.set(dev_rtt)
+    max_rtt_gauge.set(max_rtt)
+


 def generate_gw_states():
@@ -318,6 +328,7 @@ def ping_thread(gw: Gateway):
            result = ping(target.addr, ['-c', '1', '-W', '%f' % (ping_timeout / 1000.0)])

            # metrics update
+            update_max_rtt()
            up_now = result.status_code == 0 and result.sent == result.recv
            prev_known = target.known
            prev_up = target.up
@@ -344,6 +355,26 @@ def ping_thread(gw: Gateway):
            exit(1)
        

+# for max_rtt calcultions
+est_rtt = 0
+dev_rtt = 0
+max_rtt = 0
+
+
+def update_max_rtt():
+    global est_rtt
+    global dev_rtt
+    global max_rtt
+
+    avg_rtts = [gw.mon_addr.rtt_samples.last() for gw in gateways]
+    sample_rtt = avg_ignore_nan(avg_rtts)
+    if math.isnan(sample_rtt):
+        return
+
+    est_rtt = est_rtt_alpha * sample_rtt + (1 - est_rtt_alpha) * est_rtt
+    dev_rtt = dev_rtt_beta * abs(sample_rtt - est_rtt) + (1 - dev_rtt_beta) * dev_rtt
+    max_rtt = est_rtt + max_rtt_safezone * dev_rtt
+

 cur_gw_biases = []

@@ -354,7 +385,7 @@ def main_loop():
    tag = ('main loop',)
    while running:
        try:
-            time.sleep(0.5)
+            time.sleep(update_interval / 1000)

            update_metrics()

@@ -47,6 +47,21 @@ def summarize(samples: [float]) -> dict:
    }


+def avg_ignore_nan(samples: [float]):
+    quantity = 0
+    total = 0
+    for s in samples:
+        if math.isnan(s):
+            continue
+        total += s
+        quantity += 1
+    
+    if quantity == 0:
+        return nan
+    return total / quantity
+
+
+
 def lerp(i, start, end):
    return start * (1.0 - i) + end * i

@@ -69,6 +84,11 @@ class SampleData(object):
    def samples(self):
        return self._samples.copy()

+    def last(self):
+        if len(self._samples) == 0:
+            return nan
+        return self._samples[-1]
+
    def clear(self):
        self._samples = []