initial commit for public eval

2026-05-27 21:00:28 +02:00
commit d77a1bf412
36 changed files with 5414 additions and 0 deletions
@@ -0,0 +1,60 @@
+"""RTT per (experiment, solution): mean, SD, quantiles, spread.
+
+Also emits tso-pacing's variance-reduction ratios vs the other solutions.
+"""
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+def _stats(x: np.ndarray) -> dict[str, str]:
+    q05, q25, q50, q75, q95 = np.percentile(x, [5, 25, 50, 75, 95])
+    return {
+        "mean-ms":        f"{x.mean():.2f}",
+        "sd-ms":          f"{x.std(ddof=1):.2f}",
+        "median-ms":      f"{q50:.2f}",
+        "iqr-ms":         f"{q75 - q25:.2f}",
+        "p05-ms":         f"{q05:.2f}",
+        "p95-ms":         f"{q95:.2f}",
+        "spread-5-95-ms": f"{q95 - q05:.2f}",
+        "n-samples":      str(x.size),
+    }
+
+
+def compute(derived: Path) -> tuple[dict[str, str], list[Path]]:
+    out: dict[str, str] = {}
+    sources: list[Path] = []
+    for exp_dir in sorted(p for p in derived.iterdir() if p.is_dir()):
+        rtts = exp_dir / "rtts.csv"
+        if not rtts.exists():
+            continue
+        sources.append(rtts)
+        df = pd.read_csv(rtts)
+        per_sol: dict[str, dict[str, str]] = {}
+        for sol, sub in df.groupby("solution", sort=True):
+            x = sub["rtt_us"].to_numpy() / 1000.0
+            stats = _stats(x)
+            per_sol[sol] = stats
+            for k, v in stats.items():
+                out[f"{exp_dir.name}/rtt/{sol}/{k}"] = v
+
+        if "tso-pacing" in per_sol:
+            pac_sd     = float(per_sol["tso-pacing"]["sd-ms"])
+            pac_spread = float(per_sol["tso-pacing"]["spread-5-95-ms"])
+            pac_iqr    = float(per_sol["tso-pacing"]["iqr-ms"])
+            for other in ("no-tso", "tso", "cake"):
+                if other not in per_sol:
+                    continue
+                o = per_sol[other]
+                o_sd     = float(o["sd-ms"])
+                o_spread = float(o["spread-5-95-ms"])
+                o_iqr    = float(o["iqr-ms"])
+                base = f"{exp_dir.name}/rtt/tso-pacing-vs-{other}"
+                out[f"{base}/sd-ratio-pct"]         = f"{100 * pac_sd / o_sd:.1f}"
+                out[f"{base}/spread-ratio-pct"]     = f"{100 * pac_spread / o_spread:.1f}"
+                out[f"{base}/iqr-ratio-pct"]        = f"{100 * pac_iqr / o_iqr:.1f}"
+                out[f"{base}/sd-reduction-pct"]     = f"{100 * (1 - pac_sd / o_sd):.1f}"
+                out[f"{base}/spread-reduction-pct"] = f"{100 * (1 - pac_spread / o_spread):.1f}"
+                out[f"{base}/iqr-reduction-pct"]    = f"{100 * (1 - pac_iqr / o_iqr):.1f}"
+    return out, sources