"""Run the regression routes on a build, score them and compare with a base build. python tools/regress.py build/v121 --repeat 3 --out build/regress/v121 python tools/regress.py build/v121 --routes island --cold python tools/regress.py build/v121 --routes field --validation Each run writes //-/ (log.txt, shots/, score.json); the summary prints median and worst per metric and route for every build, and with --base a diff table (ok, better, worse). Runs are nondeterministic: a verdict compares medians. A candidate build without a pipeline cache, pipeline-keys.bin or shader-cache gets the base build's copy before its first warm run. Every run works on a private copy of the pipeline cache, so the builds' own files stay as they are (TOTK_SHADER_CACHE still points at the build's directory, which the game extends). --cold starts the runs with an empty pipeline cache and key file; translated shaders stay. """ import argparse import datetime import json import re import shutil import statistics import subprocess import sys from pathlib import Path from PIL import Image, ImageDraw, ImageStat sys.path.insert(0, str(Path(__file__).resolve().parent)) from regress_score import far_gradient, is_scored, parse_perf, score_frames, vuid_counts # noqa: E402 ROOT = Path(__file__).resolve().parent.parent SAVES = ROOT / 'build' / 'save-backup' SLOW_MS = 50 SHOT_STEP = 1 / 12 # seconds of world clock between shots TAIL_FRAMES = 30 # shots examined for a dead or stuck ending TAIL_BLACK = 0.9 BLACK_LUMA = 12 END_SLACK = 2 # seconds of world clock the log may stop short of the requested run # name: (save, script, seconds of world clock to run, scored window in world seconds) WORLD = re.compile(r'perf: frame (\d+): interval avg .*; world (-?[0-9.]+) s') ROUTES = { 'fight': ('user-save-2026-10-21-monsters', 'tools/input/fight.txt', 18, (1, 18)), 'rock': ('user-save-2026-10-21-monsters', 'build/recordings/rock-2026-10-22.txt', 30, (0, 30)), 'island': ('user-save-2026-10-22-island', 'tools/input/island.txt', 35, (0, 35)), 'field': ('user-save-2026-10-21-field', 'tools/input/load_save.txt', 14, (1, 14)), } # metric: (label, lower is better, absolute tolerance, relative tolerance) METRICS = { 'fps_mean': ('fps', False, 1.0, 0.03), 'max_frame_ms': ('max frame ms', True, 5.0, 0.15), 'slow_count': (f'frames >{SLOW_MS} ms', True, 1, 0.15), 'render_busy_ms': ('render busy ms', True, 0.5, 0.10), 'replay_ms': ('replay ms', True, 0.5, 0.10), 'black_frames': ('black-block frames', True, 0, 0), 'flat_frames': ('flat frames', True, 0, 0), 'vuid_total': ('validation messages', True, 0, 0), } CACHE_FILES = ('pipeline-cache.bin', 'pipeline-keys.bin') def seed_caches(build, base): """Copy the base build's caches into a candidate that lacks them; returns what was copied.""" seeded = [] if build.resolve() == base.resolve(): return seeded for name in (*CACHE_FILES, 'shader-cache'): src, dst = base / name, build / name if dst.exists() or not src.exists(): continue if src.is_dir(): shutil.copytree(src, dst) else: shutil.copy2(src, dst) seeded.append(name) return seeded def run_route(build, route, i, out, cold, validation): save, script, seconds, (lo, hi) = ROUTES[route] run = out / build.name / f'{route}-{i}' shutil.rmtree(run, ignore_errors=True) shots = run / 'shots' cache = run / 'cache' shots.mkdir(parents=True) cache.mkdir() if not cold: for name in CACHE_FILES: if (build / name).exists(): shutil.copy2(build / name, cache / name) env = { 'TOTK_TRACE': 'perf,sched', 'TOTK_SHOTS': f'w{lo}-{hi}:{SHOT_STEP:.4f}', 'TOTK_SHOTS_DIR': str(shots), 'TOTK_PIPELINE_CACHE': str(cache / 'pipeline-cache.bin'), } if (build / 'shader-cache').is_dir(): env['TOTK_SHADER_CACHE'] = str(build / 'shader-cache') if validation: env['TOTK_VK_VALIDATION'] = '1' log = run / 'log.txt' cmd = [sys.executable, str(ROOT / 'tools' / 'run_game.py'), str(build), '--save', save, '--script', str(ROOT / script), '--seconds', str(seconds), '--quiet', '--tag', f'rg-{route}-{i}', '--log', str(log)] for k, v in env.items(): cmd += ['--env', f'{k}={v}'] rc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode score = score_run(log, shots, lo, hi, seconds, rc) score['run'] = str(run) (run / 'score.json').write_text(json.dumps(score, indent=1)) return score def black_fraction(img): grey = img.convert('L') return sum(grey.histogram()[:BLACK_LUMA]) / (grey.width * grey.height) def world_frames(text, lo, hi): """The presents of the world-clock window [lo, hi] seconds, from the perf lines (one per 60 presents), and the world clock of the last one.""" lo_n = hi_n = None last = -1.0 for m in WORLD.finditer(text): n, w = int(m[1]), float(m[2]) last = w if w >= lo and lo_n is None: lo_n = n - 60 if w <= hi: hi_n = n return (lo_n or 0), (hi_n or 0), last def score_run(log, shots, lo, hi, seconds, rc): text = log.read_text(encoding='utf-8', errors='replace') if log.exists() else '' lo, hi, last_world = world_frames(text, lo, hi) perf = parse_perf(log, lo, hi) paths = {int(p.stem[1:]): p for p in shots.glob('f*.png')} fr = score_frames(paths) if paths else {'black_counts': {}, 'flat_frames': [], 'scored': 0, 'total': 0} vuids = vuid_counts(text) last = [n for n in sorted(paths)][-TAIL_FRAMES:] tail_black = 0.0 if last: fractions = [] for n in last: with Image.open(paths[n]) as im: fractions.append(black_fraction(im.convert('RGB'))) tail_black = statistics.mean(fractions) reasons = [] if tail_black > TAIL_BLACK: reasons.append(f'last {len(last)} shots {tail_black:.0%} black (game over?)') if perf['crashed']: reasons.append('crashed or no perf lines') if rc != 0: reasons.append(f'exit code {rc}') if last_world < seconds - END_SLACK: reasons.append(f'log stops at {last_world:.1f} s of world clock of {seconds}') return { 'rc': rc, 'fps_mean': perf['fps_mean'], 'max_frame_ms': perf['max_frame_ms'], 'slow_count': sum(1 for _, ms, _ in perf['slow_frames'] if ms >= SLOW_MS), 'slow_frames': [(n, ms, cls) for n, ms, cls in perf['slow_frames'] if ms >= SLOW_MS], 'render_busy_ms': perf['render_busy_ms'], 'replay_ms': perf['replay_ms'], 'black_frames': len(fr['black_counts']), 'black_counts': fr['black_counts'], 'flat_frames': len(fr['flat_frames']), 'flat_list': fr['flat_frames'], 'scored': fr['scored'], 'shots': fr['total'], 'vuids': vuids, 'vuid_total': sum(vuids.values()), 'route_failed': bool(reasons), 'fail_reasons': reasons, } def contact_sheet(scores, path): """Up to 6 worst frames over the runs of one route: black blocks first, then flat, then the lowest far gradient.""" cands = [] for s in scores: shots = Path(s['run']) / 'shots' for n, c in s['black_counts'].items(): cands.append((0, -c, s['run'], int(n), f'black x{c}')) for n in s['flat_list']: cands.append((1, 0, s['run'], int(n), 'flat')) cands.sort() chosen = cands[:6] if len(chosen) < 6 and scores: have = {(r, n) for _, _, r, n, _ in chosen} rest = [] for p in sorted((Path(scores[0]['run']) / 'shots').glob('f*.png')): with Image.open(p) as im: im = im.convert('RGB') if is_scored(im): rest.append((far_gradient(im), scores[0]['run'], int(p.stem[1:]))) rest.sort() for g, r, n in rest: if len(chosen) == 6: break if (r, n) not in have: chosen.append((2, g, r, n, f'lowest far gradient {g:.1f}')) if not chosen: return False sheet = Image.new('RGB', (3 * 480, 2 * 270)) for k, (_, _, run, n, label) in enumerate(chosen): with Image.open(Path(run) / 'shots' / f'f{n:05d}.png') as im: tile = im.convert('RGB') d = ImageDraw.Draw(tile) d.rectangle((0, 0, 330, 14), fill=(0, 0, 0)) d.text((2, 2), f'{Path(run).name} f{n} {label}', fill=(255, 255, 0)) sheet.paste(tile, ((k % 3) * 480, (k // 3) * 270)) sheet.save(path) return True def stat(values, lower_is_better): """(median, worst) over the runs that report the metric; None when none does.""" vals = [v for v in values if v is not None] if not vals: return None, None return statistics.median(vals), (max(vals) if lower_is_better else min(vals)) def fmt(v): return 'n/a' if v is None else (f'{v:.1f}' if isinstance(v, float) else str(v)) def summarize(name, results): """results: {route: [score]} -> {route: {metric: (median, worst)}}; prints the table.""" table = {r: {m: stat([s[m] for s in scs], METRICS[m][1]) for m in METRICS} for r, scs in results.items()} print(f'\n== {name}: median / worst of {max(len(s) for s in results.values())} runs') print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS) + ' failed') for r, row in table.items(): failed = sum(s['route_failed'] for s in results[r]) print(f'{r:8}' + ''.join(f'{fmt(row[m][0]) + " / " + fmt(row[m][1]):>24}' for m in METRICS) + f' {failed}') for r, scs in results.items(): for s in scs: if s['route_failed']: print(f' ROUTE FAILED {s["run"]}: {"; ".join(s["fail_reasons"])}') return table def verdict(m, cand, base): if cand is None or base is None: return 'n/a' _, lower, tol_abs, tol_rel = METRICS[m] delta = cand - base if lower else base - cand slack = max(tol_abs, tol_rel * abs(base)) if delta > slack: return 'worse' if delta < -slack: return 'better' return 'ok' def diff_table(cand, base, cand_results, base_results): print('\n== diff against base (median; verdict)') print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS)) for r in cand: cells = [] for m in METRICS: c, b = cand[r][m][0], base.get(r, {}).get(m, (None,))[0] cells.append(f'{fmt(c)} vs {fmt(b)} {verdict(m, c, b)}') print(f'{r:8}' + ''.join(f'{c:>24}' for c in cells)) new = set().union(*(s['vuids'] for s in cand_results[r])) - set().union(*(s['vuids'] for s in base_results.get(r, []))) if new: print(f' new validation ids in {r}: {", ".join(sorted(new))}') def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('build', type=Path) ap.add_argument('--base', type=Path, help='frozen build to compare with (e.g. build/v121-base)') ap.add_argument('--routes', default='fight,rock,island,field') ap.add_argument('--repeat', type=int, default=1) ap.add_argument('--cold', action='store_true', help='empty pipeline cache and key file') ap.add_argument('--validation', action='store_true', help='TOTK_VK_VALIDATION=1; counts VUIDs') ap.add_argument('--out', type=Path) a = ap.parse_args() out = (a.out or ROOT / 'build' / 'regress' / datetime.datetime.now().strftime('%Y%m%d-%H%M%S')).resolve() routes = a.routes.split(',') for r in routes: if r not in ROUTES: sys.exit(f'unknown route {r}; choose from {", ".join(ROUTES)}') if not (SAVES / ROUTES[r][0]).is_dir() or not (ROOT / ROUTES[r][1]).exists(): sys.exit(f'route {r}: missing save or script') builds = [a.build.resolve()] + ([a.base.resolve()] if a.base else []) if a.base and not a.cold: seeded = seed_caches(builds[0], builds[1]) print('caches seeded from base:', ', '.join(seeded) if seeded else 'none (the build had its own)') tables, all_results = {}, {} for build in builds: results = {} for r in routes: results[r] = [run_route(build, r, i, out, a.cold, a.validation) for i in range(a.repeat)] sheet = out / build.name / f'{r}-worst.png' contact_sheet(results[r], sheet) all_results[build.name] = results tables[build.name] = summarize(build.name, results) if a.base: diff_table(tables[builds[0].name], tables[builds[1].name], all_results[builds[0].name], all_results[builds[1].name]) (out / 'summary.json').write_text(json.dumps(tables, indent=1)) print(f'\nwrote {out}') bad = any(s['route_failed'] for res in all_results.values() for scs in res.values() for s in scs) sys.exit(1 if bad else 0) if __name__ == '__main__': main()