Files
totk-pc-recomp/tools/regress.py
T

304 lines
13 KiB
Python

"""Run the regression routes on a build, score them and compare with a base build.
python tools/regress.py build/v121 --repeat 3 --out build/regress/v121
python tools/regress.py build/v121 --routes island --cold
python tools/regress.py build/v121 --routes field --validation
Each run writes <out>/<build name>/<route>-<i>/ (log.txt, shots/, score.json); the summary prints median and worst per
metric and route for every build, and with --base a diff table (ok, better, worse). Runs are nondeterministic: a
verdict compares medians. A candidate build without a pipeline cache, pipeline-keys.bin or shader-cache gets the
base build's copy before its first warm run. Every run works on a private copy of the pipeline cache, so the builds'
own files stay as they are (TOTK_SHADER_CACHE still points at the build's directory, which the game extends).
--cold starts the runs with an empty pipeline cache and key file; translated shaders stay.
"""
import argparse
import datetime
import json
import re
import shutil
import statistics
import subprocess
import sys
from pathlib import Path
from PIL import Image, ImageDraw, ImageStat
sys.path.insert(0, str(Path(__file__).resolve().parent))
from regress_score import far_gradient, is_scored, parse_perf, score_frames, vuid_counts # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
SAVES = ROOT / 'build' / 'save-backup'
SLOW_MS = 50
SHOT_STEP = 1 / 12 # seconds of world clock between shots
TAIL_FRAMES = 30 # shots examined for a dead or stuck ending
TAIL_BLACK = 0.9
BLACK_LUMA = 12
END_SLACK = 2 # seconds of world clock the log may stop short of the requested run
# name: (save, script, seconds of world clock to run, scored window in world seconds)
WORLD = re.compile(r'perf: frame (\d+): interval avg .*; world (-?[0-9.]+) s')
ROUTES = {
'fight': ('user-save-2026-10-21-monsters', 'tools/input/fight.txt', 18, (1, 18)),
'rock': ('user-save-2026-10-21-monsters', 'build/recordings/rock-2026-10-22.txt', 30, (0, 30)),
'island': ('user-save-2026-10-22-island', 'tools/input/island.txt', 35, (0, 35)),
'field': ('user-save-2026-10-21-field', 'tools/input/load_save.txt', 14, (1, 14)),
}
# metric: (label, lower is better, absolute tolerance, relative tolerance)
METRICS = {
'fps_mean': ('fps', False, 1.0, 0.03),
'max_frame_ms': ('max frame ms', True, 5.0, 0.15),
'slow_count': (f'frames >{SLOW_MS} ms', True, 1, 0.15),
'render_busy_ms': ('render busy ms', True, 0.5, 0.10),
'replay_ms': ('replay ms', True, 0.5, 0.10),
'black_frames': ('black-block frames', True, 0, 0),
'flat_frames': ('flat frames', True, 0, 0),
'vuid_total': ('validation messages', True, 0, 0),
}
CACHE_FILES = ('pipeline-cache.bin', 'pipeline-keys.bin')
def seed_caches(build, base):
"""Copy the base build's caches into a candidate that lacks them; returns what was copied."""
seeded = []
if build.resolve() == base.resolve():
return seeded
for name in (*CACHE_FILES, 'shader-cache'):
src, dst = base / name, build / name
if dst.exists() or not src.exists():
continue
if src.is_dir():
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
seeded.append(name)
return seeded
def run_route(build, route, i, out, cold, validation):
save, script, seconds, (lo, hi) = ROUTES[route]
run = out / build.name / f'{route}-{i}'
shutil.rmtree(run, ignore_errors=True)
shots = run / 'shots'
cache = run / 'cache'
shots.mkdir(parents=True)
cache.mkdir()
if not cold:
for name in CACHE_FILES:
if (build / name).exists():
shutil.copy2(build / name, cache / name)
env = {
'TOTK_TRACE': 'perf,sched',
'TOTK_SHOTS': f'w{lo}-{hi}:{SHOT_STEP:.4f}',
'TOTK_SHOTS_DIR': str(shots),
'TOTK_PIPELINE_CACHE': str(cache / 'pipeline-cache.bin'),
}
if (build / 'shader-cache').is_dir():
env['TOTK_SHADER_CACHE'] = str(build / 'shader-cache')
if validation:
env['TOTK_VK_VALIDATION'] = '1'
log = run / 'log.txt'
cmd = [sys.executable, str(ROOT / 'tools' / 'run_game.py'), str(build), '--save', save,
'--script', str(ROOT / script), '--seconds', str(seconds), '--quiet', '--tag', f'rg-{route}-{i}',
'--log', str(log)]
for k, v in env.items():
cmd += ['--env', f'{k}={v}']
rc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
score = score_run(log, shots, lo, hi, seconds, rc)
score['run'] = str(run)
(run / 'score.json').write_text(json.dumps(score, indent=1))
return score
def black_fraction(img):
grey = img.convert('L')
return sum(grey.histogram()[:BLACK_LUMA]) / (grey.width * grey.height)
def world_frames(text, lo, hi):
"""The presents of the world-clock window [lo, hi] seconds, from the perf lines (one per 60 presents), and the
world clock of the last one."""
lo_n = hi_n = None
last = -1.0
for m in WORLD.finditer(text):
n, w = int(m[1]), float(m[2])
last = w
if w >= lo and lo_n is None:
lo_n = n - 60
if w <= hi:
hi_n = n
return (lo_n or 0), (hi_n or 0), last
def score_run(log, shots, lo, hi, seconds, rc):
text = log.read_text(encoding='utf-8', errors='replace') if log.exists() else ''
lo, hi, last_world = world_frames(text, lo, hi)
perf = parse_perf(log, lo, hi)
paths = {int(p.stem[1:]): p for p in shots.glob('f*.png')}
fr = score_frames(paths) if paths else {'black_counts': {}, 'flat_frames': [], 'scored': 0, 'total': 0}
vuids = vuid_counts(text)
last = [n for n in sorted(paths)][-TAIL_FRAMES:]
tail_black = 0.0
if last:
fractions = []
for n in last:
with Image.open(paths[n]) as im:
fractions.append(black_fraction(im.convert('RGB')))
tail_black = statistics.mean(fractions)
reasons = []
if tail_black > TAIL_BLACK:
reasons.append(f'last {len(last)} shots {tail_black:.0%} black (game over?)')
if perf['crashed']:
reasons.append('crashed or no perf lines')
if rc != 0:
reasons.append(f'exit code {rc}')
if last_world < seconds - END_SLACK:
reasons.append(f'log stops at {last_world:.1f} s of world clock of {seconds}')
return {
'rc': rc, 'fps_mean': perf['fps_mean'], 'max_frame_ms': perf['max_frame_ms'],
'slow_count': sum(1 for _, ms, _ in perf['slow_frames'] if ms >= SLOW_MS),
'slow_frames': [(n, ms, cls) for n, ms, cls in perf['slow_frames'] if ms >= SLOW_MS],
'render_busy_ms': perf['render_busy_ms'], 'replay_ms': perf['replay_ms'],
'black_frames': len(fr['black_counts']), 'black_counts': fr['black_counts'],
'flat_frames': len(fr['flat_frames']), 'flat_list': fr['flat_frames'],
'scored': fr['scored'], 'shots': fr['total'],
'vuids': vuids, 'vuid_total': sum(vuids.values()),
'route_failed': bool(reasons), 'fail_reasons': reasons,
}
def contact_sheet(scores, path):
"""Up to 6 worst frames over the runs of one route: black blocks first, then flat, then the lowest far gradient."""
cands = []
for s in scores:
shots = Path(s['run']) / 'shots'
for n, c in s['black_counts'].items():
cands.append((0, -c, s['run'], int(n), f'black x{c}'))
for n in s['flat_list']:
cands.append((1, 0, s['run'], int(n), 'flat'))
cands.sort()
chosen = cands[:6]
if len(chosen) < 6 and scores:
have = {(r, n) for _, _, r, n, _ in chosen}
rest = []
for p in sorted((Path(scores[0]['run']) / 'shots').glob('f*.png')):
with Image.open(p) as im:
im = im.convert('RGB')
if is_scored(im):
rest.append((far_gradient(im), scores[0]['run'], int(p.stem[1:])))
rest.sort()
for g, r, n in rest:
if len(chosen) == 6:
break
if (r, n) not in have:
chosen.append((2, g, r, n, f'lowest far gradient {g:.1f}'))
if not chosen:
return False
sheet = Image.new('RGB', (3 * 480, 2 * 270))
for k, (_, _, run, n, label) in enumerate(chosen):
with Image.open(Path(run) / 'shots' / f'f{n:05d}.png') as im:
tile = im.convert('RGB')
d = ImageDraw.Draw(tile)
d.rectangle((0, 0, 330, 14), fill=(0, 0, 0))
d.text((2, 2), f'{Path(run).name} f{n} {label}', fill=(255, 255, 0))
sheet.paste(tile, ((k % 3) * 480, (k // 3) * 270))
sheet.save(path)
return True
def stat(values, lower_is_better):
"""(median, worst) over the runs that report the metric; None when none does."""
vals = [v for v in values if v is not None]
if not vals:
return None, None
return statistics.median(vals), (max(vals) if lower_is_better else min(vals))
def fmt(v):
return 'n/a' if v is None else (f'{v:.1f}' if isinstance(v, float) else str(v))
def summarize(name, results):
"""results: {route: [score]} -> {route: {metric: (median, worst)}}; prints the table."""
table = {r: {m: stat([s[m] for s in scs], METRICS[m][1]) for m in METRICS} for r, scs in results.items()}
print(f'\n== {name}: median / worst of {max(len(s) for s in results.values())} runs')
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS) + ' failed')
for r, row in table.items():
failed = sum(s['route_failed'] for s in results[r])
print(f'{r:8}' + ''.join(f'{fmt(row[m][0]) + " / " + fmt(row[m][1]):>24}' for m in METRICS) + f' {failed}')
for r, scs in results.items():
for s in scs:
if s['route_failed']:
print(f' ROUTE FAILED {s["run"]}: {"; ".join(s["fail_reasons"])}')
return table
def verdict(m, cand, base):
if cand is None or base is None:
return 'n/a'
_, lower, tol_abs, tol_rel = METRICS[m]
delta = cand - base if lower else base - cand
slack = max(tol_abs, tol_rel * abs(base))
if delta > slack:
return 'worse'
if delta < -slack:
return 'better'
return 'ok'
def diff_table(cand, base, cand_results, base_results):
print('\n== diff against base (median; verdict)')
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS))
for r in cand:
cells = []
for m in METRICS:
c, b = cand[r][m][0], base.get(r, {}).get(m, (None,))[0]
cells.append(f'{fmt(c)} vs {fmt(b)} {verdict(m, c, b)}')
print(f'{r:8}' + ''.join(f'{c:>24}' for c in cells))
new = set().union(*(s['vuids'] for s in cand_results[r])) - set().union(*(s['vuids'] for s in base_results.get(r, [])))
if new:
print(f' new validation ids in {r}: {", ".join(sorted(new))}')
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('build', type=Path)
ap.add_argument('--base', type=Path, help='frozen build to compare with (e.g. build/v121-base)')
ap.add_argument('--routes', default='fight,rock,island,field')
ap.add_argument('--repeat', type=int, default=1)
ap.add_argument('--cold', action='store_true', help='empty pipeline cache and key file')
ap.add_argument('--validation', action='store_true', help='TOTK_VK_VALIDATION=1; counts VUIDs')
ap.add_argument('--out', type=Path)
a = ap.parse_args()
out = (a.out or ROOT / 'build' / 'regress' / datetime.datetime.now().strftime('%Y%m%d-%H%M%S')).resolve()
routes = a.routes.split(',')
for r in routes:
if r not in ROUTES:
sys.exit(f'unknown route {r}; choose from {", ".join(ROUTES)}')
if not (SAVES / ROUTES[r][0]).is_dir() or not (ROOT / ROUTES[r][1]).exists():
sys.exit(f'route {r}: missing save or script')
builds = [a.build.resolve()] + ([a.base.resolve()] if a.base else [])
if a.base and not a.cold:
seeded = seed_caches(builds[0], builds[1])
print('caches seeded from base:', ', '.join(seeded) if seeded else 'none (the build had its own)')
tables, all_results = {}, {}
for build in builds:
results = {}
for r in routes:
results[r] = [run_route(build, r, i, out, a.cold, a.validation) for i in range(a.repeat)]
sheet = out / build.name / f'{r}-worst.png'
contact_sheet(results[r], sheet)
all_results[build.name] = results
tables[build.name] = summarize(build.name, results)
if a.base:
diff_table(tables[builds[0].name], tables[builds[1].name], all_results[builds[0].name], all_results[builds[1].name])
(out / 'summary.json').write_text(json.dumps(tables, indent=1))
print(f'\nwrote {out}')
bad = any(s['route_failed'] for res in all_results.values() for scs in res.values() for s in scs)
sys.exit(1 if bad else 0)
if __name__ == '__main__':
main()