mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 12:08:38 +00:00
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""Run the regression routes on a build, score them and compare with a base build.
|
|
|
|
python tools/regress.py build/v121 --repeat 3 --out build/regress/v121
|
|
python tools/regress.py build/v121 --routes island --cold
|
|
python tools/regress.py build/v121 --routes field --validation
|
|
|
|
Each run writes <out>/<build name>/<route>-<i>/ (log.txt, shots/, score.json); the summary prints median and worst per
|
|
metric and route for every build, and with --base a diff table (ok, better, worse). Runs are nondeterministic: a
|
|
verdict compares medians. A candidate build without a pipeline cache, pipeline-keys.bin or shader-cache gets the
|
|
base build's copy before its first warm run. Every run works on a private copy of the pipeline cache, so the builds'
|
|
own files stay as they are (TOTK_SHADER_CACHE still points at the build's directory, which the game extends).
|
|
--cold starts the runs with an empty pipeline cache and key file; translated shaders stay.
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from PIL import Image, ImageDraw, ImageStat
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from regress_score import far_gradient, is_scored, parse_perf, score_frames, vuid_counts # noqa: E402
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SAVES = ROOT / 'build' / 'save-backup'
|
|
SLOW_MS = 50
|
|
SHOT_STEP = 5
|
|
TAIL_FRAMES = 30 # shots examined for a dead or stuck ending
|
|
TAIL_BLACK = 0.9
|
|
BLACK_LUMA = 12
|
|
END_SLACK = 120 # frames the log may stop short of the requested count
|
|
|
|
# name: (save, script, frames, scored window)
|
|
ROUTES = {
|
|
'fight': ('user-save-2026-10-21-monsters', 'tools/input/fight.txt', 2600, (1560, 2600)),
|
|
'rock': ('user-save-2026-10-21-monsters', 'build/recordings/rock-2026-10-22.txt', 3300, (1500, 3300)),
|
|
'island': ('user-save-2026-10-22-island', 'tools/input/island.txt', 3600, (1500, 3600)),
|
|
'field': ('user-save-2026-10-21-field', 'tools/input/load_save.txt', 2400, (1560, 2400)),
|
|
}
|
|
|
|
# metric: (label, lower is better, absolute tolerance, relative tolerance)
|
|
METRICS = {
|
|
'fps_mean': ('fps', False, 1.0, 0.03),
|
|
'max_frame_ms': ('max frame ms', True, 5.0, 0.15),
|
|
'slow_count': (f'frames >{SLOW_MS} ms', True, 1, 0.15),
|
|
'render_busy_ms': ('render busy ms', True, 0.5, 0.10),
|
|
'replay_ms': ('replay ms', True, 0.5, 0.10),
|
|
'black_frames': ('black-block frames', True, 0, 0),
|
|
'flat_frames': ('flat frames', True, 0, 0),
|
|
'vuid_total': ('validation messages', True, 0, 0),
|
|
}
|
|
CACHE_FILES = ('pipeline-cache.bin', 'pipeline-keys.bin')
|
|
|
|
|
|
def seed_caches(build, base):
|
|
"""Copy the base build's caches into a candidate that lacks them; returns what was copied."""
|
|
seeded = []
|
|
if build.resolve() == base.resolve():
|
|
return seeded
|
|
for name in (*CACHE_FILES, 'shader-cache'):
|
|
src, dst = base / name, build / name
|
|
if dst.exists() or not src.exists():
|
|
continue
|
|
if src.is_dir():
|
|
shutil.copytree(src, dst)
|
|
else:
|
|
shutil.copy2(src, dst)
|
|
seeded.append(name)
|
|
return seeded
|
|
|
|
|
|
def run_route(build, route, i, out, cold, validation):
|
|
save, script, frames, (lo, hi) = ROUTES[route]
|
|
run = out / build.name / f'{route}-{i}'
|
|
shutil.rmtree(run, ignore_errors=True)
|
|
shots = run / 'shots'
|
|
cache = run / 'cache'
|
|
shots.mkdir(parents=True)
|
|
cache.mkdir()
|
|
if not cold:
|
|
for name in CACHE_FILES:
|
|
if (build / name).exists():
|
|
shutil.copy2(build / name, cache / name)
|
|
env = {
|
|
'TOTK_TRACE': 'perf,sched',
|
|
'TOTK_SHOTS': f'{lo}-{hi}:{SHOT_STEP}',
|
|
'TOTK_SHOTS_DIR': str(shots),
|
|
'TOTK_PIPELINE_CACHE': str(cache / 'pipeline-cache.bin'),
|
|
}
|
|
if (build / 'shader-cache').is_dir():
|
|
env['TOTK_SHADER_CACHE'] = str(build / 'shader-cache')
|
|
if validation:
|
|
env['TOTK_VK_VALIDATION'] = '1'
|
|
log = run / 'log.txt'
|
|
cmd = [sys.executable, str(ROOT / 'tools' / 'run_game.py'), str(build), '--save', save,
|
|
'--script', str(ROOT / script), '--frames', str(frames), '--quiet', '--tag', f'rg-{route}-{i}',
|
|
'--log', str(log)]
|
|
for k, v in env.items():
|
|
cmd += ['--env', f'{k}={v}']
|
|
rc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
|
score = score_run(log, shots, lo, hi, frames, rc)
|
|
score['run'] = str(run)
|
|
(run / 'score.json').write_text(json.dumps(score, indent=1))
|
|
return score
|
|
|
|
|
|
def black_fraction(img):
|
|
grey = img.convert('L')
|
|
return sum(grey.histogram()[:BLACK_LUMA]) / (grey.width * grey.height)
|
|
|
|
|
|
def score_run(log, shots, lo, hi, frames, rc):
|
|
perf = parse_perf(log, lo, hi)
|
|
text = log.read_text(encoding='utf-8', errors='replace') if log.exists() else ''
|
|
paths = {int(p.stem[1:]): p for p in shots.glob('f*.png')}
|
|
fr = score_frames(paths) if paths else {'black_counts': {}, 'flat_frames': [], 'scored': 0, 'total': 0}
|
|
vuids = vuid_counts(text)
|
|
last = [n for n in sorted(paths)][-TAIL_FRAMES:]
|
|
tail_black = 0.0
|
|
if last:
|
|
fractions = []
|
|
for n in last:
|
|
with Image.open(paths[n]) as im:
|
|
fractions.append(black_fraction(im.convert('RGB')))
|
|
tail_black = statistics.mean(fractions)
|
|
last_frame = 0
|
|
for line in text.splitlines():
|
|
if 'perf: frame ' in line:
|
|
last_frame = int(line.split('perf: frame ')[1].split(':')[0])
|
|
reasons = []
|
|
if tail_black > TAIL_BLACK:
|
|
reasons.append(f'last {len(last)} shots {tail_black:.0%} black (game over?)')
|
|
if perf['crashed']:
|
|
reasons.append('crashed or no perf lines')
|
|
if rc != 0:
|
|
reasons.append(f'exit code {rc}')
|
|
if last_frame < frames - END_SLACK:
|
|
reasons.append(f'log stops at frame {last_frame} of {frames}')
|
|
return {
|
|
'rc': rc, 'fps_mean': perf['fps_mean'], 'max_frame_ms': perf['max_frame_ms'],
|
|
'slow_count': sum(1 for _, ms, _ in perf['slow_frames'] if ms >= SLOW_MS),
|
|
'slow_frames': [(n, ms, cls) for n, ms, cls in perf['slow_frames'] if ms >= SLOW_MS],
|
|
'render_busy_ms': perf['render_busy_ms'], 'replay_ms': perf['replay_ms'],
|
|
'black_frames': len(fr['black_counts']), 'black_counts': fr['black_counts'],
|
|
'flat_frames': len(fr['flat_frames']), 'flat_list': fr['flat_frames'],
|
|
'scored': fr['scored'], 'shots': fr['total'],
|
|
'vuids': vuids, 'vuid_total': sum(vuids.values()),
|
|
'route_failed': bool(reasons), 'fail_reasons': reasons,
|
|
}
|
|
|
|
|
|
def contact_sheet(scores, path):
|
|
"""Up to 6 worst frames over the runs of one route: black blocks first, then flat, then the lowest far gradient."""
|
|
cands = []
|
|
for s in scores:
|
|
shots = Path(s['run']) / 'shots'
|
|
for n, c in s['black_counts'].items():
|
|
cands.append((0, -c, s['run'], int(n), f'black x{c}'))
|
|
for n in s['flat_list']:
|
|
cands.append((1, 0, s['run'], int(n), 'flat'))
|
|
cands.sort()
|
|
chosen = cands[:6]
|
|
if len(chosen) < 6 and scores:
|
|
have = {(r, n) for _, _, r, n, _ in chosen}
|
|
rest = []
|
|
for p in sorted((Path(scores[0]['run']) / 'shots').glob('f*.png')):
|
|
with Image.open(p) as im:
|
|
im = im.convert('RGB')
|
|
if is_scored(im):
|
|
rest.append((far_gradient(im), scores[0]['run'], int(p.stem[1:])))
|
|
rest.sort()
|
|
for g, r, n in rest:
|
|
if len(chosen) == 6:
|
|
break
|
|
if (r, n) not in have:
|
|
chosen.append((2, g, r, n, f'lowest far gradient {g:.1f}'))
|
|
if not chosen:
|
|
return False
|
|
sheet = Image.new('RGB', (3 * 480, 2 * 270))
|
|
for k, (_, _, run, n, label) in enumerate(chosen):
|
|
with Image.open(Path(run) / 'shots' / f'f{n:05d}.png') as im:
|
|
tile = im.convert('RGB')
|
|
d = ImageDraw.Draw(tile)
|
|
d.rectangle((0, 0, 330, 14), fill=(0, 0, 0))
|
|
d.text((2, 2), f'{Path(run).name} f{n} {label}', fill=(255, 255, 0))
|
|
sheet.paste(tile, ((k % 3) * 480, (k // 3) * 270))
|
|
sheet.save(path)
|
|
return True
|
|
|
|
|
|
def stat(values, lower_is_better):
|
|
"""(median, worst) over the runs that report the metric; None when none does."""
|
|
vals = [v for v in values if v is not None]
|
|
if not vals:
|
|
return None, None
|
|
return statistics.median(vals), (max(vals) if lower_is_better else min(vals))
|
|
|
|
|
|
def fmt(v):
|
|
return 'n/a' if v is None else (f'{v:.1f}' if isinstance(v, float) else str(v))
|
|
|
|
|
|
def summarize(name, results):
|
|
"""results: {route: [score]} -> {route: {metric: (median, worst)}}; prints the table."""
|
|
table = {r: {m: stat([s[m] for s in scs], METRICS[m][1]) for m in METRICS} for r, scs in results.items()}
|
|
print(f'\n== {name}: median / worst of {max(len(s) for s in results.values())} runs')
|
|
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS) + ' failed')
|
|
for r, row in table.items():
|
|
failed = sum(s['route_failed'] for s in results[r])
|
|
print(f'{r:8}' + ''.join(f'{fmt(row[m][0]) + " / " + fmt(row[m][1]):>24}' for m in METRICS) + f' {failed}')
|
|
for r, scs in results.items():
|
|
for s in scs:
|
|
if s['route_failed']:
|
|
print(f' ROUTE FAILED {s["run"]}: {"; ".join(s["fail_reasons"])}')
|
|
return table
|
|
|
|
|
|
def verdict(m, cand, base):
|
|
if cand is None or base is None:
|
|
return 'n/a'
|
|
_, lower, tol_abs, tol_rel = METRICS[m]
|
|
delta = cand - base if lower else base - cand
|
|
slack = max(tol_abs, tol_rel * abs(base))
|
|
if delta > slack:
|
|
return 'worse'
|
|
if delta < -slack:
|
|
return 'better'
|
|
return 'ok'
|
|
|
|
|
|
def diff_table(cand, base, cand_results, base_results):
|
|
print('\n== diff against base (median; verdict)')
|
|
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS))
|
|
for r in cand:
|
|
cells = []
|
|
for m in METRICS:
|
|
c, b = cand[r][m][0], base.get(r, {}).get(m, (None,))[0]
|
|
cells.append(f'{fmt(c)} vs {fmt(b)} {verdict(m, c, b)}')
|
|
print(f'{r:8}' + ''.join(f'{c:>24}' for c in cells))
|
|
new = set().union(*(s['vuids'] for s in cand_results[r])) - set().union(*(s['vuids'] for s in base_results.get(r, [])))
|
|
if new:
|
|
print(f' new validation ids in {r}: {", ".join(sorted(new))}')
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument('build', type=Path)
|
|
ap.add_argument('--base', type=Path, help='frozen build to compare with (build/v121-m4)')
|
|
ap.add_argument('--routes', default='fight,rock,island,field')
|
|
ap.add_argument('--repeat', type=int, default=1)
|
|
ap.add_argument('--cold', action='store_true', help='empty pipeline cache and key file')
|
|
ap.add_argument('--validation', action='store_true', help='TOTK_VK_VALIDATION=1; counts VUIDs')
|
|
ap.add_argument('--out', type=Path)
|
|
a = ap.parse_args()
|
|
out = (a.out or ROOT / 'build' / 'regress' / datetime.datetime.now().strftime('%Y%m%d-%H%M%S')).resolve()
|
|
routes = a.routes.split(',')
|
|
for r in routes:
|
|
if r not in ROUTES:
|
|
sys.exit(f'unknown route {r}; choose from {", ".join(ROUTES)}')
|
|
if not (SAVES / ROUTES[r][0]).is_dir() or not (ROOT / ROUTES[r][1]).exists():
|
|
sys.exit(f'route {r}: missing save or script')
|
|
builds = [a.build.resolve()] + ([a.base.resolve()] if a.base else [])
|
|
if a.base and not a.cold:
|
|
seeded = seed_caches(builds[0], builds[1])
|
|
print('caches seeded from base:', ', '.join(seeded) if seeded else 'none (the build had its own)')
|
|
tables, all_results = {}, {}
|
|
for build in builds:
|
|
results = {}
|
|
for r in routes:
|
|
results[r] = [run_route(build, r, i, out, a.cold, a.validation) for i in range(a.repeat)]
|
|
sheet = out / build.name / f'{r}-worst.png'
|
|
contact_sheet(results[r], sheet)
|
|
all_results[build.name] = results
|
|
tables[build.name] = summarize(build.name, results)
|
|
if a.base:
|
|
diff_table(tables[builds[0].name], tables[builds[1].name], all_results[builds[0].name], all_results[builds[1].name])
|
|
(out / 'summary.json').write_text(json.dumps(tables, indent=1))
|
|
print(f'\nwrote {out}')
|
|
bad = any(s['route_failed'] for res in all_results.values() for scs in res.values() for s in scs)
|
|
sys.exit(1 if bad else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|