mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-11 01:41:02 +00:00
Keep the regression routes and the input scripts made for the maintainer's own saves out of the repository: regress.py, its scoring and test, and the fight, island and backrun scripts work only with those saves; load_save.txt stays as run_game.py's default
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed
1 parent
8ea1b5721f
commit
758b46460f
7 files changed
+6
-586
No files matched your search
@@ -1,59 +0,0 @@
|
||||
r"""Plain asserts: .venv\Scripts\python tests\regress_score_test.py"""
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'tools'))
|
||||
from regress_score import black_blocks, flat_frames, parse_perf_text, vuid_counts # noqa: E402
|
||||
|
||||
|
||||
def noise_image(seed):
|
||||
rng = random.Random(seed)
|
||||
img = Image.new('L', (480, 270))
|
||||
img.putdata([rng.randrange(60, 200) for _ in range(480 * 270)])
|
||||
return img.convert('RGB')
|
||||
|
||||
|
||||
def test_parse_perf_slow_frames():
|
||||
log = ("perf: slow frame 1601: 93.4 ms; blocked pipeline 61.0 xlate 0.0 texture 2.0 astc 0.0 fence 0.0 pacing 0.0\n"
|
||||
"perf: frame 1620: interval avg 33.40 ms max 93.40 ms (29.9 fps); render thread blocked: pacing 60% ... ; render busy avg 12.5 ms; replay avg 9.5 ms\n")
|
||||
r = parse_perf_text(log, 1560, 2600)
|
||||
assert r["slow_frames"] == [(1601, 93.4, "pipeline")]
|
||||
assert r["max_frame_ms"] == 93.4 and r["render_busy_ms"] == 12.5 and r["replay_ms"] == 9.5 and not r["crashed"]
|
||||
|
||||
|
||||
def test_parse_perf_other_and_crash():
|
||||
log = "perf: slow frame 1700: 60.0 ms; blocked pipeline 1.0 xlate 0.0 texture 2.0 astc 0.0 fence 0.0 pacing 0.0\n"
|
||||
r = parse_perf_text(log, 1560, 2600)
|
||||
assert r["slow_frames"] == [(1700, 60.0, "other")] and r["crashed"]
|
||||
fatal = "perf: frame 1620: interval avg 33.0 ms max 40.0 ms (30.0 fps)\nrt_fatal: x\n"
|
||||
assert parse_perf_text(fatal, 1560, 2600)["crashed"]
|
||||
|
||||
|
||||
def test_black_blocks_counts_hole_not_dark_frame():
|
||||
img = Image.new("RGB", (480, 270), (120, 120, 120)); img.paste((0, 0, 0), (160, 96, 176, 112))
|
||||
assert black_blocks(img) == 1
|
||||
assert black_blocks(Image.new("RGB", (480, 270), (0, 0, 0))) == 0
|
||||
|
||||
|
||||
def test_flat_frame_detected():
|
||||
sharp = noise_image(seed=1); flat = sharp.copy(); flat.paste((128, 128, 128), (0, 0, 480, 135))
|
||||
frames = [(i, sharp) for i in range(10)] + [(10, flat)] + [(i, sharp) for i in range(11, 21)]
|
||||
assert flat_frames(frames) == [10]
|
||||
|
||||
|
||||
def test_dark_frames_not_scored():
|
||||
sharp = noise_image(seed=2); dark = Image.new("RGB", (480, 270), (10, 10, 10))
|
||||
assert flat_frames([(i, sharp) for i in range(5)] + [(5, dark)]) == []
|
||||
|
||||
|
||||
def test_vuid_counts():
|
||||
assert vuid_counts("x VUID-vkCmdDraw-None-08600 y\nVUID-vkCmdDraw-None-08600\n") == {"VUID-vkCmdDraw-None-08600": 2}
|
||||
|
||||
|
||||
for name, fn in list(globals().items()):
|
||||
if name.startswith('test_'):
|
||||
fn()
|
||||
print('regress_score_test: passed')
|
||||
@@ -1,10 +0,0 @@
|
||||
# Pillars save (build/save-backup/user-save-2026-10-23-pillars): Hyrule field near standing stone pillars, about 17:40,
|
||||
# a Zonai construct beside Link. Title screen to gameplay as load_save.txt; gameplay from about frame 1500.
|
||||
# Phases (frames): 1550-2150 Link runs backward (stick down, about 20 s); 2160-2460 forward. Phantom rocks and cliffs
|
||||
# (wrong models in front of the distant mountains) appeared for the user while running backward.
|
||||
300 A 4
|
||||
520 A 4
|
||||
700 A 4
|
||||
900 A 4
|
||||
1550 LSTICK 0 -32767 600
|
||||
2160 LSTICK 0 32767 300
|
||||
@@ -1,64 +0,0 @@
|
||||
# Monsters save (build/save-backup/user-save-2026-10-21-monsters, slot 3): the fight starts right after loading.
|
||||
# Title screen to gameplay as load_save.txt; gameplay starts around frame 1500, enemies are ahead on the right.
|
||||
300 A 4
|
||||
520 A 4
|
||||
700 A 4
|
||||
900 A 4
|
||||
# Lock on (ZL held) and attack in bursts of three, with a short step toward the target between bursts.
|
||||
1540 ZL 1100
|
||||
1550 Y 4
|
||||
1570 Y 4
|
||||
1590 Y 4
|
||||
1605 LSTICK 0 32767 15
|
||||
1630 Y 4
|
||||
1650 Y 4
|
||||
1670 Y 4
|
||||
1685 LSTICK 0 32767 15
|
||||
1710 Y 4
|
||||
1730 Y 4
|
||||
1750 Y 4
|
||||
1765 LSTICK 0 32767 15
|
||||
1790 Y 4
|
||||
1810 Y 4
|
||||
1830 Y 4
|
||||
1845 LSTICK 0 32767 15
|
||||
1870 Y 4
|
||||
1890 Y 4
|
||||
1910 Y 4
|
||||
1925 LSTICK 0 32767 15
|
||||
1950 Y 4
|
||||
1970 Y 4
|
||||
1990 Y 4
|
||||
2005 LSTICK 0 32767 15
|
||||
2030 Y 4
|
||||
2050 Y 4
|
||||
2070 Y 4
|
||||
2085 LSTICK 0 32767 15
|
||||
2110 Y 4
|
||||
2130 Y 4
|
||||
2150 Y 4
|
||||
2165 LSTICK 0 32767 15
|
||||
2190 Y 4
|
||||
2210 Y 4
|
||||
2230 Y 4
|
||||
2245 LSTICK 0 32767 15
|
||||
2270 Y 4
|
||||
2290 Y 4
|
||||
2310 Y 4
|
||||
2325 LSTICK 0 32767 15
|
||||
2350 Y 4
|
||||
2370 Y 4
|
||||
2390 Y 4
|
||||
2405 LSTICK 0 32767 15
|
||||
2430 Y 4
|
||||
2450 Y 4
|
||||
2470 Y 4
|
||||
2485 LSTICK 0 32767 15
|
||||
2510 Y 4
|
||||
2530 Y 4
|
||||
2550 Y 4
|
||||
2565 LSTICK 0 32767 15
|
||||
2590 Y 4
|
||||
2610 Y 4
|
||||
2630 Y 4
|
||||
2645 LSTICK 0 32767 15
|
||||
@@ -1,13 +0,0 @@
|
||||
# Island save (build/save-backup/user-save-2026-10-22-island, slot 0): Link stands at the edge of a sky island, a Zonai
|
||||
# construct beside him. Title screen to gameplay as load_save.txt; gameplay from about frame 1500.
|
||||
# Phases (frames): 1550 stick forward, Link runs off the edge; skydive from about 1680 (the prompt "B Paraglider"
|
||||
# shows); the ground (a green slope) arrives at about frame 2030; B opens the paraglider at 1975, before the ground
|
||||
# (X does nothing in a dive; an early B glides for minutes and never lands); stick forward until 2400 to land and walk.
|
||||
# Without the paraglider Link dies at about frame 2160 (game over screen).
|
||||
300 A 4
|
||||
520 A 4
|
||||
700 A 4
|
||||
900 A 4
|
||||
1550 LSTICK 0 32767 210
|
||||
1975 B 4
|
||||
1790 LSTICK 0 32767 610
|
||||
@@ -1,303 +0,0 @@
|
||||
"""Run the regression routes on a build, score them and compare with a base build.
|
||||
|
||||
python tools/regress.py build/v121 --repeat 3 --out build/regress/v121
|
||||
python tools/regress.py build/v121 --routes island --cold
|
||||
python tools/regress.py build/v121 --routes field --validation
|
||||
|
||||
Each run writes <out>/<build name>/<route>-<i>/ (log.txt, shots/, score.json); the summary prints median and worst per
|
||||
metric and route for every build, and with --base a diff table (ok, better, worse). Runs are nondeterministic: a
|
||||
verdict compares medians. A candidate build without a pipeline cache, pipeline-keys.bin or shader-cache gets the
|
||||
base build's copy before its first warm run. Every run works on a private copy of the pipeline cache, so the builds'
|
||||
own files stay as they are (TOTK_SHADER_CACHE still points at the build's directory, which the game extends).
|
||||
--cold starts the runs with an empty pipeline cache and key file; translated shaders stay.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageDraw, ImageStat
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from regress_score import far_gradient, is_scored, parse_perf, score_frames, vuid_counts # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAVES = ROOT / 'build' / 'save-backup'
|
||||
SLOW_MS = 50
|
||||
SHOT_STEP = 1 / 12 # seconds of world clock between shots
|
||||
TAIL_FRAMES = 30 # shots examined for a dead or stuck ending
|
||||
TAIL_BLACK = 0.9
|
||||
BLACK_LUMA = 12
|
||||
END_SLACK = 2 # seconds of world clock the log may stop short of the requested run
|
||||
|
||||
# name: (save, script, seconds of world clock to run, scored window in world seconds)
|
||||
WORLD = re.compile(r'perf: frame (\d+): interval avg .*; world (-?[0-9.]+) s')
|
||||
|
||||
ROUTES = {
|
||||
'fight': ('user-save-2026-10-21-monsters', 'tools/input/fight.txt', 18, (1, 18)),
|
||||
'rock': ('user-save-2026-10-21-monsters', 'build/recordings/rock-2026-10-22.txt', 30, (0, 30)),
|
||||
'island': ('user-save-2026-10-22-island', 'tools/input/island.txt', 35, (0, 35)),
|
||||
'field': ('user-save-2026-10-21-field', 'tools/input/load_save.txt', 14, (1, 14)),
|
||||
}
|
||||
|
||||
# metric: (label, lower is better, absolute tolerance, relative tolerance)
|
||||
METRICS = {
|
||||
'fps_mean': ('fps', False, 1.0, 0.03),
|
||||
'max_frame_ms': ('max frame ms', True, 5.0, 0.15),
|
||||
'slow_count': (f'frames >{SLOW_MS} ms', True, 1, 0.15),
|
||||
'render_busy_ms': ('render busy ms', True, 0.5, 0.10),
|
||||
'replay_ms': ('replay ms', True, 0.5, 0.10),
|
||||
'black_frames': ('black-block frames', True, 0, 0),
|
||||
'flat_frames': ('flat frames', True, 0, 0),
|
||||
'vuid_total': ('validation messages', True, 0, 0),
|
||||
}
|
||||
CACHE_FILES = ('pipeline-cache.bin', 'pipeline-keys.bin')
|
||||
|
||||
|
||||
def seed_caches(build, base):
|
||||
"""Copy the base build's caches into a candidate that lacks them; returns what was copied."""
|
||||
seeded = []
|
||||
if build.resolve() == base.resolve():
|
||||
return seeded
|
||||
for name in (*CACHE_FILES, 'shader-cache'):
|
||||
src, dst = base / name, build / name
|
||||
if dst.exists() or not src.exists():
|
||||
continue
|
||||
if src.is_dir():
|
||||
shutil.copytree(src, dst)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
seeded.append(name)
|
||||
return seeded
|
||||
|
||||
|
||||
def run_route(build, route, i, out, cold, validation):
|
||||
save, script, seconds, (lo, hi) = ROUTES[route]
|
||||
run = out / build.name / f'{route}-{i}'
|
||||
shutil.rmtree(run, ignore_errors=True)
|
||||
shots = run / 'shots'
|
||||
cache = run / 'cache'
|
||||
shots.mkdir(parents=True)
|
||||
cache.mkdir()
|
||||
if not cold:
|
||||
for name in CACHE_FILES:
|
||||
if (build / name).exists():
|
||||
shutil.copy2(build / name, cache / name)
|
||||
env = {
|
||||
'TOTK_TRACE': 'perf,sched',
|
||||
'TOTK_SHOTS': f'w{lo}-{hi}:{SHOT_STEP:.4f}',
|
||||
'TOTK_SHOTS_DIR': str(shots),
|
||||
'TOTK_PIPELINE_CACHE': str(cache / 'pipeline-cache.bin'),
|
||||
}
|
||||
if (build / 'shader-cache').is_dir():
|
||||
env['TOTK_SHADER_CACHE'] = str(build / 'shader-cache')
|
||||
if validation:
|
||||
env['TOTK_VK_VALIDATION'] = '1'
|
||||
log = run / 'log.txt'
|
||||
cmd = [sys.executable, str(ROOT / 'tools' / 'run_game.py'), str(build), '--save', save,
|
||||
'--script', str(ROOT / script), '--seconds', str(seconds), '--quiet', '--tag', f'rg-{route}-{i}',
|
||||
'--log', str(log)]
|
||||
for k, v in env.items():
|
||||
cmd += ['--env', f'{k}={v}']
|
||||
rc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
||||
score = score_run(log, shots, lo, hi, seconds, rc)
|
||||
score['run'] = str(run)
|
||||
(run / 'score.json').write_text(json.dumps(score, indent=1))
|
||||
return score
|
||||
|
||||
|
||||
def black_fraction(img):
|
||||
grey = img.convert('L')
|
||||
return sum(grey.histogram()[:BLACK_LUMA]) / (grey.width * grey.height)
|
||||
|
||||
|
||||
def world_frames(text, lo, hi):
|
||||
"""The presents of the world-clock window [lo, hi] seconds, from the perf lines (one per 60 presents), and the
|
||||
world clock of the last one."""
|
||||
lo_n = hi_n = None
|
||||
last = -1.0
|
||||
for m in WORLD.finditer(text):
|
||||
n, w = int(m[1]), float(m[2])
|
||||
last = w
|
||||
if w >= lo and lo_n is None:
|
||||
lo_n = n - 60
|
||||
if w <= hi:
|
||||
hi_n = n
|
||||
return (lo_n or 0), (hi_n or 0), last
|
||||
|
||||
|
||||
def score_run(log, shots, lo, hi, seconds, rc):
|
||||
text = log.read_text(encoding='utf-8', errors='replace') if log.exists() else ''
|
||||
lo, hi, last_world = world_frames(text, lo, hi)
|
||||
perf = parse_perf(log, lo, hi)
|
||||
paths = {int(p.stem[1:]): p for p in shots.glob('f*.png')}
|
||||
fr = score_frames(paths) if paths else {'black_counts': {}, 'flat_frames': [], 'scored': 0, 'total': 0}
|
||||
vuids = vuid_counts(text)
|
||||
last = [n for n in sorted(paths)][-TAIL_FRAMES:]
|
||||
tail_black = 0.0
|
||||
if last:
|
||||
fractions = []
|
||||
for n in last:
|
||||
with Image.open(paths[n]) as im:
|
||||
fractions.append(black_fraction(im.convert('RGB')))
|
||||
tail_black = statistics.mean(fractions)
|
||||
reasons = []
|
||||
if tail_black > TAIL_BLACK:
|
||||
reasons.append(f'last {len(last)} shots {tail_black:.0%} black (game over?)')
|
||||
if perf['crashed']:
|
||||
reasons.append('crashed or no perf lines')
|
||||
if rc != 0:
|
||||
reasons.append(f'exit code {rc}')
|
||||
if last_world < seconds - END_SLACK:
|
||||
reasons.append(f'log stops at {last_world:.1f} s of world clock of {seconds}')
|
||||
return {
|
||||
'rc': rc, 'fps_mean': perf['fps_mean'], 'max_frame_ms': perf['max_frame_ms'],
|
||||
'slow_count': sum(1 for _, ms, _ in perf['slow_frames'] if ms >= SLOW_MS),
|
||||
'slow_frames': [(n, ms, cls) for n, ms, cls in perf['slow_frames'] if ms >= SLOW_MS],
|
||||
'render_busy_ms': perf['render_busy_ms'], 'replay_ms': perf['replay_ms'],
|
||||
'black_frames': len(fr['black_counts']), 'black_counts': fr['black_counts'],
|
||||
'flat_frames': len(fr['flat_frames']), 'flat_list': fr['flat_frames'],
|
||||
'scored': fr['scored'], 'shots': fr['total'],
|
||||
'vuids': vuids, 'vuid_total': sum(vuids.values()),
|
||||
'route_failed': bool(reasons), 'fail_reasons': reasons,
|
||||
}
|
||||
|
||||
|
||||
def contact_sheet(scores, path):
|
||||
"""Up to 6 worst frames over the runs of one route: black blocks first, then flat, then the lowest far gradient."""
|
||||
cands = []
|
||||
for s in scores:
|
||||
shots = Path(s['run']) / 'shots'
|
||||
for n, c in s['black_counts'].items():
|
||||
cands.append((0, -c, s['run'], int(n), f'black x{c}'))
|
||||
for n in s['flat_list']:
|
||||
cands.append((1, 0, s['run'], int(n), 'flat'))
|
||||
cands.sort()
|
||||
chosen = cands[:6]
|
||||
if len(chosen) < 6 and scores:
|
||||
have = {(r, n) for _, _, r, n, _ in chosen}
|
||||
rest = []
|
||||
for p in sorted((Path(scores[0]['run']) / 'shots').glob('f*.png')):
|
||||
with Image.open(p) as im:
|
||||
im = im.convert('RGB')
|
||||
if is_scored(im):
|
||||
rest.append((far_gradient(im), scores[0]['run'], int(p.stem[1:])))
|
||||
rest.sort()
|
||||
for g, r, n in rest:
|
||||
if len(chosen) == 6:
|
||||
break
|
||||
if (r, n) not in have:
|
||||
chosen.append((2, g, r, n, f'lowest far gradient {g:.1f}'))
|
||||
if not chosen:
|
||||
return False
|
||||
sheet = Image.new('RGB', (3 * 480, 2 * 270))
|
||||
for k, (_, _, run, n, label) in enumerate(chosen):
|
||||
with Image.open(Path(run) / 'shots' / f'f{n:05d}.png') as im:
|
||||
tile = im.convert('RGB')
|
||||
d = ImageDraw.Draw(tile)
|
||||
d.rectangle((0, 0, 330, 14), fill=(0, 0, 0))
|
||||
d.text((2, 2), f'{Path(run).name} f{n} {label}', fill=(255, 255, 0))
|
||||
sheet.paste(tile, ((k % 3) * 480, (k // 3) * 270))
|
||||
sheet.save(path)
|
||||
return True
|
||||
|
||||
|
||||
def stat(values, lower_is_better):
|
||||
"""(median, worst) over the runs that report the metric; None when none does."""
|
||||
vals = [v for v in values if v is not None]
|
||||
if not vals:
|
||||
return None, None
|
||||
return statistics.median(vals), (max(vals) if lower_is_better else min(vals))
|
||||
|
||||
|
||||
def fmt(v):
|
||||
return 'n/a' if v is None else (f'{v:.1f}' if isinstance(v, float) else str(v))
|
||||
|
||||
|
||||
def summarize(name, results):
|
||||
"""results: {route: [score]} -> {route: {metric: (median, worst)}}; prints the table."""
|
||||
table = {r: {m: stat([s[m] for s in scs], METRICS[m][1]) for m in METRICS} for r, scs in results.items()}
|
||||
print(f'\n== {name}: median / worst of {max(len(s) for s in results.values())} runs')
|
||||
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS) + ' failed')
|
||||
for r, row in table.items():
|
||||
failed = sum(s['route_failed'] for s in results[r])
|
||||
print(f'{r:8}' + ''.join(f'{fmt(row[m][0]) + " / " + fmt(row[m][1]):>24}' for m in METRICS) + f' {failed}')
|
||||
for r, scs in results.items():
|
||||
for s in scs:
|
||||
if s['route_failed']:
|
||||
print(f' ROUTE FAILED {s["run"]}: {"; ".join(s["fail_reasons"])}')
|
||||
return table
|
||||
|
||||
|
||||
def verdict(m, cand, base):
|
||||
if cand is None or base is None:
|
||||
return 'n/a'
|
||||
_, lower, tol_abs, tol_rel = METRICS[m]
|
||||
delta = cand - base if lower else base - cand
|
||||
slack = max(tol_abs, tol_rel * abs(base))
|
||||
if delta > slack:
|
||||
return 'worse'
|
||||
if delta < -slack:
|
||||
return 'better'
|
||||
return 'ok'
|
||||
|
||||
|
||||
def diff_table(cand, base, cand_results, base_results):
|
||||
print('\n== diff against base (median; verdict)')
|
||||
print(f'{"route":8}' + ''.join(f'{METRICS[m][0]:>24}' for m in METRICS))
|
||||
for r in cand:
|
||||
cells = []
|
||||
for m in METRICS:
|
||||
c, b = cand[r][m][0], base.get(r, {}).get(m, (None,))[0]
|
||||
cells.append(f'{fmt(c)} vs {fmt(b)} {verdict(m, c, b)}')
|
||||
print(f'{r:8}' + ''.join(f'{c:>24}' for c in cells))
|
||||
new = set().union(*(s['vuids'] for s in cand_results[r])) - set().union(*(s['vuids'] for s in base_results.get(r, [])))
|
||||
if new:
|
||||
print(f' new validation ids in {r}: {", ".join(sorted(new))}')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('build', type=Path)
|
||||
ap.add_argument('--base', type=Path, help='frozen build to compare with (e.g. build/v121-base)')
|
||||
ap.add_argument('--routes', default='fight,rock,island,field')
|
||||
ap.add_argument('--repeat', type=int, default=1)
|
||||
ap.add_argument('--cold', action='store_true', help='empty pipeline cache and key file')
|
||||
ap.add_argument('--validation', action='store_true', help='TOTK_VK_VALIDATION=1; counts VUIDs')
|
||||
ap.add_argument('--out', type=Path)
|
||||
a = ap.parse_args()
|
||||
out = (a.out or ROOT / 'build' / 'regress' / datetime.datetime.now().strftime('%Y%m%d-%H%M%S')).resolve()
|
||||
routes = a.routes.split(',')
|
||||
for r in routes:
|
||||
if r not in ROUTES:
|
||||
sys.exit(f'unknown route {r}; choose from {", ".join(ROUTES)}')
|
||||
if not (SAVES / ROUTES[r][0]).is_dir() or not (ROOT / ROUTES[r][1]).exists():
|
||||
sys.exit(f'route {r}: missing save or script')
|
||||
builds = [a.build.resolve()] + ([a.base.resolve()] if a.base else [])
|
||||
if a.base and not a.cold:
|
||||
seeded = seed_caches(builds[0], builds[1])
|
||||
print('caches seeded from base:', ', '.join(seeded) if seeded else 'none (the build had its own)')
|
||||
tables, all_results = {}, {}
|
||||
for build in builds:
|
||||
results = {}
|
||||
for r in routes:
|
||||
results[r] = [run_route(build, r, i, out, a.cold, a.validation) for i in range(a.repeat)]
|
||||
sheet = out / build.name / f'{r}-worst.png'
|
||||
contact_sheet(results[r], sheet)
|
||||
all_results[build.name] = results
|
||||
tables[build.name] = summarize(build.name, results)
|
||||
if a.base:
|
||||
diff_table(tables[builds[0].name], tables[builds[1].name], all_results[builds[0].name], all_results[builds[1].name])
|
||||
(out / 'summary.json').write_text(json.dumps(tables, indent=1))
|
||||
print(f'\nwrote {out}')
|
||||
bad = any(s['route_failed'] for res in all_results.values() for scs in res.values() for s in scs)
|
||||
sys.exit(1 if bad else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,134 +0,0 @@
|
||||
"""Score a game log and its TOTK_SHOTS frames for tools/regress.py.
|
||||
|
||||
Log metrics come from the TOTK_TRACE=perf lines (runtime/prof_win.c `perf_present`); frame metrics look for the two
|
||||
glitches it targets: near-black blocks inside a lit picture and a far field that lost its detail.
|
||||
"""
|
||||
import re
|
||||
import statistics
|
||||
from collections import Counter
|
||||
|
||||
from PIL import Image, ImageChops, ImageStat
|
||||
|
||||
SLOW = re.compile(r'perf: slow frame (\d+): ([\d.]+) ms; blocked pipeline ([\d.]+) xlate ([\d.]+) texture ([\d.]+) '
|
||||
r'astc ([\d.]+) fence ([\d.]+) pacing ([\d.]+)')
|
||||
FRAME = re.compile(r'perf: frame (\d+): interval avg ([\d.]+) ms max ([\d.]+) ms \(([\d.]+) fps\)')
|
||||
BUSY = re.compile(r'render busy avg ([\d.]+) ms')
|
||||
REPLAY = re.compile(r'replay avg ([\d.]+) ms')
|
||||
FATAL = re.compile(r'FATAL|rt_fatal')
|
||||
VUID = re.compile(r'VUID-[A-Za-z0-9_-]+')
|
||||
|
||||
BLOCK = 16
|
||||
BLACK_MAX = 12 # a block is black when no channel reaches this
|
||||
NEIGHBOUR_LUMA = 60 # and lit neighbours are brighter than this
|
||||
LUMA_LO, LUMA_HI = 25, 240 # frames outside are loading screens, menus, game over or white-outs: not scored
|
||||
FLAT_RATIO = 0.4
|
||||
FLAT_NEIGHBOURS = 10
|
||||
BLOCKED_MIN_MS = 5.0 # a slow frame with less blocked time in every class is "other"
|
||||
|
||||
|
||||
def parse_perf_text(text, lo, hi):
|
||||
"""Metrics of the frames in [lo, hi] from a log's text."""
|
||||
fps, maxes, busy, replay, slow = [], [], [], [], []
|
||||
seen_after_lo = False
|
||||
crashed = False
|
||||
for line in text.splitlines():
|
||||
if FATAL.search(line):
|
||||
crashed = True
|
||||
if m := SLOW.search(line):
|
||||
n = int(m[1])
|
||||
if lo <= n <= hi:
|
||||
ms = float(m[2])
|
||||
classes = {'pipeline': float(m[3]), 'xlate': float(m[4]), 'texture': float(m[5]), 'fence': float(m[7])}
|
||||
name, blocked = max(classes.items(), key=lambda kv: kv[1])
|
||||
slow.append((n, ms, name if blocked >= BLOCKED_MIN_MS else 'other'))
|
||||
elif m := FRAME.search(line):
|
||||
n = int(m[1])
|
||||
if n > lo:
|
||||
seen_after_lo = True
|
||||
if lo < n <= hi:
|
||||
fps.append(float(m[4]))
|
||||
maxes.append(float(m[3]))
|
||||
if b := BUSY.search(line):
|
||||
busy.append(float(b[1]))
|
||||
if r := REPLAY.search(line):
|
||||
replay.append(float(r[1]))
|
||||
max_ms = max([ms for _, ms, _ in slow] + maxes, default=0.0)
|
||||
return {
|
||||
'fps_mean': statistics.mean(fps) if fps else 0.0,
|
||||
'max_frame_ms': max_ms,
|
||||
'slow_frames': slow,
|
||||
'render_busy_ms': statistics.mean(busy) if busy else None,
|
||||
'replay_ms': statistics.mean(replay) if replay else None,
|
||||
'crashed': crashed or not seen_after_lo,
|
||||
}
|
||||
|
||||
|
||||
def parse_perf(path, lo, hi):
|
||||
with open(path, encoding='utf-8', errors='replace') as f:
|
||||
return parse_perf_text(f.read(), lo, hi)
|
||||
|
||||
|
||||
def vuid_counts(log):
|
||||
return dict(Counter(VUID.findall(log)))
|
||||
|
||||
|
||||
def luma_mean(img):
|
||||
return ImageStat.Stat(img.convert('L')).mean[0]
|
||||
|
||||
|
||||
def is_scored(img):
|
||||
return LUMA_LO <= luma_mean(img) <= LUMA_HI
|
||||
|
||||
|
||||
def black_blocks(img):
|
||||
"""16x16 blocks that are black while at least 3 of their 4 neighbours are lit."""
|
||||
rgb = img.convert('RGB')
|
||||
grey = rgb.convert('L')
|
||||
cols, rows = rgb.width // BLOCK, rgb.height // BLOCK
|
||||
black = [[False] * cols for _ in range(rows)]
|
||||
luma = [[0.0] * cols for _ in range(rows)]
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
box = (c * BLOCK, r * BLOCK, (c + 1) * BLOCK, (r + 1) * BLOCK)
|
||||
black[r][c] = max(hi for _, hi in rgb.crop(box).getextrema()) < BLACK_MAX
|
||||
luma[r][c] = ImageStat.Stat(grey.crop(box)).mean[0]
|
||||
count = 0
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
if not black[r][c]:
|
||||
continue
|
||||
lit = sum(1 for rr, cc in ((r - 1, c), (r + 1, c), (r, c - 1), (r, c + 1))
|
||||
if 0 <= rr < rows and 0 <= cc < cols and luma[rr][cc] > NEIGHBOUR_LUMA)
|
||||
count += lit >= 3
|
||||
return count
|
||||
|
||||
|
||||
def far_gradient(img):
|
||||
"""Mean absolute horizontal plus vertical luma difference over the top half (sky and far terrain)."""
|
||||
top = img.convert('L').crop((0, 0, img.width, img.height // 2))
|
||||
w, h = top.size
|
||||
dx = ImageChops.difference(top.crop((0, 0, w - 1, h)), top.crop((1, 0, w, h)))
|
||||
dy = ImageChops.difference(top.crop((0, 0, w, h - 1)), top.crop((0, 1, w, h)))
|
||||
return ImageStat.Stat(dx).mean[0] + ImageStat.Stat(dy).mean[0]
|
||||
|
||||
|
||||
def flat_frames(frames):
|
||||
"""Frame numbers whose far_gradient is under FLAT_RATIO x the median of the nearest scored frames."""
|
||||
scored = [(n, far_gradient(img)) for n, img in frames if is_scored(img)]
|
||||
flat = []
|
||||
for i, (n, g) in enumerate(scored):
|
||||
near = sorted((j for j in range(len(scored)) if j != i), key=lambda j: abs(j - i))[:FLAT_NEIGHBOURS]
|
||||
if near and g < FLAT_RATIO * statistics.median(scored[j][1] for j in near):
|
||||
flat.append(n)
|
||||
return flat
|
||||
|
||||
|
||||
def score_frames(paths):
|
||||
"""`paths`: {frame number: png path}. Returns per-frame black-block counts, flat frames and how many were scored."""
|
||||
frames = []
|
||||
for n in sorted(paths):
|
||||
with Image.open(paths[n]) as im:
|
||||
frames.append((n, im.convert('RGB')))
|
||||
scored = [(n, im) for n, im in frames if is_scored(im)]
|
||||
black = {n: c for n, im in scored if (c := black_blocks(im))}
|
||||
return {'black_counts': black, 'flat_frames': flat_frames(scored), 'scored': len(scored), 'total': len(frames)}
|
||||
+6
-3
@@ -4,9 +4,12 @@ The user's profile (build/profile) is never written: each run copies it to <exe
|
||||
backup, and deletes the copy afterwards. The log records how many other totk.exe and ninja.exe processes ran, because
|
||||
timings taken next to them are not comparable.
|
||||
|
||||
python tools/run_game.py build/v121 --save user-save-2026-10-21-field --seconds 15 --tag idle --env TOTK_TRACE=perf
|
||||
python tools/run_game.py build/v121 --save user-save-2026-10-21-monsters --script tools/input/fight.txt \
|
||||
--seconds 18 --tag fight --quiet --env TOTK_TRACE=perf,sched
|
||||
python tools/run_game.py build/v143 --save <backup> --seconds 15 --tag idle --env TOTK_TRACE=perf
|
||||
python tools/run_game.py build/v143 --save <backup> --script <input.txt> --seconds 18 --tag fight --quiet \
|
||||
--env TOTK_TRACE=perf,sched
|
||||
|
||||
<backup> is a folder of build/save-backup; --script defaults to tools/input/load_save.txt (title screen to the first
|
||||
save slot), and a recording made with TOTK_INPUT_RECORD replays the same way.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
|
||||
Reference in new issue
Block a user