mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-18 20:15:56 +00:00
FN_ENTER (runtime/cpu.h) opened all 282,702 lifted functions of 1.4.3 with a load of g_fnprof and a branch, for one reader: runtime/resolution.c res_fn_hit, which waits for main+0x585bd0 (SET_RENDER_SIZE). A `hook` line of the patch table (recomp/patches.py) now names the functions that keep it; patches-v143.txt hooks main+0x585bd0, and gen.py stops on a hook that is no function start. Tables without hook lines (1.2.1, 1.1.0) keep FN_ENTER in every function, and gen.py --fn-enter-all puts it everywhere for a whole-game function-entry profile. The hooked addresses of a chunk are in its lift cache key (tests/lift_cache_test.py test_hooks_are_in_the_key). On 1.4.3 the generated C is otherwise byte-identical to dev's (282,701 FN_ENTER lines fewer, nothing else differs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
247 lines
13 KiB
Python
247 lines
13 KiB
Python
"""recomp/gen.py's lift cache (<mod>/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses
|
|
every chunk, a changed patch site, spin wait or function hook lifts only the chunk that holds it, a chunk file or cache entry changed
|
|
behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to
|
|
the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt
|
|
included. No game files.
|
|
|
|
Run with .venv (lz4 and numpy, as recomp/): python tests/lift_cache_test.py
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'tools'))
|
|
from check_content import GENERATED_MARKER # noqa: E402
|
|
# The jump table module and the NSO writer of tests/exefs_patches_test.py; a change there changes this test's module,
|
|
# and test_reuse_and_invalidation checks that it still has its jump table (folded.txt 1020 1024).
|
|
from exefs_patches_test import JT_DATA, JT_RO, JT_TEXT, MOV_W8_100, MOV_W8_200, RET, nso # noqa: E402
|
|
|
|
MOV_W8_300 = 0x52806008
|
|
|
|
|
|
def bl(pc, target):
|
|
return 0x94000000 | (((target - pc) >> 2) & 0x3FFFFFF)
|
|
|
|
|
|
# The jump table module of tests/exefs_patches_test.py (fn at 0x8, its table in .rodata) and two more functions after
|
|
# it that call each other, so both are function starts: g: bl h; ret, h: mov w8, #0x100; bl g; ret. With --chunk-fns 1
|
|
# each of fn, g and h is a chunk of its own; h's mov is the patch site.
|
|
G = len(JT_TEXT)
|
|
H = G + 8
|
|
TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET)
|
|
# fn's `b.hi Ldefault` at 0xc, as a spin wait back-edge to Ldefault (0x48): patches.check accepts it, the lift gates it
|
|
SPIN = 'spin main 48 c'
|
|
|
|
|
|
def patch_file(*replacements, first=None, spin=False, hooks=()):
|
|
"""A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before
|
|
them, which moves their indices; spin: SPIN too; hooks: a hook line with these function offsets."""
|
|
lines = [f'set {first} TEST_{first.upper()} 0'] if first else []
|
|
for i, w in enumerate(replacements):
|
|
lines += [f'set a{i} TEST_A{i} 0', f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}']
|
|
if spin:
|
|
lines.append(SPIN)
|
|
if hooks:
|
|
lines.append('hook main ' + ' '.join(f'{o:x}' for o in hooks))
|
|
return '\n'.join(lines) + '\n'
|
|
|
|
|
|
class LiftCache(unittest.TestCase):
|
|
def setUp(self):
|
|
self.d = Path(tempfile.mkdtemp())
|
|
(self.d / 'exefs').mkdir()
|
|
(self.d / 'exefs' / 'main').write_bytes(nso(TEXT, JT_RO, JT_DATA))
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.d, ignore_errors=True)
|
|
|
|
def gen(self, patches, out='gen', *extra, env=None):
|
|
p = self.d / 'patches.txt'
|
|
p.write_text(patches)
|
|
env = {**os.environ, 'TOTK_LIFT_CACHE': '', **(env or {})}
|
|
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
|
|
'--out', str(self.d / out), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1',
|
|
'--patches', str(p), *extra], capture_output=True, text=True, env=env)
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
self.stderr = r.stderr
|
|
return json.loads((self.d / out / 'main' / 'manifest.json').read_text())['stats']
|
|
|
|
def files(self, out='gen'):
|
|
"""{name: bytes} of everything gen.py wrote except the manifest (timings) and the cache."""
|
|
root = self.d / out
|
|
return {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob('*'))
|
|
if p.is_file() and p.name not in ('manifest.json', 'lift_cache.json')}
|
|
|
|
def assert_like_fresh(self, patches):
|
|
"""The output under gen/ equals a --no-cache lift of the same inputs into an empty directory."""
|
|
stats = self.gen(patches, 'fresh', '--no-cache')
|
|
self.assertEqual(stats['chunks_reused'], 0)
|
|
self.assertEqual(self.files(), self.files('fresh'))
|
|
shutil.rmtree(self.d / 'fresh')
|
|
|
|
def chunk(self, n):
|
|
return self.d / 'gen' / 'main' / f'main_{n:04d}.c'
|
|
|
|
def test_reuse_and_invalidation(self):
|
|
stats = self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual((stats['emitted_functions'], stats['chunks_reused']), (3, 0))
|
|
cache = json.loads((self.d / 'gen' / 'main' / 'lift_cache.json').read_text())
|
|
self.assertEqual(cache['marker'], GENERATED_MARKER)
|
|
self.assertEqual(sorted(cache['chunks']), ['main_0000.c', 'main_0001.c', 'main_0002.c'])
|
|
self.assertIn('rt_patch_on[0]', self.chunk(2).read_text())
|
|
folded = (self.d / 'gen' / 'main' / 'folded.txt').read_text()
|
|
self.assertIn('1020 1024', folded)
|
|
|
|
# Same inputs: nothing lifted, nothing written, the jump table's bytes still in folded.txt (replayed from the
|
|
# cache, not read again) and the statistics unchanged.
|
|
mtimes = [self.chunk(n).stat().st_mtime_ns for n in range(3)]
|
|
again = self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual((again['chunks_reused'], again['chunks_written']), (3, 0))
|
|
self.assertEqual({k: v for k, v in again.items() if not k.startswith('chunks_')},
|
|
{k: v for k, v in stats.items() if not k.startswith('chunks_')})
|
|
self.assertEqual([self.chunk(n).stat().st_mtime_ns for n in range(3)], mtimes)
|
|
self.assertEqual((self.d / 'gen' / 'main' / 'folded.txt').read_text(), folded)
|
|
self.assert_like_fresh(patch_file(MOV_W8_200))
|
|
|
|
# Another replacement word: only h's chunk is lifted.
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
|
|
self.assert_like_fresh(patch_file(MOV_W8_300))
|
|
|
|
# A set defined before the site's moves its index (rt_patch_on[1]): h's chunk again.
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_300, first='z'))['chunks_reused'], 2)
|
|
self.assertIn('rt_patch_on[1]', self.chunk(2).read_text())
|
|
self.assert_like_fresh(patch_file(MOV_W8_300, first='z'))
|
|
|
|
# A second alternative at the site.
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_300, MOV_W8_200))['chunks_reused'], 2)
|
|
self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200))
|
|
|
|
def test_changed_file_is_lifted_again(self):
|
|
"""A chunk whose bytes changed while its size and mtime stayed (a flipped bit) is lifted and rewritten."""
|
|
self.gen(patch_file(MOV_W8_200))
|
|
p = self.chunk(0)
|
|
st = p.stat()
|
|
data = bytearray(p.read_bytes())
|
|
data[-2] ^= 0x01
|
|
p.write_bytes(bytes(data))
|
|
os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns))
|
|
stats = self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual((stats['chunks_reused'], stats['chunks_written']), (2, 1))
|
|
self.assert_like_fresh(patch_file(MOV_W8_200))
|
|
|
|
def test_deleted_file_and_cache(self):
|
|
self.gen(patch_file(MOV_W8_200))
|
|
self.chunk(1).unlink()
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
|
|
(self.d / 'gen' / 'main' / 'lift_cache.json').write_text('{not json')
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 0)
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--no-cache')['chunks_reused'], 0)
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3)
|
|
self.assert_like_fresh(patch_file(MOV_W8_200))
|
|
|
|
def test_spin_wait_is_in_the_key(self):
|
|
self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200, spin=True))['chunks_reused'], 2)
|
|
self.assertIn('rt_spin_gate', self.chunk(0).read_text())
|
|
self.assert_like_fresh(patch_file(MOV_W8_200, spin=True))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
|
|
self.assertNotIn('rt_spin_gate', self.chunk(0).read_text())
|
|
|
|
def test_hooks_are_in_the_key(self):
|
|
"""No hook line: FN_ENTER in every function. The first hook line takes it out of every other function, so
|
|
every chunk is lifted; one more hooked function lifts only its chunk; --fn-enter-all puts it back everywhere;
|
|
a hook on an address that is no function start stops the lift."""
|
|
fn_enter = lambda: ['FN_ENTER(' in self.chunk(n).read_text() for n in range(3)] # noqa: E731
|
|
self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual(fn_enter(), [True, True, True])
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]))['chunks_reused'], 0)
|
|
self.assertEqual(fn_enter(), [False, True, False])
|
|
self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G]))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G, H]))['chunks_reused'], 2)
|
|
self.assertEqual(fn_enter(), [False, True, True])
|
|
self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G, H]))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]), 'gen', '--fn-enter-all')['chunks_reused'], 0)
|
|
self.assertEqual(fn_enter(), [True, True, True])
|
|
p = self.d / 'patches.txt'
|
|
p.write_text(patch_file(MOV_W8_200, hooks=[G + 4]))
|
|
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
|
|
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--patches', str(p)],
|
|
capture_output=True, text=True)
|
|
self.assertNotEqual(r.returncode, 0)
|
|
self.assertIn('is no function start', r.stderr)
|
|
|
|
def test_changed_cache_entry_is_not_taken(self):
|
|
"""A changed result in lift_cache.json (a folded range, here) fails the entry's digest: the chunk is lifted
|
|
and folded.txt keeps the jump table's real bytes."""
|
|
self.gen(patch_file(MOV_W8_200))
|
|
p = self.d / 'gen' / 'main' / 'lift_cache.json'
|
|
data = json.loads(p.read_text())
|
|
self.assertEqual(data['chunks']['main_0000.c']['folded'], [[0x1020, 0x1024]])
|
|
data['chunks']['main_0000.c']['folded'] = [[0x1060, 0x1064]]
|
|
p.write_text(json.dumps(data))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
|
|
self.assertIn('1020 1024', (self.d / 'gen' / 'main' / 'folded.txt').read_text())
|
|
self.assert_like_fresh(patch_file(MOV_W8_200))
|
|
|
|
def test_unwritable_cache_only_warns(self):
|
|
"""A cache that cannot be replaced leaves the lift successful, its manifest written; the old cache's entry for
|
|
the chunk the run rewrote no longer matches it."""
|
|
self.gen(patch_file(MOV_W8_200))
|
|
p = self.d / 'gen' / 'main' / 'lift_cache.json'
|
|
os.chmod(p, stat.S_IREAD)
|
|
try:
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
|
|
self.assertIn('lift_cache.json not written', self.stderr)
|
|
finally:
|
|
os.chmod(p, stat.S_IREAD | stat.S_IWRITE)
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
|
|
self.assert_like_fresh(patch_file(MOV_W8_300))
|
|
|
|
def test_environment_switch(self):
|
|
self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200), env={'TOTK_LIFT_CACHE': '0'})['chunks_reused'], 0)
|
|
self.assertIn('lifting every chunk (TOTK_LIFT_CACHE=0)', self.stderr)
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3)
|
|
|
|
def test_emitter_options_are_in_the_key(self):
|
|
self.gen(patch_file(MOV_W8_200))
|
|
self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--reg-locals', '0')['chunks_reused'], 0)
|
|
|
|
def test_generator_sources_are_in_the_key(self):
|
|
"""A copy of the generator lifts everything again after a comment in its lifter.py changes, and nothing
|
|
after a patch table beside it (recomp/patches-*.txt) changes."""
|
|
copy = self.d / 'repo'
|
|
shutil.copytree(ROOT / 'recomp', copy / 'recomp', ignore=shutil.ignore_patterns('__pycache__'))
|
|
(copy / 'tools').mkdir()
|
|
for name in ('check_content.py', 'hostmem.py'):
|
|
shutil.copy2(ROOT / 'tools' / name, copy / 'tools' / name)
|
|
|
|
def run():
|
|
r = subprocess.run([sys.executable, str(copy / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
|
|
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1'],
|
|
capture_output=True, text=True, env={**os.environ, 'TOTK_LIFT_CACHE': ''})
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
return json.loads((self.d / 'gen' / 'main' / 'manifest.json').read_text())['stats']['chunks_reused']
|
|
|
|
self.assertEqual(run(), 0)
|
|
self.assertEqual(run(), 3)
|
|
table = next((copy / 'recomp').glob('patches-*.txt'))
|
|
table.write_text(table.read_text() + '# another line\n')
|
|
self.assertEqual(run(), 3)
|
|
lifter = copy / 'recomp' / 'lifter.py'
|
|
lifter.write_text(lifter.read_text() + '# another comment\n')
|
|
self.assertEqual(run(), 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|