Files
totk-pc-recomp/tests/lift_cache_test.py
T
Wes 5f6fc13712 Lift again only the chunks whose inputs changed
gen.py lifted every chunk on each run and only skipped rewriting the ones whose text came out the same, so a rebuild
after one changed code cheat still spent the whole lift (102 s on 12 cores, 178 s with 4 workers) to recompile two
chunks. A chunk's text depends only on the generator's sources and options, the module and the function starts
discovered from it, the chunk's number and functions, and the patch sites and spin waits inside its address range, so
the SHA-256 of these is now its key. <mod>/lift_cache.json records each chunk's key with the SHA-256, size and mtime of
its file and the statistics and folded ranges the manifest and folded.txt need, under a digest of the entry. A chunk
whose entry is intact, whose key matches and whose file is unchanged is reused without a worker; no pool starts when
nothing is left to lift. The cache is written after the manifest, and a failed write only warns. --no-cache, or
TOTK_LIFT_CACHE=0 for the installer's lift, lifts everything.

On 1.4.3 with the installed mods' cheats, a run with unchanged inputs takes 4 s and one with a changed cheat word lifts
one chunk in 7 s. Its output equals a --no-cache lift into an empty folder and the previous gen.py's output byte for
byte. tests/lift_cache_test.py checks reuse and invalidation on a module it makes itself.
2026-11-10 13:03:38 -06:00

222 lines
11 KiB
Python

"""recomp/gen.py's lift cache (<mod>/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses
every chunk, a changed patch site or spin wait lifts only the chunk that holds it, a chunk file or cache entry changed
behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to
the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt
included. No game files.
Run with .venv (lz4 and numpy, as recomp/): python tests/lift_cache_test.py
"""
import json
import os
import shutil
import stat
import struct
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'tools'))
from check_content import GENERATED_MARKER # noqa: E402
# The jump table module and the NSO writer of tests/exefs_patches_test.py; a change there changes this test's module,
# and test_reuse_and_invalidation checks that it still has its jump table (folded.txt 1020 1024).
from exefs_patches_test import JT_DATA, JT_RO, JT_TEXT, MOV_W8_100, MOV_W8_200, RET, nso # noqa: E402
MOV_W8_300 = 0x52806008
def bl(pc, target):
return 0x94000000 | (((target - pc) >> 2) & 0x3FFFFFF)
# The jump table module of tests/exefs_patches_test.py (fn at 0x8, its table in .rodata) and two more functions after
# it that call each other, so both are function starts: g: bl h; ret, h: mov w8, #0x100; bl g; ret. With --chunk-fns 1
# each of fn, g and h is a chunk of its own; h's mov is the patch site.
G = len(JT_TEXT)
H = G + 8
TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET)
# fn's `b.hi Ldefault` at 0xc, as a spin wait back-edge to Ldefault (0x48): patches.check accepts it, the lift gates it
SPIN = 'spin main 48 c'
def patch_file(*replacements, first=None, spin=False):
"""A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before
them, which moves their indices; spin: SPIN too."""
lines = [f'set {first} TEST_{first.upper()} 0'] if first else []
for i, w in enumerate(replacements):
lines += [f'set a{i} TEST_A{i} 0', f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}']
if spin:
lines.append(SPIN)
return '\n'.join(lines) + '\n'
class LiftCache(unittest.TestCase):
def setUp(self):
self.d = Path(tempfile.mkdtemp())
(self.d / 'exefs').mkdir()
(self.d / 'exefs' / 'main').write_bytes(nso(TEXT, JT_RO, JT_DATA))
def tearDown(self):
shutil.rmtree(self.d, ignore_errors=True)
def gen(self, patches, out='gen', *extra, env=None):
p = self.d / 'patches.txt'
p.write_text(patches)
env = {**os.environ, 'TOTK_LIFT_CACHE': '', **(env or {})}
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
'--out', str(self.d / out), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1',
'--patches', str(p), *extra], capture_output=True, text=True, env=env)
self.assertEqual(r.returncode, 0, r.stderr)
self.stderr = r.stderr
return json.loads((self.d / out / 'main' / 'manifest.json').read_text())['stats']
def files(self, out='gen'):
"""{name: bytes} of everything gen.py wrote except the manifest (timings) and the cache."""
root = self.d / out
return {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob('*'))
if p.is_file() and p.name not in ('manifest.json', 'lift_cache.json')}
def assert_like_fresh(self, patches):
"""The output under gen/ equals a --no-cache lift of the same inputs into an empty directory."""
stats = self.gen(patches, 'fresh', '--no-cache')
self.assertEqual(stats['chunks_reused'], 0)
self.assertEqual(self.files(), self.files('fresh'))
shutil.rmtree(self.d / 'fresh')
def chunk(self, n):
return self.d / 'gen' / 'main' / f'main_{n:04d}.c'
def test_reuse_and_invalidation(self):
stats = self.gen(patch_file(MOV_W8_200))
self.assertEqual((stats['emitted_functions'], stats['chunks_reused']), (3, 0))
cache = json.loads((self.d / 'gen' / 'main' / 'lift_cache.json').read_text())
self.assertEqual(cache['marker'], GENERATED_MARKER)
self.assertEqual(sorted(cache['chunks']), ['main_0000.c', 'main_0001.c', 'main_0002.c'])
self.assertIn('rt_patch_on[0]', self.chunk(2).read_text())
folded = (self.d / 'gen' / 'main' / 'folded.txt').read_text()
self.assertIn('1020 1024', folded)
# Same inputs: nothing lifted, nothing written, the jump table's bytes still in folded.txt (replayed from the
# cache, not read again) and the statistics unchanged.
mtimes = [self.chunk(n).stat().st_mtime_ns for n in range(3)]
again = self.gen(patch_file(MOV_W8_200))
self.assertEqual((again['chunks_reused'], again['chunks_written']), (3, 0))
self.assertEqual({k: v for k, v in again.items() if not k.startswith('chunks_')},
{k: v for k, v in stats.items() if not k.startswith('chunks_')})
self.assertEqual([self.chunk(n).stat().st_mtime_ns for n in range(3)], mtimes)
self.assertEqual((self.d / 'gen' / 'main' / 'folded.txt').read_text(), folded)
self.assert_like_fresh(patch_file(MOV_W8_200))
# Another replacement word: only h's chunk is lifted.
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
self.assert_like_fresh(patch_file(MOV_W8_300))
# A set defined before the site's moves its index (rt_patch_on[1]): h's chunk again.
self.assertEqual(self.gen(patch_file(MOV_W8_300, first='z'))['chunks_reused'], 2)
self.assertIn('rt_patch_on[1]', self.chunk(2).read_text())
self.assert_like_fresh(patch_file(MOV_W8_300, first='z'))
# A second alternative at the site.
self.assertEqual(self.gen(patch_file(MOV_W8_300, MOV_W8_200))['chunks_reused'], 2)
self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200))
def test_changed_file_is_lifted_again(self):
"""A chunk whose bytes changed while its size and mtime stayed (a flipped bit) is lifted and rewritten."""
self.gen(patch_file(MOV_W8_200))
p = self.chunk(0)
st = p.stat()
data = bytearray(p.read_bytes())
data[-2] ^= 0x01
p.write_bytes(bytes(data))
os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns))
stats = self.gen(patch_file(MOV_W8_200))
self.assertEqual((stats['chunks_reused'], stats['chunks_written']), (2, 1))
self.assert_like_fresh(patch_file(MOV_W8_200))
def test_deleted_file_and_cache(self):
self.gen(patch_file(MOV_W8_200))
self.chunk(1).unlink()
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
(self.d / 'gen' / 'main' / 'lift_cache.json').write_text('{not json')
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 0)
self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--no-cache')['chunks_reused'], 0)
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3)
self.assert_like_fresh(patch_file(MOV_W8_200))
def test_spin_wait_is_in_the_key(self):
self.gen(patch_file(MOV_W8_200))
self.assertEqual(self.gen(patch_file(MOV_W8_200, spin=True))['chunks_reused'], 2)
self.assertIn('rt_spin_gate', self.chunk(0).read_text())
self.assert_like_fresh(patch_file(MOV_W8_200, spin=True))
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
self.assertNotIn('rt_spin_gate', self.chunk(0).read_text())
def test_changed_cache_entry_is_not_taken(self):
"""A changed result in lift_cache.json (a folded range, here) fails the entry's digest: the chunk is lifted
and folded.txt keeps the jump table's real bytes."""
self.gen(patch_file(MOV_W8_200))
p = self.d / 'gen' / 'main' / 'lift_cache.json'
data = json.loads(p.read_text())
self.assertEqual(data['chunks']['main_0000.c']['folded'], [[0x1020, 0x1024]])
data['chunks']['main_0000.c']['folded'] = [[0x1060, 0x1064]]
p.write_text(json.dumps(data))
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2)
self.assertIn('1020 1024', (self.d / 'gen' / 'main' / 'folded.txt').read_text())
self.assert_like_fresh(patch_file(MOV_W8_200))
def test_unwritable_cache_only_warns(self):
"""A cache that cannot be replaced leaves the lift successful, its manifest written; the old cache's entry for
the chunk the run rewrote no longer matches it."""
self.gen(patch_file(MOV_W8_200))
p = self.d / 'gen' / 'main' / 'lift_cache.json'
os.chmod(p, stat.S_IREAD)
try:
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
self.assertIn('lift_cache.json not written', self.stderr)
finally:
os.chmod(p, stat.S_IREAD | stat.S_IWRITE)
self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2)
self.assert_like_fresh(patch_file(MOV_W8_300))
def test_environment_switch(self):
self.gen(patch_file(MOV_W8_200))
self.assertEqual(self.gen(patch_file(MOV_W8_200), env={'TOTK_LIFT_CACHE': '0'})['chunks_reused'], 0)
self.assertIn('lifting every chunk (TOTK_LIFT_CACHE=0)', self.stderr)
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3)
def test_emitter_options_are_in_the_key(self):
self.gen(patch_file(MOV_W8_200))
self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--reg-locals', '0')['chunks_reused'], 0)
def test_generator_sources_are_in_the_key(self):
"""A copy of the generator lifts everything again after a comment in its lifter.py changes, and nothing
after a patch table beside it (recomp/patches-*.txt) changes."""
copy = self.d / 'repo'
shutil.copytree(ROOT / 'recomp', copy / 'recomp', ignore=shutil.ignore_patterns('__pycache__'))
(copy / 'tools').mkdir()
for name in ('check_content.py', 'hostmem.py'):
shutil.copy2(ROOT / 'tools' / name, copy / 'tools' / name)
def run():
r = subprocess.run([sys.executable, str(copy / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1'],
capture_output=True, text=True, env={**os.environ, 'TOTK_LIFT_CACHE': ''})
self.assertEqual(r.returncode, 0, r.stderr)
return json.loads((self.d / 'gen' / 'main' / 'manifest.json').read_text())['stats']['chunks_reused']
self.assertEqual(run(), 0)
self.assertEqual(run(), 3)
table = next((copy / 'recomp').glob('patches-*.txt'))
table.write_text(table.read_text() + '# another line\n')
self.assertEqual(run(), 3)
lifter = copy / 'recomp' / 'lifter.py'
lifter.write_text(lifter.read_text() + '# another comment\n')
self.assertEqual(run(), 0)
if __name__ == '__main__':
unittest.main()