"""recomp/gen.py's lift cache (/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses every chunk, a changed patch site, spin wait or function hook lifts only the chunk that holds it, a site two ordered sets contest tests its pick byte (recomp/patches.py `ordered`) while one ordered set alone is emitted as an unordered one, a chunk file or cache entry changed behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt included. No game files. Run with .venv (lz4 and numpy, as recomp/): python tests/lift_cache_test.py """ import json import os import shutil import stat import struct import subprocess import sys import tempfile import unittest from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / 'tests')) sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import patches # noqa: E402 from check_content import GENERATED_MARKER # noqa: E402 # The jump table module and the NSO writer of tests/exefs_patches_test.py; a change there changes this test's module, # and test_reuse_and_invalidation checks that it still has its jump table (folded.txt 1020 1024). from exefs_patches_test import JT_DATA, JT_RO, JT_TEXT, MOV_W8_100, MOV_W8_200, RET, nso # noqa: E402 MOV_W8_300 = 0x52806008 def bl(pc, target): return 0x94000000 | (((target - pc) >> 2) & 0x3FFFFFF) # The jump table module of tests/exefs_patches_test.py (fn at 0x8, its table in .rodata) and two more functions after # it that call each other, so both are function starts: g: bl h; ret, h: mov w8, #0x100; bl g; ret. With --chunk-fns 1 # each of fn, g and h is a chunk of its own; h's mov is the patch site. G = len(JT_TEXT) H = G + 8 TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET) # fn's `b.hi Ldefault` at 0xc, as a spin wait back-edge to Ldefault (0x48): patches.check accepts it, the lift gates it SPIN = 'spin main 48 c' def patch_file(*replacements, first=None, spin=False, hooks=(), ordered=False, cheat=None): """A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before them, which moves their indices; spin: SPIN too; hooks: a hook line with these function offsets; ordered: the a sets are ordered (recomp/patches.py); cheat: an unordered set c with this replacement at the site, before them.""" lines = [f'set {first} TEST_{first.upper()} 0'] if first else [] if cheat: lines += ['set c TEST_C 0', f'site c main {H:x} {MOV_W8_100:08x} {cheat:08x}'] for i, w in enumerate(replacements): lines += [f'set a{i} TEST_A{i} 0' + (' ordered' if ordered else ''), f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}'] if spin: lines.append(SPIN) if hooks: lines.append('hook main ' + ' '.join(f'{o:x}' for o in hooks)) return '\n'.join(lines) + '\n' class LiftCache(unittest.TestCase): def setUp(self): self.d = Path(tempfile.mkdtemp()) (self.d / 'exefs').mkdir() (self.d / 'exefs' / 'main').write_bytes(nso(TEXT, JT_RO, JT_DATA)) def tearDown(self): shutil.rmtree(self.d, ignore_errors=True) def gen(self, patches, out='gen', *extra, env=None): p = self.d / 'patches.txt' p.write_text(patches) env = {**os.environ, 'TOTK_LIFT_CACHE': '', **(env or {})} r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), '--out', str(self.d / out), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1', '--patches', str(p), *extra], capture_output=True, text=True, env=env) self.assertEqual(r.returncode, 0, r.stderr) self.stderr = r.stderr return json.loads((self.d / out / 'main' / 'manifest.json').read_text())['stats'] def files(self, out='gen'): """{name: bytes} of everything gen.py wrote except the manifest (timings) and the cache.""" root = self.d / out return {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob('*')) if p.is_file() and p.name not in ('manifest.json', 'lift_cache.json')} def assert_like_fresh(self, patches): """The output under gen/ equals a --no-cache lift of the same inputs into an empty directory.""" stats = self.gen(patches, 'fresh', '--no-cache') self.assertEqual(stats['chunks_reused'], 0) self.assertEqual(self.files(), self.files('fresh')) shutil.rmtree(self.d / 'fresh') def chunk(self, n): return self.d / 'gen' / 'main' / f'main_{n:04d}.c' def test_reuse_and_invalidation(self): stats = self.gen(patch_file(MOV_W8_200)) self.assertEqual((stats['emitted_functions'], stats['chunks_reused']), (3, 0)) cache = json.loads((self.d / 'gen' / 'main' / 'lift_cache.json').read_text()) self.assertEqual(cache['marker'], GENERATED_MARKER) self.assertEqual(sorted(cache['chunks']), ['main_0000.c', 'main_0001.c', 'main_0002.c']) self.assertIn('rt_patch_on[0]', self.chunk(2).read_text()) folded = (self.d / 'gen' / 'main' / 'folded.txt').read_text() self.assertIn('1020 1024', folded) # Same inputs: nothing lifted, nothing written, the jump table's bytes still in folded.txt (replayed from the # cache, not read again) and the statistics unchanged. mtimes = [self.chunk(n).stat().st_mtime_ns for n in range(3)] again = self.gen(patch_file(MOV_W8_200)) self.assertEqual((again['chunks_reused'], again['chunks_written']), (3, 0)) self.assertEqual({k: v for k, v in again.items() if not k.startswith('chunks_')}, {k: v for k, v in stats.items() if not k.startswith('chunks_')}) self.assertEqual([self.chunk(n).stat().st_mtime_ns for n in range(3)], mtimes) self.assertEqual((self.d / 'gen' / 'main' / 'folded.txt').read_text(), folded) self.assert_like_fresh(patch_file(MOV_W8_200)) # Another replacement word: only h's chunk is lifted. self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) self.assert_like_fresh(patch_file(MOV_W8_300)) # A set defined before the site's moves its index (rt_patch_on[1]): h's chunk again. self.assertEqual(self.gen(patch_file(MOV_W8_300, first='z'))['chunks_reused'], 2) self.assertIn('rt_patch_on[1]', self.chunk(2).read_text()) self.assert_like_fresh(patch_file(MOV_W8_300, first='z')) # A second alternative at the site. self.assertEqual(self.gen(patch_file(MOV_W8_300, MOV_W8_200))['chunks_reused'], 2) self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200)) def test_changed_file_is_lifted_again(self): """A chunk whose bytes changed while its size and mtime stayed (a flipped bit) is lifted and rewritten.""" self.gen(patch_file(MOV_W8_200)) p = self.chunk(0) st = p.stat() data = bytearray(p.read_bytes()) data[-2] ^= 0x01 p.write_bytes(bytes(data)) os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns)) stats = self.gen(patch_file(MOV_W8_200)) self.assertEqual((stats['chunks_reused'], stats['chunks_written']), (2, 1)) self.assert_like_fresh(patch_file(MOV_W8_200)) def test_deleted_file_and_cache(self): self.gen(patch_file(MOV_W8_200)) self.chunk(1).unlink() self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) (self.d / 'gen' / 'main' / 'lift_cache.json').write_text('{not json') self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 0) self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--no-cache')['chunks_reused'], 0) self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3) self.assert_like_fresh(patch_file(MOV_W8_200)) def test_ordered_sets(self): """Two ordered sets on h's mov make it a contested site: their alternatives test rt_patch_pick[0] (1 for a0, 2 for a1) and patches.h lists the site; the cheat before them keeps its switch and its place. One ordered set alone emits what an unordered one does, header included. The pick is in the key: only h's chunk again.""" h = lambda: self.chunk(2).read_text() # noqa: E731 header = lambda: (self.d / 'gen' / 'patches.h').read_text() # noqa: E731 self.gen(patch_file(MOV_W8_300)) plain, plain_header = h(), header() self.assertEqual(self.gen(patch_file(MOV_W8_300, ordered=True))['chunks_reused'], 3) self.assertEqual((h(), header()), (plain, plain_header)) self.assertNotIn('RT_PATCH_PICK', plain_header) self.gen(patch_file(MOV_W8_300, MOV_W8_200, cheat=MOV_W8_200)) self.assertNotIn('rt_patch_pick', h()) stats = self.gen(patch_file(MOV_W8_300, MOV_W8_200, ordered=True, cheat=MOV_W8_200)) self.assertEqual(stats['chunks_reused'], 2) text = h() self.assertIn('extern uint8_t rt_patch_pick[];', text) self.assertIn('extern uint8_t rt_patch_on[];', text) on, pick1, pick2 = (text.index(s) for s in ('if (rt_patch_on[0])', 'if (rt_patch_pick[0] == 1)', 'if (rt_patch_pick[0] == 2)')) self.assertLess(on, pick1) self.assertLess(pick1, pick2) self.assertNotIn('rt_patch_on[1]', text) self.assertIn('#define RT_PATCH_PICK_COUNT 1', header()) self.assertIn(f'X(0, "main", {H:#x}, 1, 2)', header()) self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200, ordered=True, cheat=MOV_W8_200)) def test_spin_wait_is_in_the_key(self): self.gen(patch_file(MOV_W8_200)) self.assertEqual(self.gen(patch_file(MOV_W8_200, spin=True))['chunks_reused'], 2) self.assertIn('rt_spin_gate', self.chunk(0).read_text()) self.assert_like_fresh(patch_file(MOV_W8_200, spin=True)) self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) self.assertNotIn('rt_spin_gate', self.chunk(0).read_text()) def test_hooks_are_in_the_key(self): """No hook line: FN_ENTER in every function. The first hook line takes it out of every other function, so every chunk is lifted; one more hooked function lifts only its chunk; --fn-enter-all puts it back everywhere; a hook on an address that is no function start stops the lift.""" fn_enter = lambda: ['FN_ENTER(' in self.chunk(n).read_text() for n in range(3)] # noqa: E731 self.gen(patch_file(MOV_W8_200)) self.assertEqual(fn_enter(), [True, True, True]) self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]))['chunks_reused'], 0) self.assertEqual(fn_enter(), [False, True, False]) self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G])) self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G, H]))['chunks_reused'], 2) self.assertEqual(fn_enter(), [False, True, True]) self.assert_like_fresh(patch_file(MOV_W8_200, hooks=[G, H])) self.assertEqual(self.gen(patch_file(MOV_W8_200, hooks=[G]), 'gen', '--fn-enter-all')['chunks_reused'], 0) self.assertEqual(fn_enter(), [True, True, True]) p = self.d / 'patches.txt' p.write_text(patch_file(MOV_W8_200, hooks=[G + 4])) r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), '--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--patches', str(p)], capture_output=True, text=True) self.assertNotEqual(r.returncode, 0) self.assertIn('is no function start', r.stderr) def test_changed_cache_entry_is_not_taken(self): """A changed result in lift_cache.json (a folded range, here) fails the entry's digest: the chunk is lifted and folded.txt keeps the jump table's real bytes.""" self.gen(patch_file(MOV_W8_200)) p = self.d / 'gen' / 'main' / 'lift_cache.json' data = json.loads(p.read_text()) self.assertEqual(data['chunks']['main_0000.c']['folded'], [[0x1020, 0x1024]]) data['chunks']['main_0000.c']['folded'] = [[0x1060, 0x1064]] p.write_text(json.dumps(data)) self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) self.assertIn('1020 1024', (self.d / 'gen' / 'main' / 'folded.txt').read_text()) self.assert_like_fresh(patch_file(MOV_W8_200)) def test_unwritable_cache_only_warns(self): """A cache that cannot be replaced leaves the lift successful, its manifest written; the old cache's entry for the chunk the run rewrote no longer matches it.""" self.gen(patch_file(MOV_W8_200)) p = self.d / 'gen' / 'main' / 'lift_cache.json' os.chmod(p, stat.S_IREAD) try: self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) self.assertIn('lift_cache.json not written', self.stderr) finally: os.chmod(p, stat.S_IREAD | stat.S_IWRITE) self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) self.assert_like_fresh(patch_file(MOV_W8_300)) def test_environment_switch(self): self.gen(patch_file(MOV_W8_200)) self.assertEqual(self.gen(patch_file(MOV_W8_200), env={'TOTK_LIFT_CACHE': '0'})['chunks_reused'], 0) self.assertIn('lifting every chunk (TOTK_LIFT_CACHE=0)', self.stderr) self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3) def test_emitter_options_are_in_the_key(self): self.gen(patch_file(MOV_W8_200)) self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--reg-locals', '0')['chunks_reused'], 0) def test_generator_sources_are_in_the_key(self): """A copy of the generator lifts everything again after a comment in its lifter.py changes, and nothing after a patch table beside it (recomp/patches-*.txt) changes.""" copy = self.d / 'repo' shutil.copytree(ROOT / 'recomp', copy / 'recomp', ignore=shutil.ignore_patterns('__pycache__')) (copy / 'tools').mkdir() for name in ('check_content.py', 'hostmem.py'): shutil.copy2(ROOT / 'tools' / name, copy / 'tools' / name) def run(): r = subprocess.run([sys.executable, str(copy / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), '--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1'], capture_output=True, text=True, env={**os.environ, 'TOTK_LIFT_CACHE': ''}) self.assertEqual(r.returncode, 0, r.stderr) return json.loads((self.d / 'gen' / 'main' / 'manifest.json').read_text())['stats']['chunks_reused'] self.assertEqual(run(), 0) self.assertEqual(run(), 3) table = next((copy / 'recomp').glob('patches-*.txt')) table.write_text(table.read_text() + '# another line\n') self.assertEqual(run(), 3) lifter = copy / 'recomp' / 'lifter.py' lifter.write_text(lifter.read_text() + '# another comment\n') self.assertEqual(run(), 0) class Picks(unittest.TestCase): """recomp/patches.py: which sites are contested and how the patch file declares ordered sets.""" def table(self, text): with tempfile.TemporaryDirectory() as d: p = Path(d) / 'patches.txt' p.write_text(text) return patches.load(p, [('main', 0x8000000, 0x100000), ('sdk', 0x9000000, 0x100000)]) def test_contested_sites(self): t = self.table('set b E_B 1\nset m1 E_M1 0 ordered\nset m2 E_M2 0 ordered\nset m3 E_M3 0 ordered\n' 'site b main 20 d503201f d503201f\nsite m2 main 20 d503201f d503201f\n' 'site m1 main 20 d503201f d503201f\nsite m3 main 10 d503201f d503201f\n' 'site m3 sdk 30 d503201f d503201f\nsite m1 sdk 30 d503201f d503201f\n') self.assertEqual(t.ordered, {1, 2, 3}) # by module name and offset; the sets in file order of their site lines; main+0x10 has one ordered set self.assertEqual(t.picks(), [('main', 0x20, [2, 1]), ('sdk', 0x30, [3, 1])]) self.assertEqual(t.picks_for('sdk', 0x9000000), {0x9000030: (1, {3: 1, 1: 2})}) self.assertIn('X(0, "main", 0x20, 2, 1) X(1, "sdk", 0x30, 3, 1)', patches.header_text(t, 'm')) def test_bad_keyword(self): with self.assertRaises(patches.PatchError): self.table('set m E_M 0 sorted\n') if __name__ == '__main__': unittest.main()