Files
totk-pc-recomp/tests/exefs_patches_test.py
T
Valiantsin DzerakhandClaude Opus 5.5 4d2e29aa79 The generator lets the run time pick which ordered patch set runs at a site several of them patch
A patch set marked `ordered` (recomp/patches.py) belongs to something the player orders; tools/exefs_patches.py marks
the mods' ExeFS patch sets so. A site two or more ordered sets patch is contested: its ordered alternatives test one
byte, rt_patch_pick[<site>], instead of each set's switch, and patches.h lists the contested sites with their sets for
the runtime. Other alternatives keep their place and switch, so a cheat before the mods' sets still wins while it is
on. Sites with at most one ordered set, and tables without contested sites, are emitted as before, patches.h
included. The pick goes into the lift cache key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 20:11:27 +03:00

259 lines
13 KiB
Python

"""tools/exefs_patches.py on modules it makes itself: code writes become sites, data writes become runs of the data file
as emulators apply a patch file before the dynamic linker (plain bytes, DT_RELR words rebased, writes into words an
explicit-addend relocation fills dropped), the writes it refuses (outside the image, into the linker's tables or a
lazily bound pointer), the status and data files, and the check after the lift against the bytes recomp/gen.py lists in
folded.txt, with gen.py run on a module with one jump table. No game files.
Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_patches_test.py
"""
import struct
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tools'))
import exefs_patches as ep # noqa: E402
BUILD_ID = bytes.fromhex('0123456789abcdef') + bytes(24)
BID_NAME = '0123456789ABCDEF' # an IPS file name may leave the build id's trailing zeros out
MOV_W8_100, MOV_W8_200, RET = 0x52802008, 0x52804008, 0xD65F03C0
R_RELATIVE, R_GLOB_DAT, R_JUMP_SLOT = 1027, 1025, 1026
def nso(text, ro, data, bss=0x1000):
"""An uncompressed NSO with .text at 0, .rodata at 0x1000 and .data at 0x2000."""
segs = [(0x0, text), (0x1000, ro), (0x2000, data)]
hdr = bytearray(0x100)
hdr[0:4] = b'NSO0'
off = 0x100
for i, (mem, d) in enumerate(segs):
struct.pack_into('<IIII', hdr, 0x10 + 0x10 * i, off, mem, len(d), bss if i == 2 else 0)
struct.pack_into('<I', hdr, 0x60 + 4 * i, len(d))
off += len(d)
hdr[0x40:0x60] = BUILD_ID
return bytes(hdr) + b''.join(d for _, d in segs)
def put(buf, off, data):
buf[off:off + len(data)] = data
def reloc_module():
"""fn at 0x8 (mov w8, #0x100; ret); .rodata: constants at 0x1000, MOD0 at 0x1100 and the relocation tables at
0x1200 (RELA: a RELATIVE word at 0x2100, a GLOB_DAT word at 0x2108), 0x1300 (JMPREL: a JUMP_SLOT word at 0x2110)
and 0x1400 (RELR: the word at 0x2118, addend 0x30); .data: .dynamic at 0x2000, plain bytes at 0x2200."""
text = struct.pack('<4I', 0x14000002, 0x1100, MOV_W8_100, RET)
ro = bytearray(0x500)
put(ro, 0x000, b'constant bytes!!')
put(ro, 0x100, b'MOD0' + struct.pack('<6i', 0x2000 - 0x1100, 0, 0, -0x1100, -0x1100, 0))
put(ro, 0x200, struct.pack('<QQq', 0x2100, R_RELATIVE, 0x8) + struct.pack('<QQq', 0x2108, R_GLOB_DAT, 0))
put(ro, 0x300, struct.pack('<QQq', 0x2110, R_JUMP_SLOT, 0))
put(ro, 0x400, struct.pack('<Q', 0x2118))
data = bytearray(0x300)
dyn = [(7, 0x1200), (8, 48), (23, 0x1300), (2, 24), (0x24, 0x1400), (0x23, 8), (0, 0)]
put(data, 0x000, b''.join(struct.pack('<QQ', t, v) for t, v in dyn))
put(data, 0x118, struct.pack('<Q', 0x30))
put(data, 0x200, b'ABCDEFGH')
return nso(text, bytes(ro), bytes(data))
# A module with one jump table (clang's compressed form), assembled from:
# b fn ; 0x0
# .word mod0 ; 0x4: the MOD0 offset
# fn: cmp w0, #3 ; 0x8
# b.hi Ldefault
# adrp x8, table ; table = 0x1020: .byte 0, 2, 4, 6
# add x8, x8, #0x20
# adr x9, L0
# ldrb w10, [x8, w0, uxtw]
# add x9, x9, x10, lsl #2
# br x9
# L0..L3: mov w0, #10/20/30/40; ret ; Ldefault: mov w0, #0; ret
# .rodata: 32 bytes of text before the table, 21 after it, MOD0 at 0x1040 and a RELATIVE relocation to fn (the
# lifter's function seed); .data: .dynamic (DT_RELA) and the relocated word.
JT_TEXT = bytes.fromhex(
'02000014401000001f0c0071e8010054080000b008810091890000100a49603829090a8b20011fd640018052c0035fd680028052'
'c0035fd6c0038052c0035fd600058052c0035fd600008052c0035fd6')
JT_RO = bytes.fromhex(
'756e746f7563686564206279746573206265666f726520746865207461626c6500020406627974657320616674657220746865'
'207461626c65000000000000004d4f4430c00f00000000000000000000c0efffffc0efffff00000000000000003020000000000000'
'03040000000000000800000000000000')
JT_DATA = struct.pack('<7Q', 7, 0x1060, 8, 24, 0, 0, 0) # DT_RELA, DT_RELASZ, DT_NULL; the word the RELATIVE fills
def ips32(writes):
"""An IPS32 file of {image offset: bytes} (offsets count the NSO header, as emulators read them)."""
body = b''.join((off + ep.NSO_HEADER).to_bytes(4, 'big') + len(d).to_bytes(2, 'big') + d
for off, d in sorted(writes.items()))
return b'IPS32' + body + b'EEOF'
class Classify(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
path = Path(cls.tmp.name) / 'main'
path.write_bytes(reloc_module())
cls.m = ep.Main(path)
cls.md = ep.MainData(cls.m)
@classmethod
def tearDownClass(cls):
cls.tmp.cleanup()
def run_file(self, writes, md=True):
return ep.classify(self.m, self.md if md else None, self.m.nso.build_id, Path(f'{BID_NAME}.ips'),
ips32(writes))
def test_relocated_words(self):
self.assertEqual(self.md.rela, {0x2100: R_RELATIVE, 0x2108: R_GLOB_DAT, 0x2110: R_JUMP_SLOT})
self.assertEqual(self.md.relr, {0x2118})
def test_code_only(self):
kind, note, sites, runs, dropped = self.run_file({0x8: struct.pack('<I', MOV_W8_200)})
self.assertEqual((kind, note, runs, dropped), ('code', '', [], 0))
self.assertEqual(sites, {0x8: [MOV_W8_200]})
def test_data_only(self):
kind, note, sites, runs, dropped = self.run_file({0x1000: b'\xAA\xBB', 0x2200: b'XY'})
self.assertEqual((kind, sites, dropped), ('data', {}, 0))
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
def test_code_and_data(self):
kind, _, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), 0x1004: b'\x01'})
self.assertEqual(kind, 'code')
self.assertEqual(sites, {0x8: [MOV_W8_200]})
self.assertEqual(runs, [(0x1004, b'\x01', False)])
def test_same_bytes_are_nothing(self):
kind, _, _, runs, _ = self.run_file({0x1000: b'constant'})
self.assertEqual((kind, runs), ('empty', []))
def test_explicit_addend_words_change_nothing(self):
kind, _, _, runs, dropped = self.run_file({0x2100: b'\x11\x22\x33\x44', 0x2108: b'\x55'})
self.assertEqual((kind, runs, dropped), ('empty', [], 5))
def test_relr_word_is_rebased(self):
kind, _, _, runs, _ = self.run_file({0x2118: b'\x31'})
self.assertEqual(kind, 'data')
self.assertEqual(runs, [(0x2118, struct.pack('<Q', 0x31), True)])
def test_long_runs_are_split(self):
_, _, _, runs, _ = self.run_file({0x2200: bytes(i % 255 + 1 for i in range(ep.RUN_MAX)) + b'\xFF'})
self.assertEqual([(o, len(d)) for o, d, _ in runs], [(0x2200, ep.RUN_MAX), (0x2200 + ep.RUN_MAX, 1)])
def test_refused_writes(self):
for writes, why in (({0x2110: b'\x01'}, 'binds lazily'), ({0x2000: b'\x01'}, 'linker\'s tables'),
({0x1208: b'\x01'}, 'linker\'s tables'), ({0x4000: b'\x01'}, 'outside main\'s image'),
({-0x80: b'\x01'}, 'outside main\'s image')):
kind, note, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), **writes})
self.assertEqual((kind, sites, runs), ('unsupported', {}, []), note)
self.assertIn(why, note)
def test_data_without_a_data_file(self):
kind, note, _, _, _ = self.run_file({0x1000: b'\xAA'}, md=False)
self.assertEqual(kind, 'unsupported')
self.assertIn('only code patches are supported', note)
def test_pchtxt(self):
text = (f'@nsobid-{BID_NAME}\n@flag offset_shift 0x100\n@enabled\n'
f'00001000 AABB\n00002200 "XY"\n@disabled\n00002204 CC\n')
kind, _, _, runs, _ = ep.classify(self.m, self.md, self.m.nso.build_id, Path('x.pchtxt'), text.encode())
self.assertEqual(kind, 'data')
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
class Tool(unittest.TestCase):
"""The command line: build, then the check after the lift."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.d = Path(self.tmp.name)
(self.d / 'exefs').mkdir()
(self.d / 'patches.txt').write_text('# no sets of the port\n')
def tearDown(self):
self.tmp.cleanup()
def mod_file(self, mod, writes):
f = self.d / 'mods' / mod / 'exefs' / f'{BID_NAME}.ips'
f.parent.mkdir(parents=True, exist_ok=True)
f.write_bytes(ips32(writes))
def tool(self, *args):
r = subprocess.run([sys.executable, str(ROOT / 'tools' / 'exefs_patches.py'), '--exefs', str(self.d / 'exefs'),
*map(str, args)], capture_output=True, text=True)
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
return r.stdout
def build(self):
self.tool('--mods', self.d / 'mods', '--patches', self.d / 'patches.txt', '--out', self.d / 'out.txt',
'--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data')
status = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
data = [line.split('\t') for line in (self.d / 'exefs.data').read_text().splitlines()
if not line.startswith('#')]
return status, data
def key(self, mod):
return ep.fnv(f'{mod}/exefs/{BID_NAME}.ips'.encode())
def test_build_and_check(self):
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
self.mod_file('Code', {0x8: struct.pack('<I', MOV_W8_200), 0x2200: b'Q'})
self.mod_file('Data', {0x1000: b'\xAA', 0x2118: b'\x31'})
self.mod_file('Bad', {0x2110: b'\x01'})
status, data = self.build()
code, dat, bad = self.key('Code'), self.key('Data'), self.key('Bad')
self.assertEqual([status[k][2] for k in (code, dat, bad)], ['code', 'data', 'unsupported'])
self.assertEqual([status[k][4] for k in (code, dat)], ['data 1', 'data 9']) # bytes its data lines hold
self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0 ordered', (self.d / 'out.txt').read_text())
self.assertEqual(sorted((f[0], f[2], f[3], f[4:]) for f in data),
sorted([(code, '2200', '51', []), (dat, '1000', 'aa', []),
(dat, '2118', '3100000000000000', ['base'])]))
# after the lift: .rodata bytes the lifted code read make the Data file unsupported; the Code file's data is
# in .data, which the lifter never reads
folded = self.d / 'folded.txt'
folded.write_text('# test\n1000 1001\n')
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
status2 = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
self.assertEqual(status2[code][2], 'code')
self.assertEqual(status2[dat][2:4], ['unsupported', '-'])
self.assertIn('jump table', status2[dat][4])
left = [line.split('\t')[0] for line in (self.d / 'exefs.data').read_text().splitlines()
if not line.startswith('#')]
self.assertEqual(left, [code])
def test_check_without_the_lift_list(self):
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
self.mod_file('Ro', {0x1000: b'\xAA'})
self.mod_file('Rw', {0x2200: b'Q'})
self.build()
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded',
self.d / 'missing.txt')
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
self.assertEqual((status[self.key('Ro')], status[self.key('Rw')]), ('unsupported', 'data'))
def test_jump_table_through_the_lift(self):
"""recomp/gen.py lists the jump table's bytes in folded.txt; a write into them is refused, one beside them
is kept."""
(self.d / 'exefs' / 'main').write_bytes(nso(JT_TEXT, JT_RO, JT_DATA))
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1'],
capture_output=True, text=True)
self.assertEqual(r.returncode, 0, r.stderr)
folded = self.d / 'gen' / 'main' / 'folded.txt'
self.assertEqual(ep.read_ranges(folded), [(0x1020, 0x1024)])
self.mod_file('Table', {0x1022: b'\x00'})
self.mod_file('Beside', {0x1024: b'B'})
self.build()
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
self.assertEqual((status[self.key('Table')], status[self.key('Beside')]), ('unsupported', 'data'))
if __name__ == '__main__':
unittest.main()