mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Store the data writes of mods' ExeFS patch files into main before the game runs
An IPS, IPS32 or pchtxt file for main that writes main's .rodata, .data or .bss was refused as a whole. Such writes need no lifting: tools/exefs_patches.py now puts them in build-cache/exefs.data, and runtime/mods.c stores them into the loaded main before any game code runs, as emulators and Atmosphere apply a patch file to the module before its dynamic linker runs. A DT_RELR word gets main's base added; bytes over words an explicit-addend relocation fills change nothing and are dropped; a write into the dynamic linker's tables, a lazily bound function pointer or outside main's image keeps the file unsupported. A mod whose data lines do not all fit the build keeps its code patches off too. The mods are applied from the bottom one up (mods_exefs_apply), so where two mods write the same bytes the higher one wins, as it does for files. --probe-mods checks it with two mods writing one byte. The lifted code reads some of main's bytes at lift time (the jump tables). recomp/gen.py lists those ranges in <gen>/main/folded.txt, and the setup checks the data writes against them after the lift: a file that writes them becomes unsupported, since the change would not reach the generated code. Generated C without mods is unchanged. This is the data part of #790 (ericdallo's ExeFS work), split out on its own; the plugin part follows separately. Co-Authored-By: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df9ba65a91
commit
fa94e36549
13 files changed
+1001
-102
No files matched your search
@@ -0,0 +1,258 @@
|
||||
"""tools/exefs_patches.py on modules it makes itself: code writes become sites, data writes become runs of the data file
|
||||
as emulators apply a patch file before the dynamic linker (plain bytes, DT_RELR words rebased, writes into words an
|
||||
explicit-addend relocation fills dropped), the writes it refuses (outside the image, into the linker's tables or a
|
||||
lazily bound pointer), the status and data files, and the check after the lift against the bytes recomp/gen.py lists in
|
||||
folded.txt, with gen.py run on a module with one jump table. No game files.
|
||||
|
||||
Run with .venv (lz4 and numpy, as recomp/): python tests/exefs_patches_test.py
|
||||
"""
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import exefs_patches as ep # noqa: E402
|
||||
|
||||
BUILD_ID = bytes.fromhex('0123456789abcdef') + bytes(24)
|
||||
BID_NAME = '0123456789ABCDEF' # an IPS file name may leave the build id's trailing zeros out
|
||||
MOV_W8_100, MOV_W8_200, RET = 0x52802008, 0x52804008, 0xD65F03C0
|
||||
R_RELATIVE, R_GLOB_DAT, R_JUMP_SLOT = 1027, 1025, 1026
|
||||
|
||||
|
||||
def nso(text, ro, data, bss=0x1000):
|
||||
"""An uncompressed NSO with .text at 0, .rodata at 0x1000 and .data at 0x2000."""
|
||||
segs = [(0x0, text), (0x1000, ro), (0x2000, data)]
|
||||
hdr = bytearray(0x100)
|
||||
hdr[0:4] = b'NSO0'
|
||||
off = 0x100
|
||||
for i, (mem, d) in enumerate(segs):
|
||||
struct.pack_into('<IIII', hdr, 0x10 + 0x10 * i, off, mem, len(d), bss if i == 2 else 0)
|
||||
struct.pack_into('<I', hdr, 0x60 + 4 * i, len(d))
|
||||
off += len(d)
|
||||
hdr[0x40:0x60] = BUILD_ID
|
||||
return bytes(hdr) + b''.join(d for _, d in segs)
|
||||
|
||||
|
||||
def put(buf, off, data):
|
||||
buf[off:off + len(data)] = data
|
||||
|
||||
|
||||
def reloc_module():
|
||||
"""fn at 0x8 (mov w8, #0x100; ret); .rodata: constants at 0x1000, MOD0 at 0x1100 and the relocation tables at
|
||||
0x1200 (RELA: a RELATIVE word at 0x2100, a GLOB_DAT word at 0x2108), 0x1300 (JMPREL: a JUMP_SLOT word at 0x2110)
|
||||
and 0x1400 (RELR: the word at 0x2118, addend 0x30); .data: .dynamic at 0x2000, plain bytes at 0x2200."""
|
||||
text = struct.pack('<4I', 0x14000002, 0x1100, MOV_W8_100, RET)
|
||||
ro = bytearray(0x500)
|
||||
put(ro, 0x000, b'constant bytes!!')
|
||||
put(ro, 0x100, b'MOD0' + struct.pack('<6i', 0x2000 - 0x1100, 0, 0, -0x1100, -0x1100, 0))
|
||||
put(ro, 0x200, struct.pack('<QQq', 0x2100, R_RELATIVE, 0x8) + struct.pack('<QQq', 0x2108, R_GLOB_DAT, 0))
|
||||
put(ro, 0x300, struct.pack('<QQq', 0x2110, R_JUMP_SLOT, 0))
|
||||
put(ro, 0x400, struct.pack('<Q', 0x2118))
|
||||
data = bytearray(0x300)
|
||||
dyn = [(7, 0x1200), (8, 48), (23, 0x1300), (2, 24), (0x24, 0x1400), (0x23, 8), (0, 0)]
|
||||
put(data, 0x000, b''.join(struct.pack('<QQ', t, v) for t, v in dyn))
|
||||
put(data, 0x118, struct.pack('<Q', 0x30))
|
||||
put(data, 0x200, b'ABCDEFGH')
|
||||
return nso(text, bytes(ro), bytes(data))
|
||||
|
||||
|
||||
# A module with one jump table (clang's compressed form), assembled from:
|
||||
# b fn ; 0x0
|
||||
# .word mod0 ; 0x4: the MOD0 offset
|
||||
# fn: cmp w0, #3 ; 0x8
|
||||
# b.hi Ldefault
|
||||
# adrp x8, table ; table = 0x1020: .byte 0, 2, 4, 6
|
||||
# add x8, x8, #0x20
|
||||
# adr x9, L0
|
||||
# ldrb w10, [x8, w0, uxtw]
|
||||
# add x9, x9, x10, lsl #2
|
||||
# br x9
|
||||
# L0..L3: mov w0, #10/20/30/40; ret ; Ldefault: mov w0, #0; ret
|
||||
# .rodata: 32 bytes of text before the table, 21 after it, MOD0 at 0x1040 and a RELATIVE relocation to fn (the
|
||||
# lifter's function seed); .data: .dynamic (DT_RELA) and the relocated word.
|
||||
JT_TEXT = bytes.fromhex(
|
||||
'02000014401000001f0c0071e8010054080000b008810091890000100a49603829090a8b20011fd640018052c0035fd680028052'
|
||||
'c0035fd6c0038052c0035fd600058052c0035fd600008052c0035fd6')
|
||||
JT_RO = bytes.fromhex(
|
||||
'756e746f7563686564206279746573206265666f726520746865207461626c6500020406627974657320616674657220746865'
|
||||
'207461626c65000000000000004d4f4430c00f00000000000000000000c0efffffc0efffff00000000000000003020000000000000'
|
||||
'03040000000000000800000000000000')
|
||||
JT_DATA = struct.pack('<7Q', 7, 0x1060, 8, 24, 0, 0, 0) # DT_RELA, DT_RELASZ, DT_NULL; the word the RELATIVE fills
|
||||
|
||||
|
||||
def ips32(writes):
|
||||
"""An IPS32 file of {image offset: bytes} (offsets count the NSO header, as emulators read them)."""
|
||||
body = b''.join((off + ep.NSO_HEADER).to_bytes(4, 'big') + len(d).to_bytes(2, 'big') + d
|
||||
for off, d in sorted(writes.items()))
|
||||
return b'IPS32' + body + b'EEOF'
|
||||
|
||||
|
||||
class Classify(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
path = Path(cls.tmp.name) / 'main'
|
||||
path.write_bytes(reloc_module())
|
||||
cls.m = ep.Main(path)
|
||||
cls.md = ep.MainData(cls.m)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def run_file(self, writes, md=True):
|
||||
return ep.classify(self.m, self.md if md else None, self.m.nso.build_id, Path(f'{BID_NAME}.ips'),
|
||||
ips32(writes))
|
||||
|
||||
def test_relocated_words(self):
|
||||
self.assertEqual(self.md.rela, {0x2100: R_RELATIVE, 0x2108: R_GLOB_DAT, 0x2110: R_JUMP_SLOT})
|
||||
self.assertEqual(self.md.relr, {0x2118})
|
||||
|
||||
def test_code_only(self):
|
||||
kind, note, sites, runs, dropped = self.run_file({0x8: struct.pack('<I', MOV_W8_200)})
|
||||
self.assertEqual((kind, note, runs, dropped), ('code', '', [], 0))
|
||||
self.assertEqual(sites, {0x8: [MOV_W8_200]})
|
||||
|
||||
def test_data_only(self):
|
||||
kind, note, sites, runs, dropped = self.run_file({0x1000: b'\xAA\xBB', 0x2200: b'XY'})
|
||||
self.assertEqual((kind, sites, dropped), ('data', {}, 0))
|
||||
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
|
||||
|
||||
def test_code_and_data(self):
|
||||
kind, _, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), 0x1004: b'\x01'})
|
||||
self.assertEqual(kind, 'code')
|
||||
self.assertEqual(sites, {0x8: [MOV_W8_200]})
|
||||
self.assertEqual(runs, [(0x1004, b'\x01', False)])
|
||||
|
||||
def test_same_bytes_are_nothing(self):
|
||||
kind, _, _, runs, _ = self.run_file({0x1000: b'constant'})
|
||||
self.assertEqual((kind, runs), ('empty', []))
|
||||
|
||||
def test_explicit_addend_words_change_nothing(self):
|
||||
kind, _, _, runs, dropped = self.run_file({0x2100: b'\x11\x22\x33\x44', 0x2108: b'\x55'})
|
||||
self.assertEqual((kind, runs, dropped), ('empty', [], 5))
|
||||
|
||||
def test_relr_word_is_rebased(self):
|
||||
kind, _, _, runs, _ = self.run_file({0x2118: b'\x31'})
|
||||
self.assertEqual(kind, 'data')
|
||||
self.assertEqual(runs, [(0x2118, struct.pack('<Q', 0x31), True)])
|
||||
|
||||
def test_long_runs_are_split(self):
|
||||
_, _, _, runs, _ = self.run_file({0x2200: bytes(i % 255 + 1 for i in range(ep.RUN_MAX)) + b'\xFF'})
|
||||
self.assertEqual([(o, len(d)) for o, d, _ in runs], [(0x2200, ep.RUN_MAX), (0x2200 + ep.RUN_MAX, 1)])
|
||||
|
||||
def test_refused_writes(self):
|
||||
for writes, why in (({0x2110: b'\x01'}, 'binds lazily'), ({0x2000: b'\x01'}, 'linker\'s tables'),
|
||||
({0x1208: b'\x01'}, 'linker\'s tables'), ({0x4000: b'\x01'}, 'outside main\'s image'),
|
||||
({-0x80: b'\x01'}, 'outside main\'s image')):
|
||||
kind, note, sites, runs, _ = self.run_file({0x8: struct.pack('<I', MOV_W8_200), **writes})
|
||||
self.assertEqual((kind, sites, runs), ('unsupported', {}, []), note)
|
||||
self.assertIn(why, note)
|
||||
|
||||
def test_data_without_a_data_file(self):
|
||||
kind, note, _, _, _ = self.run_file({0x1000: b'\xAA'}, md=False)
|
||||
self.assertEqual(kind, 'unsupported')
|
||||
self.assertIn('only code patches are supported', note)
|
||||
|
||||
def test_pchtxt(self):
|
||||
text = (f'@nsobid-{BID_NAME}\n@flag offset_shift 0x100\n@enabled\n'
|
||||
f'00001000 AABB\n00002200 "XY"\n@disabled\n00002204 CC\n')
|
||||
kind, _, _, runs, _ = ep.classify(self.m, self.md, self.m.nso.build_id, Path('x.pchtxt'), text.encode())
|
||||
self.assertEqual(kind, 'data')
|
||||
self.assertEqual(runs, [(0x1000, b'\xAA\xBB', False), (0x2200, b'XY', False)])
|
||||
|
||||
|
||||
class Tool(unittest.TestCase):
|
||||
"""The command line: build, then the check after the lift."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.d = Path(self.tmp.name)
|
||||
(self.d / 'exefs').mkdir()
|
||||
(self.d / 'patches.txt').write_text('# no sets of the port\n')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def mod_file(self, mod, writes):
|
||||
f = self.d / 'mods' / mod / 'exefs' / f'{BID_NAME}.ips'
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
f.write_bytes(ips32(writes))
|
||||
|
||||
def tool(self, *args):
|
||||
r = subprocess.run([sys.executable, str(ROOT / 'tools' / 'exefs_patches.py'), '--exefs', str(self.d / 'exefs'),
|
||||
*map(str, args)], capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||||
return r.stdout
|
||||
|
||||
def build(self):
|
||||
self.tool('--mods', self.d / 'mods', '--patches', self.d / 'patches.txt', '--out', self.d / 'out.txt',
|
||||
'--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data')
|
||||
status = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
data = [line.split('\t') for line in (self.d / 'exefs.data').read_text().splitlines()
|
||||
if not line.startswith('#')]
|
||||
return status, data
|
||||
|
||||
def key(self, mod):
|
||||
return ep.fnv(f'{mod}/exefs/{BID_NAME}.ips'.encode())
|
||||
|
||||
def test_build_and_check(self):
|
||||
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
|
||||
self.mod_file('Code', {0x8: struct.pack('<I', MOV_W8_200), 0x2200: b'Q'})
|
||||
self.mod_file('Data', {0x1000: b'\xAA', 0x2118: b'\x31'})
|
||||
self.mod_file('Bad', {0x2110: b'\x01'})
|
||||
status, data = self.build()
|
||||
code, dat, bad = self.key('Code'), self.key('Data'), self.key('Bad')
|
||||
self.assertEqual([status[k][2] for k in (code, dat, bad)], ['code', 'data', 'unsupported'])
|
||||
self.assertEqual([status[k][4] for k in (code, dat)], ['data 1', 'data 9']) # bytes its data lines hold
|
||||
self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0', (self.d / 'out.txt').read_text())
|
||||
self.assertEqual(sorted((f[0], f[2], f[3], f[4:]) for f in data),
|
||||
sorted([(code, '2200', '51', []), (dat, '1000', 'aa', []),
|
||||
(dat, '2118', '3100000000000000', ['base'])]))
|
||||
# after the lift: .rodata bytes the lifted code read make the Data file unsupported; the Code file's data is
|
||||
# in .data, which the lifter never reads
|
||||
folded = self.d / 'folded.txt'
|
||||
folded.write_text('# test\n1000 1001\n')
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
|
||||
status2 = {f[0]: f for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual(status2[code][2], 'code')
|
||||
self.assertEqual(status2[dat][2:4], ['unsupported', '-'])
|
||||
self.assertIn('jump table', status2[dat][4])
|
||||
left = [line.split('\t')[0] for line in (self.d / 'exefs.data').read_text().splitlines()
|
||||
if not line.startswith('#')]
|
||||
self.assertEqual(left, [code])
|
||||
|
||||
def test_check_without_the_lift_list(self):
|
||||
(self.d / 'exefs' / 'main').write_bytes(reloc_module())
|
||||
self.mod_file('Ro', {0x1000: b'\xAA'})
|
||||
self.mod_file('Rw', {0x2200: b'Q'})
|
||||
self.build()
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded',
|
||||
self.d / 'missing.txt')
|
||||
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual((status[self.key('Ro')], status[self.key('Rw')]), ('unsupported', 'data'))
|
||||
|
||||
def test_jump_table_through_the_lift(self):
|
||||
"""recomp/gen.py lists the jump table's bytes in folded.txt; a write into them is refused, one beside them
|
||||
is kept."""
|
||||
(self.d / 'exefs' / 'main').write_bytes(nso(JT_TEXT, JT_RO, JT_DATA))
|
||||
r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'),
|
||||
'--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1'],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
folded = self.d / 'gen' / 'main' / 'folded.txt'
|
||||
self.assertEqual(ep.read_ranges(folded), [(0x1020, 0x1024)])
|
||||
self.mod_file('Table', {0x1022: b'\x00'})
|
||||
self.mod_file('Beside', {0x1024: b'B'})
|
||||
self.build()
|
||||
self.tool('--status', self.d / 'exefs.status', '--data', self.d / 'exefs.data', '--folded', folded)
|
||||
status = {f[0]: f[2] for f in (line.split('\t') for line in (self.d / 'exefs.status').read_text().splitlines())}
|
||||
self.assertEqual((status[self.key('Table')], status[self.key('Beside')]), ('unsupported', 'data'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -315,6 +315,75 @@ class LiftKeyTest(unittest.TestCase):
|
||||
self.assertEqual(self.key(), before)
|
||||
|
||||
|
||||
class LiftRunTest(unittest.TestCase):
|
||||
"""Lift.run with fake tools: the ExeFS patch stage writes the status and data files, the lift lists the bytes it
|
||||
read, and the data writes are checked against that list after it; a failed lift or check drops both files."""
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.dest, temp = base / 'install', base / 'temp'
|
||||
self.dest.mkdir()
|
||||
temp.mkdir()
|
||||
layout = SimpleNamespace(recomp=ROOT / 'recomp', tools=ROOT / 'tools',
|
||||
patches=lambda version: base / 'patches-v143.txt')
|
||||
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, layout=layout, keys={'lift': 'k'},
|
||||
state={'version': '1.4.3'}, reporter=Reporter(out=io.StringIO()))
|
||||
self.addCleanup(self.ctx.reporter.close)
|
||||
mod = self.dest / 'mods' / 'SQ' / 'exefs'
|
||||
mod.mkdir(parents=True)
|
||||
(mod / '277178B7DBA1B6D4.ips').write_bytes(b'IPS32EEOF')
|
||||
self.calls, self.fail = [], set()
|
||||
saved = steps.run_tool
|
||||
self.addCleanup(lambda: setattr(steps, 'run_tool', saved))
|
||||
steps.run_tool = self.run_tool
|
||||
|
||||
def run_tool(self, ctx, step, script, args, parse=None, cwd=None):
|
||||
args = [str(a) for a in args]
|
||||
name = Path(script).name + (' --folded' if '--folded' in args else '')
|
||||
self.calls.append((name, args))
|
||||
log = ctx.temp / 'tool.log'
|
||||
log.write_text('')
|
||||
if name in self.fail:
|
||||
return 1, log
|
||||
if name == 'exefs_patches.py':
|
||||
for flag in ('--status', '--data', '--out'):
|
||||
p = Path(args[args.index(flag) + 1])
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(flag)
|
||||
elif name == 'gen.py':
|
||||
out = Path(args[args.index('--out') + 1]) / 'main'
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / 'folded.txt').write_text('# test\n')
|
||||
return 0, log
|
||||
|
||||
def outputs(self):
|
||||
return [p.is_file() for p in (steps.exefs_status(self.ctx), steps.exefs_data(self.ctx))]
|
||||
|
||||
def test_data_file_and_the_check_after_the_lift(self):
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual([c[0] for c in self.calls], ['exefs_patches.py', 'gen.py', 'exefs_patches.py --folded'])
|
||||
build = self.calls[0][1]
|
||||
self.assertEqual(build[build.index('--data') + 1], str(steps.exefs_data(self.ctx)))
|
||||
check = self.calls[2][1]
|
||||
self.assertEqual(check[check.index('--folded') + 1], str(self.ctx.temp / 'gen' / 'main' / 'folded.txt'))
|
||||
self.assertEqual(check[check.index('--status') + 1], str(steps.exefs_status(self.ctx)))
|
||||
self.assertEqual(self.outputs(), [True, True])
|
||||
|
||||
def test_failed_lift_drops_the_data_file(self):
|
||||
self.fail = {'gen.py'}
|
||||
with self.assertRaises(steps.InstallError):
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual(self.outputs(), [False, False])
|
||||
|
||||
def test_failed_check_drops_both_files(self):
|
||||
self.fail = {'exefs_patches.py --folded'}
|
||||
steps.Lift().run(self.ctx)
|
||||
self.assertEqual(self.outputs(), [False, False])
|
||||
self.assertIn('warn.exefs_skipped', self.ctx.reporter.out.getvalue())
|
||||
|
||||
|
||||
class IniTest(unittest.TestCase):
|
||||
EXAMPLE = '[display]\n# vk_present = mailbox\n\n[language]\n# locale = en-US\n# lang = en-US\n# x = 1\n'
|
||||
|
||||
|
||||
Reference in new issue
Block a user