Store the data writes of mods' ExeFS patch files into main before the game runs

An IPS, IPS32 or pchtxt file for main that writes main's .rodata, .data or .bss was refused as a whole. Such writes
need no lifting: tools/exefs_patches.py now puts them in build-cache/exefs.data, and runtime/mods.c stores them into
the loaded main before any game code runs, as emulators and Atmosphere apply a patch file to the module before its
dynamic linker runs. A DT_RELR word gets main's base added; bytes over words an explicit-addend relocation fills change
nothing and are dropped; a write into the dynamic linker's tables, a lazily bound function pointer or outside main's
image keeps the file unsupported. A mod whose data lines do not all fit the build keeps its code patches off too.

The mods are applied from the bottom one up (mods_exefs_apply), so where two mods write the same bytes the higher one
wins, as it does for files. --probe-mods checks it with two mods writing one byte.

The lifted code reads some of main's bytes at lift time (the jump tables). recomp/gen.py lists those ranges in
<gen>/main/folded.txt, and the setup checks the data writes against them after the lift: a file that writes them
becomes unsupported, since the change would not reach the generated code. Generated C without mods is unchanged.

This is the data part of #790 (ericdallo's ExeFS work), split out on its own; the plugin part follows separately.

Co-Authored-By: itaskaev2 <165683571+itaskaev2@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and itaskaev2 committed 2026-11-06 15:51:42 +01:00
1 parent df9ba65a91
commit fa94e36549
13 files changed
+1001 -102

No files matched your search

+41 -2
View File
@@ -9,6 +9,9 @@ Output, under --out:
<mod>/<mod>_NNNN.c one `void f_<addr>(Cpu *c)` per guest function, about --chunk-bytes of C per file
<mod>/<mod>_table.c sorted <mod>_fn_addrs[] / <mod>_fn_ptrs[] / <mod>_fn_count for rt_dispatch
<mod>/manifest.json chunk list, function count, statistics and timings
<mod>/folded.txt the module's bytes the generated code took as constants: what the jump tables read at
lift time, as module offsets [start, end); tools/exefs_patches.py keeps a mod's data
writes off them
patches.h code patch sets for the runtime (recomp/patches.py; no sets without --patches)
Function discovery runs in this process, vectorised over the module's .text words. Function starts are the module
@@ -555,7 +558,8 @@ def _retry(fn, *args):
# ------------------------------------------------------------------------------------------------ workers
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter},
# 'reads' {name: [(start, end)]} module bytes read since the last chunk
def _worker_init(mods, starts, table, opts, jobs):
@@ -565,6 +569,7 @@ def _worker_init(mods, starts, table, opts, jobs):
_W['patches'] = table
_W['opts'] = opts # Emitter keyword arguments
_W['emitters'] = {}
_W['reads'] = {}
def _emitter(name):
@@ -572,12 +577,31 @@ def _emitter(name):
if e is None:
path, base = _W['mods'][name]
mod = Module(name, path, base, 'c')
# Module.read is how the jump tables (recomp/jumptables.py _eval) read .text and .rodata at lift time: the
# bytes it returns become part of the generated code, so <mod>/folded.txt lists them
read, reads = mod.read, _W['reads'].setdefault(name, [])
def recorded(a, n, signed=False):
reads.append((a - base, a - base + n))
return read(a, n, signed=signed)
mod.read = recorded
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
_W['patches'].for_module(name, base),
spins=_W['patches'].spins_for(name, base), **_W['opts'])
return e
def merge_ranges(ranges):
"""Sorted [start, end) ranges with overlapping and touching ones joined."""
out = []
for s, e in sorted(ranges):
if out and s <= out[-1][1]:
out[-1][1] = max(out[-1][1], e)
else:
out.append([s, e])
return out
def _emit_chunk(task):
"""Worker: write one chunk. Returns its statistics."""
name, number, path, fns = task
@@ -595,9 +619,12 @@ def _emit_chunk(task):
written = write_if_changed(Path(path), text)
except OSError as ex:
written, error = False, f'{path}: {ex}'
reads = _W['reads'].get(name, [])
folded = merge_ranges(reads)
reads.clear()
return {'module': name, 'number': number, 'file': Path(path).name, 'functions': len(fns), 'bytes': len(text),
'written': written, 'error': error, 'seconds': time.perf_counter() - t0, 'stats': dict(e.stats),
'unsupported': dict(e.unsupported)}
'unsupported': dict(e.unsupported), 'folded': folded}
# ------------------------------------------------------------------------------------------------ driver
@@ -613,6 +640,14 @@ def table_text(name, addrs):
return ''.join(lines)
def folded_text(name, ranges):
"""<mod>/folded.txt: the module offsets [start, end) of the bytes the jump tables read at lift time, in hex."""
lines = [f'# {GENERATED_MARKER}: {name} bytes the generated code took as constants (jump tables), module offsets '
f'[start, end), generated by recomp/gen.py from game code. Do not edit or commit.']
lines += [f'{s:x} {e:x}' for s, e in ranges]
return '\n'.join(lines) + '\n'
def remove_stale(moddir, name, nchunks, errors):
"""Delete chunks an earlier run wrote beyond this run's count, and temporary files an interrupted run left."""
pat = re.compile(re.escape(name) + r'_(\d{4})\.c')
@@ -714,6 +749,7 @@ def main():
# Largest chunks first, so the pool does not end on a long tail.
tasks.sort(key=lambda t: -t[0])
results = {name: [] for name in info}
folded = {name: [] for name in info}
done_at, errors = {}, []
t_emit = time.perf_counter()
ctx = mp.get_context('spawn')
@@ -724,9 +760,11 @@ def main():
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
try:
# Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again.
# Its folded bytes come with its result: a chunk emitted again reads them again in its new worker.
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool:
for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1):
results[r['module']].append(r)
folded[r['module']] += r['folded']
if r['error']:
errors.append(r['error'])
done_at[r['module']] = time.perf_counter() - t_emit
@@ -753,6 +791,7 @@ def main():
unsupported.update(r['unsupported'])
t = time.perf_counter()
table_written = write_if_changed(moddir / f'{name}_table.c', table_text(name, d['emitted']))
write_if_changed(moddir / 'folded.txt', folded_text(name, merge_ranges(folded[name])))
removed = remove_stale(moddir, name, d['chunks'], errors)
manifest = {
'module': name,