mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 12:55:57 +00:00
Store the data writes of mods' ExeFS patch files into main before the game runs
An IPS, IPS32 or pchtxt file for main that writes main's .rodata, .data or .bss was refused as a whole. Such writes need no lifting: tools/exefs_patches.py now puts them in build-cache/exefs.data, and runtime/mods.c stores them into the loaded main before any game code runs, as emulators and Atmosphere apply a patch file to the module before its dynamic linker runs. A DT_RELR word gets main's base added; bytes over words an explicit-addend relocation fills change nothing and are dropped; a write into the dynamic linker's tables, a lazily bound function pointer or outside main's image keeps the file unsupported. A mod whose data lines do not all fit the build keeps its code patches off too. The mods are applied from the bottom one up (mods_exefs_apply), so where two mods write the same bytes the higher one wins, as it does for files. --probe-mods checks it with two mods writing one byte. The lifted code reads some of main's bytes at lift time (the jump tables). recomp/gen.py lists those ranges in <gen>/main/folded.txt, and the setup checks the data writes against them after the lift: a file that writes them becomes unsupported, since the change would not reach the generated code. Generated C without mods is unchanged. This is the data part of #790 (ericdallo's ExeFS work), split out on its own; the plugin part follows separately. Co-Authored-By: itaskaev2 <165683571+itaskaev2@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
df9ba65a91
commit
fa94e36549
13 files changed
+1001
-102
No files matched your search
+41
-2
@@ -9,6 +9,9 @@ Output, under --out:
|
||||
<mod>/<mod>_NNNN.c one `void f_<addr>(Cpu *c)` per guest function, about --chunk-bytes of C per file
|
||||
<mod>/<mod>_table.c sorted <mod>_fn_addrs[] / <mod>_fn_ptrs[] / <mod>_fn_count for rt_dispatch
|
||||
<mod>/manifest.json chunk list, function count, statistics and timings
|
||||
<mod>/folded.txt the module's bytes the generated code took as constants: what the jump tables read at
|
||||
lift time, as module offsets [start, end); tools/exefs_patches.py keeps a mod's data
|
||||
writes off them
|
||||
patches.h code patch sets for the runtime (recomp/patches.py; no sets without --patches)
|
||||
|
||||
Function discovery runs in this process, vectorised over the module's .text words. Function starts are the module
|
||||
@@ -555,7 +558,8 @@ def _retry(fn, *args):
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ workers
|
||||
|
||||
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
|
||||
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter},
|
||||
# 'reads' {name: [(start, end)]} module bytes read since the last chunk
|
||||
|
||||
|
||||
def _worker_init(mods, starts, table, opts, jobs):
|
||||
@@ -565,6 +569,7 @@ def _worker_init(mods, starts, table, opts, jobs):
|
||||
_W['patches'] = table
|
||||
_W['opts'] = opts # Emitter keyword arguments
|
||||
_W['emitters'] = {}
|
||||
_W['reads'] = {}
|
||||
|
||||
|
||||
def _emitter(name):
|
||||
@@ -572,12 +577,31 @@ def _emitter(name):
|
||||
if e is None:
|
||||
path, base = _W['mods'][name]
|
||||
mod = Module(name, path, base, 'c')
|
||||
# Module.read is how the jump tables (recomp/jumptables.py _eval) read .text and .rodata at lift time: the
|
||||
# bytes it returns become part of the generated code, so <mod>/folded.txt lists them
|
||||
read, reads = mod.read, _W['reads'].setdefault(name, [])
|
||||
|
||||
def recorded(a, n, signed=False):
|
||||
reads.append((a - base, a - base + n))
|
||||
return read(a, n, signed=signed)
|
||||
mod.read = recorded
|
||||
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
|
||||
_W['patches'].for_module(name, base),
|
||||
spins=_W['patches'].spins_for(name, base), **_W['opts'])
|
||||
return e
|
||||
|
||||
|
||||
def merge_ranges(ranges):
|
||||
"""Sorted [start, end) ranges with overlapping and touching ones joined."""
|
||||
out = []
|
||||
for s, e in sorted(ranges):
|
||||
if out and s <= out[-1][1]:
|
||||
out[-1][1] = max(out[-1][1], e)
|
||||
else:
|
||||
out.append([s, e])
|
||||
return out
|
||||
|
||||
|
||||
def _emit_chunk(task):
|
||||
"""Worker: write one chunk. Returns its statistics."""
|
||||
name, number, path, fns = task
|
||||
@@ -595,9 +619,12 @@ def _emit_chunk(task):
|
||||
written = write_if_changed(Path(path), text)
|
||||
except OSError as ex:
|
||||
written, error = False, f'{path}: {ex}'
|
||||
reads = _W['reads'].get(name, [])
|
||||
folded = merge_ranges(reads)
|
||||
reads.clear()
|
||||
return {'module': name, 'number': number, 'file': Path(path).name, 'functions': len(fns), 'bytes': len(text),
|
||||
'written': written, 'error': error, 'seconds': time.perf_counter() - t0, 'stats': dict(e.stats),
|
||||
'unsupported': dict(e.unsupported)}
|
||||
'unsupported': dict(e.unsupported), 'folded': folded}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ driver
|
||||
@@ -613,6 +640,14 @@ def table_text(name, addrs):
|
||||
return ''.join(lines)
|
||||
|
||||
|
||||
def folded_text(name, ranges):
|
||||
"""<mod>/folded.txt: the module offsets [start, end) of the bytes the jump tables read at lift time, in hex."""
|
||||
lines = [f'# {GENERATED_MARKER}: {name} bytes the generated code took as constants (jump tables), module offsets '
|
||||
f'[start, end), generated by recomp/gen.py from game code. Do not edit or commit.']
|
||||
lines += [f'{s:x} {e:x}' for s, e in ranges]
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def remove_stale(moddir, name, nchunks, errors):
|
||||
"""Delete chunks an earlier run wrote beyond this run's count, and temporary files an interrupted run left."""
|
||||
pat = re.compile(re.escape(name) + r'_(\d{4})\.c')
|
||||
@@ -714,6 +749,7 @@ def main():
|
||||
# Largest chunks first, so the pool does not end on a long tail.
|
||||
tasks.sort(key=lambda t: -t[0])
|
||||
results = {name: [] for name in info}
|
||||
folded = {name: [] for name in info}
|
||||
done_at, errors = {}, []
|
||||
t_emit = time.perf_counter()
|
||||
ctx = mp.get_context('spawn')
|
||||
@@ -724,9 +760,11 @@ def main():
|
||||
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
|
||||
try:
|
||||
# Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again.
|
||||
# Its folded bytes come with its result: a chunk emitted again reads them again in its new worker.
|
||||
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool:
|
||||
for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1):
|
||||
results[r['module']].append(r)
|
||||
folded[r['module']] += r['folded']
|
||||
if r['error']:
|
||||
errors.append(r['error'])
|
||||
done_at[r['module']] = time.perf_counter() - t_emit
|
||||
@@ -753,6 +791,7 @@ def main():
|
||||
unsupported.update(r['unsupported'])
|
||||
t = time.perf_counter()
|
||||
table_written = write_if_changed(moddir / f'{name}_table.c', table_text(name, d['emitted']))
|
||||
write_if_changed(moddir / 'folded.txt', folded_text(name, merge_ranges(folded[name])))
|
||||
removed = remove_stale(moddir, name, d['chunks'], errors)
|
||||
manifest = {
|
||||
'module': name,
|
||||
|
||||
Reference in new issue
Block a user