Files
totk-pc-recomp/recomp/patches.py
T
Wes 5f6fc13712 Lift again only the chunks whose inputs changed
gen.py lifted every chunk on each run and only skipped rewriting the ones whose text came out the same, so a rebuild
after one changed code cheat still spent the whole lift (102 s on 12 cores, 178 s with 4 workers) to recompile two
chunks. A chunk's text depends only on the generator's sources and options, the module and the function starts
discovered from it, the chunk's number and functions, and the patch sites and spin waits inside its address range, so
the SHA-256 of these is now its key. <mod>/lift_cache.json records each chunk's key with the SHA-256, size and mtime of
its file and the statistics and folded ranges the manifest and folded.txt need, under a digest of the entry. A chunk
whose entry is intact, whose key matches and whose file is unchanged is reused without a worker; no pool starts when
nothing is left to lift. The cache is written after the manifest, and a failed write only warns. --no-cache, or
TOTK_LIFT_CACHE=0 for the installer's lift, lifts everything.

On 1.4.3 with the installed mods' cheats, a run with unchanged inputs takes 4 s and one with a changed cheat word lifts
one chunk in 7 s. Its output equals a --no-cache lift into an empty folder and the previous gen.py's output byte for
byte. tests/lift_cache_test.py checks reuse and invalidation on a module it makes itself.
2026-11-10 13:03:38 -06:00

160 lines
8.1 KiB
Python

"""Code patch table for the generator: guest instructions replaced in the lifted C, switchable at run time.
Lifted code cannot be byte-patched when the game runs, so a patch is applied when the C is generated. A patch file
(gen.py --patches) has two kinds of lines; `#` starts a comment, fields are separated by whitespace:
set <name> <env> <default> a patch set: a run-time switch <env>=0/1, <default> when it is unset
site <set> <module> <address> <original> <replacement> [<replacement> ...]
spin <module> <head> <back-edge> [<back-edge> ...]
<address> is a module offset (hex), <original> the instruction word the module has there (checked; a mismatch stops
the generator), each <replacement> an instruction word (hex) or `@page:x<n>`, which is `adrp x<n>, <patch page>`
encoded for the site's address. The patch page is the first 4 KiB page after main's image (layout.py leaves the rest
of the 2 MiB-aligned module slot free); the runtime commits it and owns its contents. Words in use, by byte offset:
+0..15 the frame-rate step (runtime/frame_rate.c), +16..55 the output, swapchain and scene sizes
(runtime/resolution.c res_init), +56..59 the camera's aspect ratio (runtime/resolution.c res_init, set aspect), +60..63
the field-of-view factor (runtime/resolution.c res_fov_live, set fov), +92..95 the sun shadow map size
(runtime/resolution.c shadow_init, set shadow). A new set takes words outside these.
The replacement words are lifted at the site's address in place of the original instruction, and the generated C reads
`if (rt_patch_on[<set>]) { replacement } else { original }`: one byte load and a branch per executed site. Several
sets may patch one address (cheats that write different values there): `if (a) {..} else if (b) {..} else {..}`, the
first set in file order wins. The original may be any instruction but br, blr and ret; a replacement's last word may be
a bl (lifted with the site's return address), a b or b.cond / cbz / tbz (its target joins the function's labels when
inside it) or a ret; earlier words must not branch. Every alternative of a site writes the same flags. The generator
writes patches.h (set names, switches, the page address) next to layout.h for the runtime.
A `spin` line names a spin wait found by tools/spin_waits.py: a loop with head <head> that only another thread can end,
and the direct branches back to it (module offsets). Each back-edge (checked: a b, b.cond, cbz / cbnz or tbz / tbnz to
<head>) is lifted as `rt_spin_gate(c, <head>); goto <head>`, which keeps every register and only adds time
(runtime/kernel.c kern_spin_gate).
"""
from pathlib import Path
PAGE = 0x1000
class PatchError(ValueError):
pass
def adrp(pc, target, rd):
imm = (target >> 12) - (pc >> 12)
if not -(1 << 20) <= imm < (1 << 20):
raise PatchError(f'{pc:#x}: patch page {target:#x} out of ADRP range')
imm &= 0x1FFFFF
return 0x90000000 | (imm & 3) << 29 | (imm >> 2) << 5 | rd
class Table:
"""sets: [(name, env, default)]; sites: {module: {offset: [(set index, [word | ('page', reg)], original)]}} (the
alternatives of a site in file order); page. What the generated C takes from here (for_module, spins_for) is in
the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too."""
def __init__(self, sets=(), sites=None, page=0, spins=None):
self.sets = list(sets)
self.sites = sites or {}
self.page = page
self.spins = spins or {} # {module: {back-edge offset: head offset}}
def spins_for(self, name, base):
"""{absolute back-edge address: absolute head address}."""
return {base + b: base + h for b, h in self.spins.get(name, {}).items()}
def for_module(self, name, base):
"""{absolute address: [(set index, [words])]} with @page resolved."""
out = {}
for off, alts in self.sites.get(name, {}).items():
a = base + off
out[a] = [(k, [adrp(a, self.page, w[1]) if isinstance(w, tuple) else w for w in repl])
for k, repl, _ in alts]
return out
def load(path, mods):
"""Table from a patch file; mods is layout() output [(name, base, size)]."""
sets, index, sites, spins = [], {}, {}, {}
bases = {n: (b, s) for n, b, s in mods}
for no, raw in enumerate(Path(path).read_text().splitlines(), 1):
f = raw.split('#', 1)[0].split()
if not f:
continue
where = f'{path}:{no}'
if f[0] == 'set' and len(f) == 4:
if f[1] in index:
raise PatchError(f'{where}: set {f[1]} defined twice')
index[f[1]] = len(sets)
sets.append((f[1], f[2], int(f[3])))
elif f[0] == 'site' and len(f) >= 6:
if f[1] not in index:
raise PatchError(f'{where}: unknown set {f[1]}')
if f[2] not in bases:
raise PatchError(f'{where}: module {f[2]} not in the layout')
off, orig = int(f[3], 16), int(f[4], 16)
repl = []
for w in f[5:]:
if w.startswith('@page:x'):
repl.append(('page', int(w[7:])))
else:
repl.append(int(w, 16))
alts = sites.setdefault(f[2], {}).setdefault(off, [])
if any(k == index[f[1]] for k, _, _ in alts):
raise PatchError(f'{where}: {f[2]}+{off:#x} patched twice by set {f[1]}')
if alts and alts[0][2] != orig:
raise PatchError(f'{where}: {f[2]}+{off:#x} expects {orig:08x}, an earlier line {alts[0][2]:08x}')
alts.append((index[f[1]], repl, orig))
elif f[0] == 'spin' and len(f) >= 4:
if f[1] not in bases:
raise PatchError(f'{where}: module {f[1]} not in the layout')
mod, head = spins.setdefault(f[1], {}), int(f[2], 16)
for b in f[3:]:
b = int(b, 16)
if b in mod:
raise PatchError(f'{where}: {f[1]}+{b:#x} is a back-edge twice')
mod[b] = head
else:
raise PatchError(f'{where}: cannot parse {raw!r}')
base, size = bases['main'][0], bases['main'][1]
page = (base + size + PAGE - 1) & ~(PAGE - 1)
nxt = min((b for _, b, _ in mods if b > base), default=None)
if nxt is not None and page + PAGE > nxt:
raise PatchError(f'no room for the patch page after main ({page:#x})')
return Table(sets, sites, page, spins)
def check(table, mod, name):
"""Stops on a site whose original word differs from the module's."""
for off, [(_, _, want), *_] in table.sites.get(name, {}).items():
a = mod.base + off
if not mod.in_text(a):
raise PatchError(f'{name}+{off:#x}: patch site outside .text')
have = mod.word(a)
if have != want:
raise PatchError(f'{name}+{off:#x}: module has {have:08x}, the patch expects {want:08x}')
for b, head in table.spins.get(name, {}).items():
a = mod.base + b
if not mod.in_text(a) or branch_target(mod.word(a), a) != mod.base + head:
raise PatchError(f'{name}+{b:#x}: not a direct branch to the spin wait head {name}+{head:#x}')
def branch_target(w, a):
"""Target of the direct branch w at a (b, b.cond, cbz / cbnz, tbz / tbnz), or None."""
if w >> 26 == 0x05:
return a + ((((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) << 2)
if w & 0xFF000010 == 0x54000000 or w & 0x7E000000 == 0x34000000:
return a + ((((w >> 5) & 0x7FFFF) ^ 0x40000) - 0x40000 << 2)
if w & 0x7E000000 == 0x36000000:
return a + ((((w >> 5) & 0x3FFF) ^ 0x2000) - 0x2000 << 2)
return None
def header_text(table, marker):
lines = [f'// {marker}: code patch sets, generated by recomp/gen.py --patches. Do not edit or commit.',
'#pragma once', '', f'#define RT_PATCH_COUNT {len(table.sets)}',
f'#define RT_PATCH_PAGE {table.page:#x}ull',
'// X(index, name, env switch, default)',
'#define RT_PATCH_SETS(X) ' + ' '.join(f'X({i}, {n}, "{e}", {d})' for i, (n, e, d) in
enumerate(table.sets)), '']
return '\n'.join(lines)