The generator lets the run time pick which ordered patch set runs at a site several of them patch

A patch set marked `ordered` (recomp/patches.py) belongs to something the player orders; tools/exefs_patches.py marks
the mods' ExeFS patch sets so. A site two or more ordered sets patch is contested: its ordered alternatives test one
byte, rt_patch_pick[<site>], instead of each set's switch, and patches.h lists the contested sites with their sets for
the runtime. Other alternatives keep their place and switch, so a cheat before the mods' sets still wins while it is
on. Sites with at most one ordered set, and tables without contested sites, are emitted as before, patches.h
included. The pick goes into the lift cache key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-16 20:11:27 +03:00
1 parent 8e098e95a8
commit 4d2e29aa79
5 files changed
+144 -31

No files matched your search

+25 -14
View File
@@ -229,10 +229,11 @@ class Emitter:
the C (an argument here, or anything else it reads) must go into the lift cache's key: chunk_key."""
def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None,
v_locals=vlocals.LOCALS, v_trust=False, hooks=None, fn_enter_all=False):
v_locals=vlocals.LOCALS, v_trust=False, hooks=None, fn_enter_all=False, picks=None):
self.mod = mod
self.starts = starts
self.patched = patched or {} # {address: [(set index, [replacement words])]} (recomp/patches.py)
self.picks = picks or {} # {contested site: (pick index, {ordered set: value})} (recomp/patches.py)
self.spins = spins or {} # {spin wait back-edge: its head} (recomp/patches.py `spin`)
self.reg_locals = reg_locals # recomp/regs.py: registers per function in C locals
self.kept = regs.CALLEE_SAVED if callee_saved else frozenset() # not reloaded after a call
@@ -241,6 +242,7 @@ class Emitter:
# Functions that start with FN_ENTER (recomp/patches.py `hook`): None is every function
self.hooks = None if fn_enter_all else hooks
self.uses_patches = False
self.uses_picks = False
self.stats = Counter()
self.unsupported = Counter()
@@ -454,13 +456,17 @@ class Emitter:
raise patches.PatchError(f'{a:#x}: patched instruction is a {r.kind}')
def patch(i, ins, a):
"""if (switch) { replacement } else if ... else { original } for a patched site (recomp/patches.py)."""
"""if (switch) { replacement } else if ... else { original } for a patched site (recomp/patches.py); an
ordered set of a contested site tests the site's pick byte instead of its switch."""
if ins.kind not in ('op', 'bl', 'b', 'bcond'):
raise patches.PatchError(f'{a:#x}: patched instruction is a {ins.kind}')
line = ' '
n, values = self.picks.get(a, (None, {}))
for (k, _), words in zip(self.patched[a], self.replacements(a)):
line += f' if (rt_patch_on[{k}]) {{ {" ".join(one(i, r, a) for r in words)} }} else'
test = f'rt_patch_pick[{n}] == {values[k]}' if k in values else f'rt_patch_on[{k}]'
line += f' if ({test}) {{ {" ".join(one(i, r, a) for r in words)} }} else'
self.uses_patches = True
self.uses_picks |= bool(values)
self.stats['patched_sites'] += 1
return f'{line} {{ {one(i, ins, a)} }}'
@@ -537,7 +543,7 @@ class Emitter:
def chunk_text(name, number, emitter, fns):
"""C text of one chunk. fns: [(entry, end)]."""
body, refs = [], set()
emitter.uses_patches = False
emitter.uses_patches = emitter.uses_picks = False
for entry, end in fns:
lines, r = emitter.function(entry, end)
body.append('\n'.join(lines))
@@ -546,6 +552,8 @@ def chunk_text(name, number, emitter, fns):
protos = ''.join(f'void {fname(r)}(Cpu *c);\n' for r in sorted(refs))
if emitter.uses_patches:
protos += 'extern uint8_t rt_patch_on[];\n'
if emitter.uses_picks:
protos += 'extern uint8_t rt_patch_pick[];\n'
vdef = '#define RT_VLOCALS 1\n' if emitter.v_locals else '' # runtime/simd.h: the V-locals accessors
return (header(f'{name} chunk {number:04d}') + vdef + '#include "cpu.h"\n#include "simd.h"\n\n' + protos
+ '\n' + '\n\n'.join(body) + '\n')
@@ -618,7 +626,8 @@ def _emitter(name):
e = _W['emitters'][name] = Emitter(mod, set(_W['starts'][name].tolist()),
_W['patches'].for_module(name, base),
spins=_W['patches'].spins_for(name, base),
hooks=_W['patches'].hooks_for(name, base), **_W['opts'])
hooks=_W['patches'].hooks_for(name, base),
picks=_W['patches'].picks_for(name, base), **_W['opts'])
return e
@@ -733,16 +742,18 @@ def in_span(addrs, lo, hi):
return addrs[bisect.bisect_left(addrs, lo):bisect.bisect_left(addrs, hi)]
def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs):
def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks=None):
"""Key of one chunk: its module, number (in its header) and functions, and the patch sites (set index and
replacement words, @page resolved, so the patch page is in them), spin waits and hooked functions inside its
address range; hook_addrs None (FN_ENTER in every function) keys apart from an empty list. Every input of the
chunk text that is not in generator_key or module_key must be added here, with a case in
tests/lift_cache_test.py."""
replacement words, @page resolved, so the patch page is in them), the pick index and values of its contested
sites, spin waits and hooked functions inside its address range; hook_addrs None (FN_ENTER in every function) keys
apart from an empty list. Every input of the chunk text that is not in generator_key or module_key must be added
here, with a case in tests/lift_cache_test.py."""
lo, hi = fns[0][0], fns[-1][1]
sites = [[a, [[k, words] for k, words in patched[a]]] for a in in_span(site_addrs, lo, hi)]
in_chunk = in_span(site_addrs, lo, hi)
sites = [[a, [[k, words] for k, words in patched[a]]] for a in in_chunk]
hooks = None if hook_addrs is None else in_span(hook_addrs, lo, hi)
return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)], hooks)
contested = [[a, picks[a][0], sorted(picks[a][1].items())] for a in in_chunk if a in (picks or {})]
return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)], hooks, contested)
def _entry_check(file, entry):
@@ -902,7 +913,7 @@ def main():
moddir = args.out / name
moddir.mkdir(parents=True, exist_ok=True)
mkey = module_key(gen_key, name, path, base, abs_starts)
patched, spins = table.for_module(name, base), table.spins_for(name, base)
patched, spins, picks = table.for_module(name, base), table.spins_for(name, base), table.picks_for(name, base)
site_addrs, spin_addrs = sorted(patched), sorted(spins)
hooks = None if args.fn_enter_all else table.hooks_for(name, base)
stray = sorted(set(hooks or ()) - set(abs_starts.tolist()))
@@ -912,7 +923,7 @@ def main():
hook_addrs = None if hooks is None else sorted(hooks)
for number, (i0, i1) in enumerate(chunks):
fns = list(zip(abs_starts[i0:i1].tolist(), ends[i0:i1].tolist()))
key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs)
key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs, hook_addrs, picks)
tasks.append((int(est[i0:i1].sum()), (name, number, str(moddir / f'{name}_{number:04d}.c'), fns, key)))
paths[name] = (str(path), base)
starts_abs[name] = abs_starts
+50 -9
View File
@@ -3,7 +3,7 @@
Lifted code cannot be byte-patched when the game runs, so a patch is applied when the C is generated. A patch file
(gen.py --patches) has these kinds of lines; `#` starts a comment, fields are separated by whitespace:
set <name> <env> <default> a patch set: a run-time switch <env>=0/1, <default> when it is unset
set <name> <env> <default> [ordered] a patch set: a run-time switch <env>=0/1, <default> when it is unset
site <set> <module> <address> <original> <replacement> [<replacement> ...]
spin <module> <head> <back-edge> [<back-edge> ...]
@@ -22,10 +22,22 @@ the field-of-view factor (runtime/resolution.c res_fov_live, set fov), +92..95 t
The replacement words are lifted at the site's address in place of the original instruction, and the generated C reads
`if (rt_patch_on[<set>]) { replacement } else { original }`: one byte load and a branch per executed site. Several
sets may patch one address (cheats that write different values there): `if (a) {..} else if (b) {..} else {..}`, the
first set in file order wins. The original may be any instruction but br, blr and ret; a replacement's last word may be
first set in file order wins.
An `ordered` set belongs to something the player puts in an order (a mod of the Mods tab: tools/exefs_patches.py marks
its sets so). A site that two or more ordered sets patch is contested: the run time, not the file order, picks which
of them runs. Each contested site gets a byte rt_patch_pick[<n>] (n counts the contested sites, by module name and
offset), and each of its ordered alternatives tests `rt_patch_pick[<n>] == <j>`, j being the alternative's place
among the site's ordered sets in file order (1 up; 0 runs none of them): one byte load for the whole group. The
runtime fills the bytes from the sets that are on and their ranks (runtime/patches.c rt_patch_picks: the lowest rank
wins, a tie goes to the set later in the file). The other alternatives of a contested site keep their place and
their rt_patch_on test, so a cheat before the mods' sets still wins while it is on. A site with at most one ordered
set, and a table without contested sites, is emitted exactly as without the keyword.
The original may be any instruction but br, blr and ret; a replacement's last word may be
a bl (lifted with the site's return address), a b or b.cond / cbz / tbz (its target joins the function's labels when
inside it) or a ret; earlier words must not branch. Every alternative of a site writes the same flags. The generator
writes patches.h (set names, switches, the page address) next to layout.h for the runtime.
writes patches.h (set names, switches, the page address, the contested sites) next to layout.h for the runtime.
A `spin` line names a spin wait found by tools/spin_waits.py: a loop with head <head> that only another thread can end,
and the direct branches back to it (module offsets). Each back-edge (checked: a b, b.cond, cbz / cbnz or tbz / tbnz to
@@ -40,6 +52,7 @@ without any keeps it in every function, as does gen.py --fn-enter-all (a whole-g
from pathlib import Path
PAGE = 0x1000
PICK_MAX = 255 # ordered sets on one site: rt_patch_pick is a byte, 0 for none
class PatchError(ValueError):
@@ -56,15 +69,35 @@ def adrp(pc, target, rd):
class Table:
"""sets: [(name, env, default)]; sites: {module: {offset: [(set index, [word | ('page', reg)], original)]}} (the
alternatives of a site in file order); page. What the generated C takes from here (for_module, spins_for, hooks_for)
is in the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too."""
alternatives of a site in file order); page; ordered: the indices of the ordered sets. What the generated C takes
from here (for_module, picks_for, spins_for, hooks_for) is in the lift cache's key (recomp/gen.py chunk_key); a new
kind of line the Emitter reads must go there too."""
def __init__(self, sets=(), sites=None, page=0, spins=None, hooks=None):
def __init__(self, sets=(), sites=None, page=0, spins=None, hooks=None, ordered=()):
self.sets = list(sets)
self.sites = sites or {}
self.page = page
self.spins = spins or {} # {module: {back-edge offset: head offset}}
self.hooks = hooks or {} # {module: {function offset}}; empty: FN_ENTER in every function
self.ordered = set(ordered)
def picks(self):
"""The contested sites, by module name and offset: [(module, offset, [ordered set index in file order])]; the
list index is the site's rt_patch_pick byte."""
out = []
for name in sorted(self.sites):
for off in sorted(self.sites[name]):
sets = [k for k, _, _ in self.sites[name][off] if k in self.ordered]
if len(sets) > 1:
if len(sets) > PICK_MAX:
raise PatchError(f'{name}+{off:#x}: {len(sets)} ordered sets, at most {PICK_MAX}')
out.append((name, off, sets))
return out
def picks_for(self, name, base):
"""{absolute address of a contested site: (its pick index, {ordered set index: the value that runs it})}."""
return {base + off: (n, {k: j + 1 for j, k in enumerate(sets)})
for n, (mod, off, sets) in enumerate(self.picks()) if mod == name}
def hooks_for(self, name, base):
"""The absolute addresses of the module's hooked functions, or None when the table has no hook line at all
@@ -89,17 +122,19 @@ class Table:
def load(path, mods):
"""Table from a patch file; mods is layout() output [(name, base, size)]."""
sets, index, sites, spins, hooks = [], {}, {}, {}, {}
sets, index, sites, spins, hooks, ordered = [], {}, {}, {}, {}, set()
bases = {n: (b, s) for n, b, s in mods}
for no, raw in enumerate(Path(path).read_text().splitlines(), 1):
f = raw.split('#', 1)[0].split()
if not f:
continue
where = f'{path}:{no}'
if f[0] == 'set' and len(f) == 4:
if f[0] == 'set' and (len(f) == 4 or len(f) == 5 and f[4] == 'ordered'):
if f[1] in index:
raise PatchError(f'{where}: set {f[1]} defined twice')
index[f[1]] = len(sets)
if len(f) == 5:
ordered.add(len(sets))
sets.append((f[1], f[2], int(f[3])))
elif f[0] == 'site' and len(f) >= 6:
if f[1] not in index:
@@ -139,7 +174,7 @@ def load(path, mods):
nxt = min((b for _, b, _ in mods if b > base), default=None)
if nxt is not None and page + PAGE > nxt:
raise PatchError(f'no room for the patch page after main ({page:#x})')
return Table(sets, sites, page, spins, hooks)
return Table(sets, sites, page, spins, hooks, ordered)
def check(table, mod, name):
@@ -175,4 +210,10 @@ def header_text(table, marker):
'// X(index, name, env switch, default)',
'#define RT_PATCH_SETS(X) ' + ' '.join(f'X({i}, {n}, "{e}", {d})' for i, (n, e, d) in
enumerate(table.sets)), '']
picks = table.picks()
if picks: # none: the header is what it was before ordered sets
lines += [f'#define RT_PATCH_PICK_COUNT {len(picks)}',
'// X(index, module, module offset, its ordered sets in the order its C tests them: rt_patch_pick 1 up)',
'#define RT_PATCH_PICKS(X) ' + ' '.join(f'X({i}, "{m}", {o:#x}, {", ".join(map(str, s))})'
for i, (m, o, s) in enumerate(picks)), '']
return '\n'.join(lines)
+1 -1
View File
@@ -209,7 +209,7 @@ class Tool(unittest.TestCase):
code, dat, bad = self.key('Code'), self.key('Data'), self.key('Bad')
self.assertEqual([status[k][2] for k in (code, dat, bad)], ['code', 'data', 'unsupported'])
self.assertEqual([status[k][4] for k in (code, dat)], ['data 1', 'data 9']) # bytes its data lines hold
self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0', (self.d / 'out.txt').read_text())
self.assertIn(f'set mod_{code.lower()} TOTK_MODPATCH_{code} 0 ordered', (self.d / 'out.txt').read_text())
self.assertEqual(sorted((f[0], f[2], f[3], f[4:]) for f in data),
sorted([(code, '2200', '51', []), (dat, '1000', 'aa', []),
(dat, '2118', '3100000000000000', ['base'])]))
+65 -5
View File
@@ -1,6 +1,7 @@
"""recomp/gen.py's lift cache (<mod>/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses
every chunk, a changed patch site, spin wait or function hook lifts only the chunk that holds it, a chunk file or cache entry changed
behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to
every chunk, a changed patch site, spin wait or function hook lifts only the chunk that holds it, a site two ordered
sets contest tests its pick byte (recomp/patches.py `ordered`) while one ordered set alone is emitted as an unordered
one, a chunk file or cache entry changed behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to
the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt
included. No game files.
@@ -20,6 +21,8 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'tools'))
sys.path.insert(0, str(ROOT / 'recomp'))
import patches # noqa: E402
from check_content import GENERATED_MARKER # noqa: E402
# The jump table module and the NSO writer of tests/exefs_patches_test.py; a change there changes this test's module,
# and test_reuse_and_invalidation checks that it still has its jump table (folded.txt 1020 1024).
@@ -42,12 +45,16 @@ TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET
SPIN = 'spin main 48 c'
def patch_file(*replacements, first=None, spin=False, hooks=()):
def patch_file(*replacements, first=None, spin=False, hooks=(), ordered=False, cheat=None):
"""A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before
them, which moves their indices; spin: SPIN too; hooks: a hook line with these function offsets."""
them, which moves their indices; spin: SPIN too; hooks: a hook line with these function offsets; ordered: the a
sets are ordered (recomp/patches.py); cheat: an unordered set c with this replacement at the site, before them."""
lines = [f'set {first} TEST_{first.upper()} 0'] if first else []
if cheat:
lines += ['set c TEST_C 0', f'site c main {H:x} {MOV_W8_100:08x} {cheat:08x}']
for i, w in enumerate(replacements):
lines += [f'set a{i} TEST_A{i} 0', f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}']
lines += [f'set a{i} TEST_A{i} 0' + (' ordered' if ordered else ''),
f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}']
if spin:
lines.append(SPIN)
if hooks:
@@ -148,6 +155,34 @@ class LiftCache(unittest.TestCase):
self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3)
self.assert_like_fresh(patch_file(MOV_W8_200))
def test_ordered_sets(self):
"""Two ordered sets on h's mov make it a contested site: their alternatives test rt_patch_pick[0] (1 for a0, 2
for a1) and patches.h lists the site; the cheat before them keeps its switch and its place. One ordered set
alone emits what an unordered one does, header included. The pick is in the key: only h's chunk again."""
h = lambda: self.chunk(2).read_text() # noqa: E731
header = lambda: (self.d / 'gen' / 'patches.h').read_text() # noqa: E731
self.gen(patch_file(MOV_W8_300))
plain, plain_header = h(), header()
self.assertEqual(self.gen(patch_file(MOV_W8_300, ordered=True))['chunks_reused'], 3)
self.assertEqual((h(), header()), (plain, plain_header))
self.assertNotIn('RT_PATCH_PICK', plain_header)
self.gen(patch_file(MOV_W8_300, MOV_W8_200, cheat=MOV_W8_200))
self.assertNotIn('rt_patch_pick', h())
stats = self.gen(patch_file(MOV_W8_300, MOV_W8_200, ordered=True, cheat=MOV_W8_200))
self.assertEqual(stats['chunks_reused'], 2)
text = h()
self.assertIn('extern uint8_t rt_patch_pick[];', text)
self.assertIn('extern uint8_t rt_patch_on[];', text)
on, pick1, pick2 = (text.index(s) for s in ('if (rt_patch_on[0])', 'if (rt_patch_pick[0] == 1)',
'if (rt_patch_pick[0] == 2)'))
self.assertLess(on, pick1)
self.assertLess(pick1, pick2)
self.assertNotIn('rt_patch_on[1]', text)
self.assertIn('#define RT_PATCH_PICK_COUNT 1', header())
self.assertIn(f'X(0, "main", {H:#x}, 1, 2)', header())
self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200, ordered=True, cheat=MOV_W8_200))
def test_spin_wait_is_in_the_key(self):
self.gen(patch_file(MOV_W8_200))
self.assertEqual(self.gen(patch_file(MOV_W8_200, spin=True))['chunks_reused'], 2)
@@ -242,5 +277,30 @@ class LiftCache(unittest.TestCase):
self.assertEqual(run(), 0)
class Picks(unittest.TestCase):
"""recomp/patches.py: which sites are contested and how the patch file declares ordered sets."""
def table(self, text):
with tempfile.TemporaryDirectory() as d:
p = Path(d) / 'patches.txt'
p.write_text(text)
return patches.load(p, [('main', 0x8000000, 0x100000), ('sdk', 0x9000000, 0x100000)])
def test_contested_sites(self):
t = self.table('set b E_B 1\nset m1 E_M1 0 ordered\nset m2 E_M2 0 ordered\nset m3 E_M3 0 ordered\n'
'site b main 20 d503201f d503201f\nsite m2 main 20 d503201f d503201f\n'
'site m1 main 20 d503201f d503201f\nsite m3 main 10 d503201f d503201f\n'
'site m3 sdk 30 d503201f d503201f\nsite m1 sdk 30 d503201f d503201f\n')
self.assertEqual(t.ordered, {1, 2, 3})
# by module name and offset; the sets in file order of their site lines; main+0x10 has one ordered set
self.assertEqual(t.picks(), [('main', 0x20, [2, 1]), ('sdk', 0x30, [3, 1])])
self.assertEqual(t.picks_for('sdk', 0x9000000), {0x9000030: (1, {3: 1, 1: 2})})
self.assertIn('X(0, "main", 0x20, 2, 1) X(1, "sdk", 0x30, 3, 1)', patches.header_text(t, 'm'))
def test_bad_keyword(self):
with self.assertRaises(patches.PatchError):
self.table('set m E_M 0 sorted\n')
if __name__ == '__main__':
unittest.main()
+3 -2
View File
@@ -19,7 +19,8 @@ offsets count the NSO header (0x100 bytes) before main's image; a pchtxt offset
A pchtxt patch line applies between @enabled and @disabled; @stop ends the file.
--out gets the --patches file followed by one set per file with sites (`mod_<hash>`, switch TOTK_MODPATCH_<HASH>,
default off) and its sites. --status gets one line per file, mods by name, tab separated:
default off, `ordered`: where sets of several files patch one site, the runtime runs the one of the mod highest on the
Mods tab, recomp/patches.py) and its sites. --status gets one line per file, mods by name, tab separated:
<file hash> <content hash> <kind> <built-in sets> <note>
@@ -330,7 +331,7 @@ def build(args):
if file_sites:
name = f'mod_{key.lower()}'
label = f'{mod}/{rel}'.encode('ascii', 'replace').decode()
sets.append(f'set {name} TOTK_MODPATCH_{key} 0 # {label}')
sets.append(f'set {name} TOTK_MODPATCH_{key} 0 ordered # {label}')
for a, repl in sorted(file_sites.items()):
sites.append(f'site {name} main {a:x} {m.word(a):08x} {" ".join(f"{w:08x}" for w in repl)}')
over += [s for s in builtin.get(a, []) if s not in over]