diff --git a/docs/building.md b/docs/building.md index 8f2ab0dd..5332c22a 100644 --- a/docs/building.md +++ b/docs/building.md @@ -100,7 +100,7 @@ of them. objects before it links: any AVX, BMI or other instruction above x86-64-v2 outside a function listed there as dispatched by a runtime CPU check stops the build. - `gen.py` sizes its emit pool from the CPU count and the free memory; `--jobs N` sets it, and so does the environment variable `TOTK_LIFT_JOBS=N`, which the installer's lift step honours too (it has no `--jobs`). When a worker runs out of memory, the pool restarts with half the workers (`emit: out of memory with N workers, retrying with M`), down to one; only an out-of-memory error with one worker fails the step. + `gen.py` sizes its emit pool from the CPU count and the free memory; `--jobs N` sets it, and so does the environment variable `TOTK_LIFT_JOBS=N`, which the installer's lift step honours too (it has no `--jobs`). When a worker runs out of memory, the pool restarts with half the workers (`emit: out of memory with N workers, retrying with M`), down to one; only an out-of-memory error with one worker fails the step. A later lift into the same `--out` lifts again only the chunks whose inputs changed (`/lift_cache.json`, see the top of `recomp/gen.py`); `--no-cache`, or `TOTK_LIFT_CACHE=0` (which the installer's lift step honours too), lifts every chunk. Lifting takes about 15 s, a clean compile about 200 s on 24 threads; later builds recompile only what changed. A compiler cache in front of clang shortens it further: with `SCCACHE` set to an sccache executable, `build_game.py` @@ -222,7 +222,12 @@ when a setting is described neither in `totk.ini.example` nor in [settings.md](s `.venv\Scripts\python tests\exefs_patches_test.py`, which makes its own modules and runs `gen.py` on one with a jump table (no game files), and `totk.exe --probe-mods`. A change to the cheat VM or the cheat opcode tables (`runtime/cheats.c`, `tools/cheats.py`) passes `.venv\Scripts\python tests\cheats_vm_check.py`, which runs both on -cheats of its own with clang from PATH (`--cc` names another). +cheats of its own with clang from PATH (`--cc` names another). A change to what a generated chunk depends on, or to +the lift cache of `recomp/gen.py` (`/lift_cache.json`), passes `.venv\Scripts\python tests\lift_cache_test.py`, +which lifts a module it makes itself with and without the cache and compares the output (no game files); a new input +of the chunk text goes into `chunk_key` with a case there. On game files, a lift with the cache into a used output +folder and a `--no-cache` lift into an empty one give the same files apart from `manifest.json` and +`lift_cache.json`. The lifter (`recomp/`) and the CPU headers (`runtime/cpu.h`, `runtime/simd.h`) are checked against Unicorn. Every lifter change passes these, all run with `.venv`: diff --git a/docs/installer.md b/docs/installer.md index 9a6e0bfb..09fadff4 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -106,9 +106,13 @@ A cancelled or failed setup resumes from the last finished step (`totk-install.j ## Rebuilding for the mods -The generated C and the objects stay in the temp folder, so the next lift rewrites only the chunks whose C changed and -Ninja recompiles only those: a new or edited code cheat costs the lift (under a minute on 24 cores, a few minutes on 4) -and a few dozen chunks, not a full compile, and the shaders stay prepared. +The generated C and the objects stay in the temp folder. The next lift lifts again only the chunks whose inputs +changed (the lift cache of `recomp/gen.py`: the patch sites inside a chunk, with their set indices) and rewrites only +those whose C changed, and Ninja recompiles only those. A new or edited code cheat costs a few seconds of lift +(discovery and a hash of the kept chunks; 7 s for one changed cheat word on 12 cores, against 102 s for the whole lift) +and the compile of the chunks that hold its sites, not a full compile, and the shaders stay prepared. A cheat set added +or removed before others in the patch file moves their set indices, so the chunks of their sites are lifted too. A +change to `recomp/*.py` lifts everything. `build-cache/rebuild.txt` holds the command that opens the setup window in its rebuild mode on the install folder (`installer/gui.py --rebuild `, started through `python -c` so the release folder is on `sys.path` diff --git a/installer/steps.py b/installer/steps.py index c0bfb809..3888dfee 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -14,9 +14,10 @@ screen shapes and the prompts style this player needs, installer/screens.py), to shader state of totk.exe --prepare-shaders, totk.ini, totk-install.json. Temp folder (/-): check/ (ExeFS of both inputs, the host check), unpacked/ (the NSP of an NSZ or XCZ input until the extraction is done), gen/ (C), build/ (objects), logs/. The cleanup keeps gen/, build/ and -logs/: the next run lifts into the same gen/, where recomp/gen.py rewrites only the chunks whose C changed, and Ninja -in build/ recompiles only those. A change of the mods' code cheats then costs the lift (about a minute) and the chunks -it touches, not a full compile. +logs/: the next run lifts into the same gen/, where recomp/gen.py lifts again only the chunks whose inputs changed (its +lift cache, gen//lift_cache.json) and rewrites only those whose C changed, and Ninja in build/ recompiles only +those. A change of the mods' code cheats then costs a few seconds of lift and the chunks holding its sites, not a full +lift or compile. Install folder build-cache/: cheats.status (tools/cheats.py), exefs.status and exefs.data (tools/exefs_patches.py) and rebuild.txt, the command that starts this setup in its rebuild mode on the install folder (installer/gui.py --rebuild), @@ -900,8 +901,9 @@ class Lift(Step): return (gen / 'layout.json').is_file() and stamp.is_file() and stamp.read_text().strip() == ctx.keys['lift'] def run(self, ctx): - # gen/ of the last run stays: gen.py rewrites only the chunks whose C changed (and removes the ones a smaller - # partition no longer has), so Ninja recompiles only those + # gen/ of the last run stays: gen.py lifts again only the chunks whose inputs changed + # (gen//lift_cache.json; TOTK_LIFT_CACHE=0 lifts all), rewrites only those whose C changed (and removes + # the ones a smaller partition no longer has), so Ninja recompiles only those layout, rep, gen = ctx.layout, ctx.reporter, ctx.temp / 'gen' (gen / STAMP).unlink(missing_ok=True) rep.progress('lift', 'discover', force=True) diff --git a/recomp/gen.py b/recomp/gen.py index 88a73f2a..e9f3e16c 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -3,12 +3,14 @@ Usage: gen.py --exefs build/extracted/v121/exefs --out generated/v121 [--modules main,subsdk0,sdk] [--jobs N] [--limit N] [--chunk-fns N] [--chunk-bytes B] [--big-fn B] [--patches FILE] [--v-locals K [--v-trust]] + [--no-cache] Output, under --out: layout.h, layout.json module bases (recomp/layout.py) /_NNNN.c one `void f_(Cpu *c)` per guest function, about --chunk-bytes of C per file /_table.c sorted _fn_addrs[] / _fn_ptrs[] / _fn_count for rt_dispatch /manifest.json chunk list, function count, statistics and timings + /lift_cache.json per chunk: the key of its inputs, its file's SHA-256, size and mtime, and its results /folded.txt the module's bytes the generated code took as constants: what the jump tables read at lift time, as module offsets [start, end); tools/exefs_patches.py keeps a mod's data writes off them @@ -24,8 +26,20 @@ Emission runs in worker processes, one chunk per task. Each worker loads the mod time and resolves its jump tables (recomp/jumptables.py), so memory per worker does not grow with the module. Chunks are sized from a per-instruction estimate of the C they produce, so their names and contents only depend on the input; a chunk whose text did not change is not rewritten and keeps its mtime. + +A chunk is not lifted again when its inputs did not change. Its text depends only on the generator (recomp/*.py, +GENERATED_MARKER, the Emitter options), the module (its file, its base, and the function starts discovered from it), +the chunk's number and functions, and the patch sites and spin waits inside the chunk's address range: the per-module +caches of recomp/jumptables.py are derived from the module's bytes and base, and its per-function state is reset for +each function. The SHA-256 of all of these is the chunk's key. /lift_cache.json records the key of each chunk +with the SHA-256, size and mtime of the file it wrote and the results the manifest and folded.txt need, under a +digest of the entry. A chunk whose entry is intact, whose key matches and whose file still has that size, mtime and +hash is reused as it is; any other chunk is lifted. So a rebuild after a change to one cheat lifts only the chunks +holding its sites. --no-cache, or TOTK_LIFT_CACHE=0 in the environment, lifts every chunk. """ import argparse +import bisect +import hashlib import json import multiprocessing as mp import os @@ -33,6 +47,7 @@ import re import sys import time from collections import Counter +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import numpy as np @@ -210,7 +225,8 @@ def partition(est, max_fns, max_bytes, big_fn): # ------------------------------------------------------------------------------------------------ emission class Emitter: - """C for the functions of one module. starts is the set of every function start of the module.""" + """C for the functions of one module. starts is the set of every function start of the module. A new input of + the C (an argument here, or anything else it reads) must go into the lift cache's key: chunk_key.""" def __init__(self, mod, starts, patched=None, reg_locals=regs.LOCALS, callee_saved=True, spins=None, v_locals=vlocals.LOCALS, v_trust=False): @@ -532,9 +548,10 @@ def chunk_text(name, number, emitter, fns): + '\n' + '\n\n'.join(body) + '\n') -def write_if_changed(path, text): +def write_if_changed(path, text, wait=True): """Returns True if the file was written. An unchanged file keeps its mtime, so the build skips it. A changed one - is written under a temporary name and renamed into place, so a reader never sees it half written.""" + is written under a temporary name and renamed into place, so a reader never sees it half written. wait: retry + the rename while another process holds the file (_retry); without it the first failure raises.""" data = text.encode() try: if path.stat().st_size == len(data) and path.read_bytes() == data: @@ -544,7 +561,10 @@ def write_if_changed(path, text): tmp = path.with_name(f'.{path.name}.{os.getpid()}.tmp') tmp.write_bytes(data) try: - _retry(os.replace, tmp, path) + if wait: + _retry(os.replace, tmp, path) + else: + os.replace(tmp, path) except OSError: tmp.unlink(missing_ok=True) raise @@ -610,8 +630,8 @@ def merge_ranges(ranges): def _emit_chunk(task): - """Worker: write one chunk. Returns its statistics.""" - name, number, path, fns = task + """Worker: write one chunk. Returns its statistics, and the SHA-256, size and mtime of its file for the cache.""" + name, number, path, fns, key = task t0 = time.perf_counter() # Test hook: a pool wider than this raises the MemoryError a busy machine does, so the retry can be exercised. limit = os.environ.get('TOTK_LIFT_TEST_OOM_ABOVE') @@ -621,17 +641,20 @@ def _emit_chunk(task): e.stats.clear() e.unsupported.clear() text = chunk_text(name, number, e, fns) - error = None + error, st = None, None try: written = write_if_changed(Path(path), text) + st = os.stat(path) except OSError as ex: written, error = False, f'{path}: {ex}' reads = _W['reads'].get(name, []) folded = merge_ranges(reads) reads.clear() return {'module': name, 'number': number, 'file': Path(path).name, 'functions': len(fns), 'bytes': len(text), - 'written': written, 'error': error, 'seconds': time.perf_counter() - t0, 'stats': dict(e.stats), - 'unsupported': dict(e.unsupported), 'folded': folded} + 'written': written, 'reused': False, 'error': error, 'seconds': time.perf_counter() - t0, + 'stats': dict(e.stats), 'unsupported': dict(e.unsupported), 'folded': folded, 'key': key, + 'sha256': _sha256(text.encode()), 'size': st.st_size if st else None, + 'mtime_ns': st.st_mtime_ns if st else None} # ------------------------------------------------------------------------------------------------ driver @@ -670,6 +693,120 @@ def remove_stale(moddir, name, nchunks, errors): return removed +# ------------------------------------------------------------------------------------------------ lift cache + +CACHE_FILE = 'lift_cache.json' +CACHE_VERSION = 1 # raise when the cache's format or the meaning of its key changes +# Per-chunk results the cache keeps beside the key: what the manifest, folded.txt and the reuse check need. +CACHED = ('functions', 'bytes', 'stats', 'unsupported', 'folded', 'sha256', 'size', 'mtime_ns') + + +def _sha256(data): + return hashlib.sha256(data).hexdigest() + + +def _key(*parts): + """SHA-256 of JSON-encodable parts.""" + return _sha256(json.dumps(parts, separators=(',', ':')).encode()) + + +def generator_key(opts): + """Key of everything the chunk text depends on besides the module and the patches: the generator's sources + (recomp/*.py, not the patch tables beside them), the marker in each header, the Emitter options, the Python + version that runs them.""" + srcs = [[p.name, _sha256(p.read_bytes())] for p in sorted((ROOT / 'recomp').glob('*.py'))] + return _key('gen', CACHE_VERSION, list(sys.version_info[:2]), GENERATED_MARKER, sorted(opts.items()), srcs) + + +def module_key(gen_key, name, path, base, abs_starts): + """Key of one module's inputs: the generator, the module's file and base, and the function starts discovered + from them (they decide which branches become direct calls).""" + return _key(gen_key, name, _sha256(Path(path).read_bytes()), base, _sha256(abs_starts.tobytes())) + + +def in_span(addrs, lo, hi): + """The addresses of the sorted list addrs in [lo, hi).""" + return addrs[bisect.bisect_left(addrs, lo):bisect.bisect_left(addrs, hi)] + + +def chunk_key(mod_key, number, fns, patched, site_addrs, spins, spin_addrs): + """Key of one chunk: its module, number (in its header) and functions, and the patch sites (set index and + replacement words, @page resolved, so the patch page is in them) and spin waits inside its address range. Every + input of the chunk text that is not in generator_key or module_key must be added here, with a case in + tests/lift_cache_test.py.""" + lo, hi = fns[0][0], fns[-1][1] + sites = [[a, [[k, words] for k, words in patched[a]]] for a in in_span(site_addrs, lo, hi)] + return _key(mod_key, number, fns, sites, [[b, spins[b]] for b in in_span(spin_addrs, lo, hi)]) + + +def _entry_check(file, entry): + """Digest of a cache entry: its chunk file name, key and results. The results go into the manifest and folded.txt + as they are, so a changed value in the cache file must not be taken (tools/exefs_patches.py --folded keeps a + mod's data writes off the folded bytes).""" + return _key(file, entry['key'], [entry[k] for k in CACHED]) + + +def load_cache(moddir): + """{chunk file name: entry} from /lift_cache.json: the entries whose digest matches; empty when the file is + missing or unreadable.""" + try: + data = json.loads((moddir / CACHE_FILE).read_text()) + if data.get('version') != CACHE_VERSION or not isinstance(data.get('chunks'), dict): + return {} + out = {} + for file, e in data['chunks'].items(): + try: + if e['check'] == _entry_check(file, e): + out[file] = e + except (KeyError, TypeError): + pass + return out + except (OSError, ValueError, AttributeError): + return {} + + +def _intact(path, entry): + """True when the file at path still is the one the entry recorded: same size and mtime, then same SHA-256 (a bit + flipped in a kept file would otherwise stay forever, as tools/build_game.py check_objects says of objects).""" + try: + st = path.stat() + if st.st_size != entry['size'] or st.st_mtime_ns != entry['mtime_ns']: + return False + return _sha256(path.read_bytes()) == entry['sha256'] + except (OSError, KeyError, TypeError): + return False + + +def reuse(tasks, caches): + """The results of the tasks whose chunk can be reused: its recorded key equals the task's and its file is intact. + The files are hashed on threads (hashlib releases the GIL), about 1 GB in all for the three modules.""" + cands = [] + for name, number, path, fns, key in tasks: + e = caches.get(name, {}).get(Path(path).name) + if e is not None and e['key'] == key: + cands.append((name, number, Path(path), key, e)) + with ThreadPoolExecutor(min(16, os.cpu_count() or 4)) as ex: + ok = list(ex.map(lambda c: _intact(c[2], c[4]), cands)) + out = [] + for (name, number, path, key, e), good in zip(cands, ok): + if good: + out.append({'module': name, 'number': number, 'file': path.name, 'written': False, 'reused': True, + 'error': None, 'seconds': 0.0, 'key': key, **{k: e[k] for k in CACHED}}) + return out + + +def cache_text(name, results): + """/lift_cache.json for this run's chunks; a chunk that could not be written gets no entry. Keys keep their + order (no sort_keys), so a reused chunk's `unsupported` merges into the census as it did when it was lifted.""" + chunks = {} + for r in results: + if not r['error'] and r['size'] is not None: + e = {'key': r['key'], **{k: r[k] for k in CACHED}} + chunks[r['file']] = {**e, 'check': _entry_check(r['file'], e)} + return json.dumps({'module': name, 'marker': GENERATED_MARKER, 'version': CACHE_VERSION, 'chunks': chunks}, + separators=(',', ':')) + '\n' + + # Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB. WORKER_MEM = 1536 << 20 MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs @@ -718,9 +855,16 @@ def main(): ap.add_argument('--v-trust', action='store_true', help='with --v-locals: do not reload V8-V15 after a call (AAPCS64: the callee preserves their low ' '64 bits)') + ap.add_argument('--no-cache', action='store_true', + help=f'lift every chunk, also those whose inputs match /{CACHE_FILE} (it is written anew); ' + 'so does the environment variable TOTK_LIFT_CACHE=0') args = ap.parse_args() + # The installer's lift step passes no --no-cache; the environment reaches it, as TOTK_LIFT_JOBS does (pick_jobs) + no_cache = ('--no-cache' if args.no_cache else + 'TOTK_LIFT_CACHE=0' if os.environ.get('TOTK_LIFT_CACHE', '').strip() == '0' else None) opts = {'reg_locals': args.reg_locals, 'callee_saved': not args.no_callee_saved_trust, 'v_locals': args.v_locals, 'v_trust': args.v_trust} + gen_key = generator_key(opts) t0 = time.perf_counter() mods = layout(args.exefs) @@ -748,9 +892,13 @@ def main(): chunks = partition(est[:n], args.chunk_fns, args.chunk_bytes, args.big_fn) moddir = args.out / name moddir.mkdir(parents=True, exist_ok=True) + mkey = module_key(gen_key, name, path, base, abs_starts) + patched, spins = table.for_module(name, base), table.spins_for(name, base) + site_addrs, spin_addrs = sorted(patched), sorted(spins) for number, (i0, i1) in enumerate(chunks): fns = list(zip(abs_starts[i0:i1].tolist(), ends[i0:i1].tolist())) - tasks.append((int(est[i0:i1].sum()), (name, number, str(moddir / f'{name}_{number:04d}.c'), fns))) + key = chunk_key(mkey, number, fns, patched, site_addrs, spins, spin_addrs) + tasks.append((int(est[i0:i1].sum()), (name, number, str(moddir / f'{name}_{number:04d}.c'), fns, key))) paths[name] = (str(path), base) starts_abs[name] = abs_starts info[name] = {'base': base, 'lo': mod.lo, 'hi': mod.hi, 'emitted': abs_starts[:n].tolist(), @@ -769,9 +917,23 @@ def main(): ctx = mp.get_context('spawn') jobs, why = pick_jobs(args.jobs) pending = [t for _, t in tasks] - emitted = 0 - while True: - print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True) + reused = [] if no_cache else reuse(pending, {name: load_cache(d['moddir']) for name, d in info.items()}) + for r in reused: + results[r['module']].append(r) + folded[r['module']] += r['folded'] + skip = {(r['module'], r['number']) for r in reused} + pending = [t for t in pending if (t[0], t[1]) not in skip] + emitted = len(reused) + print(f'reusing {len(reused)} of {len(tasks)} chunks (inputs unchanged), {time.perf_counter() - t_emit:.1f}s' + if not no_cache else f'lifting every chunk ({no_cache})', file=sys.stderr, flush=True) + if reused or not pending: + # installer/steps.py EMITTED reads these lines for its progress bar: the reused chunks count as done, and a + # run with nothing to lift still reaches M/M + print(f'emitted {emitted}/{len(tasks)} chunks, 0s', file=sys.stderr, flush=True) + if len(pending) < jobs: + jobs, why = max(1, len(pending)), f'{len(pending)} chunk{"s" if len(pending) != 1 else ""} to lift' + while pending: + print(f'emitting with {jobs} worker{"s" if jobs != 1 else ""} ({why})', file=sys.stderr, flush=True) try: # Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again. # Its folded bytes come with its result: a chunk emitted again reads them again in its new worker. @@ -841,6 +1003,7 @@ def main(): 'c_bytes': sum(r['bytes'] for r in rs), 'largest_chunk_bytes': max((r['bytes'] for r in rs), default=0), 'chunks_written': sum(r['written'] for r in rs) + table_written, + 'chunks_reused': sum(r['reused'] for r in rs), 'stale_chunks_removed': removed, **{k: v for k, v in sorted(stats.items()) if k.startswith('v_')}, # --v-locals only }, @@ -853,8 +1016,17 @@ def main(): }, } write_if_changed(moddir / 'manifest.json', json.dumps(manifest, indent=1) + '\n') + # The cache goes last, after every output it describes. It only saves time, so a failed write warns and the + # lift still succeeds, without waiting for a holder of the file: the previous cache stays, and its entries + # for the chunks this run rewrote no longer match their files' size, mtime and hash. A run stopped before + # here leaves it the same way. + try: + write_if_changed(moddir / CACHE_FILE, cache_text(name, rs), wait=False) + except OSError as ex: + print(f'{name}: {CACHE_FILE} not written, the next lift reuses less: {ex}', file=sys.stderr, flush=True) s = manifest['stats'] - print(f'{name}: {s["emitted_functions"]} functions, {len(rs)} chunks, {s["c_bytes"] / 1e6:.0f} MB of C, ' + print(f'{name}: {s["emitted_functions"]} functions, {len(rs)} chunks ({s["chunks_reused"]} reused), ' + f'{s["c_bytes"] / 1e6:.0f} MB of C, ' f'{s["jump_tables"]} jump tables, {s["all_label_fallbacks"]} all-label fallbacks, ' f'{s["br_dispatch"]} br via dispatch, {s["traps"]} traps ({s["unsupported"]} unsupported)', file=sys.stderr) diff --git a/recomp/patches.py b/recomp/patches.py index 544e5086..0228c52e 100644 --- a/recomp/patches.py +++ b/recomp/patches.py @@ -49,7 +49,8 @@ def adrp(pc, target, rd): class Table: """sets: [(name, env, default)]; sites: {module: {offset: [(set index, [word | ('page', reg)], original)]}} (the - alternatives of a site in file order); page.""" + alternatives of a site in file order); page. What the generated C takes from here (for_module, spins_for) is in + the lift cache's key (recomp/gen.py chunk_key); a new kind of line the Emitter reads must go there too.""" def __init__(self, sets=(), sites=None, page=0, spins=None): self.sets = list(sets) diff --git a/tests/lift_cache_test.py b/tests/lift_cache_test.py new file mode 100644 index 00000000..b8ba5ec4 --- /dev/null +++ b/tests/lift_cache_test.py @@ -0,0 +1,221 @@ +"""recomp/gen.py's lift cache (/lift_cache.json) on a module it makes itself: a run with unchanged inputs reuses +every chunk, a changed patch site or spin wait lifts only the chunk that holds it, a chunk file or cache entry changed +behind the cache's back is not taken, a cache that cannot be written only warns, TOTK_LIFT_CACHE=0 and a change to +the generator's sources or options lift everything, and the output always equals a --no-cache lift's, folded.txt +included. No game files. + +Run with .venv (lz4 and numpy, as recomp/): python tests/lift_cache_test.py +""" +import json +import os +import shutil +import stat +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tests')) +sys.path.insert(0, str(ROOT / 'tools')) +from check_content import GENERATED_MARKER # noqa: E402 +# The jump table module and the NSO writer of tests/exefs_patches_test.py; a change there changes this test's module, +# and test_reuse_and_invalidation checks that it still has its jump table (folded.txt 1020 1024). +from exefs_patches_test import JT_DATA, JT_RO, JT_TEXT, MOV_W8_100, MOV_W8_200, RET, nso # noqa: E402 + +MOV_W8_300 = 0x52806008 + + +def bl(pc, target): + return 0x94000000 | (((target - pc) >> 2) & 0x3FFFFFF) + + +# The jump table module of tests/exefs_patches_test.py (fn at 0x8, its table in .rodata) and two more functions after +# it that call each other, so both are function starts: g: bl h; ret, h: mov w8, #0x100; bl g; ret. With --chunk-fns 1 +# each of fn, g and h is a chunk of its own; h's mov is the patch site. +G = len(JT_TEXT) +H = G + 8 +TEXT = JT_TEXT + struct.pack('<5I', bl(G, H), RET, MOV_W8_100, bl(H + 4, G), RET) +# fn's `b.hi Ldefault` at 0xc, as a spin wait back-edge to Ldefault (0x48): patches.check accepts it, the lift gates it +SPIN = 'spin main 48 c' + + +def patch_file(*replacements, first=None, spin=False): + """A patch file with set a patching h's mov with each replacement (sets a0, a1, ...); first: a set defined before + them, which moves their indices; spin: SPIN too.""" + lines = [f'set {first} TEST_{first.upper()} 0'] if first else [] + for i, w in enumerate(replacements): + lines += [f'set a{i} TEST_A{i} 0', f'site a{i} main {H:x} {MOV_W8_100:08x} {w:08x}'] + if spin: + lines.append(SPIN) + return '\n'.join(lines) + '\n' + + +class LiftCache(unittest.TestCase): + def setUp(self): + self.d = Path(tempfile.mkdtemp()) + (self.d / 'exefs').mkdir() + (self.d / 'exefs' / 'main').write_bytes(nso(TEXT, JT_RO, JT_DATA)) + + def tearDown(self): + shutil.rmtree(self.d, ignore_errors=True) + + def gen(self, patches, out='gen', *extra, env=None): + p = self.d / 'patches.txt' + p.write_text(patches) + env = {**os.environ, 'TOTK_LIFT_CACHE': '', **(env or {})} + r = subprocess.run([sys.executable, str(ROOT / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), + '--out', str(self.d / out), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1', + '--patches', str(p), *extra], capture_output=True, text=True, env=env) + self.assertEqual(r.returncode, 0, r.stderr) + self.stderr = r.stderr + return json.loads((self.d / out / 'main' / 'manifest.json').read_text())['stats'] + + def files(self, out='gen'): + """{name: bytes} of everything gen.py wrote except the manifest (timings) and the cache.""" + root = self.d / out + return {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob('*')) + if p.is_file() and p.name not in ('manifest.json', 'lift_cache.json')} + + def assert_like_fresh(self, patches): + """The output under gen/ equals a --no-cache lift of the same inputs into an empty directory.""" + stats = self.gen(patches, 'fresh', '--no-cache') + self.assertEqual(stats['chunks_reused'], 0) + self.assertEqual(self.files(), self.files('fresh')) + shutil.rmtree(self.d / 'fresh') + + def chunk(self, n): + return self.d / 'gen' / 'main' / f'main_{n:04d}.c' + + def test_reuse_and_invalidation(self): + stats = self.gen(patch_file(MOV_W8_200)) + self.assertEqual((stats['emitted_functions'], stats['chunks_reused']), (3, 0)) + cache = json.loads((self.d / 'gen' / 'main' / 'lift_cache.json').read_text()) + self.assertEqual(cache['marker'], GENERATED_MARKER) + self.assertEqual(sorted(cache['chunks']), ['main_0000.c', 'main_0001.c', 'main_0002.c']) + self.assertIn('rt_patch_on[0]', self.chunk(2).read_text()) + folded = (self.d / 'gen' / 'main' / 'folded.txt').read_text() + self.assertIn('1020 1024', folded) + + # Same inputs: nothing lifted, nothing written, the jump table's bytes still in folded.txt (replayed from the + # cache, not read again) and the statistics unchanged. + mtimes = [self.chunk(n).stat().st_mtime_ns for n in range(3)] + again = self.gen(patch_file(MOV_W8_200)) + self.assertEqual((again['chunks_reused'], again['chunks_written']), (3, 0)) + self.assertEqual({k: v for k, v in again.items() if not k.startswith('chunks_')}, + {k: v for k, v in stats.items() if not k.startswith('chunks_')}) + self.assertEqual([self.chunk(n).stat().st_mtime_ns for n in range(3)], mtimes) + self.assertEqual((self.d / 'gen' / 'main' / 'folded.txt').read_text(), folded) + self.assert_like_fresh(patch_file(MOV_W8_200)) + + # Another replacement word: only h's chunk is lifted. + self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) + self.assert_like_fresh(patch_file(MOV_W8_300)) + + # A set defined before the site's moves its index (rt_patch_on[1]): h's chunk again. + self.assertEqual(self.gen(patch_file(MOV_W8_300, first='z'))['chunks_reused'], 2) + self.assertIn('rt_patch_on[1]', self.chunk(2).read_text()) + self.assert_like_fresh(patch_file(MOV_W8_300, first='z')) + + # A second alternative at the site. + self.assertEqual(self.gen(patch_file(MOV_W8_300, MOV_W8_200))['chunks_reused'], 2) + self.assert_like_fresh(patch_file(MOV_W8_300, MOV_W8_200)) + + def test_changed_file_is_lifted_again(self): + """A chunk whose bytes changed while its size and mtime stayed (a flipped bit) is lifted and rewritten.""" + self.gen(patch_file(MOV_W8_200)) + p = self.chunk(0) + st = p.stat() + data = bytearray(p.read_bytes()) + data[-2] ^= 0x01 + p.write_bytes(bytes(data)) + os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns)) + stats = self.gen(patch_file(MOV_W8_200)) + self.assertEqual((stats['chunks_reused'], stats['chunks_written']), (2, 1)) + self.assert_like_fresh(patch_file(MOV_W8_200)) + + def test_deleted_file_and_cache(self): + self.gen(patch_file(MOV_W8_200)) + self.chunk(1).unlink() + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) + (self.d / 'gen' / 'main' / 'lift_cache.json').write_text('{not json') + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 0) + self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--no-cache')['chunks_reused'], 0) + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3) + self.assert_like_fresh(patch_file(MOV_W8_200)) + + def test_spin_wait_is_in_the_key(self): + self.gen(patch_file(MOV_W8_200)) + self.assertEqual(self.gen(patch_file(MOV_W8_200, spin=True))['chunks_reused'], 2) + self.assertIn('rt_spin_gate', self.chunk(0).read_text()) + self.assert_like_fresh(patch_file(MOV_W8_200, spin=True)) + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) + self.assertNotIn('rt_spin_gate', self.chunk(0).read_text()) + + def test_changed_cache_entry_is_not_taken(self): + """A changed result in lift_cache.json (a folded range, here) fails the entry's digest: the chunk is lifted + and folded.txt keeps the jump table's real bytes.""" + self.gen(patch_file(MOV_W8_200)) + p = self.d / 'gen' / 'main' / 'lift_cache.json' + data = json.loads(p.read_text()) + self.assertEqual(data['chunks']['main_0000.c']['folded'], [[0x1020, 0x1024]]) + data['chunks']['main_0000.c']['folded'] = [[0x1060, 0x1064]] + p.write_text(json.dumps(data)) + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 2) + self.assertIn('1020 1024', (self.d / 'gen' / 'main' / 'folded.txt').read_text()) + self.assert_like_fresh(patch_file(MOV_W8_200)) + + def test_unwritable_cache_only_warns(self): + """A cache that cannot be replaced leaves the lift successful, its manifest written; the old cache's entry for + the chunk the run rewrote no longer matches it.""" + self.gen(patch_file(MOV_W8_200)) + p = self.d / 'gen' / 'main' / 'lift_cache.json' + os.chmod(p, stat.S_IREAD) + try: + self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) + self.assertIn('lift_cache.json not written', self.stderr) + finally: + os.chmod(p, stat.S_IREAD | stat.S_IWRITE) + self.assertEqual(self.gen(patch_file(MOV_W8_300))['chunks_reused'], 2) + self.assert_like_fresh(patch_file(MOV_W8_300)) + + def test_environment_switch(self): + self.gen(patch_file(MOV_W8_200)) + self.assertEqual(self.gen(patch_file(MOV_W8_200), env={'TOTK_LIFT_CACHE': '0'})['chunks_reused'], 0) + self.assertIn('lifting every chunk (TOTK_LIFT_CACHE=0)', self.stderr) + self.assertEqual(self.gen(patch_file(MOV_W8_200))['chunks_reused'], 3) + + def test_emitter_options_are_in_the_key(self): + self.gen(patch_file(MOV_W8_200)) + self.assertEqual(self.gen(patch_file(MOV_W8_200), 'gen', '--reg-locals', '0')['chunks_reused'], 0) + + def test_generator_sources_are_in_the_key(self): + """A copy of the generator lifts everything again after a comment in its lifter.py changes, and nothing + after a patch table beside it (recomp/patches-*.txt) changes.""" + copy = self.d / 'repo' + shutil.copytree(ROOT / 'recomp', copy / 'recomp', ignore=shutil.ignore_patterns('__pycache__')) + (copy / 'tools').mkdir() + for name in ('check_content.py', 'hostmem.py'): + shutil.copy2(ROOT / 'tools' / name, copy / 'tools' / name) + + def run(): + r = subprocess.run([sys.executable, str(copy / 'recomp' / 'gen.py'), '--exefs', str(self.d / 'exefs'), + '--out', str(self.d / 'gen'), '--modules', 'main', '--jobs', '1', '--chunk-fns', '1'], + capture_output=True, text=True, env={**os.environ, 'TOTK_LIFT_CACHE': ''}) + self.assertEqual(r.returncode, 0, r.stderr) + return json.loads((self.d / 'gen' / 'main' / 'manifest.json').read_text())['stats']['chunks_reused'] + + self.assertEqual(run(), 0) + self.assertEqual(run(), 3) + table = next((copy / 'recomp').glob('patches-*.txt')) + table.write_text(table.read_text() + '# another line\n') + self.assertEqual(run(), 3) + lifter = copy / 'recomp' / 'lifter.py' + lifter.write_text(lifter.read_text() + '# another comment\n') + self.assertEqual(run(), 0) + + +if __name__ == '__main__': + unittest.main()