"""Compile the generated C for one game version and link it with the runtime into build//totk.exe. Writes build//build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is recompiled only when it, a header it includes or its flags change. .venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/totk.exe .venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others) .venv\\Scripts\\python tools\\build_game.py --jobs 8 .venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory .venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-/llvm-mingw--ucrt-x86_64 \\ --out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64 Every runtime/*.c is compiled. build//modules.c carries the layout, the linked function tables, the expected SHA-256 of each ExeFS file (tools/data/expected-.json) and the default game paths (--exefs, --romfs, --profile), which totk.exe uses unless its own command line names others. tools/romfs_index.py writes build//romfs-index.bin and romfs_config.h from the RomFS directory's index.json (runtime/romfs.c reads the index next to totk.exe). Private builds reuse the main build's objects. When --out differs from build/, the main build's generated objects are up to date (checked with a dry run of its build.ninja) and every header they include from runtime/ is byte-identical in --runtime (the list comes from the main build's Ninja deps log), build_game.py hardlinks those objects into /obj and compiles only runtime/*.c. A hardlink is a snapshot: clang and Ninja replace an object by renaming a new file over it, so a later main rebuild leaves the linked copies alone. Otherwise (a header differs, a main object is missing or stale, another --gen) it compiles everything and prints one line saying why. --full-build skips the check. Object manifest. A flipped bit in an object written by an earlier build stays forever, because Ninja rebuilds an object only when its inputs change, and forks hardlink the object into every private build. So each build dir keeps obj/objects.sha256 (SHA-256, size and mtime of every object it compiled). After Ninja finishes and before the link, build_game.py hashes the objects, compares each one whose size and mtime match its manifest entry, and records the objects Ninja just rebuilt (new mtime). A mismatch deletes the object, Ninja recompiles it, and the build prints the summary line. A fork checks the hardlinked generated objects against build//obj/objects.sha256 (read-only; a bad shared object stops the fork and asks for a main build run, which recompiles it and breaks the shared inode). Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB and 80% of RAM) at JOB_MEM per job, so a big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory of its processes (Windows job object accounting: committed private bytes). --ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used. --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++ runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects. Split build. --make-runtime-lib compiles only runtime/ into /libtotkrt.a (what CI prebuilds). --runtime-lib PATH then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's game hashes in) and links them with that library. The library holds no game-derived data. Standard library only. """ import argparse import ctypes import ctypes.wintypes as wt import hashlib import json import os import shutil import subprocess import sys import time from pathlib import Path import romfs_index from check_content import GENERATED_MARKER ROOT = Path(__file__).resolve().parent.parent # In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows # them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root. MAIN_ROOT = (ROOT / 'build').resolve().parent UCRT = Path(r'C:\msys64\ucrt64\bin') CLANG = UCRT / 'clang.exe' NINJA = UCRT / 'ninja.exe' SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] # Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering. GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '"-Wl,--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies) VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator # --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lvulkan-1', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a'] DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw' INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library LIB_NAME = 'libtotkrt.a' AR = UCRT / 'llvm-ar.exe' WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain DEPS = None # the deps directory of an llvm-mingw build # tools/data/expected-.json for each build version name. DISPLAY_VERSIONS = {'v0': '1.0.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'} GiB = 1 << 30 # Peak commit of one compile, 1.2.1 at -O1 (2026-10-20): 120-180 MiB per ~3 MiB chunk, 568 MiB for main_table.c # (295k-entry tables). 768 MiB leaves headroom for the largest file. JOB_MEM = 768 << 20 MEM_BUDGET = 50 * GiB # total for all compile jobs def below_root(p): """Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else absolute.""" p = Path(p).resolve() for root in (ROOT, MAIN_ROOT): if p.is_relative_to(root): return p.relative_to(root) return p def ninja_path(p): """A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped.""" return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:') def flag_path(p): return below_root(p).as_posix() def cmd_arg(s): """s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and lld split their Windows command line at unquoted spaces.""" s = s.replace('$', '$$') return f'"{s}"' if ' ' in s else s def cmd_path(p): return cmd_arg(flag_path(p)) def read_layout(gen): """[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency).""" data = json.loads((gen / 'layout.json').read_text()) return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] def chunk_files(gen, mod): """The C files of one module: the chunks listed in /manifest.json, then _table.c.""" d = gen / mod data = json.loads((d / 'manifest.json').read_text()) entries = data if isinstance(data, list) else data.get('chunks', data.get('files')) if entries is None: sys.exit(f'{d / "manifest.json"}: no "chunks" or "files" list') names = [] for e in entries: name = e if isinstance(e, str) else e.get('file', e.get('path', e.get('name'))) names.append(Path(name).name) table = f'{mod}_table.c' if table not in names: names.append(table) files = [d / n for n in names] missing = [str(f) for f in files if not f.is_file()] if missing: sys.exit(f'{mod}: {len(missing)} files listed in manifest.json are missing, first {missing[0]}') return files def write_if_changed(path, text): """Keep the mtime of an unchanged file so Ninja does not rebuild from it.""" if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text: return path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII def c_string(s): return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"' RUNTIME = ROOT / 'runtime' # --runtime: a copy of runtime/ lets a task build without touching the shared tree def runtime_sources(skip=()): """Every runtime/*.c, runtime/platform/*.c and runtime/*.rc except the names in skip, so a task adds a file without editing this script.""" pats = ('*.c', '*.cpp', 'platform/*.c', '*.rc') return sorted(p for pat in pats for p in RUNTIME.glob(pat) if p.name not in skip) def runtime_object(out, src): """obj/runtime/.o, so platform/win32.c cannot collide with a top-level win32.c.""" src = Path(src) rel = src.relative_to(RUNTIME) if src.is_relative_to(RUNTIME) else Path(src.name) return out / 'obj' / 'runtime' / rel.with_suffix('.o') def module_glue(layout, linked, hashes, paths): """C for build//modules.c: the layout table, the linked modules' function tables (dispatch.h), the expected ExeFS hashes and the default game paths (host.h).""" out = [f'// Module glue written by tools/build_game.py; do not edit. {GENERATED_MARKER}', '#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] for name, base, size in layout: u = name.upper() out.append(f'_Static_assert(MOD_{u}_BASE == {base:#x}ull && MOD_{u}_SIZE == {size:#x}ull, ' f'"layout.h and layout.json disagree on {name}");') out += ['', 'const RtLayoutEntry rt_layout[] = {'] out += [f' {{"{n}", MOD_{n.upper()}_BASE, MOD_{n.upper()}_SIZE}},' for n, _, _ in layout] out += ['};', f'const int rt_layout_count = {len(layout)};', ''] for name in linked: out += [f'extern const uint64_t {name}_fn_addrs[];', f'extern const GuestFn {name}_fn_ptrs[];', f'extern const uint64_t {name}_fn_count;'] out += ['', 'int rt_linked_modules(RtModule *out, int max) {', ' int n = 0;'] for name in linked: u = name.upper() out.append(f' if (n < max) out[n++] = (RtModule){{"{name}", MOD_{u}_BASE, MOD_{u}_SIZE, {name}_fn_addrs, ' f'{name}_fn_ptrs, {name}_fn_count}};') out += [' return n;', '}', ''] out += ['const RtFileHash rt_exefs_sha256[] = {'] out += [f' {{{c_string(f)}, {c_string(h)}}},' for f, h in sorted(hashes.items())] out += ['};', f'const int rt_exefs_sha256_count = {len(hashes)};', ''] for name in ('exefs', 'romfs', 'profile'): out.append(f'const char rt_default_{name}[] = {c_string(paths[name].as_posix())};') out.append('') return '\n'.join(out) def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=None, make_lib=False): """build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE.""" inc = f'-I {cmd_path(RUNTIME)} -I {cmd_path(gen)}' vma = f'{cmd_path(DEPS / "include")} -I {cmd_path(DEPS / "include" / "vma")}' if DEPS else cmd_path(VMA_INCLUDE) rt_inc = f'{inc} -I {cmd_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers) libs = [f'-L {cmd_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS lines = [ '# Written by tools/build_game.py; do not edit.', 'ninja_required_version = 1.10', f'builddir = {ninja_path(out)}', f'cc = {cmd_arg(CLANG.as_posix())}', f'genflags = {" ".join(GEN_FLAGS)} {inc}', f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}', f'ldflags = {" ".join(LINK_FLAGS)}', f'libs = {" ".join(libs)}', '', 'pool gen', f' depth = {pool_depth}', '', 'rule cc_gen', ' command = $cc $genflags -MD -MF $out.d -c $in -o $out', ' deps = gcc', ' depfile = $out.d', ' description = CC $in', '', 'rule cc_rt', ' command = $cc $rtflags -MD -MF $out.d -c $in -o $out', ' deps = gcc', ' depfile = $out.d', ' description = CC $in', '', 'rule rc', f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out', ' description = RC $in', '', 'rule link', ' command = $cc $ldflags @$out.rsp $libs -o $out', ' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out', '', ] objs = {} for mod, files in mods.items(): objs[mod] = [] for f in files: o = out / 'obj' / mod / (f.stem + '.o') if not prebuilt: lines.append(f'build {ninja_path(o)}: cc_gen {ninja_path(f)}') lines.append(' pool = gen') objs[mod].append(o) rt = [] sources = runtime_sources(skip) if make_lib: sources = [p for p in sources if p.name not in INSTALL_SIDE] elif runtime_lib: sources = [p for p in sources if p.name in INSTALL_SIDE] + [out / 'modules.c'] else: sources += [out / 'modules.c'] for src in sources: o = runtime_object(out, src) lines.append(f'build {ninja_path(o)}: {"rc" if src.suffix == ".rc" else "cc_rt"} {ninja_path(src)}') rt.append(o) objs['runtime'] = rt every = [o for v in objs.values() for o in v] exe = out / 'totk.exe' if make_lib: exe = out / LIB_NAME lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] return '\n'.join(lines), objs, exe rule = 'link' if runtime_lib: # Whole archive: the weak probe stubs in gfx_stubs.c would otherwise satisfy references to the probes before the # linker looks into the library, and the real probes would stay out of totk.exe. rule = 'link_lib' lines += ['', 'rule link_lib', f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {cmd_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out', ' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out'] lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) + (f' | {ninja_path(runtime_lib)}' if runtime_lib else ''), 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] return '\n'.join(lines), objs, exe def main_objects(gen, main_out, rt_dir, mods): """({relative object path: the main build's object}, '') when the main build's generated objects can stand in for ours, else (None, why). That holds when the main build.ninja compiles the same sources with the same flags, its objects are up to date, and every header they include from runtime/ is byte-identical in rt_dir.""" main_ninja = main_out / 'build.ninja' if not main_ninja.is_file(): return None, f'{flag_path(main_ninja)} not found' if gen != (ROOT / 'generated' / main_out.name).resolve(): return None, 'generated C directory is not the main one' lines = main_ninja.read_text(encoding='utf-8').splitlines() sources = dict(l[6:].split(': cc_gen ') for l in lines if l.startswith('build ') and ': cc_gen ' in l) flags = next((l for l in lines if l.startswith('genflags = ')), '') if not flags.startswith('genflags = ' + ' '.join(GEN_FLAGS) + ' -I '): return None, 'main build uses other compile flags' pairs = {} for mod, files in mods.items(): for f in files: o = Path('obj') / mod / (f.stem + '.o') theirs = main_out / o if sources.get(ninja_path(theirs)) != ninja_path(f): return None, f'main build has no edge for {f.name}' if not theirs.is_file(): return None, f'main object {flag_path(theirs)} is missing' pairs[o] = theirs # Ninja runs in the main checkout: the main build.ninja is relative to it, and the worktree's runtime/ files carry newer # mtimes that would make every object look stale. Header equality is checked byte for byte below. env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', '')) targets = [ninja_path(t) for t in pairs.values()] for i in range(0, len(targets), 200): r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-n', '-d', 'explain'] + targets[i:i + 200], cwd=MAIN_ROOT, env=env, capture_output=True, text=True) if r.returncode or 'no work to do' not in r.stdout: return None, f'main generated objects are stale ({flag_path(main_out)} needs a build)' r = subprocess.run([str(NINJA), '-f', str(main_ninja), '-t', 'deps'], cwd=MAIN_ROOT, env=env, capture_output=True, text=True) ours = {flag_path(t) for t in pairs.values()} headers, cur = set(), False for l in r.stdout.splitlines(): if l.startswith(' '): if cur: headers.add(l.strip()) else: cur = l.split(': #deps')[0] in ours main_rt = MAIN_ROOT / 'runtime' for h in sorted(headers): p = Path(h) if Path(h).is_absolute() else MAIN_ROOT / h try: rel = p.resolve().relative_to(main_rt) except ValueError: continue mine = rt_dir / rel if not mine.is_file() or mine.read_bytes() != p.read_bytes(): return None, f'header {rel.as_posix()} differs in {flag_path(rt_dir)}' return pairs, '' def link_objects(out, pairs): """Hardlink the main build's objects into out. Clang and Ninja replace an object by renaming a new file over it, so a link keeps the old bytes after a main rebuild. Copies where linking fails.""" for o, theirs in pairs.items(): dst = out / o dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists(): if os.path.samefile(dst, theirs): continue dst.unlink() try: os.link(theirs, dst) except OSError: shutil.copy2(theirs, dst) def unlink_shared(out, mods): """Drop generated objects still hardlinked to another build, so the next compile writes a new file.""" for mod, files in mods.items(): for f in files: o = out / 'obj' / mod / (f.stem + '.o') if o.exists() and o.stat().st_nlink > 1: o.unlink() MANIFEST = 'objects.sha256' def file_sha256(path): h = hashlib.sha256() with open(path, 'rb', buffering=0) as f: buf = bytearray(1 << 20) while n := f.readinto(buf): h.update(memoryview(buf)[:n]) return h.hexdigest() def read_manifest(path): """{object path below obj/: (sha256, size, mtime_ns)}; empty when the file is missing.""" man = {} if path.is_file(): for line in path.read_text(encoding='utf-8').splitlines(): sha, size, mtime, rel = line.split(' ', 3) man[rel] = (sha, int(size), int(mtime)) return man def write_manifest(path, man): write_if_changed(path, ''.join(f'{sha} {size} {mtime} {rel}\n' for rel, (sha, size, mtime) in sorted(man.items()))) def check_objects(out, objects, man, record): """Compare objects with the manifest man. An object whose size and mtime match its entry is hashed and must equal it; any other object was just (re)built, so with record its hash enters man. Returns (verified, bad, unrecorded): the objects with a wrong hash, and with record False the ones without a current entry.""" verified, bad, unrecorded = 0, [], [] for o in objects: rel = o.relative_to(out / 'obj').as_posix() st = o.stat() e = man.get(rel) if e and e[1] == st.st_size and e[2] == st.st_mtime_ns: if file_sha256(o) == e[0]: verified += 1 else: bad.append(o) elif record: man[rel] = (file_sha256(o), st.st_size, st.st_mtime_ns) verified += 1 else: unrecorded.append(o) return verified, bad, unrecorded def ninja_build(base, targets, env): for i in range(0, len(targets), 200): code = subprocess.run(base + targets[i:i + 200], cwd=ROOT, env=env).returncode if code: sys.exit(code) def verify_objects(out, objs, borrowed, main_out, pairs, base, env): """Verify every object before the link; see the module docstring. borrowed: the hardlinked generated objects of main_out (checked against its manifest, never recorded); the others belong to out.""" t0 = time.monotonic() owned = [o for mod, v in objs.items() for o in v if not (borrowed and mod != 'runtime')] shared = [o for mod, v in objs.items() for o in v if borrowed and mod != 'runtime'] man_path = out / 'obj' / MANIFEST man = read_manifest(man_path) verified, bad, _ = check_objects(out, owned, man, True) recompiled = 0 if bad: for o in bad: print(f'build_game: object {flag_path(o)} does not match its hash; deleting it to recompile') o.unlink() ninja_build(base, [ninja_path(o) for o in bad], env) recompiled = len(bad) verified += check_objects(out, bad, man, True)[0] write_manifest(man_path, man) if shared: main_man = read_manifest(main_out / 'obj' / MANIFEST) theirs = [pairs[o.relative_to(out)] for o in shared] v, bad, unrecorded = check_objects(main_out, theirs, main_man, False) verified += v if bad: for b in bad: print(f'build_game: shared object {flag_path(b)} does not match its hash in ' f'{flag_path(main_out / "obj" / MANIFEST)}', file=sys.stderr) sys.exit(f'build_game: run tools/build_game.py for {flag_path(main_out)} (it recompiles them), then rebuild ' f'{flag_path(out)}') if unrecorded: print(f'build_game: warning: {len(unrecorded)} shared objects have no current hash in ' f'{flag_path(main_out / "obj" / MANIFEST)}; build {flag_path(main_out)} once to record them') print(f'objects verified: {verified}, recompiled after a hash mismatch: {recompiled} ({time.monotonic() - t0:.1f} s)') # ---- Windows job object: peak memory of a process tree class IoCounters(ctypes.Structure): _fields_ = [(n, ctypes.c_ulonglong) for n in ('r_ops', 'w_ops', 'o_ops', 'r_bytes', 'w_bytes', 'o_bytes')] class BasicLimits(ctypes.Structure): _fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong), ('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t), ('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD), ('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD)] class ExtendedLimits(ctypes.Structure): _fields_ = [('BasicLimitInformation', BasicLimits), ('IoInfo', IoCounters), ('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t), ('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)] JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000 CREATE_SUSPENDED = 0x4 def run_measured(cmd, env): """Run cmd in its own job object. Returns (exit code, seconds, peak bytes of one process, peak bytes of all).""" k32 = ctypes.WinDLL('kernel32', use_last_error=True) k32.CreateJobObjectW.restype = wt.HANDLE k32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wt.LPCWSTR] k32.SetInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD] k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p] k32.AssignProcessToJobObject.argtypes = [wt.HANDLE, wt.HANDLE] k32.CloseHandle.argtypes = [wt.HANDLE] ntdll = ctypes.WinDLL('ntdll') ntdll.NtResumeProcess.argtypes = [wt.HANDLE] job = k32.CreateJobObjectW(None, None) info = ExtendedLimits() info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE # Ctrl+C stops every compile k32.SetInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), ctypes.sizeof(info)) t0 = time.monotonic() # Suspended until it is in the job, so the compilers it starts are counted from the first one. proc = subprocess.Popen(cmd, cwd=ROOT, env=env, creationflags=CREATE_SUSPENDED) if not k32.AssignProcessToJobObject(job, int(proc._handle)): print(f'build_game: cannot track memory (error {ctypes.get_last_error()})', file=sys.stderr) ntdll.NtResumeProcess(int(proc._handle)) try: code = proc.wait() except KeyboardInterrupt: k32.CloseHandle(job) raise dt = time.monotonic() - t0 k32.QueryInformationJobObject(job, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), ctypes.sizeof(info), None) k32.CloseHandle(job) return code, dt, info.PeakProcessMemoryUsed, info.PeakJobMemoryUsed def total_ram(): class MemStatus(ctypes.Structure): _fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \ [(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')] s = MemStatus() s.dwLength = ctypes.sizeof(s) ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(s)) return s.total def pe_sections(path): """{section name: virtual size} and SizeOfImage of a PE32+ file.""" data = path.read_bytes()[:4096] pe = int.from_bytes(data[0x3C:0x40], 'little') nsec = int.from_bytes(data[pe + 6:pe + 8], 'little') opt = pe + 24 image = int.from_bytes(data[opt + 56:opt + 60], 'little') table = opt + int.from_bytes(data[pe + 20:pe + 22], 'little') out = {} for i in range(nsec): s = data[table + 40 * i:table + 40 * (i + 1)] out[s[:8].rstrip(b'\0').decode(errors='replace')] = int.from_bytes(s[8:12], 'little') return out, image def mib(n): return f'{n / (1 << 20):,.1f} MiB' def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--version', default='v121', help='game version (default v121)') ap.add_argument('--gen', type=Path, help='generated C directory (default generated/)') ap.add_argument('--out', type=Path, help='build directory (default build/)') ap.add_argument('--modules', help='comma-separated subset of the layout modules (default all)') ap.add_argument('--jobs', type=int, help='parallel compiles (default: by memory budget and CPU count)') ap.add_argument('--compile-only', action='store_true', help='compile, do not link') ap.add_argument('--runtime', type=Path, help='runtime source directory (default runtime)') ap.add_argument('--full-build', action='store_true', help="compile the generated C even where build/'s objects could be reused") ap.add_argument('--skip-runtime', default='', help='comma-separated runtime/*.c names to leave out (the link needs another definition of what they ' 'define)') ap.add_argument('--exefs', type=Path, help="totk.exe's default ExeFS directory (default build/extracted//exefs)") ap.add_argument('--romfs', type=Path, help="totk.exe's default RomFS directory with romfs.bin and index.json " '(default build/extracted//romfs)') ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile)") ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of ' 'MSYS2 UCRT64 clang') ap.add_argument('--ninja', type=Path, help='ninja.exe to build with (default: MSYS2 UCRT64, else bin/ninja.exe of the ' 'release folder)') ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx ' f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})') ap.add_argument('--make-runtime-lib', action='store_true', help=f'build only /{LIB_NAME} from runtime/ (no generated C, no totk.exe)') ap.add_argument('--runtime-lib', type=Path, help=f'link against this {LIB_NAME} instead of compiling runtime/ ' f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)') ap.add_argument('--expected', type=Path, help='expected hashes (default tools/data/expected-.json)') args = ap.parse_args() gen = (args.gen or ROOT / 'generated' / args.version).resolve() out = (args.out or ROOT / 'build' / args.version).resolve() extracted = ROOT / 'build' / 'extracted' / args.version paths = {'exefs': (args.exefs or extracted / 'exefs').resolve(), 'romfs': (args.romfs or extracted / 'romfs').resolve(), 'profile': (args.profile or ROOT / 'build' / 'profile').resolve()} expected = args.expected if not expected: if args.version not in DISPLAY_VERSIONS: sys.exit(f'no expected hashes known for {args.version}; pass --expected') expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json' hashes = json.loads(Path(expected).read_text())['exefs_sha256'] global CLANG, AR, WINDRES, NINJA, TOOLCHAIN_BIN, DEPS if args.ninja: NINJA = args.ninja.resolve() elif not NINJA.is_file() and (ROOT / 'bin' / 'ninja.exe').is_file(): NINJA = ROOT / 'bin' / 'ninja.exe' # the release folder (tools/make_release.py) dlls = SDL3_DLLS if args.toolchain: TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve() CLANG = TOOLCHAIN_BIN / 'clang.exe' AR = TOOLCHAIN_BIN / 'llvm-ar.exe' WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe' DEPS = (args.deps or DEFAULT_DEPS).resolve() dlls = sorted((DEPS / 'bin').glob('*.dll')) for tool in (CLANG, NINJA): if not tool.is_file(): sys.exit(f'{tool} not found: install MSYS2 UCRT64 clang, lld and ninja (docs/toolchain-msys2-python.md)') if not (gen / 'layout.json').is_file(): sys.exit(f'{gen / "layout.json"} not found: run the generator first') layout = read_layout(gen) names = [n for n, _, _ in layout] wanted = args.modules.split(',') if args.modules else names unknown = [m for m in wanted if m not in names] if unknown: sys.exit(f'unknown modules {unknown}; the layout has {names}') linked = [m for m in names if m in wanted] mods = {m: chunk_files(gen, m) for m in linked} cpus = os.cpu_count() or 1 budget = min(MEM_BUDGET, int(total_ram() * 0.8)) jobs = args.jobs or max(1, min(cpus, budget // JOB_MEM)) missing = [n for n in names if n not in hashes] if missing: sys.exit(f'{expected}: no exefs_sha256 for {missing}') write_if_changed(out / 'modules.c', module_glue(layout, linked, hashes, paths)) if (paths['romfs'] / 'index.json').is_file(): count, size, dt = romfs_index.write(paths['romfs'], out) print(f'romfs index: {count} files, {mib(size)}, {dt:.1f} s') else: romfs_index.write_placeholder(out) print(f'build_game: warning: no {paths["romfs"] / "index.json"}; totk.exe will have no RomFS', file=sys.stderr) global RUNTIME RUNTIME = (args.runtime or RUNTIME).resolve() skip = {s for s in args.skip_runtime.split(',') if s} main_out = (ROOT / 'build' / args.version).resolve() pairs = None if args.make_runtime_lib: mods, linked, why = {}, [], '' elif args.full_build: why = '--full-build' elif args.toolchain: why = f'--toolchain {flag_path(args.toolchain)}' elif out == main_out: why = '' else: pairs, why = main_objects(gen, main_out, RUNTIME, mods) if pairs is not None: link_objects(out, pairs) print(f'build_game: reusing {len(pairs)} generated objects of {flag_path(main_out)} (hardlinks)') else: if out != main_out: # forks hold hardlinks to the main build's objects; the main build keeps its own unlink_shared(out, mods) if why: print(f'build_game: compiling all generated C: {why}') runtime_lib = args.runtime_lib.resolve() if args.runtime_lib else None if runtime_lib and not runtime_lib.is_file(): sys.exit(f'{runtime_lib} not found: build it with --make-runtime-lib') text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None, runtime_lib, args.make_runtime_lib) write_if_changed(out / 'build.ninja', text) env = dict(os.environ, PATH=str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', '')) base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)] nfiles = sum(len(v) for v in mods.values()) print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True) code, dt, peak_one, peak_all = run_measured(base + ['objects'], env) print(f'compile: {dt:.1f} s, peak {mib(peak_one)} per process, {mib(peak_all)} all processes') if code: sys.exit(code) for mod, files in objs.items(): size = sum(o.stat().st_size for o in files) print(f' {mod:8} {len(files):5} objects {mib(size):>14}') verify_objects(out, objs, pairs is not None, main_out, pairs, base, env) if args.compile_only: return if args.make_runtime_lib: exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env) print(f'link: {dt:.1f} s, peak {mib(peak_one)}') if code: print(f'build_game: link failed, {flag_path(exe)} is the old one (a running totk.exe from this dir locks it)', file=sys.stderr) sys.exit(code) if args.make_runtime_lib: print(f'{below_root(exe)}: {mib(exe.stat().st_size)}') return for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe shutil.copy2(dll, exe.parent / dll.name) sections, image = pe_sections(exe) print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, ' f'image {mib(image)}; ' + ', '.join(f'{k} {mib(v)}' for k, v in sections.items() if k.startswith('.') and v)) if image > 3 * GiB // 2: print('build_game: warning: image over 1.5 GiB; RIP-relative code reaches only 2 GiB', file=sys.stderr) if __name__ == '__main__': main()