"""Compile the generated C for one game version and link it with the runtime into build//totk.exe. Writes build//build.ninja and runs MSYS2 UCRT64 Ninja on it. Ninja keeps the build incremental: a chunk is recompiled only when it, a header it includes or its flags change. On Linux the same steps make build//totk with the clang, lld and ninja on PATH (or the clang and lld in --toolchain DIR/bin): the runtime's *_sdl.* and platform/linux.c take the place of its *_win.*, platform/win32.c and totk.rc, the libraries are the system's (Vulkan, SDL3, Opus, libvpx, zstd; libstdc++ static), and the executable finds a libSDL3.so.0 beside it first (rpath $ORIGIN). docs/building.md lists the packages. .venv\\Scripts\\python tools\\build_game.py # generated/v121 -> build/v121/totk.exe .venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others) .venv\\Scripts\\python tools\\build_game.py --jobs 8 .venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory .venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-/llvm-mingw--ucrt-x86_64 \\ --out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64 Every runtime/*.c is compiled. build//modules.c carries the layout, the linked function tables, the expected SHA-256 of each ExeFS file (tools/data/expected-.json) and the default game paths (--exefs, --romfs, --profile), which totk.exe uses unless its own command line names others. --exefs and --romfs default to extracted/ in the work folder: the [work] dir of config.local.toml, where tools/extract.py writes them, else build/. tools/romfs_index.py writes build//romfs-index.bin and romfs_config.h from the RomFS directory's index.json (runtime/romfs.c reads the index next to totk.exe). Without that index.json the build stops: the exe would stop at start. --no-romfs builds it anyway. Private builds reuse the main build's objects. When --out differs from build/, the main build's generated objects are up to date (checked with a dry run of its build.ninja) and every header they include from runtime/ is byte-identical in --runtime (the list comes from the main build's Ninja deps log), build_game.py hardlinks those objects into /obj and compiles only runtime/*.c. A hardlink is a snapshot: clang and Ninja replace an object by renaming a new file over it, so a later main rebuild leaves the linked copies alone. Otherwise (a header differs, a main object is missing or stale, another --gen) it compiles everything and prints one line saying why. --full-build skips the check. Object manifest. A flipped bit in an object written by an earlier build stays forever, because Ninja rebuilds an object only when its inputs change, and forks hardlink the object into every private build. So each build dir keeps obj/objects.sha256 (SHA-256, size and mtime of every object it compiled). After Ninja finishes and before the link, build_game.py hashes the objects, compares each one whose size and mtime match its manifest entry, and records the objects Ninja just rebuilt (new mtime). A mismatch deletes the object, Ninja recompiles it, and the build prints the summary line. A fork checks the hardlinked generated objects against build//obj/objects.sha256 (read-only; a bad shared object stops the fork and asks for a main build run, which recompiles it and breaks the shared inode). Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB and 80% of RAM) at JOB_MEM per job, so a big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory of its processes (Windows job object accounting: committed private bytes). --ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used. --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs copied next to totk.exe: SDL3.dll, libopus-0.dll and, when SDL3.dll comes from MSYS2, the libiconv-2.dll it imports, without which the exe stops at start with 0xc000007b or a missing-DLL error), and the C++ runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects. Split build. --make-runtime-lib compiles only runtime/ into /libtotkrt.a (what CI prebuilds). --runtime-lib PATH then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's game hashes and version in) and links them with that library. The library holds no game-derived data. Standard library only. """ import argparse import ctypes import ctypes.wintypes as wt # the job-object structures below; importing it works on every OS import hashlib import json import os import shutil import subprocess import sys import time import urllib.request from pathlib import Path import check_isa import romfs_index from build_shaders import stale as stale_shaders from check_content import GENERATED_MARKER WIN = os.name == 'nt' if not WIN: import resource ROOT = Path(__file__).resolve().parent.parent # In a git worktree (tools/worktree.py) build/ and generated/ are junctions into the main checkout, and resolve() follows # them. MAIN_ROOT is where they land, so resolved paths still print and compare relative to a root. MAIN_ROOT = (ROOT / 'build').resolve().parent UCRT = Path(r'C:\msys64\ucrt64\bin') EXE = 'totk.exe' if WIN else 'totk' def on_path(name): """name on PATH, else a path that does not exist (the tool check below names it).""" found = shutil.which(name) return Path(found) if found else Path('/usr/bin') / name CLANG = UCRT / 'clang.exe' if WIN else on_path('clang') NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja') SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else [] # Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering. # --cpu v2 replaces the level for CPUs without AVX2/FMA; runtime/fma.c then supplies the correctly rounded fma the # lowering needs (runtime/simd.h). GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] def set_cpu(cpu): """Target x86-64-v3 (the default) or x86-64-v2. RT_CPU_LEVEL lets runtime/main.c's self-test check the same instruction set the build was compiled for.""" march = f'-march=x86-64-{cpu}' GEN_FLAGS[GEN_FLAGS.index('-march=x86-64-v3')] = march RT_FLAGS[RT_FLAGS.index('-march=x86-64-v3')] = march RT_FLAGS.append(f'-DRT_CPU_LEVEL={3 if cpu == "v3" else 2}') # Profile-guided optimisation (--pgo). Both modes compile the generated code at -O2, the level the profile's inlining and # hot/cold layout work at. A static function's profile name carries its source path by default; the base name alone # keeps the names equal between the training build and a build from another tree (a worktree, an install). PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false'] PGO_INPUT = None # the .profdata of --pgo FILE: every compile depends on it, so a new profile rebuilds them def set_pgo(pgo, out): """--pgo gen: instrument the generated code and the runtime; totk.exe adds its counts to out/pgo/default_.profraw every minute and at exit (runtime/platform win32.c), across runs. --pgo FILE: compile with that merged profile (tools/pgo.py merge). Functions the profile never saw run are optimised for size.""" global PGO_INPUT GEN_FLAGS[GEN_FLAGS.index('-O1')] = '-O2' if pgo == 'gen': # absolute: the instrumented exe resolves the directory against its own working directory flags = [f'-fprofile-generate={cmd_arg((out / "pgo").resolve().as_posix())}'] RT_FLAGS.append('-DTOTK_PGO_GEN') LINK_FLAGS.append('-fprofile-generate') # links clang_rt.profile else: PGO_INPUT = Path(pgo).resolve() if not PGO_INPUT.is_file(): sys.exit(f'--pgo {pgo}: no such profile') # unprofiled: chunks the training never entered; out-of-date and backend-plugin: functions whose code changed # since the training, which then compile without a profile flags = [f'-fprofile-use={cmd_path(PGO_INPUT)}', '-Wno-profile-instr-unprofiled', '-Wno-profile-instr-out-of-date', '-Wno-backend-plugin'] GEN_FLAGS.extend(flags + PGO_NAMES) RT_FLAGS.extend(flags + PGO_NAMES) if WIN: # _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain) # to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call. RT_FLAGS.append('-D_WIN32_WINNT=0x0A00') # -Xlinker passes the map path as one argument; -Wl, would split it at any comma in the build dir path LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Xlinker', '"--Map=$builddir/totk.map"'] # the map feeds tools/prof_report.py LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end; libvpx static (movies); libzstd static (resolution.c) VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator else: # -no-pie: the lifted modules' function tables (hundreds of thousands of pointers) need no load-time relocation. # $ORIGIN: a libSDL3.so.0 next to totk comes before the system's (the release's portable folder). LINK_FLAGS = ['-fuse-ld=lld', '-no-pie', '-Wl,-z,noexecstack', "'-Wl,-rpath,$$ORIGIN'", '"-Wl,-Map=$builddir/totk.map"'] LINK_LIBS = ['-lvulkan', '-lSDL3', '-lopus', '-lvpx', '-lzstd', '-l:libstdc++.a', '-lm', '-lpthread', '-ldl'] VMA_INCLUDE = Path('/usr/include') # vk_mem_alloc.h; Arch (AUR) vulkan-memory-allocator, Debian libvulkan-memory-allocator-dev # Without a system vk_mem_alloc.h, the header the Windows release pins (tools/make_release.py, VMA 3.4.0) is fetched once # into toolchain/vma: one MIT header (THIRD_PARTY_NOTICES.md), checked against its hash. VMA_FETCH = ('https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/v3.4.0/include/vk_mem_alloc.h', '8487b7995ad3b263eb73bc5b9a77d71aa69b6bef5d58a715c02d2663afd81f1a') VMA_LOCAL = ROOT / 'toolchain' / 'vma' def vma_include(): """The directory holding vk_mem_alloc.h: the system's, else toolchain/vma, fetched there on first use.""" if (VMA_INCLUDE / 'vk_mem_alloc.h').is_file(): return VMA_INCLUDE dst = VMA_LOCAL / 'vk_mem_alloc.h' if not dst.is_file(): url, sha = VMA_FETCH print(f'build_game: no {VMA_INCLUDE / "vk_mem_alloc.h"}; fetching VMA 3.4.0 into {VMA_LOCAL}') try: data = urllib.request.urlopen(url, timeout=60).read() except OSError as e: sys.exit(f'build_game: fetching {url} failed ({e}); install VMA or put vk_mem_alloc.h in {VMA_LOCAL}') if hashlib.sha256(data).hexdigest() != sha: sys.exit(f'build_game: {url} does not match the pinned hash') VMA_LOCAL.mkdir(parents=True, exist_ok=True) dst.write_bytes(data) return VMA_LOCAL # --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead. LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-limm32', '-lwinmm', '-lole32', '-lshell32', '-luuid', '-lvulkan-1', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libzstd.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a'] DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw' # Runtime sources compiled per install, not in the library: romfs.c includes romfs_config.h, patches.c and ov_zstd.c # the patches.h of the player's game version (its patch page is the page after that version's main; ov_zstd.c's host # functions sit on it). mouse_direct.c and ov_ability.c also register host callbacks relative to that version's patch # page. A library source that includes one of PER_VERSION_HEADERS stops the library build (library_sources). INSTALL_SIDE = {'romfs.c', 'patches.c', 'ov_zstd.c', 'mouse_direct.c', 'ov_ability.c'} PER_VERSION_HEADERS = ('"patches.h"', '"romfs_config.h"') LIB_NAME = 'libtotkrt.a' AR = UCRT / 'llvm-ar.exe' if WIN else on_path('llvm-ar') WINDRES = UCRT / 'windres.exe' # runtime/totk.rc: the manifest of totk.exe TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain DEPS = None # the deps directory of an llvm-mingw build # tools/data/expected-.json for each build version name. DISPLAY_VERSIONS = {'v0': '1.0.0', 'v110': '1.1.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'} GiB = 1 << 30 # Peak commit of one compile, 1.2.1 at -O1 (2026-10-20): 120-180 MiB per ~3 MiB chunk, 568 MiB for main_table.c # (295k-entry tables). 768 MiB leaves headroom for the largest file. JOB_MEM = 768 << 20 MEM_BUDGET = 50 * GiB # total for all compile jobs def below_root(p): """Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else absolute.""" p = Path(p).resolve() for root in (ROOT, MAIN_ROOT): if p.is_relative_to(root): return p.relative_to(root) return p def ninja_path(p): """A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped.""" return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:') def flag_path(p): return below_root(p).as_posix() def cmd_arg(s): """s for a Ninja command line: Ninja-escaped, and quoted when it holds a space (a user or folder name), as clang and lld split their Windows command line at unquoted spaces.""" s = s.replace('$', '$$') return f'"{s}"' if ' ' in s else s def cmd_path(p): return cmd_arg(flag_path(p)) def read_layout(gen): """[(name, base, size)] from layout.json (same format as recomp/layout.py read(), without its NSO dependency).""" data = json.loads((gen / 'layout.json').read_text()) return [(m['name'], int(m['base'], 16), int(m['size'], 16)) for m in data] def chunk_files(gen, mod): """The C files of one module: the chunks listed in /manifest.json, then _table.c.""" d = gen / mod data = json.loads((d / 'manifest.json').read_text()) entries = data if isinstance(data, list) else data.get('chunks', data.get('files')) if entries is None: sys.exit(f'{d / "manifest.json"}: no "chunks" or "files" list') names = [] for e in entries: name = e if isinstance(e, str) else e.get('file', e.get('path', e.get('name'))) names.append(Path(name).name) table = f'{mod}_table.c' if table not in names: names.append(table) files = [d / n for n in names] missing = [str(f) for f in files if not f.is_file()] if missing: sys.exit(f'{mod}: {len(missing)} files listed in manifest.json are missing, first {missing[0]}') return files def write_if_changed(path, text): """Keep the mtime of an unchanged file so Ninja does not rebuild from it.""" if path.exists() and path.read_text(encoding='utf-8', errors='replace') == text: return path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding='utf-8', newline='\n') # Ninja and clang read UTF-8; paths may be non-ASCII def copy_romfs_patches(version, out): """totk.exe loads romfs-patches beside romfs-index.bin (runtime/romfs.c overlay_init). tools/make_romfs_patches.py writes them into build/ only, so a build in another directory ran without the 60 fps overlay (the quick menus' captured background turned black). Copies every romfs-patches* folder of build/ that has a manifest, the same way when it differs: romfs-patches, romfs-patches-xbox, romfs-patches-xbox-label and romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts, make_romfs_patches.py --prompts) and the ones of another shape, romfs-patches-x and romfs-patches-