Merge branch 'task/llvmmingw' into task/installer

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-24 08:06:04 +03:00
commit fd14250314
1 file changed
+83 -10
+83 -10
View File
@@ -7,6 +7,8 @@ recompiled only when it, a header it includes or its flags change.
.venv\\Scripts\\python tools\\build_game.py --modules subsdk0 # one module (links if it has no calls into others)
.venv\\Scripts\\python tools\\build_game.py --jobs 8
.venv\\Scripts\\python tools\\build_game.py --out build/v121-t2 # a separate build directory
.venv\\Scripts\\python tools\\build_game.py --toolchain toolchain/llvm-mingw-<version>/llvm-mingw-<version>-ucrt-x86_64 \\
--out build/v121-llvmmingw # llvm-mingw instead of MSYS2 UCRT64
Every runtime/*.c is compiled. build/<version>/modules.c carries the layout, the linked function tables, the
expected SHA-256 of each ExeFS file (tools/data/expected-<display version>.json) and the default game paths
@@ -34,6 +36,15 @@ Compile jobs share a memory budget (MEM_BUDGET, default the smaller of 50 GiB an
big machine does not run out of memory on the largest chunks. Each phase reports its wall time and the peak memory
of its processes (Windows job object accounting: committed private bytes).
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
Split build. --make-runtime-lib compiles only runtime/ into <out>/libtotkrt.a (what CI prebuilds). --runtime-lib PATH
then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's
game hashes in) and links them with that library. The library holds no game-derived data.
Standard library only.
"""
import argparse
@@ -67,6 +78,15 @@ RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3']
LINK_FLAGS = ['-fuse-ld=lld', '-Wl,--stack,16777216', '-Wl,--Map=$builddir/totk.map'] # the map feeds tools/prof_report.py
LINK_LIBS = ['-lpsapi', '-lbcrypt', '-lvulkan-1', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lSDL3', '-lopus', '-l:libvpx.a', '-l:libstdc++.a', '-l:libwinpthread.a'] # Vulkan back end of M4; libvpx static (movies)
VMA_INCLUDE = UCRT.parent / 'include' / 'vma' # vk_mem_alloc.h; MSYS2 package vulkan-memory-allocator
# --toolchain: an llvm-mingw tree. The libraries, headers and DLLs MSYS2 supplies come from the deps directory instead.
LLVM_LINK_LIBS = ['-lpsapi', '-lbcrypt', '-luser32', '-lgdi32', '-ldwmapi', '-lwinmm', '-lvulkan-1', '-lSDL3', '-lopus',
'-l:libvpx.a', '-l:libc++.a', '-l:libc++abi.a', '-l:libunwind.a', '-l:libwinpthread.a']
DEFAULT_DEPS = ROOT / 'toolchain' / 'deps-mingw'
INSTALL_SIDE = {'romfs.c'} # runtime sources that include romfs_config.h: compiled per install, not in the library
LIB_NAME = 'libtotkrt.a'
AR = UCRT / 'llvm-ar.exe'
TOOLCHAIN_BIN = None # llvm-mingw bin directory, set by main() for --toolchain
DEPS = None # the deps directory of an llvm-mingw build
# tools/data/expected-<display version>.json for each build version name.
DISPLAY_VERSIONS = {'v0': '1.0.0', 'v112': '1.1.2', 'v121': '1.2.1', 'v143': '1.4.3'}
@@ -182,11 +202,14 @@ def module_glue(layout, linked, hashes, paths):
return '\n'.join(out)
def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False):
def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=None, make_lib=False):
"""build.ninja text, and the object lists per module. With prebuilt the generated objects already exist in
out/obj and are link inputs without a compile edge."""
out/obj and are link inputs without a compile edge. runtime_lib: a prebuilt library replaces the runtime objects
except INSTALL_SIDE. make_lib: build only that library, from the runtime objects except INSTALL_SIDE."""
inc = f'-I {flag_path(RUNTIME)} -I {flag_path(gen)}'
rt_inc = f'{inc} -I {flag_path(out)} -I {flag_path(VMA_INCLUDE)}' # romfs_config.h, vk_mem_alloc.h
vma = f'{flag_path(DEPS / "include")} -I {flag_path(DEPS / "include" / "vma")}' if DEPS else flag_path(VMA_INCLUDE)
rt_inc = f'{inc} -I {flag_path(out)} -I {vma}' # romfs_config.h, vk_mem_alloc.h (and the deps headers)
libs = [f'-L{flag_path(DEPS / "lib")}'] + LLVM_LINK_LIBS if DEPS else LINK_LIBS
lines = [
'# Written by tools/build_game.py; do not edit.',
'ninja_required_version = 1.10',
@@ -195,7 +218,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False):
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
f'ldflags = {" ".join(LINK_FLAGS)}',
f'libs = {" ".join(LINK_LIBS)}',
f'libs = {" ".join(libs)}',
'',
'pool gen',
f' depth = {pool_depth}',
@@ -229,14 +252,35 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False):
lines.append(' pool = gen')
objs[mod].append(o)
rt = []
for src in runtime_sources(skip) + [out / 'modules.c']:
sources = runtime_sources(skip)
if make_lib:
sources = [p for p in sources if p.name not in INSTALL_SIDE]
elif runtime_lib:
sources = [p for p in sources if p.name in INSTALL_SIDE] + [out / 'modules.c']
else:
sources += [out / 'modules.c']
for src in sources:
o = runtime_object(out, src)
lines.append(f'build {ninja_path(o)}: cc_rt {ninja_path(src)}')
rt.append(o)
objs['runtime'] = rt
every = [o for v in objs.values() for o in v]
exe = out / 'totk.exe'
lines += ['', f'build {ninja_path(exe)}: link ' + ' '.join(ninja_path(o) for o in every),
if make_lib:
exe = out / LIB_NAME
lines += ['', 'rule ar', f' command = {AR.as_posix()} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
rule = 'link'
if runtime_lib:
# Whole archive: the weak probe stubs in gfx_stubs.c would otherwise satisfy references to the probes before the
# linker looks into the library, and the real probes would stay out of totk.exe.
rule = 'link_lib'
lines += ['', 'rule link_lib',
f' command = $cc $ldflags @$out.rsp -Wl,--whole-archive {flag_path(runtime_lib)} -Wl,--no-whole-archive $libs -o $out',
' rspfile = $out.rsp', ' rspfile_content = $in_newline', ' description = LINK $out']
lines += ['', f'build {ninja_path(exe)}: {rule} ' + ' '.join(ninja_path(o) for o in every) +
(f' | {ninja_path(runtime_lib)}' if runtime_lib else ''),
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
return '\n'.join(lines), objs, exe
@@ -517,6 +561,14 @@ def main():
ap.add_argument('--romfs', type=Path, help="totk.exe's default RomFS directory with romfs.bin and index.json "
'(default build/extracted/<version>/romfs)')
ap.add_argument('--profile', type=Path, help="totk.exe's default profile directory (default build/profile)")
ap.add_argument('--toolchain', type=Path, help='llvm-mingw directory (with bin/clang.exe) to build with instead of '
'MSYS2 UCRT64 clang')
ap.add_argument('--deps', type=Path, help=f'with --toolchain: include/, lib/ and bin/ of Vulkan, SDL3, Opus, libvpx '
f'and VMA (default {DEFAULT_DEPS.relative_to(ROOT).as_posix()})')
ap.add_argument('--make-runtime-lib', action='store_true',
help=f'build only <out>/{LIB_NAME} from runtime/ (no generated C, no totk.exe)')
ap.add_argument('--runtime-lib', type=Path, help=f'link against this {LIB_NAME} instead of compiling runtime/ '
f'({", ".join(sorted(INSTALL_SIDE))} and modules.c still compile)')
ap.add_argument('--expected', type=Path,
help='expected hashes (default tools/data/expected-<display version>.json)')
args = ap.parse_args()
@@ -533,6 +585,14 @@ def main():
sys.exit(f'no expected hashes known for {args.version}; pass --expected')
expected = ROOT / 'tools' / 'data' / f'expected-v{DISPLAY_VERSIONS[args.version]}.json'
hashes = json.loads(Path(expected).read_text())['exefs_sha256']
global CLANG, AR, TOOLCHAIN_BIN, DEPS
dlls = SDL3_DLLS
if args.toolchain:
TOOLCHAIN_BIN = (args.toolchain / 'bin').resolve()
CLANG = TOOLCHAIN_BIN / 'clang.exe'
AR = TOOLCHAIN_BIN / 'llvm-ar.exe'
DEPS = (args.deps or DEFAULT_DEPS).resolve()
dlls = sorted((DEPS / 'bin').glob('*.dll'))
for tool in (CLANG, NINJA):
if not tool.is_file():
sys.exit(f'{tool} not found: install MSYS2 UCRT64 clang, lld and ninja (docs/toolchain-msys2-python.md)')
@@ -566,8 +626,12 @@ def main():
skip = {s for s in args.skip_runtime.split(',') if s}
main_out = (ROOT / 'build' / args.version).resolve()
pairs = None
if args.full_build:
if args.make_runtime_lib:
mods, linked, why = {}, [], ''
elif args.full_build:
why = '--full-build'
elif args.toolchain:
why = f'--toolchain {flag_path(args.toolchain)}'
elif out == main_out:
why = ''
else:
@@ -580,10 +644,14 @@ def main():
unlink_shared(out, mods)
if why:
print(f'build_game: compiling all generated C: {why}')
text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None)
runtime_lib = args.runtime_lib.resolve() if args.runtime_lib else None
if runtime_lib and not runtime_lib.is_file():
sys.exit(f'{runtime_lib} not found: build it with --make-runtime-lib')
text, objs, exe = ninja_file(gen, out, mods, jobs, skip, pairs is not None, runtime_lib,
args.make_runtime_lib)
write_if_changed(out / 'build.ninja', text)
env = dict(os.environ, PATH=str(UCRT) + os.pathsep + os.environ.get('PATH', ''))
env = dict(os.environ, PATH=str(TOOLCHAIN_BIN or UCRT) + os.pathsep + os.environ.get('PATH', ''))
base = [str(NINJA), '-f', str(out / 'build.ninja'), '-j', str(jobs)]
nfiles = sum(len(v) for v in mods.values())
print(f'build_game: {nfiles} generated files in {len(linked)} modules ({", ".join(linked)}), {jobs} jobs', flush=True)
@@ -598,13 +666,18 @@ def main():
verify_objects(out, objs, pairs is not None, main_out, pairs, base, env)
if args.compile_only:
return
if args.make_runtime_lib:
exe.unlink(missing_ok=True) # ar rcs would keep members of deleted sources
code, dt, peak_one, _ = run_measured(base + [flag_path(exe)], env)
print(f'link: {dt:.1f} s, peak {mib(peak_one)}')
if code:
print(f'build_game: link failed, {flag_path(exe)} is the old one (a running totk.exe from this dir locks it)',
file=sys.stderr)
sys.exit(code)
for dll in SDL3_DLLS: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe
if args.make_runtime_lib:
print(f'{below_root(exe)}: {mib(exe.stat().st_size)}')
return
for dll in dlls: # dynamic SDL3 (zlib) and its libiconv (LGPL, unmodified DLL), found next to totk.exe
shutil.copy2(dll, exe.parent / dll.name)
sections, image = pe_sections(exe)
print(f'{below_root(exe)}: file {mib(exe.stat().st_size)}, '