From 34b4faee7c6930127a25f173e2ea6bd8bdaec52c Mon Sep 17 00:00:00 2001 From: CHEN Feng Date: Sun, 1 Nov 2026 07:09:13 +0800 Subject: [PATCH] Run the game on x86-64-v2 CPUs without AVX2 -march=x86-64-v3 kept the port and the installer to CPUs with AVX2 and FMA. --cpu v2 compiles the generated code and the runtime for SSE4.2 instead. Without FMA3 the FP lowering calls fma/fmaf, so runtime/fma.c provides correctly rounded ones (musl, MIT) that honour MXCSR's rounding mode, FTZ and DAZ; the UCRT's software fallback rounds incorrectly by up to half an ulp. simd.h's fast paths keep their assumptions. The installer's host check asks for x86-64-v2 and selects libtotkrt.a or libtotkrt-v2.a; both release scripts build and ship the pair. The differential tests compile fma.c as well, so a --cflags=-march=x86-64-v2 run tests it. Tested on an i7-3770K (no AVX2, FMA or BMI) with an RTX 3060 Ti, game 1.4.3: the installer ran end to end, --selftest passed and play holds 40-60 fps at 1080p. fma and fmaf matched 152,288 exact vectors over two seeds, including ties, subnormals and every rounding mode, FTZ and DAZ combination. The v3 build is unchanged: fma.c is an empty translation unit there. --- README.md | 2 +- THIRD_PARTY_NOTICES.md | 1 + docs/building.md | 12 +- docs/installer.md | 14 +- docs/tester-guide.md | 3 +- installer/hostcheck.c | 19 ++- installer/layout.py | 27 +++- installer/steps.py | 35 +++-- installer/text/en.json | 2 +- installer/text/ru.json | 2 +- runtime/cpu.h | 2 +- runtime/fma.c | 246 ++++++++++++++++++++++++++++++++++++ runtime/main.c | 11 ++ runtime/simd.h | 2 +- tests/diff/common.py | 3 +- tests/diff/func.py | 3 +- tools/build_game.py | 14 ++ tools/make_release.py | 31 +++-- tools/make_release_linux.py | 31 +++-- 19 files changed, 402 insertions(+), 58 deletions(-) create mode 100644 runtime/fma.c diff --git a/README.md b/README.md index cd851018..ccc24da1 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). ## Installing -Requirements: Windows 10 or later, an x86-64 CPU with AVX2, a GPU with Vulkan 1.3 and `VK_EXT_external_memory_host`, about 36 GiB free during setup (20 GiB after). +Requirements: Windows 10 or later, an x86-64 CPU with SSE4.2 (AVX2/FMA machines get a faster build), a GPU with Vulkan 1.3 and `VK_EXT_external_memory_host`, about 36 GiB free during setup (20 GiB after). 1. Download `TotK-Setup--windows.zip` from [Releases](../../releases/latest) and unpack it. 2. Start `TotK-Setup.exe`. If SmartScreen warns about an unsigned app, choose *More info*, then *Run anyway*. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 86d96747..99dcca40 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -37,6 +37,7 @@ Code copied or adapted from another project is listed here with its source, the | AMD FidelityFX Super Resolution 1.0 (`ffx_a.h`, `ffx_fsr1.h`, unmodified; licence text in `runtime/shaders/ffx/LICENSE.txt`) | https://github.com/GPUOpen-Effects/FidelityFX-FSR @ `a21ffb8f6c13233ba336352bdff293894c706575`, Copyright (c) 2021 Advanced Micro Devices, Inc. | MIT | `runtime/shaders/ffx/`, `runtime/shaders/fsr_easu.comp`, `runtime/shaders/fsr_rcas.comp` (compiled into `runtime/fsr_easu_spv.h`, `runtime/fsr_rcas_spv.h`) | | AMD Display Library (ADL) declarations: `AdapterInfo`, `ADLSingleSensorData`, `ADLPMLogDataOutput`, the `ADL_PMLOG_*` sensor numbers and the `ADL2_*` function signatures | https://github.com/GPUOpen-LibrariesAndSDKs/display-library @ `84c881cb2136f04287eb3af12de00a9e2867aa9e` (`include/adl_structures.h`, `include/adl_defines.h`, `include/adl_sdk.h`), Copyright (c) 2016 - 2022 Advanced Micro Devices, Inc. | MIT | `runtime/perfmon.c` | | Zstandard 1.5.7 (MSYS2 `mingw-w64-ucrt-x86_64-zstd`), static link | https://github.com/facebook/zstd, Copyright (c) Meta Platforms, Inc. and affiliates | BSD-3-Clause | `runtime/resolution.c` (the HeapDef table), `runtime/ov_zstd.c` (the game's zstd frames decoded on the host) | +| musl `fma.c`, `fmaf.c` and `scalbn.c`, adapted into one file for the x86-64-v2 build | https://github.com/kraj/musl @ `5a56ad13e1e0790c06122978aad2a14d6b9efc1a` (mirror of https://git.musl-libc.org/cgit/musl), Copyright (c) 2005-2020 Rich Felker and contributors | MIT | `runtime/fma.c` (the correctly rounded fma/fmaf the FP lowering needs on CPUs without FMA3) | ## Compiler runtime in a source build diff --git a/docs/building.md b/docs/building.md index e7d48480..6ebd0f3d 100644 --- a/docs/building.md +++ b/docs/building.md @@ -1,7 +1,9 @@ # Building from source -Windows 10 or later, an x86-64 CPU with AVX2, and a GPU with Vulkan 1.3 and `VK_EXT_external_memory_host`. Allow about -60 GB for the work folder. Extraction wants about 55 GB free while it runs and keeps about 35 GB afterwards. +Windows 10 or later, an x86-64 CPU with SSE4.2, and a GPU with Vulkan 1.3 and `VK_EXT_external_memory_host`. The +default build targets x86-64-v3 (AVX2/FMA); a CPU without them builds with `--cpu v2` instead and runs slower (see +"Build"). Allow about 60 GB for the work folder. Extraction wants about 55 GB free while it runs and keeps about 35 GB +afterwards. Players who only want to play can use the [installer](installer.md) instead: it builds the game on their machine without this toolchain. @@ -89,6 +91,9 @@ From the repository root, in PowerShell. Every step that depends on the game ver .venv\Scripts\python tools\build_game.py --version ``` + `--cpu v2` targets x86-64-v2 (SSE4.2, no AVX2/FMA) for older CPUs; the default `v3` needs AVX2. The v2 build links + `runtime/fma.c`, a correctly rounded software fma the FP lowering needs, and runs slower. + Lifting takes about 15 s, a clean compile about 200 s on 24 threads; later builds recompile only what changed. A compiler cache in front of clang shortens it further: with `SCCACHE` set to an sccache executable, `build_game.py` writes it into the ninja file ahead of `cc`, and every lift rewrites the generated chunks with the same contents, so @@ -195,6 +200,9 @@ change passes these, all run with `.venv`: | `tests\excl_mt.py` | the exclusive monitor helpers under contention and ABA, on several threads | no | | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | +A `--cpu v2` build passes the same differential tests with `--cflags=-march=x86-64-v2`; they compile `runtime/fma.c` +too, so they check the software fma the v2 build links rather than the C runtime's. + Runtime probes, each a mode of `totk.exe` (the list is at the top of `runtime/main.c`): `--probe-kernel`, `--probe-vkexec`, `--probe-nvn-tex` and `--probe-shader` cover most runtime changes. `--probe-shader` and the draw part of `--probe-vkexec` translate shaders of the census below: they look for `build//shaders/jobs.json` of the diff --git a/docs/installer.md b/docs/installer.md index 327f2972..c53bba62 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -41,14 +41,15 @@ keeps `profile/`. Saves carry over both ways: 1.4.3 converts a 1.2.1 save on loa 1. **Check** before writing anything: the files open with the keys, the update is 1.2.1 or 1.4.3 (by hash, as `tools/extract.py`), the keys are new enough for the update, enough space is free on the target disk (about 36 GiB at the peak of a - first install, 20 GiB after it), the CPU has - AVX2 (`-march=x86-64-v3`), a Vulkan 1.3 GPU exists. Each failure is one clear message. + first install, 20 GiB after it), the CPU is at least x86-64-v2 (SSE4.2; with AVX2/FMA the faster `v3` build is used, + otherwise the `v2` one), a Vulkan 1.3 GPU exists. Each failure is one clear message. 2. **Extract** ExeFS and RomFS into the install folder and verify the hashes. 3. **Lift** the ExeFS to C (`recomp/gen.py`, `recomp/patches-v121.txt` or `patches-v143.txt`) into a temp folder next to the installer. 4. **Compile** the generated C, `modules.c`, `romfs.c`, `patches.c` and `ov_zstd.c` (they bake the per-install layout, RomFS hashes and the game version's patch page) and link - them with the prebuilt runtime library into `totk.exe` (`build_game.py --runtime-lib`; about 3 min on 24 cores). + them with the prebuilt runtime library for the host's level (`libtotkrt.a` or `libtotkrt-v2.a`) into `totk.exe` + (`build_game.py --runtime-lib`; about 3 min on 24 cores). 5. **Prepare** the RomFS patches and the shaders (`tools/make_romfs_patches.py`, `tools/prepare_shaders.py`, `totk.exe --prepare-shaders`) in the install folder, then fold the shader cache's loose files (four per shader, about 117,000) into `shader-cache/shader-cache.pack` (`tools/pack_shader_cache.py`). totk.exe reads the pack first and @@ -97,7 +98,8 @@ installer/ the installer: steps, CLI, window, en/ru text catalog python/ Python 3.14.9 embeddable + numpy, lz4, cryptography, cffi, pycparser (+ tkinter for the window) recomp/, tools/ the lifter and the build, extraction and shader scripts the steps run (copied from the repo) llvm/ llvm-mingw, x86_64-only subset (Apache-2.0 with LLVM exception; mingw-w64 parts permissive) -runtime/ libtotkrt.a and the headers the generated code and modules.c/romfs.c/patches.c include +runtime/ libtotkrt.a, libtotkrt-v2.a and the headers the generated code and modules.c/romfs.c/patches.c + include bin/ totk_xlate.exe, hactool.exe, SDL3.dll, libopus-0.dll, ninja.exe data/ shader_list.tsv (MacNativeGames, MIT) THIRD_PARTY_NOTICES.md @@ -124,8 +126,8 @@ python/ python-build-standalone 3.14 (with tkinter) + the installe tools/ and recomp/ on sys.path llvm/ LLVM 23 (clang, lld, llvm-ar) and llvm/sysroot: the files of the Steam Runtime 3 "sniper" SDK the game's compile and link read (about 130 files) -runtime/ libtotkrt.a, headers, the install-side sources, totk.png; deps/ with the Vulkan headers, VMA and a - static zstd +runtime/ libtotkrt.a, libtotkrt-v2.a, headers, the install-side sources, totk.png; deps/ with the Vulkan + headers, VMA and a static zstd bin/ totk_xlate, hactool, ninja, libSDL3.so.0 ``` diff --git a/docs/tester-guide.md b/docs/tester-guide.md index 67a9fc70..8fe92ba8 100644 --- a/docs/tester-guide.md +++ b/docs/tester-guide.md @@ -6,7 +6,8 @@ from it may be shared. ## What you need -- Windows 10 or later, an x86-64 CPU with AVX2, about 60 GB free. +- Windows 10 or later, an x86-64 CPU with SSE4.2 (AVX2/FMA recommended: they build the default, faster version), about + 60 GB free. - A GPU with Vulkan 1.3 and `VK_EXT_external_memory_host`, with a current driver. - *Tears of the Kingdom* base game (1.0.0) and the 1.2.1 or 1.4.3 update, dumped from your own Switch as NSP or XCI. - `prod.keys` dumped from your own Switch. Never send it to anyone, and never paste it into a log or an issue. diff --git a/installer/hostcheck.c b/installer/hostcheck.c index 6cd08332..d3099271 100644 --- a/installer/hostcheck.c +++ b/installer/hostcheck.c @@ -1,5 +1,5 @@ // Host check of the installer's first step, compiled with the bundled clang at install time: does the CPU run -// -march=x86-64-v3 code, and does a Vulkan 1.3 device exist. Prints one JSON object on stdout. +// -march=x86-64-v3 code (or only x86-64-v2), and does a Vulkan 1.3 device exist. Prints one JSON object on stdout. // The Vulkan loader is opened at run time, so a machine without one gets "loader": false instead of a failed start. #include #include @@ -25,10 +25,24 @@ static void json_string(const char *s) { putchar('"'); } +// The x86-64-v2 level from CPUID: SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, CMPXCHG16B and LAHF/SAHF. Every CPU with +// SSE4.2 has the others, but the level is what the v2 build is compiled for, so check them all. +static int x86_64_v2(void) { + unsigned a, b, c, d; + if (!__get_cpuid(1, &a, &b, &c, &d)) return 0; + const unsigned sse3 = 1u << 0, ssse3 = 1u << 9, cx16 = 1u << 13, sse41 = 1u << 19, sse42 = 1u << 20, + popcnt = 1u << 23; + if ((c & (sse3 | ssse3 | cx16 | sse41 | sse42 | popcnt)) != (sse3 | ssse3 | cx16 | sse41 | sse42 | popcnt)) + return 0; + if (!__get_cpuid(0x80000001, &a, &b, &c, &d)) return 0; + return (c & (1u << 0)) != 0; // LAHF/SAHF +} + // The x86-64-v3 level from CPUID: AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT and MOVBE, with the OS saving the YMM state. // Not __builtin_cpu_supports("x86-64-v3"): clang reads that from the C runtime's CPU model, and libgcc before GCC 12 // (the Linux release's sysroot has GCC 10) has no level bits, so it answers false on every CPU. static int x86_64_v3(void) { + if (!x86_64_v2()) return 0; unsigned a, b, c, d; if (!__get_cpuid(1, &a, &b, &c, &d)) return 0; const unsigned fma = 1u << 12, movbe = 1u << 22, osxsave = 1u << 27, avx = 1u << 28, f16c = 1u << 29; @@ -44,7 +58,8 @@ static int x86_64_v3(void) { } int main(void) { - printf("{\"x86_64_v3\": %s, ", x86_64_v3() ? "true" : "false"); + printf("{\"x86_64_v2\": %s, \"x86_64_v3\": %s, ", x86_64_v2() ? "true" : "false", + x86_64_v3() ? "true" : "false"); void *lib = open_lib(); PFN_vkGetInstanceProcAddr gipa = lib ? (PFN_vkGetInstanceProcAddr)lib_sym(lib, "vkGetInstanceProcAddr") : NULL; if (!gipa) { diff --git a/installer/layout.py b/installer/layout.py index 23c7137b..7ccf3091 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -7,7 +7,8 @@ Release (docs/installer.md "Release layout"), root = the unpacked folder: recomp/, tools/ the lifter and the scripts; recomp/patches-.txt, tools/data/expected-v.json for each supported update (SUPPORTED) llvm/ llvm-mingw (bin/clang.exe) - runtime/ libtotkrt.a, the headers, install-side sources (romfs.c, patches.c, ov_zstd.c) and totk.ico; runtime/deps/ holds + runtime/ libtotkrt.a and libtotkrt-v2.a (x86-64-v3 / v2), the headers, install-side sources (romfs.c, + patches.c, ov_zstd.c) and totk.ico; runtime/deps/ holds include/ + lib/ of Vulkan, SDL3, Opus, libvpx and VMA (build_game.py --deps) bin/ totk_xlate.exe (+ totk_xlate.id), hactool.exe, ninja.exe, SDL3.dll, libopus-0.dll data/shader_list.tsv the MacNativeGames shader list @@ -21,8 +22,8 @@ against), runtime/deps/ with include/ and the static Opus, libvpx and zstd, bin/ Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw; -runtime/ as the sources with build/install-rtlib/libtotkrt.a (tools/build_game.py --make-runtime-lib --out -build/install-rtlib); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from +runtime/ as the sources with build/install-rtlib/libtotkrt.a and libtotkrt-v2.a (tools/build_game.py --make-runtime-lib +--out build/install-rtlib, once per --cpu); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from refs/MacNativeGames; build/install-temp. The version is VERSION with -dev and the commit: 0.1.0-dev+973c626. On Linux, dev mode takes clang, lld and ninja from PATH and the system's libraries (no deps folder, no SDL3 copy). """ @@ -64,11 +65,20 @@ class Layout: ini_example: Path | None = None temp: Path | None = None sysroot: Path | None = None # Linux release: what the game links against, instead of the host's + runtime_lib_v2: Path | None = None # libtotkrt-v2.a: the runtime for CPUs without AVX2 (build_game.py --cpu v2) @property def clang(self): return self.llvm / 'bin' / ('clang' + EXE) + def runtime_lib_for(self, cpu): + """The prebuilt runtime library for cpu (v2 or v3). None in dev mode when it was not built: the runtime is + then compiled from the sources in runtime/ instead.""" + lib = self.runtime_lib_v2 if cpu == 'v2' else self.runtime_lib + if lib and lib.is_file(): + return lib + return None if self.dev else lib + def patches(self, version): """The code patch table for recomp/gen.py.""" return self.recomp / f'patches-{version_tag(version)}.txt' @@ -77,13 +87,15 @@ class Layout: """The expected hashes of the update.""" return self.tools / 'data' / f'expected-v{version}.json' - def missing(self): - """Paths the steps need that do not exist.""" + def missing(self, cpu='v3'): + """Paths the steps need that do not exist. cpu picks the runtime library the host will use.""" + lib = self.runtime_lib_for(cpu) need = [self.python, self.tools / 'extract.py', self.tools / 'build_game.py', self.tools / 'prepare_shaders.py', self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.tools / 'ultrawide_layout.py', self.recomp / 'gen.py', self.clang, - *((self.deps / 'include', self.deps / 'lib') if self.deps else ()), self.runtime, self.runtime_lib, self.ninja, self.hactool, + *((self.deps / 'include', self.deps / 'lib') if self.deps else ()), self.runtime, *([lib] if lib else ()), + self.ninja, self.hactool, self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))] if self.shader_list: need.append(self.shader_list) @@ -109,6 +121,7 @@ def release_layout(root): tools=root / 'tools', recomp=root / 'recomp', llvm=root / 'llvm', deps=root / 'runtime' / 'deps', runtime=root / 'runtime', runtime_lib=root / 'runtime' / 'libtotkrt.a', + runtime_lib_v2=root / 'runtime' / 'libtotkrt-v2.a', ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE), xlate=root / 'bin' / ('totk_xlate' + EXE), dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll') if WIN else (root / 'bin' / 'libSDL3.so.0',), @@ -134,6 +147,7 @@ def dev_layout(root): tools=root / 'tools', recomp=root / 'recomp', llvm=trees[-1] if trees else root / 'toolchain' / 'llvm-mingw-missing', deps=deps, runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a', + runtime_lib_v2=root / 'build' / 'install-rtlib' / 'libtotkrt-v2.a', ninja=UCRT / ('ninja' + EXE), hactool=root / 'toolchain' / 'hactool' / ('hactool' + EXE), xlate=root / 'build' / 'xlate' / ('totk_xlate' + EXE), dlls=tuple(sorted((deps / 'bin').glob('*.dll'))), @@ -153,6 +167,7 @@ def dev_layout_posix(root): python=Path(sys.executable), tools=root / 'tools', recomp=root / 'recomp', llvm=Path(clang).parent.parent if clang else root / 'toolchain' / 'clang-missing', deps=None, runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a', + runtime_lib_v2=root / 'build' / 'install-rtlib' / 'libtotkrt-v2.a', ninja=Path(ninja) if ninja else root / 'toolchain' / 'ninja-missing', hactool=root / 'toolchain' / 'hactool' / 'hactool', xlate=root / 'build' / 'xlate' / 'totk_xlate', shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp') diff --git a/installer/steps.py b/installer/steps.py index b03a69f1..75a4d905 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -223,9 +223,19 @@ class Check(Step): id = 'check' always = True + def key(self, ctx): + # The compile step fingerprints the CPU level, and every key is computed before a step runs: probe the host + # here, once, so both steps see the same answer (the check step itself is always run). A failed probe is + # reported by run(), where the error names this step. + try: + self.host(ctx) + except InstallError: + pass + return '' + def run(self, ctx): layout, rep, dest = ctx.layout, ctx.reporter, ctx.dest - missing = layout.missing() + missing = layout.missing(getattr(ctx, 'cpu', 'v3')) if missing: raise InstallError('check.release_incomplete', paths=missing[:5]) if dest.exists() and not dest.is_dir(): @@ -258,6 +268,8 @@ class Check(Step): return {} def host(self, ctx): + if getattr(ctx, 'host', None) is not None: + return layout, work = ctx.layout, ctx.temp / 'check' glibc = linux_glibc() if glibc and glibc < GLIBC_NEEDED and not layout.dev: @@ -282,8 +294,10 @@ class Check(Step): except ValueError: raise InstallError('check.host_probe_failed', code=r.returncode) from None ctx.reporter.log(f'host: {r.stdout.strip()}') - if not host['x86_64_v3']: - raise InstallError('check.cpu_no_avx2') + if not host['x86_64_v2']: + raise InstallError('check.cpu_too_old') + # v3 is the faster build; v2 runs on CPUs without AVX2/FMA (runtime/fma.c supplies the fused multiply-add). + ctx.cpu = 'v3' if host['x86_64_v3'] else 'v2' if not host['loader']: raise InstallError('check.vulkan_missing') devices = host['devices'] @@ -572,11 +586,12 @@ class Compile(Step): id = 'compile' def key(self, ctx): - layout = ctx.layout + layout, cpu = ctx.layout, getattr(ctx, 'cpu', 'v3') clang = subprocess.run([str(layout.clang), '--version'], capture_output=True, text=True).stdout tools = [layout.tools / n for n in ('build_game.py', 'romfs_index.py', 'check_content.py')] - return fingerprint('compile-1', ctx.keys['lift'], layout.runtime, layout.runtime_lib, layout.deps or 'system', - layout.sysroot or 'host', *tools, clang, *[d.name for d in layout.dlls]) + return fingerprint('compile-2', cpu, layout.runtime_lib_for(cpu), ctx.keys['lift'], layout.runtime, + layout.deps or 'system', layout.sysroot or 'host', *tools, clang, + *[d.name for d in layout.dlls]) def current(self, ctx, record): exe = ctx.dest / ('totk' + EXE) @@ -600,10 +615,12 @@ class Compile(Step): rep.progress('compile', 'link', force=True) rep.progress('compile', 'compile', force=True) + cpu = getattr(ctx, 'cpu', 'v3') + lib = layout.runtime_lib_for(cpu) code, log = run_tool(ctx, 'compile', layout.tools / 'build_game.py', [ - '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, + '--gen', ctx.temp / 'gen', '--out', build, '--toolchain', layout.llvm, '--cpu', cpu, *(['--deps', layout.deps] if layout.deps else []), *(['--sysroot', layout.sysroot] if layout.sysroot else []), - '--runtime', layout.runtime, '--runtime-lib', layout.runtime_lib, '--ninja', layout.ninja, + '--runtime', layout.runtime, *(['--runtime-lib', lib] if lib else ()), '--ninja', layout.ninja, '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', '--profile', dest / 'profile'], parse) @@ -617,7 +634,7 @@ class Compile(Step): replace_file(build / 'romfs-index.bin', dest / 'romfs-index.bin') for dll in layout.dlls: replace_file(dll, dest / dll.name) - return {'exe_sha256': file_sha256(dest / exe.name)} + return {'exe_sha256': file_sha256(dest / exe.name), 'cpu': cpu} # ---- 5. prepare diff --git a/installer/text/en.json b/installer/text/en.json index 19073742..34b5229d 100644 --- a/installer/text/en.json +++ b/installer/text/en.json @@ -44,7 +44,7 @@ "check.toolchain_broken": "The bundled compiler does not work: {detail}", "check.glibc_old": "This system's C library is glibc {version}; the setup's compiler needs {need} or later (Ubuntu 22.04, Debian 12, Fedora 35, SteamOS 3 and newer have it). Updating the system fixes this.", "check.host_probe_failed": "The computer check did not run (exit code {code}).", - "check.cpu_no_avx2": "This processor lacks AVX2 (x86-64-v3), which the game needs. Intel Haswell (2013) or AMD Excavator / Zen and newer have it.", + "check.cpu_too_old": "This processor lacks x86-64-v2 (SSE4.2, POPCNT and CMPXCHG16B), the oldest level the game is built for. Intel Nehalem (2008) or AMD Bulldozer (2011) and newer have it.", "check.vulkan_missing": "No Vulkan driver is installed. Install the latest driver for your graphics card.", "check.vulkan_no_device": "Vulkan finds no graphics card. Install the latest driver for your graphics card.", "check.vulkan_too_old": "The game needs a graphics card with Vulkan 1.3; found: {devices}. Update the graphics driver.", diff --git a/installer/text/ru.json b/installer/text/ru.json index 235f72d6..3d602775 100644 --- a/installer/text/ru.json +++ b/installer/text/ru.json @@ -44,7 +44,7 @@ "check.toolchain_broken": "Встроенный компилятор не работает: {detail}", "check.glibc_old": "Библиотека C этой системы — glibc {version}; компилятору установщика нужна {need} или новее (она есть в Ubuntu 22.04, Debian 12, Fedora 35, SteamOS 3 и новее). Обновление системы это исправит.", "check.host_probe_failed": "Проверка компьютера не выполнилась (код выхода {code}).", - "check.cpu_no_avx2": "Этому процессору не хватает AVX2 (x86-64-v3), который нужен игре. AVX2 есть у Intel Haswell (2013) и новее, у AMD Excavator и Zen и новее.", + "check.cpu_too_old": "Этому процессору не хватает x86-64-v2 (SSE4.2, POPCNT и CMPXCHG16B) — минимального уровня, под который собрана игра. Он есть у Intel Nehalem (2008) и AMD Bulldozer (2011) и новее.", "check.vulkan_missing": "Драйвер Vulkan не установлен. Установите свежий драйвер видеокарты.", "check.vulkan_no_device": "Vulkan не находит видеокарту. Установите свежий драйвер видеокарты.", "check.vulkan_too_old": "Игре нужна видеокарта с Vulkan 1.3; найдено: {devices}. Обновите драйвер видеокарты.", diff --git a/runtime/cpu.h b/runtime/cpu.h index a3be39f4..207f4901 100644 --- a/runtime/cpu.h +++ b/runtime/cpu.h @@ -109,7 +109,7 @@ EXCL(16, uint16_t) EXCL(32, uint32_t) EXCL(64, uint64_t) -// Exclusive pair of two X registers: one 16-byte-aligned 128-bit access (cmpxchg16b; x86-64-v3 has CX16). +// Exclusive pair of two X registers: one 16-byte-aligned 128-bit access (cmpxchg16b; x86-64-v2 has CX16). // A pair of two W registers is a 64-bit exclusive and uses ldx64/stx64. static inline __uint128_t ldxp128(Cpu *c, uint64_t a) { __uint128_t v = __atomic_load_n((__uint128_t *)(uintptr_t)a, __ATOMIC_ACQUIRE); diff --git a/runtime/fma.c b/runtime/fma.c new file mode 100644 index 00000000..dbebdc12 --- /dev/null +++ b/runtime/fma.c @@ -0,0 +1,246 @@ +// Correctly rounded fma/fmaf for x86-64 builds without FMA3: the -march=x86-64-v2 build (docs/building.md). +// +// runtime/simd.h lowers AArch64 FMADD/FMSUB and the RECPS/RSQRTS helpers to __builtin_elementwise_fma, which clang +// turns into a call to fma/fmaf when the target has no FMA instruction. The C runtime is not a substitute: on CPUs +// without FMA3 the UCRT's software fallback is documented as rounding incorrectly by up to half an ulp, while the FP +// lowering needs the single rounding the ARM instruction gives. This file supplies it. +// +// Adapted from musl src/math/fma.c and src/math/fmaf.c (MIT, Copyright (c) 2005-2020 Rich Felker and contributors). +// The result is rounded in the MXCSR rounding mode the caller set (simd.h simd_rm_enter), and MXCSR FTZ and DAZ are +// honoured as the hardware FMA would, so runtime/simd.h's fast paths keep their assumptions: denormal inputs read as +// zero and denormal results as zero under FPCR.FZ, without a trip through the slow path. +// +// The definitions exist only where the compiler would emit the calls. A -march=x86-64-v3 build defines __FMA__ and +// emits vfmadd instead, so this file contributes nothing there. +#include + +#if defined(__x86_64__) && !defined(__FMA__) + +#include +#include + +#define MXCSR_DAZ 0x0040u +#define MXCSR_FTZ 0x8000u + +static double fma_flush_d(double x) { + union { double f; uint64_t i; } u = {x}; + if (!(u.i & 0x7ff0000000000000ull) && (u.i & 0x000fffffffffffffull)) u.i &= 0x8000000000000000ull; + return u.f; +} + +static float fma_flush_f(float x) { + union { float f; uint32_t i; } u = {x}; + if (!(u.i & 0x7f800000u) && (u.i & 0x007fffffu)) u.i &= 0x80000000u; + return u.f; +} + +// ---- float: the exact product fits in a double, so one extra term and a round-to-odd step decide the float +// rounding. Halfway and tiny results are the only ones where rounding the double first can change the float result. +float fmaf(float x, float y, float z) { + unsigned csr = _mm_getcsr(); + if (csr & MXCSR_DAZ) x = fma_flush_f(x), y = fma_flush_f(y), z = fma_flush_f(z); + double xy = (double)x * y; // exact + union { double r; uint64_t i; } u = {xy + z}; + int e = u.i >> 52 & 0x7ff; + int halfway = (u.i & 0x1fffffff) == 0x10000000; + int tiny = e <= 0x3ff - 126 && e >= 0x3ff - 149; + if (halfway || tiny) { + if (e != 0x7ff) { + // r+t is the exact sum in nearest rounding; otherwise t keeps its sign and whether it is zero, which + // is all the round-to-odd step needs. An odd sticky bit makes the final conversion round once, in + // whichever mode MXCSR asks for. + int s = u.i >> 63; + double t = s == (xy < z) ? xy - u.r + z : z - u.r + xy; + if (t) { + u.i -= s ^ (t < 0); + u.i |= 1; + } + } + } + float r = (float)u.r; + if (csr & MXCSR_FTZ) r = fma_flush_f(r); + return r; +} + +// ---- double: the significands are multiplied and aligned as integers, then converted back with one rounding. +#define ASUINT64(x) ((union { double f; uint64_t i; }){x}).i +#define ZEROINFNAN (0x7ff - 0x3ff - 52 - 1) + +struct fma_num { uint64_t m; int e; int sign; }; + +static struct fma_num fma_normalize(double x) { + uint64_t ix = ASUINT64(x); + int e = ix >> 52; + int sign = e & 0x800; + e &= 0x7ff; + if (!e) { + ix = ASUINT64(x * 0x1p63); + e = ix >> 52 & 0x7ff; + e = e ? e - 63 : 0x800; + } + ix &= (1ull << 52) - 1; + ix |= 1ull << 52; + ix <<= 1; + e -= 0x3ff + 52 + 1; + return (struct fma_num){ix, e, sign}; +} + +static void fma_mul(uint64_t *hi, uint64_t *lo, uint64_t x, uint64_t y) { + uint64_t xlo = (uint32_t)x, xhi = x >> 32; + uint64_t ylo = (uint32_t)y, yhi = y >> 32; + uint64_t t1 = xlo * ylo; + uint64_t t2 = xlo * yhi + xhi * ylo; + uint64_t t3 = xhi * yhi; + *lo = t1 + (t2 << 32); + *hi = t3 + (t2 >> 32) + (t1 > *lo); +} + +// scalbn from musl src/math/scalbn.c (MIT): the fma's final scaling, kept here so the result does not depend on the +// C runtime's version. Its multiplications honour MXCSR like any other host FP operation. +static double fma_scalbn(double x, int n) { + union { double f; uint64_t i; } u; + double y = x; + if (n > 1023) { + y *= 0x1p1023; + n -= 1023; + if (n > 1023) { + y *= 0x1p1023; + n -= 1023; + if (n > 1023) n = 1023; + } + } else if (n < -1022) { + // Make sure the final n < -53, so the subnormal range rounds only once. + y *= 0x1p-1022 * 0x1p53; + n += 1022 - 53; + if (n < -1022) { + y *= 0x1p-1022 * 0x1p53; + n += 1022 - 53; + if (n < -1022) n = -1022; + } + } + u.i = (uint64_t)(0x3ff + n) << 52; + return y * u.f; +} + +double fma(double x, double y, double z) { +#pragma STDC FENV_ACCESS ON + unsigned csr = _mm_getcsr(); + if (csr & MXCSR_DAZ) x = fma_flush_d(x), y = fma_flush_d(y), z = fma_flush_d(z); + + struct fma_num nx = fma_normalize(x), ny = fma_normalize(y), nz = fma_normalize(z); + if (nx.e >= ZEROINFNAN || ny.e >= ZEROINFNAN) return x * y + z; + if (nz.e >= ZEROINFNAN) { + if (nz.e > ZEROINFNAN) return x * y; // z == 0 + return z; + } + + uint64_t rhi, rlo, zhi, zlo; + fma_mul(&rhi, &rlo, nx.m, ny.m); + + // Align the exponents: shift z left by kz and r right by kr with kz + kr == d. + int e = nx.e + ny.e; + int d = nz.e - e; + if (d > 0) { + if (d < 64) { + zlo = nz.m << d; + zhi = nz.m >> (64 - d); + } else { + zlo = 0; + zhi = nz.m; + e = nz.e - 64; + d -= 64; + if (d == 0) { + } else if (d < 64) { + rlo = rhi << (64 - d) | rlo >> d | !!(rlo << (64 - d)); + rhi = rhi >> d; + } else { + rlo = 1; + rhi = 0; + } + } + } else { + zhi = 0; + d = -d; + if (d == 0) { + zlo = nz.m; + } else if (d < 64) { + zlo = nz.m >> d | !!(nz.m << (64 - d)); + } else { + zlo = 1; + } + } + + int sign = nx.sign ^ ny.sign; + int samesign = !(sign ^ nz.sign); + int nonzero = 1; + if (samesign) { + rlo += zlo; + rhi += zhi + (rlo < zlo); + } else { + uint64_t t = rlo; + rlo -= zlo; + rhi = rhi - zhi - (t < rlo); + if (rhi >> 63) { + rlo = -rlo; + rhi = -rhi - !!rlo; + sign = !sign; + } + nonzero = !!rhi; + } + + // Keep the top 63 bits in rhi; the last bit is sticky. + if (nonzero) { + e += 64; + d = __builtin_clzll(rhi) - 1; + rhi = rhi << d | rlo >> (64 - d) | !!(rlo << d); + } else if (rlo) { + d = __builtin_clzll(rlo) - 1; + if (d < 0) + rhi = rlo >> 1 | (rlo & 1); + else + rhi = rlo << d; + } else { + return x * y + z; // exact +-0 + } + e -= d; + + int64_t i = rhi; + if (sign) i = -i; + double r = i; // |r| is in [0x1p62, 0x1p63] + + if (e < -1022 - 62) { + // The result is subnormal before rounding. + if (e == -1022 - 63) { + double c = 0x1p63; + if (sign) c = -c; + if (r == c) { + // The smallest normal after rounding: whether it underflows depends on the architecture. A double + // to float conversion imitates the before-rounding rule the guest has. + float fltmin = 0x0.ffffff8p-63 * FLT_MIN * r; + r = DBL_MIN / FLT_MIN * fltmin; + return (csr & MXCSR_FTZ) ? fma_flush_d(r) : r; + } + // One bit is lost when scaled: add another top bit so the conversion rounds only once if inexact. + if (rhi << 53) { + i = rhi >> 1 | (rhi & 1) | 1ull << 62; + if (sign) i = -i; + r = i; + r = 2 * r - c; + // Raise underflow portably, so the compiler cannot optimize the expression away. + double tiny = DBL_MIN / FLT_MIN * r; + r += (double)(tiny * tiny) * (r - r); + } + } else { + // Only round once when scaled. + d = 10; + i = (rhi >> d | !!(rhi << (64 - d))) << d; + if (sign) i = -i; + r = i; + } + } + r = fma_scalbn(r, e); + if (csr & MXCSR_FTZ) r = fma_flush_d(r); + return r; +} + +#endif diff --git a/runtime/main.c b/runtime/main.c index 893a2764..e8abf5e4 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -249,11 +249,22 @@ static double now_s(void) { return (double)plat_now_ns() / 1e9; } static volatile int host_probe_hits; static void host_probe(Cpu *c) { host_probe_hits++; c->x[0] = 42; } +// The instruction set build_game.py --cpu compiled for: v3 (AVX2/FMA/BMI2, the default) or v2 (SSE4.2/POPCNT). +#ifndef RT_CPU_LEVEL +#define RT_CPU_LEVEL 3 +#endif + static void check_cpu(void) { __builtin_cpu_init(); +#if RT_CPU_LEVEL >= 3 int ok = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") && __builtin_cpu_supports("bmi2"); printf("cpu: x86-64-v3 (AVX2, FMA, BMI2) %s\n", ok ? "present" : "MISSING"); CHECK(ok, "the lifted code is built for x86-64-v3"); +#else + int ok = __builtin_cpu_supports("sse4.2") && __builtin_cpu_supports("popcnt"); + printf("cpu: x86-64-v2 (SSE4.2, POPCNT) %s\n", ok ? "present" : "MISSING"); + CHECK(ok, "the lifted code is built for x86-64-v2"); +#endif } // Layout: ascending, no overlap, below 2^39, and inside the window mem_init reserved. diff --git a/runtime/simd.h b/runtime/simd.h index 535cb106..d8fb45c8 100644 --- a/runtime/simd.h +++ b/runtime/simd.h @@ -7,7 +7,7 @@ // tables. Not modelled: FPSR cumulative exception bits and QC, FPCR trap enables, FP16 arithmetic (ARMv8.2). // FPCR.AHP is honoured by the half-precision conversions. Host assumptions: IEEE binary32/64, MXCSR in // round-to-nearest outside the slow paths, with FTZ|DAZ equal to FPCR.FZ (cpu.h rt_fpcr_sync keeps it so); fma must be -// correctly rounded (build x86 with -mfma). +// correctly rounded: -march=x86-64-v3 has the FMA instruction, and the v2 build links runtime/fma.c for it. #pragma once #include #include diff --git a/tests/diff/common.py b/tests/diff/common.py index 648e280c..c2f796df 100644 --- a/tests/diff/common.py +++ b/tests/diff/common.py @@ -213,7 +213,8 @@ def build_harness(work, bodies, jobs, cflags): f.write(''.join(f'void t{k}(Cpu *c);\n' for k in range(len(bodies)))) f.write('GuestFn tests[] = {' + ','.join(f't{k}' for k in range(len(bodies))) + '};\n') f.write(f'int ntests = {len(bodies)};\n') - srcs += [work / 'table.c', ROOT / 'tests' / 'diff' / 'harness.c'] + # fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's. + srcs += [work / 'table.c', ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c'] objs, procs = [], [] for s in srcs: o = work / (s.stem + '.o') diff --git a/tests/diff/func.py b/tests/diff/func.py index 25a5d0bc..62c34068 100644 --- a/tests/diff/func.py +++ b/tests/diff/func.py @@ -204,7 +204,8 @@ def build(work, emitted, jobs, cflags): f'int ntests = {len(emitted)};\n' + 'void rt_stub_call(Cpu *c, uint64_t target);\n' + ''.join(f'void {gen.fname(t)}(Cpu *c) {{ rt_stub_call(c, {hex(t)}ull); }}\n' for t in sorted(called))) - srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c'] + # fma.c: a --cflags=-march=x86-64-v2 run must exercise the software fma the v2 build links, not the C runtime's. + srcs += [table, ROOT / 'tests' / 'diff' / 'harness.c', ROOT / 'runtime' / 'fma.c'] objs, procs = [], [] for s in srcs: o = work / (s.stem + '.o') diff --git a/tools/build_game.py b/tools/build_game.py index 8d88d01c..faa58469 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -92,9 +92,20 @@ NINJA = UCRT / 'ninja.exe' if WIN else on_path('ninja') SDL3_DLLS = [UCRT / 'SDL3.dll', UCRT / 'libiconv-2.dll', UCRT / 'libopus-0.dll'] if WIN else [] # Flags for generated code as LAS Recomp builds it (Makefile GENFLAGS, Windows), plus -march for the FP/SIMD lowering. +# --cpu v2 replaces the level for CPUs without AVX2/FMA; runtime/fma.c then supplies the correctly rounded fma the +# lowering needs (runtime/simd.h). GEN_FLAGS = ['-O1', '-fno-strict-aliasing', '-Wno-return-type', '-Wno-unused-label', '-fvisibility=hidden', '-march=x86-64-v3'] RT_FLAGS = ['-O2', '-g', '-Wall', '-fno-strict-aliasing', '-march=x86-64-v3'] + + +def set_cpu(cpu): + """Target x86-64-v3 (the default) or x86-64-v2. RT_CPU_LEVEL lets runtime/main.c's self-test check the same + instruction set the build was compiled for.""" + march = f'-march=x86-64-{cpu}' + GEN_FLAGS[GEN_FLAGS.index('-march=x86-64-v3')] = march + RT_FLAGS[RT_FLAGS.index('-march=x86-64-v3')] = march + RT_FLAGS.append(f'-DRT_CPU_LEVEL={3 if cpu == "v3" else 2}') if WIN: # _WIN32_WINNT: Windows 10, the port's minimum. MSYS2's headers default to it, llvm-mingw's (the release toolchain) # to Windows 7, which hides the per-monitor DPI functions gfx_window_win.c and dpi_probe_win.c call. @@ -698,6 +709,8 @@ def mib(n): def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--version', default='v121', help='game version (default v121)') + ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3', + help='target CPU level: v3 (AVX2/FMA, default) or v2 (SSE4.2, for older CPUs; slower)') ap.add_argument('--gen', type=Path, help='generated C directory (default generated/)') ap.add_argument('--out', type=Path, help='build directory (default build/)') ap.add_argument('--modules', help='comma-separated subset of the layout modules (default all)') @@ -732,6 +745,7 @@ def main(): ap.add_argument('--expected', type=Path, help='expected hashes (default tools/data/expected-.json)') args = ap.parse_args() + set_cpu(args.cpu) gen = (args.gen or ROOT / 'generated' / args.version).resolve() out = (args.out or ROOT / 'build' / args.version).resolve() diff --git a/tools/make_release.py b/tools/make_release.py index 6e57caa8..914f7421 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -10,8 +10,9 @@ cached in toolchain/release-dl/ and checked against the sha256 in PINS; a mismat files come out of the python.org installer's own component MSI, unpacked with msiexec /a and expand.exe; the installer is never run) llvm/ llvm-mingw, x86_64 only, the files clang and lld need to compile and link totk.exe - runtime/ libtotkrt.a (build_game.py --make-runtime-lib with llvm/), the runtime headers, romfs.c, patches.c and - ov_zstd.c (compiled per install), deps/ (headers and import libraries of SDL3, Opus, libvpx, Vulkan and VMA) + runtime/ libtotkrt.a and libtotkrt-v2.a (build_game.py --make-runtime-lib with llvm/, x86-64-v3 and v2), the + runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), deps/ (headers and import + libraries of SDL3, Opus, libvpx, Vulkan and VMA) bin/ totk_xlate.exe (+ .id), hactool.exe, SDL3.dll (upstream), libopus-0.dll, ninja.exe recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, totk.ini.example VERSION the repository's VERSION plus the commit, 0.1.0+973c626; the installer's release string @@ -342,15 +343,18 @@ def lifted_version(): def runtime_lib(out, llvm): - """Build libtotkrt.a with the folder's llvm/ and copy it, the headers and the install-side sources into runtime/.""" - build = MAIN_ROOT / 'build' / 'release-check' / 'rtlib' + """Build libtotkrt.a (x86-64-v3) and libtotkrt-v2.a (x86-64-v2) with the folder's llvm/, and copy them, the headers + and the install-side sources into runtime/.""" version = lifted_version() gen = MAIN_ROOT / 'generated' / version - run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--version', version, - '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', '--gen', gen, '--out', build], - env=clean_env(llvm / 'bin', out / 'bin')) rt = out / 'runtime' - shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a') + for cpu, name in (('v3', 'libtotkrt.a'), ('v2', 'libtotkrt-v2.a')): + build = MAIN_ROOT / 'build' / 'release-check' / f'rtlib-{cpu}' + run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--cpu', cpu, + '--version', version, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja.exe', + '--gen', gen, '--out', build], + env=clean_env(llvm / 'bin', out / 'bin')) + shutil.copy2(build / 'libtotkrt.a', rt / name) for p in sorted((ROOT / 'runtime').rglob('*')): if p.suffix in RUNTIME_SUFFIXES and p.is_file(): (rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True) @@ -477,7 +481,7 @@ def write_zip(out, path): print(f'{path}: {mib(path.stat().st_size)}') -def prove(out): +def prove(out, cpu='v3'): """Compile the generated C and link totk.exe with the folder's tools only, then run the probes next to it.""" out = out.resolve() version = lifted_version() @@ -486,8 +490,9 @@ def prove(out): chk = MAIN_ROOT / 'build' / 'release-check' env = clean_env(out / 'python', out / 'llvm' / 'bin', out / 'bin') t0 = time.time() - run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', - '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', '--runtime-lib', out / 'runtime' / 'libtotkrt.a', + run([out / 'python' / 'python.exe', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu, + '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', + '--runtime-lib', out / 'runtime' / ('libtotkrt-v2.a' if cpu == 'v2' else 'libtotkrt.a'), '--ninja', out / 'bin' / 'ninja.exe', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'], env=env) print(f'prove: build {time.time() - t0:.0f} s') @@ -510,10 +515,12 @@ def main(): ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)') ap.add_argument('--zip', type=Path, help='also write the folder as this zip') ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder') + ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3', + help='--prove: build and run the check for this CPU level (default v3)') args = ap.parse_args() out = args.out.resolve() if args.prove: - prove(out) + prove(out, args.cpu) return if out.exists(): shutil.rmtree(out) diff --git a/tools/make_release_linux.py b/tools/make_release_linux.py index 546c761e..c98af2cf 100644 --- a/tools/make_release_linux.py +++ b/tools/make_release_linux.py @@ -14,9 +14,9 @@ the .NET 10 SDK and Python 3.12+ with Pillow (.venv). The folder holds: file puts the folder, tools/ and recomp/ on sys.path llvm/ LLVM 23 (clang, lld, llvm-ar, the clang resource headers) and llvm/sysroot: the files of the sniper SDK sysroot the game's compile and link read, nothing else (recorded while building the runtime library) - runtime/ libtotkrt.a (build_game.py --make-runtime-lib --sysroot), the runtime headers, romfs.c, patches.c and - ov_zstd.c (compiled per install), totk.png (the window icon), deps/ (Vulkan headers 1.4.363, VMA, zstd - built static; SDL3, Opus and libvpx come from the sysroot) + runtime/ libtotkrt.a and libtotkrt-v2.a (build_game.py --make-runtime-lib --sysroot, x86-64-v3 and v2), the + runtime headers, romfs.c, patches.c and ov_zstd.c (compiled per install), totk.png (the window icon), + deps/ (Vulkan headers 1.4.363, VMA, zstd built static; SDL3, Opus and libvpx come from the sysroot) bin/ totk_xlate (+ .id), hactool, ninja, libSDL3.so.0 (the sysroot's; the game finds it beside itself) recomp/, tools/, data/, installer/, licenses/, THIRD_PARTY_NOTICES.md, totk.ini.example, VERSION AppRun, totk-setup.desktop, totk-setup.png the AppImage entry: python/bin/python3 -m installer.gui @@ -197,14 +197,16 @@ def build_bin(out, llvm, sysroot): # ---- runtime/ ---------------------------------------------------------------------------------------------------------- def runtime_lib(out, llvm, sysroot): - build = CHECK / 'rtlib' - shutil.rmtree(build, ignore_errors=True) + """libtotkrt.a (x86-64-v3) and libtotkrt-v2.a (x86-64-v2), so a machine without AVX2 can install too.""" version = mr.lifted_version() - run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, '--sysroot', sysroot, - '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja', '--version', version, - '--gen', MAIN_ROOT / 'generated' / version, '--out', build]) rt = out / 'runtime' - shutil.copy2(build / 'libtotkrt.a', rt / 'libtotkrt.a') + for cpu, name in (('v3', 'libtotkrt.a'), ('v2', 'libtotkrt-v2.a')): + build = CHECK / f'rtlib-{cpu}' + shutil.rmtree(build, ignore_errors=True) + run([sys.executable, ROOT / 'tools' / 'build_game.py', '--make-runtime-lib', '--toolchain', llvm, + '--sysroot', sysroot, '--cpu', cpu, '--deps', out / 'runtime' / 'deps', '--ninja', out / 'bin' / 'ninja', + '--version', version, '--gen', MAIN_ROOT / 'generated' / version, '--out', build]) + shutil.copy2(build / 'libtotkrt.a', rt / name) for p in sorted((ROOT / 'runtime').rglob('*')): if p.suffix in mr.RUNTIME_SUFFIXES and p.is_file(): (rt / p.relative_to(ROOT / 'runtime')).parent.mkdir(parents=True, exist_ok=True) @@ -459,7 +461,7 @@ def check_bundle(out): print('bundle check: the window built every screen in both languages and closed') -def prove(out): +def prove(out, cpu='v3'): """Compile the generated C and link totk with the folder's tools only, then run the probes beside it.""" out = out.resolve() version = mr.lifted_version() @@ -468,9 +470,10 @@ def prove(out): chk = CHECK / 'game' env = dict(os.environ, PATH=os.pathsep.join([str(out / 'llvm' / 'bin'), str(out / 'bin'), '/usr/bin', '/bin'])) t0 = time.time() - run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', + run([out / 'python' / 'bin' / 'python3', out / 'tools' / 'build_game.py', '--toolchain', out / 'llvm', '--cpu', cpu, '--sysroot', out / 'llvm' / 'sysroot', '--deps', out / 'runtime' / 'deps', '--runtime', out / 'runtime', - '--runtime-lib', out / 'runtime' / 'libtotkrt.a', '--ninja', out / 'bin' / 'ninja', '--version', version, + '--runtime-lib', out / 'runtime' / ('libtotkrt-v2.a' if cpu == 'v2' else 'libtotkrt.a'), + '--ninja', out / 'bin' / 'ninja', '--version', version, '--gen', gen, '--out', chk, '--exefs', ext / 'exefs', '--romfs', ext / 'romfs', '--profile', chk / 'profile'], env=env) print(f'prove: build {time.time() - t0:.0f} s') @@ -497,10 +500,12 @@ def main(): ap.add_argument('--out', type=Path, required=True, help='release folder (replaced)') ap.add_argument('--appimage', type=Path, help='also pack the folder as this AppImage') ap.add_argument('--prove', action='store_true', help='only run the compile-and-probe check on an existing folder') + ap.add_argument('--cpu', choices=('v2', 'v3'), default='v3', + help='--prove: build and run the check for this CPU level (default v3)') args = ap.parse_args() out = args.out.resolve() if args.prove: - prove(out) + prove(out, args.cpu) return if out.exists(): shutil.rmtree(out)