mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 04:15:57 +00:00
The setup's builds name every path relative to the build kit, so a moved game folder rebuilds incrementally (#845)
tools/build_game.py --portable writes build.ninja paths on the tools' drive relative to the kit, also through '..' (the generated C and objects in build-cache/work), names the compiler, windres and ar the same way (backslashes: CreateProcess does not find a relative program with forward slashes), and passes clang -resource-dir and --sysroot relative to the kit, because clang names its own headers by its absolute folder whatever argv[0] says. The Ninja deps log then holds no absolute path. installer/steps.py build_args passes --portable for the build kit. Measured on an install moved after a portable build: the rebuild compiled modules.c only, 3.5 s; before, the move cost a full compile of 430 objects (335 s on 12 cores). Without --portable build.ninja is byte-identical to before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8c9c18a6d2
commit
e7c901b864
5 files changed
+61
-13
No files matched your search
+7
-5
@@ -142,11 +142,13 @@ cheats or ExeFS patch files the build lacks, `totk.exe` runs it at start and exi
|
||||
failed or cancelled rebuild offers to start the game with the code it has; `build-cache/rebuild.skip` then keeps the
|
||||
same files from starting it again, until "Rebuild the game now" on the Cheats or Mods tab. A command that does not start
|
||||
at all leaves no `rebuild.skip`, so the next start tries it again. `[mods] mods_rebuild = off` turns it off. A moved
|
||||
install folder keeps working, but its first rebuild compiles everything once: Ninja's records hold the old absolute
|
||||
paths of the generated C and of the compiler's own headers. On Linux the command starts the AppImage file itself
|
||||
(`$APPIMAGE --rebuild <install folder>`), because the AppImage's mounted folder is gone once the setup exits; a moved or
|
||||
deleted AppImage means running a setup again. Installs set up by an AppImage before this change hold a command into that
|
||||
mount: running the new setup once on the install folder writes the new one.
|
||||
install folder keeps its incremental rebuild: the kit's builds run `tools/build_game.py --portable`, whose Ninja file
|
||||
and deps log name every path relative to the kit (3.5 s to rebuild after a move instead of a full compile). A build with
|
||||
a mod plugin names the plugin's file by its absolute path (`recomp/layout.py`), so after a move the game finds it gone
|
||||
and runs the rebuild once. On Linux the command starts the AppImage file itself (`$APPIMAGE --rebuild <install
|
||||
folder>`), because the AppImage's mounted folder is gone once the setup exits; a moved or deleted AppImage means running
|
||||
a setup again. Installs set up by an AppImage before this change hold a command into that mount: running the new setup
|
||||
once on the install folder writes the new one.
|
||||
|
||||
## Command line
|
||||
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ ahead of time, so these patches are built into `totk.exe`:
|
||||
3. After the rebuild the mod switches on and off from the Mods tab like any other, without another rebuild.
|
||||
|
||||
The rebuild runs from `build-kit` in the game folder, which the setup put there: the setup folder can be deleted
|
||||
and the game folder moved. The first rebuild after a move compiles the whole game once (several minutes). A game
|
||||
and the game folder moved. After a move, a game with a code plugin mod rebuilds once at its next start. A game
|
||||
installed by an older setup (no `build-kit` in its folder) still rebuilds from that setup folder until a newer
|
||||
setup runs once on it.
|
||||
`mods_rebuild = off` under `[mods]` in `totk.ini` turns the automatic rebuild off.
|
||||
|
||||
+3
-1
@@ -1191,7 +1191,9 @@ def build_args(ctx, gen):
|
||||
'--ninja', layout.ninja,
|
||||
'--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)),
|
||||
'--exefs', dest / 'exefs', '--romfs', dest / 'romfs',
|
||||
'--profile', dest / 'profile']
|
||||
'--profile', dest / 'profile',
|
||||
# the build kit and the temp folder move with the install folder: Ninja's paths relative to the kit
|
||||
*(['--portable'] if layouts.uses_kit(layout) else [])]
|
||||
|
||||
|
||||
class Compile(Step):
|
||||
|
||||
+11
-2
@@ -327,7 +327,7 @@ class LiftRunTest(unittest.TestCase):
|
||||
self.dest, temp = base / 'install', base / 'temp'
|
||||
self.dest.mkdir()
|
||||
temp.mkdir()
|
||||
layout = SimpleNamespace(recomp=ROOT / 'recomp', tools=ROOT / 'tools',
|
||||
layout = SimpleNamespace(dev=True, recomp=ROOT / 'recomp', tools=ROOT / 'tools',
|
||||
patches=lambda version: base / 'patches-v143.txt')
|
||||
self.ctx = SimpleNamespace(dest=self.dest, temp=temp, layout=layout, keys={'lift': 'k'},
|
||||
state={'version': '1.4.3'}, reporter=Reporter(out=io.StringIO()))
|
||||
@@ -478,7 +478,7 @@ class CompileLockedTest(unittest.TestCase):
|
||||
dest, temp = base / 'install', base / 'temp'
|
||||
dest.mkdir()
|
||||
temp.mkdir()
|
||||
layout = SimpleNamespace(tools=ROOT / 'tools', llvm=base / 'llvm', deps=None, sysroot=None, runtime=base,
|
||||
layout = SimpleNamespace(dev=True, tools=ROOT / 'tools', llvm=base / 'llvm', deps=None, sysroot=None, runtime=base,
|
||||
ninja=base / 'ninja', dlls=[], runtime_lib_for=lambda cpu: None,
|
||||
pgo=lambda version: None, expected=lambda version: base / 'expected.json')
|
||||
self.ctx = SimpleNamespace(dest=dest, temp=temp, layout=layout, state={'version': '1.4.3'},
|
||||
@@ -715,6 +715,15 @@ class KitTest(unittest.TestCase):
|
||||
'build-kit', '--root', 'build-kit', '--rebuild', '.'])
|
||||
self.assertFalse(any(str(self.dest) in a or str(self.setup) in a for a in cmd))
|
||||
|
||||
def test_the_kits_builds_are_portable(self):
|
||||
ctx, _ = self.install()
|
||||
ctx.temp, ctx.state = self.dest / 'build-cache' / 'work', {'version': '1.4.3'}
|
||||
self.assertIn('--portable', steps.build_args(ctx, ctx.temp / 'gen'))
|
||||
ctx.layout = Layout(root=ROOT, dev=True, release='x', python=Path('python'), tools=Path('tools'),
|
||||
recomp=Path('recomp'), llvm=Path('llvm'), deps=None, runtime=Path('rt'),
|
||||
runtime_lib=Path('lib'), ninja=Path('ninja'), hactool=Path('hactool'), xlate=Path('xlate'))
|
||||
self.assertNotIn('--portable', steps.build_args(ctx, ctx.temp / 'gen'))
|
||||
|
||||
def test_dev_mode_has_no_kit(self):
|
||||
ctx = self.ctx(SimpleNamespace(dev=True, release='x', root=ROOT))
|
||||
kit = steps.Kit()
|
||||
|
||||
+39
-4
@@ -44,6 +44,9 @@ big machine does not run out of memory on the largest chunks. Each phase reports
|
||||
of its processes (Windows job object accounting: committed private bytes).
|
||||
|
||||
--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used.
|
||||
--portable (the setup's builds, installer/steps.py build_args) writes every path on the tools' drive relative to the
|
||||
folder above tools/, through '..' where needed, so the install folder that holds the build kit, the generated C and the
|
||||
build folder can move and the next build stays incremental (PORTABLE).
|
||||
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
|
||||
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs copied next to
|
||||
@@ -117,6 +120,13 @@ def set_cpu(cpu):
|
||||
# keeps the names equal between the training build and a build from another tree (a worktree, an install).
|
||||
PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false']
|
||||
PGO_INPUT = None # the .profdata of --pgo FILE: every compile depends on it, so a new profile rebuilds them
|
||||
# --portable (the setup's builds in an install folder, installer/steps.py build_args): build.ninja names every path on
|
||||
# ROOT's drive relative to ROOT, the compiler too. clang names its own headers by its absolute folder whatever argv[0]
|
||||
# says, so with --toolchain the compiles also get -resource-dir and --sysroot relative to ROOT (the folders clang would
|
||||
# take itself); the Ninja deps log then holds no absolute path. Ninja runs in ROOT, so a folder that holds ROOT, the
|
||||
# generated C and the build folder (the install folder with build-kit/ and build-cache/work/) can move without a full
|
||||
# recompile.
|
||||
PORTABLE = False
|
||||
|
||||
|
||||
def set_pgo(pgo, out):
|
||||
@@ -210,14 +220,27 @@ MEM_BUDGET = 50 * GiB # total for all compile jobs
|
||||
|
||||
def below_root(p):
|
||||
"""Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else
|
||||
absolute."""
|
||||
absolute; with --portable relative to ROOT through '..' when it is on ROOT's drive."""
|
||||
p = Path(p).resolve()
|
||||
for root in (ROOT, MAIN_ROOT):
|
||||
if p.is_relative_to(root):
|
||||
return p.relative_to(root)
|
||||
if PORTABLE:
|
||||
try:
|
||||
return Path(os.path.relpath(p, ROOT))
|
||||
except ValueError: # another drive
|
||||
pass
|
||||
return p
|
||||
|
||||
|
||||
def tool_arg(p):
|
||||
"""A program for a Ninja command line: relative like the other paths with --portable (with backslashes on Windows:
|
||||
CreateProcess does not find a relative program named with forward slashes), else absolute."""
|
||||
if not PORTABLE:
|
||||
return cmd_arg(p.as_posix())
|
||||
return cmd_arg(str(below_root(p)) if WIN else below_root(p).as_posix())
|
||||
|
||||
|
||||
def ninja_path(p):
|
||||
"""A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped."""
|
||||
return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:')
|
||||
@@ -434,7 +457,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
'# Written by tools/build_game.py; do not edit.',
|
||||
'ninja_required_version = 1.10',
|
||||
f'builddir = {ninja_path(out)}',
|
||||
f'cc = {cache}{cmd_arg(CLANG.as_posix())}',
|
||||
f'cc = {cache}{tool_arg(CLANG)}',
|
||||
f'genflags = {" ".join(GEN_FLAGS)} {inc}',
|
||||
f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}',
|
||||
f'ldflags = {" ".join(LINK_FLAGS)}',
|
||||
@@ -456,7 +479,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
' description = CC $in',
|
||||
'',
|
||||
'rule rc',
|
||||
f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
|
||||
f' command = {tool_arg(WINDRES)} -I {cmd_path(RUNTIME)} -O coff $in -o $out',
|
||||
' description = RC $in',
|
||||
'',
|
||||
'rule link',
|
||||
@@ -498,7 +521,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib=
|
||||
exe = out / EXE
|
||||
if make_lib:
|
||||
exe = out / LIB_NAME
|
||||
lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
|
||||
lines += ['', 'rule ar', f' command = {tool_arg(AR)} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every),
|
||||
'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', '']
|
||||
return '\n'.join(lines), objs, exe
|
||||
rule = 'link'
|
||||
@@ -862,8 +885,14 @@ def main():
|
||||
help='compile the generated code at -O2 without a profile (the --pgo level), for A/B builds')
|
||||
ap.add_argument('--expected', type=Path,
|
||||
help='expected hashes (default tools/data/expected-<display version>.json)')
|
||||
ap.add_argument('--portable', action='store_true',
|
||||
help='paths in build.ninja relative to the tools\' folder, also through .. (an install folder that '
|
||||
'moves with its build kit and build folder)')
|
||||
args = ap.parse_args()
|
||||
set_cpu(args.cpu)
|
||||
if args.portable:
|
||||
global PORTABLE
|
||||
PORTABLE = True
|
||||
|
||||
gen = (args.gen or ROOT / 'generated' / args.version).resolve()
|
||||
out = (args.out or ROOT / 'build' / args.version).resolve()
|
||||
@@ -902,6 +931,12 @@ def main():
|
||||
CLANG = TOOLCHAIN_BIN / f'clang{suffix}'
|
||||
AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}'
|
||||
WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe'
|
||||
if PORTABLE and WIN:
|
||||
resource = sorted((TOOLCHAIN_BIN.parent / 'lib' / 'clang').glob('*'))
|
||||
if len(resource) != 1:
|
||||
sys.exit(f'--portable: {TOOLCHAIN_BIN.parent / "lib" / "clang"} holds {len(resource)} version folders')
|
||||
for flags in (GEN_FLAGS, RT_FLAGS):
|
||||
flags += ['-resource-dir', cmd_path(resource[0]), f'--sysroot={cmd_path(TOOLCHAIN_BIN.parent)}']
|
||||
if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw
|
||||
DEPS = (args.deps or DEFAULT_DEPS).resolve()
|
||||
dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []
|
||||
|
||||
Reference in new issue
Block a user