diff --git a/docs/installer.md b/docs/installer.md index d828f898..3fcad579 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -142,11 +142,13 @@ cheats or ExeFS patch files the build lacks, `totk.exe` runs it at start and exi failed or cancelled rebuild offers to start the game with the code it has; `build-cache/rebuild.skip` then keeps the same files from starting it again, until "Rebuild the game now" on the Cheats or Mods tab. A command that does not start at all leaves no `rebuild.skip`, so the next start tries it again. `[mods] mods_rebuild = off` turns it off. A moved -install folder keeps working, but its first rebuild compiles everything once: Ninja's records hold the old absolute -paths of the generated C and of the compiler's own headers. On Linux the command starts the AppImage file itself -(`$APPIMAGE --rebuild `), because the AppImage's mounted folder is gone once the setup exits; a moved or -deleted AppImage means running a setup again. Installs set up by an AppImage before this change hold a command into that -mount: running the new setup once on the install folder writes the new one. +install folder keeps its incremental rebuild: the kit's builds run `tools/build_game.py --portable`, whose Ninja file +and deps log name every path relative to the kit (3.5 s to rebuild after a move instead of a full compile). A build with +a mod plugin names the plugin's file by its absolute path (`recomp/layout.py`), so after a move the game finds it gone +and runs the rebuild once. On Linux the command starts the AppImage file itself (`$APPIMAGE --rebuild `), because the AppImage's mounted folder is gone once the setup exits; a moved or deleted AppImage means running +a setup again. Installs set up by an AppImage before this change hold a command into that mount: running the new setup +once on the install folder writes the new one. ## Command line diff --git a/docs/mods.md b/docs/mods.md index 285ad345..1fd24061 100644 --- a/docs/mods.md +++ b/docs/mods.md @@ -81,7 +81,7 @@ ahead of time, so these patches are built into `totk.exe`: 3. After the rebuild the mod switches on and off from the Mods tab like any other, without another rebuild. The rebuild runs from `build-kit` in the game folder, which the setup put there: the setup folder can be deleted -and the game folder moved. The first rebuild after a move compiles the whole game once (several minutes). A game +and the game folder moved. After a move, a game with a code plugin mod rebuilds once at its next start. A game installed by an older setup (no `build-kit` in its folder) still rebuilds from that setup folder until a newer setup runs once on it. `mods_rebuild = off` under `[mods]` in `totk.ini` turns the automatic rebuild off. diff --git a/installer/steps.py b/installer/steps.py index f044216f..c67c95b6 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -1191,7 +1191,9 @@ def build_args(ctx, gen): '--ninja', layout.ninja, '--version', version_tag(game_version(ctx)), '--expected', layout.expected(game_version(ctx)), '--exefs', dest / 'exefs', '--romfs', dest / 'romfs', - '--profile', dest / 'profile'] + '--profile', dest / 'profile', + # the build kit and the temp folder move with the install folder: Ninja's paths relative to the kit + *(['--portable'] if layouts.uses_kit(layout) else [])] class Compile(Step): diff --git a/tests/installer_test.py b/tests/installer_test.py index ea23c2fc..638125d0 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -327,7 +327,7 @@ class LiftRunTest(unittest.TestCase): self.dest, temp = base / 'install', base / 'temp' self.dest.mkdir() temp.mkdir() - layout = SimpleNamespace(recomp=ROOT / 'recomp', tools=ROOT / 'tools', + layout = SimpleNamespace(dev=True, recomp=ROOT / 'recomp', tools=ROOT / 'tools', patches=lambda version: base / 'patches-v143.txt') self.ctx = SimpleNamespace(dest=self.dest, temp=temp, layout=layout, keys={'lift': 'k'}, state={'version': '1.4.3'}, reporter=Reporter(out=io.StringIO())) @@ -478,7 +478,7 @@ class CompileLockedTest(unittest.TestCase): dest, temp = base / 'install', base / 'temp' dest.mkdir() temp.mkdir() - layout = SimpleNamespace(tools=ROOT / 'tools', llvm=base / 'llvm', deps=None, sysroot=None, runtime=base, + layout = SimpleNamespace(dev=True, tools=ROOT / 'tools', llvm=base / 'llvm', deps=None, sysroot=None, runtime=base, ninja=base / 'ninja', dlls=[], runtime_lib_for=lambda cpu: None, pgo=lambda version: None, expected=lambda version: base / 'expected.json') self.ctx = SimpleNamespace(dest=dest, temp=temp, layout=layout, state={'version': '1.4.3'}, @@ -715,6 +715,15 @@ class KitTest(unittest.TestCase): 'build-kit', '--root', 'build-kit', '--rebuild', '.']) self.assertFalse(any(str(self.dest) in a or str(self.setup) in a for a in cmd)) + def test_the_kits_builds_are_portable(self): + ctx, _ = self.install() + ctx.temp, ctx.state = self.dest / 'build-cache' / 'work', {'version': '1.4.3'} + self.assertIn('--portable', steps.build_args(ctx, ctx.temp / 'gen')) + ctx.layout = Layout(root=ROOT, dev=True, release='x', python=Path('python'), tools=Path('tools'), + recomp=Path('recomp'), llvm=Path('llvm'), deps=None, runtime=Path('rt'), + runtime_lib=Path('lib'), ninja=Path('ninja'), hactool=Path('hactool'), xlate=Path('xlate')) + self.assertNotIn('--portable', steps.build_args(ctx, ctx.temp / 'gen')) + def test_dev_mode_has_no_kit(self): ctx = self.ctx(SimpleNamespace(dev=True, release='x', root=ROOT)) kit = steps.Kit() diff --git a/tools/build_game.py b/tools/build_game.py index 38b356e8..66e7d5bf 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -44,6 +44,9 @@ big machine does not run out of memory on the largest chunks. Each phase reports of its processes (Windows job object accounting: committed private bytes). --ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used. +--portable (the setup's builds, installer/steps.py build_args) writes every path on the tools' drive relative to the +folder above tools/, through '..' where needed, so the install folder that holds the build kit, the generated C and the +build folder can move and the next build stays incremental (PORTABLE). --toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR (default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs copied next to @@ -117,6 +120,13 @@ def set_cpu(cpu): # keeps the names equal between the training build and a build from another tree (a worktree, an install). PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false'] PGO_INPUT = None # the .profdata of --pgo FILE: every compile depends on it, so a new profile rebuilds them +# --portable (the setup's builds in an install folder, installer/steps.py build_args): build.ninja names every path on +# ROOT's drive relative to ROOT, the compiler too. clang names its own headers by its absolute folder whatever argv[0] +# says, so with --toolchain the compiles also get -resource-dir and --sysroot relative to ROOT (the folders clang would +# take itself); the Ninja deps log then holds no absolute path. Ninja runs in ROOT, so a folder that holds ROOT, the +# generated C and the build folder (the install folder with build-kit/ and build-cache/work/) can move without a full +# recompile. +PORTABLE = False def set_pgo(pgo, out): @@ -210,14 +220,27 @@ MEM_BUDGET = 50 * GiB # total for all compile jobs def below_root(p): """Resolved p relative to ROOT, else to MAIN_ROOT (the same relative path through the worktree's junctions), else - absolute.""" + absolute; with --portable relative to ROOT through '..' when it is on ROOT's drive.""" p = Path(p).resolve() for root in (ROOT, MAIN_ROOT): if p.is_relative_to(root): return p.relative_to(root) + if PORTABLE: + try: + return Path(os.path.relpath(p, ROOT)) + except ValueError: # another drive + pass return p +def tool_arg(p): + """A program for a Ninja command line: relative like the other paths with --portable (with backslashes on Windows: + CreateProcess does not find a relative program named with forward slashes), else absolute.""" + if not PORTABLE: + return cmd_arg(p.as_posix()) + return cmd_arg(str(below_root(p)) if WIN else below_root(p).as_posix()) + + def ninja_path(p): """A path for a build line: relative to the repo root where possible, forward slashes, Ninja-escaped.""" return below_root(p).as_posix().replace('$', '$$').replace(' ', '$ ').replace(':', '$:') @@ -434,7 +457,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= '# Written by tools/build_game.py; do not edit.', 'ninja_required_version = 1.10', f'builddir = {ninja_path(out)}', - f'cc = {cache}{cmd_arg(CLANG.as_posix())}', + f'cc = {cache}{tool_arg(CLANG)}', f'genflags = {" ".join(GEN_FLAGS)} {inc}', f'rtflags = {" ".join(RT_FLAGS)} {rt_inc}', f'ldflags = {" ".join(LINK_FLAGS)}', @@ -456,7 +479,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= ' description = CC $in', '', 'rule rc', - f' command = {cmd_arg(WINDRES.as_posix())} -I {cmd_path(RUNTIME)} -O coff $in -o $out', + f' command = {tool_arg(WINDRES)} -I {cmd_path(RUNTIME)} -O coff $in -o $out', ' description = RC $in', '', 'rule link', @@ -498,7 +521,7 @@ def ninja_file(gen, out, mods, pool_depth, skip=(), prebuilt=False, runtime_lib= exe = out / EXE if make_lib: exe = out / LIB_NAME - lines += ['', 'rule ar', f' command = {cmd_arg(AR.as_posix())} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), + lines += ['', 'rule ar', f' command = {tool_arg(AR)} rcs $out $in', ' description = AR $out', '', f'build {ninja_path(exe)}: ar ' + ' '.join(ninja_path(o) for o in every), 'build objects: phony ' + ' '.join(ninja_path(o) for o in every), f'default {ninja_path(exe)}', ''] return '\n'.join(lines), objs, exe rule = 'link' @@ -862,8 +885,14 @@ def main(): help='compile the generated code at -O2 without a profile (the --pgo level), for A/B builds') ap.add_argument('--expected', type=Path, help='expected hashes (default tools/data/expected-.json)') + ap.add_argument('--portable', action='store_true', + help='paths in build.ninja relative to the tools\' folder, also through .. (an install folder that ' + 'moves with its build kit and build folder)') args = ap.parse_args() set_cpu(args.cpu) + if args.portable: + global PORTABLE + PORTABLE = True gen = (args.gen or ROOT / 'generated' / args.version).resolve() out = (args.out or ROOT / 'build' / args.version).resolve() @@ -902,6 +931,12 @@ def main(): CLANG = TOOLCHAIN_BIN / f'clang{suffix}' AR = TOOLCHAIN_BIN / f'llvm-ar{suffix}' WINDRES = TOOLCHAIN_BIN / 'llvm-windres.exe' + if PORTABLE and WIN: + resource = sorted((TOOLCHAIN_BIN.parent / 'lib' / 'clang').glob('*')) + if len(resource) != 1: + sys.exit(f'--portable: {TOOLCHAIN_BIN.parent / "lib" / "clang"} holds {len(resource)} version folders') + for flags in (GEN_FLAGS, RT_FLAGS): + flags += ['-resource-dir', cmd_path(resource[0]), f'--sysroot={cmd_path(TOOLCHAIN_BIN.parent)}'] if args.toolchain and WIN or args.deps and (args.toolchain or not WIN): # Linux: --deps only, never deps-mingw DEPS = (args.deps or DEFAULT_DEPS).resolve() dlls = sorted((DEPS / 'bin').glob('*.dll')) if WIN else []