From 77dce74028aa4a2d2fb7d9c066ac686ac6b0a8f3 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Wed, 18 Nov 2026 18:34:41 +0300 Subject: [PATCH] The RomFS patch generator renames a finished folder into place, so a folder is never half written (#180, #378) tools/make_romfs_patches.py writes into .part- and renames it to once every file and the manifest are there. With --background (for runs the game starts) it runs at below-normal priority, does nothing when already has a manifest, and drops its own result when another run finished first, so two runs for one folder do not mix their files and a folder a running game loaded is never written into. Without the flag is replaced whole, as the setup and source builds expect. A .part folder left by a stopped run goes once it is an hour old: the next run for that folder or the setup's clean-up removes it. build_game.py copy_romfs_patches skips .part folders. Co-Authored-By: Claude Opus 5.5 --- installer/steps.py | 29 ++++++- tests/installer_test.py | 15 ++++ tests/romfs_prompts_test.py | 53 +++++++++++++ tools/build_game.py | 6 +- tools/make_romfs_patches.py | 148 ++++++++++++++++++++++++++++++------ 5 files changed, 223 insertions(+), 28 deletions(-) diff --git a/installer/steps.py b/installer/steps.py index bc6888e8..12abed07 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -1410,6 +1410,28 @@ class PcPrompts(Step): SHAPED_FOLDER = re.compile(r'^romfs-patches(?:-(?:%s))?-\d+x\d+(?:-hud(?:16|21)x9)?$' % '|'.join(map(re.escape, screens.PROMPTS_STYLES))) +# make_romfs_patches.py writes .part- and renames it to when it is complete. A run that was +# stopped (the PC turned off under a generator the game started, runtime/romfs.c romfs_folders_prepare) leaves its +# folder behind: one whose files did not change for PART_STALE_S is such a leftover, a younger one may belong to a +# generator running now. SHAPED_FOLDER matches neither. +PART_FOLDER = re.compile(r'^romfs-patches[\w.-]*\.part-[\w-]+$') +PART_STALE_S = 3600 + + +def stale_parts(dest, now=None): + """The leftover .part folders of make_romfs_patches.py runs in dest.""" + now = time.time() if now is None else now + found = [] + for folder in dest.iterdir() if dest.is_dir() else (): + if not (folder.is_dir() and not folder.is_symlink() and PART_FOLDER.match(folder.name)): + continue + try: + newest = max([folder.stat().st_mtime] + [f.stat().st_mtime for f in folder.rglob('*')]) + except OSError: + continue + if now - newest > PART_STALE_S: + found.append(folder) + return found def hud_width(dest): @@ -1436,8 +1458,9 @@ class Shapes(Step): those of the connected displays and of the totk.ini output_res, one folder per shape (installer/screens.py; the game maps its resolution to the folder in runtime/romfs.c overlay_init). A setup before this step wrote all 15 sizes for every prompts style, 64 folders. The key holds the shapes, prompts style and selected HUD width, so a run after a - monitor or settings change (or its rebuild mode) writes only these again; the game's settings menu can also start - the generator for the shape in use (runtime/ui/menu.cpp row_output_res).""" + monitor or settings change (or its rebuild mode) writes only these again; the game also starts the generator for a + folder its settings need and the install lacks, at start and from its menu (runtime/romfs.c romfs_folders_prepare, + make_romfs_patches.py --background), whose unfinished .part folders the clean-up below leaves alone.""" id = 'shapes' @@ -1596,7 +1619,7 @@ class Cleanup(Step): ctx.state['lang'] = ctx.lang (dest / 'profile').mkdir(exist_ok=True) rep.progress('cleanup', 'temp', force=True) - for stale in dest.glob('update-merge-*'): + for stale in [*dest.glob('update-merge-*'), *stale_parts(dest)]: remove(stale) for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild remove(ctx.temp / name) diff --git a/tests/installer_test.py b/tests/installer_test.py index 28057377..92de8b4d 100644 --- a/tests/installer_test.py +++ b/tests/installer_test.py @@ -635,6 +635,21 @@ class CleanupTest(unittest.TestCase): lines = (self.dest / 'build-cache' / 'rebuild.txt').read_text(encoding='utf-8').splitlines() self.assertEqual(lines[1:3], ['-c', steps.REBUILD_BOOT]) + def test_removes_only_stale_part_folders(self): + # make_romfs_patches.py's unfinished folders: an hour-old one is a stopped run's, a fresh one a running one's + old, fresh = self.dest / 'romfs-patches-2560x1080.part-41', self.dest / 'romfs-patches-pc.part-stale-42' + for folder in (old, fresh): + (folder / 'UI').mkdir(parents=True) + (folder / 'UI' / 'a.zs').write_bytes(b'x') + (self.dest / 'romfs-patches-2560x1080').mkdir() + hour = steps.time.time() - steps.PART_STALE_S - 60 + for path in (old, old / 'UI', old / 'UI' / 'a.zs'): + steps.os.utime(path, (hour, hour)) + self.assertEqual(self.run_cleanup(), 'ok') + self.assertFalse(old.exists()) + self.assertTrue(fresh.is_dir() and (self.dest / 'romfs-patches-2560x1080').is_dir()) + self.assertIsNone(steps.SHAPED_FOLDER.match(old.name)) # the Shapes step's own clean-up leaves them + def test_emptied_romfs_fails(self): for p in (self.dest / 'romfs').iterdir(): p.unlink() diff --git a/tests/romfs_prompts_test.py b/tests/romfs_prompts_test.py index bc8ef04d..3a2ca33c 100644 --- a/tests/romfs_prompts_test.py +++ b/tests/romfs_prompts_test.py @@ -12,12 +12,14 @@ import hashlib import importlib.util import io import json +import os import random import shutil import struct import subprocess import sys import tempfile +import time import unittest import zlib from pathlib import Path @@ -789,6 +791,56 @@ class PromptsTest(unittest.TestCase): self.assertNotEqual((source / 'manifest.tsv').read_bytes(), (pc / 'manifest.tsv').read_bytes()) self.assertEqual({p: (self.base / 'reuse' / 'pc' / p).read_bytes() for p in manifest(pc)}, before) + def test_output_appears_whole_and_a_second_writer_keeps_it(self): + # The game starts --background runs for a folder it lacks (runtime/romfs.c romfs_folders_prepare): the menu and + # the start, or a restart while the last run goes on, can start two for one folder. Each writes its own + # .part-; the first to finish renames it into place, the other drops its own. + romfs = self.romfs['v121'] + want = (self.outs['v121', 'xbox'][0] / 'manifest.tsv').read_bytes() + base = self.base / 'publish' + out = base / 'romfs-patches-xbox' + base.mkdir() + command = [sys.executable, '-I', str(TOOL), '--version', 'v121', '--romfs', str(romfs.folder), '--out', str(out), + '--prompts', 'xbox', '--background'] + runs = [subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for _ in range(2)] + results = [(p.returncode, *out) for p in runs for out in [p.communicate()]] + for code, _, err in results: + self.assertEqual(code, 0, err) + summaries = [json.loads(o.strip().splitlines()[-1]) for _, o, _ in results] + self.assertEqual(sum('kept' not in s for s in summaries), 1, summaries) + self.assertEqual((out / 'manifest.tsv').read_bytes(), want) + self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox']) + # Once it is there a --background run leaves it as it is (a running game checks its files on every open) + stamp = (out / 'manifest.tsv').stat().st_mtime_ns + proc = romfs.run(out, '--prompts', 'xbox', '--background') + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(json.loads(proc.stdout.strip().splitlines()[-1]), {'status': 'pass', 'kept': str(out)}) + self.assertEqual((out / 'manifest.tsv').stat().st_mtime_ns, stamp) + # An unfinished folder without a manifest (an older generator's) is replaced; a stopped run's .part folder an + # hour old goes, a younger one (a run going on) stays + shutil.rmtree(out) + (out / 'UI').mkdir(parents=True) + (out / 'UI' / 'half.zs').write_bytes(b'half') + stale, young = base / 'romfs-patches-xbox.part-1', base / 'romfs-patches-xbox.part-2' + for folder in (stale, young): + folder.mkdir() + (folder / 'x').write_bytes(b'x') + hour = time.time() - 3700 + for path in (stale / 'x', stale): + os.utime(path, (hour, hour)) + proc = romfs.run(out, '--prompts', 'xbox', '--background') + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertNotIn('kept', json.loads(proc.stdout.strip().splitlines()[-1])) + self.assertEqual((out / 'manifest.tsv').read_bytes(), want) + self.assertFalse((out / 'UI' / 'half.zs').exists()) + self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox', young.name]) + # Without --background (the setup, a source build) the folder is replaced whole: no file of the old one stays + (out / 'extra.zs').write_bytes(b'old') + proc = romfs.run(out, '--prompts', 'xbox') + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual((out / 'manifest.tsv').read_bytes(), want) + self.assertFalse((out / 'extra.zs').exists()) + def test_unrecognised_romfs_fails_before_writing(self): cases = {'no text fonts': (dict(text_fonts=False), icons.TEXT_FONTS[0]), 'no size entry': (dict(restbl_skips=(REGIONAL_FONT,)), 'has no entry in the resource size table'), @@ -801,6 +853,7 @@ class PromptsTest(unittest.TestCase): self.assertNotEqual(proc.returncode, 0) self.assertIn(message, proc.stderr) self.assertFalse(out.exists(), 'a failed run writes nothing') + self.assertEqual([p.name for p in out.parent.iterdir()], ['romfs'], 'and leaves no .part folder') if __name__ == '__main__': diff --git a/tools/build_game.py b/tools/build_game.py index 212e248f..3de72efd 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -314,8 +314,10 @@ def copy_romfs_patches(version, out): manifest, the same way when it differs: romfs-patches, romfs-patches-xbox, romfs-patches-xbox-label and romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts, make_romfs_patches.py --prompts) and the ones of another shape, romfs-patches-x and romfs-patches-