The RomFS patch generator renames a finished folder into place, so a folder is never half written (#180, #378)

tools/make_romfs_patches.py writes into <out>.part-<pid> and renames it to <out> once every file and the manifest
are there. With --background (for runs the game starts) it runs at below-normal priority, does nothing when <out>
already has a manifest, and drops its own result when another run finished <out> first, so two runs for one folder
do not mix their files and a folder a running game loaded is never written into. Without the flag <out> is replaced
whole, as the setup and source builds expect. A .part folder left by a stopped run goes once it is an hour old: the
next run for that folder or the setup's clean-up removes it. build_game.py copy_romfs_patches skips .part folders.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-18 18:34:41 +03:00
1 parent bae687a10d
commit 77dce74028
5 files changed
+223 -28

No files matched your search

+26 -3
View File
@@ -1410,6 +1410,28 @@ class PcPrompts(Step):
SHAPED_FOLDER = re.compile(r'^romfs-patches(?:-(?:%s))?-\d+x\d+(?:-hud(?:16|21)x9)?$'
% '|'.join(map(re.escape, screens.PROMPTS_STYLES)))
# make_romfs_patches.py writes <folder>.part-<pid> and renames it to <folder> when it is complete. A run that was
# stopped (the PC turned off under a generator the game started, runtime/romfs.c romfs_folders_prepare) leaves its
# folder behind: one whose files did not change for PART_STALE_S is such a leftover, a younger one may belong to a
# generator running now. SHAPED_FOLDER matches neither.
PART_FOLDER = re.compile(r'^romfs-patches[\w.-]*\.part-[\w-]+$')
PART_STALE_S = 3600
def stale_parts(dest, now=None):
"""The leftover .part folders of make_romfs_patches.py runs in dest."""
now = time.time() if now is None else now
found = []
for folder in dest.iterdir() if dest.is_dir() else ():
if not (folder.is_dir() and not folder.is_symlink() and PART_FOLDER.match(folder.name)):
continue
try:
newest = max([folder.stat().st_mtime] + [f.stat().st_mtime for f in folder.rglob('*')])
except OSError:
continue
if now - newest > PART_STALE_S:
found.append(folder)
return found
def hud_width(dest):
@@ -1436,8 +1458,9 @@ class Shapes(Step):
those of the connected displays and of the totk.ini output_res, one folder per shape (installer/screens.py; the
game maps its resolution to the folder in runtime/romfs.c overlay_init). A setup before this step wrote all 15
sizes for every prompts style, 64 folders. The key holds the shapes, prompts style and selected HUD width, so a run after a
monitor or settings change (or its rebuild mode) writes only these again; the game's settings menu can also start
the generator for the shape in use (runtime/ui/menu.cpp row_output_res)."""
monitor or settings change (or its rebuild mode) writes only these again; the game also starts the generator for a
folder its settings need and the install lacks, at start and from its menu (runtime/romfs.c romfs_folders_prepare,
make_romfs_patches.py --background), whose unfinished .part folders the clean-up below leaves alone."""
id = 'shapes'
@@ -1596,7 +1619,7 @@ class Cleanup(Step):
ctx.state['lang'] = ctx.lang
(dest / 'profile').mkdir(exist_ok=True)
rep.progress('cleanup', 'temp', force=True)
for stale in dest.glob('update-merge-*'):
for stale in [*dest.glob('update-merge-*'), *stale_parts(dest)]:
remove(stale)
for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild
remove(ctx.temp / name)
+15
View File
@@ -635,6 +635,21 @@ class CleanupTest(unittest.TestCase):
lines = (self.dest / 'build-cache' / 'rebuild.txt').read_text(encoding='utf-8').splitlines()
self.assertEqual(lines[1:3], ['-c', steps.REBUILD_BOOT])
def test_removes_only_stale_part_folders(self):
# make_romfs_patches.py's unfinished folders: an hour-old one is a stopped run's, a fresh one a running one's
old, fresh = self.dest / 'romfs-patches-2560x1080.part-41', self.dest / 'romfs-patches-pc.part-stale-42'
for folder in (old, fresh):
(folder / 'UI').mkdir(parents=True)
(folder / 'UI' / 'a.zs').write_bytes(b'x')
(self.dest / 'romfs-patches-2560x1080').mkdir()
hour = steps.time.time() - steps.PART_STALE_S - 60
for path in (old, old / 'UI', old / 'UI' / 'a.zs'):
steps.os.utime(path, (hour, hour))
self.assertEqual(self.run_cleanup(), 'ok')
self.assertFalse(old.exists())
self.assertTrue(fresh.is_dir() and (self.dest / 'romfs-patches-2560x1080').is_dir())
self.assertIsNone(steps.SHAPED_FOLDER.match(old.name)) # the Shapes step's own clean-up leaves them
def test_emptied_romfs_fails(self):
for p in (self.dest / 'romfs').iterdir():
p.unlink()
+53
View File
@@ -12,12 +12,14 @@ import hashlib
import importlib.util
import io
import json
import os
import random
import shutil
import struct
import subprocess
import sys
import tempfile
import time
import unittest
import zlib
from pathlib import Path
@@ -789,6 +791,56 @@ class PromptsTest(unittest.TestCase):
self.assertNotEqual((source / 'manifest.tsv').read_bytes(), (pc / 'manifest.tsv').read_bytes())
self.assertEqual({p: (self.base / 'reuse' / 'pc' / p).read_bytes() for p in manifest(pc)}, before)
def test_output_appears_whole_and_a_second_writer_keeps_it(self):
# The game starts --background runs for a folder it lacks (runtime/romfs.c romfs_folders_prepare): the menu and
# the start, or a restart while the last run goes on, can start two for one folder. Each writes its own
# <out>.part-<pid>; the first to finish renames it into place, the other drops its own.
romfs = self.romfs['v121']
want = (self.outs['v121', 'xbox'][0] / 'manifest.tsv').read_bytes()
base = self.base / 'publish'
out = base / 'romfs-patches-xbox'
base.mkdir()
command = [sys.executable, '-I', str(TOOL), '--version', 'v121', '--romfs', str(romfs.folder), '--out', str(out),
'--prompts', 'xbox', '--background']
runs = [subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for _ in range(2)]
results = [(p.returncode, *out) for p in runs for out in [p.communicate()]]
for code, _, err in results:
self.assertEqual(code, 0, err)
summaries = [json.loads(o.strip().splitlines()[-1]) for _, o, _ in results]
self.assertEqual(sum('kept' not in s for s in summaries), 1, summaries)
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox'])
# Once it is there a --background run leaves it as it is (a running game checks its files on every open)
stamp = (out / 'manifest.tsv').stat().st_mtime_ns
proc = romfs.run(out, '--prompts', 'xbox', '--background')
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertEqual(json.loads(proc.stdout.strip().splitlines()[-1]), {'status': 'pass', 'kept': str(out)})
self.assertEqual((out / 'manifest.tsv').stat().st_mtime_ns, stamp)
# An unfinished folder without a manifest (an older generator's) is replaced; a stopped run's .part folder an
# hour old goes, a younger one (a run going on) stays
shutil.rmtree(out)
(out / 'UI').mkdir(parents=True)
(out / 'UI' / 'half.zs').write_bytes(b'half')
stale, young = base / 'romfs-patches-xbox.part-1', base / 'romfs-patches-xbox.part-2'
for folder in (stale, young):
folder.mkdir()
(folder / 'x').write_bytes(b'x')
hour = time.time() - 3700
for path in (stale / 'x', stale):
os.utime(path, (hour, hour))
proc = romfs.run(out, '--prompts', 'xbox', '--background')
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertNotIn('kept', json.loads(proc.stdout.strip().splitlines()[-1]))
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
self.assertFalse((out / 'UI' / 'half.zs').exists())
self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox', young.name])
# Without --background (the setup, a source build) the folder is replaced whole: no file of the old one stays
(out / 'extra.zs').write_bytes(b'old')
proc = romfs.run(out, '--prompts', 'xbox')
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
self.assertFalse((out / 'extra.zs').exists())
def test_unrecognised_romfs_fails_before_writing(self):
cases = {'no text fonts': (dict(text_fonts=False), icons.TEXT_FONTS[0]),
'no size entry': (dict(restbl_skips=(REGIONAL_FONT,)), 'has no entry in the resource size table'),
@@ -801,6 +853,7 @@ class PromptsTest(unittest.TestCase):
self.assertNotEqual(proc.returncode, 0)
self.assertIn(message, proc.stderr)
self.assertFalse(out.exists(), 'a failed run writes nothing')
self.assertEqual([p.name for p in out.parent.iterdir()], ['romfs'], 'and leaves no .part folder')
if __name__ == '__main__':
+4 -2
View File
@@ -314,8 +314,10 @@ def copy_romfs_patches(version, out):
manifest, the same way when it differs: romfs-patches, romfs-patches-xbox, romfs-patches-xbox-label and
romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts, make_romfs_patches.py --prompts) and
the ones of another shape, romfs-patches-<w>x<h> and romfs-patches-<style>-<w>x<h> (make_romfs_patches.py
--aspect), which runtime/romfs.c overlay_init needs for a game resolution that is not 16:9."""
names = sorted(p.name for p in (ROOT / 'build' / version).glob('romfs-patches*') if p.is_dir())
--aspect), which runtime/romfs.c overlay_init needs for a game resolution that is not 16:9. The <name>.part-<pid>
folders a make_romfs_patches.py run writes before it renames them into place are no overlay and stay behind."""
names = sorted(p.name for p in (ROOT / 'build' / version).glob('romfs-patches*')
if p.is_dir() and '.part-' not in p.name)
for name in names:
src = ROOT / 'build' / version / name
dst = out / name
+125 -23
View File
@@ -66,15 +66,27 @@ lists them. The output is byte for byte the same as without it.
Input: build/extracted/<version>/romfs (romfs.bin and index.json, verified by tools/extract.py). Output:
build/<version>/romfs-patches/<path> by default (romfs-patches-<style> with --prompts, -<w>x<h> with --aspect,
and -hud<a>x<b> with --hud), and <out>/manifest.tsv (path, size, SHA-256 of the overlay, SHA-256 of the original). Run with .venv-py314.
The files go into <out>.part-<pid> beside <out>, which is renamed to <out> once complete (publish), so a folder is
never seen half written and two runs for one folder do not mix their files. Without --background an older <out> is
removed first (the setup and a source build replace it). --background is the game's (runtime/romfs.c
romfs_folders_prepare, for a folder the settings need and the install lacks): it runs at a lower priority, does
nothing when <out> has a manifest at its start, and drops its own result when another run finished <out> first, so a
folder a running game loaded (whose files totk.exe checks again on every open) is never written into. A .part folder
left by a run that was stopped is removed by the next run for that folder once it is an hour old.
"""
import argparse
import compression.zstd as zstd
import hashlib
import importlib.util
import json
import ctypes
import glob
import os
import re
import shutil
import struct
import time
from pathlib import Path
root = Path(__file__).resolve().parents[1]
@@ -96,6 +108,8 @@ ap.add_argument('--binds', type=Path, help='--prompts pc: a totk.ini whose bind_
ap.add_argument('--xbox-prompts', dest='prompts', action='store_const', const='xbox', help='the same as --prompts xbox')
ap.add_argument('--reuse', type=Path,
help='an overlay folder made before: its files with the same contents are linked, not compressed again')
ap.add_argument('--background', action='store_true',
help='started by the game: lower priority, and an <out> that has a manifest is kept, not replaced')
args = ap.parse_args()
args.romfs = args.romfs or root / 'build/extracted' / args.version / 'romfs'
width = height = None
@@ -116,6 +130,54 @@ if not args.out:
if args.aspect: name += '-' + args.aspect
if args.hud != 'full': name += '-hud' + args.hud.replace(':', 'x')
args.out = root / 'build' / args.version / name
PART_STALE_S = 3600 # a .part folder untouched this long belongs to a run that was stopped
def finished(folder):
return (folder / 'manifest.tsv').is_file()
def newest_mtime(folder):
times = [folder.stat().st_mtime]
for path in folder.rglob('*'):
try:
times.append(path.stat().st_mtime)
except OSError:
pass
return max(times)
def remove_stale_parts(out):
"""Remove the <out>.part-* folders of runs that were stopped (none of their files changed for PART_STALE_S)."""
for part in out.parent.glob(glob.escape(out.name) + '.part-*'):
try:
if part.is_dir() and time.time() - newest_mtime(part) > PART_STALE_S:
shutil.rmtree(part, ignore_errors=True)
except OSError:
pass
def lower_priority():
"""Below-normal priority, so the game that started this run keeps its cores."""
try:
if os.name == 'nt':
kernel32 = ctypes.windll.kernel32
kernel32.SetPriorityClass(kernel32.GetCurrentProcess(), 0x4000) # BELOW_NORMAL_PRIORITY_CLASS
else:
os.nice(10)
except (OSError, AttributeError):
pass
if args.background:
if finished(args.out): # another run made it since the game looked
print(json.dumps(dict(status='pass', kept=str(args.out))))
raise SystemExit(0)
lower_priority()
args.out.parent.mkdir(parents=True, exist_ok=True)
remove_stale_parts(args.out)
part = args.out.with_name(f'{args.out.name}.part-{os.getpid()}') # the files go here first (publish)
shutil.rmtree(part, ignore_errors=True)
index = json.loads((args.romfs / 'index.json').read_text())
assert index['status'] == 'pass'
files = {f['path']: f for f in index['files']}
@@ -514,30 +576,70 @@ def reusable(path, contents, dictionary):
return data if same else None
lines, reused = [], []
for path, contents in outputs.items():
raw = originals[path]
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
target = args.out / path
target.parent.mkdir(parents=True, exist_ok=True)
# Unlinked first: a file linked from another folder (--reuse) is not written through, which would break the other
# folder's manifest.
target.unlink(missing_ok=True)
data = reusable(path, contents, dictionary)
if data is not None:
try: # a hard link takes no room; a volume without them gets a copy
os.link(args.reuse / path, target)
except OSError:
def publish(part, out, sizes, keep_existing):
"""Rename the complete folder part to out. Each file the manifest lists must be there with its size first: a
setup's clean-up that removed part's files under a running generator must not leave a manifest without them.
keep_existing (--background): an out that has a manifest by now stays and part is dropped; an out without one is
an unfinished folder of an older generator, which nothing loads. Without it out is replaced. Returns whether part
became out."""
for path, size in sizes.items():
assert (part / path).is_file() and (part / path).stat().st_size == size, f'{part / path} is missing or changed'
for attempt in range(3):
if keep_existing and finished(out):
shutil.rmtree(part, ignore_errors=True)
return False
if out.exists() and not keep_existing:
shutil.rmtree(out)
elif out.exists():
# Moved aside, never removed in place: another run may publish between the check and this rename, and a
# folder moved aside with a manifest goes back (a removed one could be the one a game just loaded).
stale = out.with_name(f'{out.name}.part-stale-{os.getpid()}')
try:
os.rename(out, stale)
except OSError:
pass
else:
if finished(stale):
os.rename(stale, out)
continue
shutil.rmtree(stale, ignore_errors=True)
try:
os.rename(part, out)
return True
except OSError: # another run renamed its folder into place after the check: the next pass looks again
if attempt == 2:
raise
shutil.rmtree(part, ignore_errors=True)
return False
lines, reused, sizes = [], [], {}
try:
for path, contents in outputs.items():
raw = originals[path]
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
target = part / path
target.parent.mkdir(parents=True, exist_ok=True)
data = reusable(path, contents, dictionary)
if data is not None:
try: # a hard link takes no room; a volume without them gets a copy
os.link(args.reuse / path, target)
except OSError:
target.write_bytes(data)
reused.append(path)
else:
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
assert zstd.decompress(data, zstd_dict=dictionary) == contents
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
target.write_bytes(data)
reused.append(path)
else:
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
assert zstd.decompress(data, zstd_dict=dictionary) == contents
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
target.write_bytes(data)
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
(args.out / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
sizes[path] = len(data)
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
(part / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
+ ''.join(line + '\n' for line in lines))
published = publish(part, args.out, sizes, args.background)
except BaseException:
shutil.rmtree(part, ignore_errors=True)
raise
print(json.dumps(dict(status='pass', path=ARCHIVE, members=patched, changed_bytes=len(diff),
ultrawide_layout=dict(aspect=args.aspect, hud=args.hud, members=len(layout)) if args.aspect else None,
prompts=prompts, reused=reused)))
prompts=prompts, reused=reused, **({} if published else {'kept': str(args.out)}))))