mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
The RomFS patch generator renames a finished folder into place, so a folder is never half written (#180, #378)
tools/make_romfs_patches.py writes into <out>.part-<pid> and renames it to <out> once every file and the manifest are there. With --background (for runs the game starts) it runs at below-normal priority, does nothing when <out> already has a manifest, and drops its own result when another run finished <out> first, so two runs for one folder do not mix their files and a folder a running game loaded is never written into. Without the flag <out> is replaced whole, as the setup and source builds expect. A .part folder left by a stopped run goes once it is an hour old: the next run for that folder or the setup's clean-up removes it. build_game.py copy_romfs_patches skips .part folders. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bae687a10d
commit
77dce74028
5 files changed
+223
-28
No files matched your search
+26
-3
@@ -1410,6 +1410,28 @@ class PcPrompts(Step):
|
||||
|
||||
SHAPED_FOLDER = re.compile(r'^romfs-patches(?:-(?:%s))?-\d+x\d+(?:-hud(?:16|21)x9)?$'
|
||||
% '|'.join(map(re.escape, screens.PROMPTS_STYLES)))
|
||||
# make_romfs_patches.py writes <folder>.part-<pid> and renames it to <folder> when it is complete. A run that was
|
||||
# stopped (the PC turned off under a generator the game started, runtime/romfs.c romfs_folders_prepare) leaves its
|
||||
# folder behind: one whose files did not change for PART_STALE_S is such a leftover, a younger one may belong to a
|
||||
# generator running now. SHAPED_FOLDER matches neither.
|
||||
PART_FOLDER = re.compile(r'^romfs-patches[\w.-]*\.part-[\w-]+$')
|
||||
PART_STALE_S = 3600
|
||||
|
||||
|
||||
def stale_parts(dest, now=None):
|
||||
"""The leftover .part folders of make_romfs_patches.py runs in dest."""
|
||||
now = time.time() if now is None else now
|
||||
found = []
|
||||
for folder in dest.iterdir() if dest.is_dir() else ():
|
||||
if not (folder.is_dir() and not folder.is_symlink() and PART_FOLDER.match(folder.name)):
|
||||
continue
|
||||
try:
|
||||
newest = max([folder.stat().st_mtime] + [f.stat().st_mtime for f in folder.rglob('*')])
|
||||
except OSError:
|
||||
continue
|
||||
if now - newest > PART_STALE_S:
|
||||
found.append(folder)
|
||||
return found
|
||||
|
||||
|
||||
def hud_width(dest):
|
||||
@@ -1436,8 +1458,9 @@ class Shapes(Step):
|
||||
those of the connected displays and of the totk.ini output_res, one folder per shape (installer/screens.py; the
|
||||
game maps its resolution to the folder in runtime/romfs.c overlay_init). A setup before this step wrote all 15
|
||||
sizes for every prompts style, 64 folders. The key holds the shapes, prompts style and selected HUD width, so a run after a
|
||||
monitor or settings change (or its rebuild mode) writes only these again; the game's settings menu can also start
|
||||
the generator for the shape in use (runtime/ui/menu.cpp row_output_res)."""
|
||||
monitor or settings change (or its rebuild mode) writes only these again; the game also starts the generator for a
|
||||
folder its settings need and the install lacks, at start and from its menu (runtime/romfs.c romfs_folders_prepare,
|
||||
make_romfs_patches.py --background), whose unfinished .part folders the clean-up below leaves alone."""
|
||||
|
||||
id = 'shapes'
|
||||
|
||||
@@ -1596,7 +1619,7 @@ class Cleanup(Step):
|
||||
ctx.state['lang'] = ctx.lang
|
||||
(dest / 'profile').mkdir(exist_ok=True)
|
||||
rep.progress('cleanup', 'temp', force=True)
|
||||
for stale in dest.glob('update-merge-*'):
|
||||
for stale in [*dest.glob('update-merge-*'), *stale_parts(dest)]:
|
||||
remove(stale)
|
||||
for name in ('check', 'unpacked', 'shader_list.tsv', 'shared-fonts'): # gen/ and build/ stay for the next rebuild
|
||||
remove(ctx.temp / name)
|
||||
|
||||
@@ -635,6 +635,21 @@ class CleanupTest(unittest.TestCase):
|
||||
lines = (self.dest / 'build-cache' / 'rebuild.txt').read_text(encoding='utf-8').splitlines()
|
||||
self.assertEqual(lines[1:3], ['-c', steps.REBUILD_BOOT])
|
||||
|
||||
def test_removes_only_stale_part_folders(self):
|
||||
# make_romfs_patches.py's unfinished folders: an hour-old one is a stopped run's, a fresh one a running one's
|
||||
old, fresh = self.dest / 'romfs-patches-2560x1080.part-41', self.dest / 'romfs-patches-pc.part-stale-42'
|
||||
for folder in (old, fresh):
|
||||
(folder / 'UI').mkdir(parents=True)
|
||||
(folder / 'UI' / 'a.zs').write_bytes(b'x')
|
||||
(self.dest / 'romfs-patches-2560x1080').mkdir()
|
||||
hour = steps.time.time() - steps.PART_STALE_S - 60
|
||||
for path in (old, old / 'UI', old / 'UI' / 'a.zs'):
|
||||
steps.os.utime(path, (hour, hour))
|
||||
self.assertEqual(self.run_cleanup(), 'ok')
|
||||
self.assertFalse(old.exists())
|
||||
self.assertTrue(fresh.is_dir() and (self.dest / 'romfs-patches-2560x1080').is_dir())
|
||||
self.assertIsNone(steps.SHAPED_FOLDER.match(old.name)) # the Shapes step's own clean-up leaves them
|
||||
|
||||
def test_emptied_romfs_fails(self):
|
||||
for p in (self.dest / 'romfs').iterdir():
|
||||
p.unlink()
|
||||
|
||||
@@ -12,12 +12,14 @@ import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
@@ -789,6 +791,56 @@ class PromptsTest(unittest.TestCase):
|
||||
self.assertNotEqual((source / 'manifest.tsv').read_bytes(), (pc / 'manifest.tsv').read_bytes())
|
||||
self.assertEqual({p: (self.base / 'reuse' / 'pc' / p).read_bytes() for p in manifest(pc)}, before)
|
||||
|
||||
def test_output_appears_whole_and_a_second_writer_keeps_it(self):
|
||||
# The game starts --background runs for a folder it lacks (runtime/romfs.c romfs_folders_prepare): the menu and
|
||||
# the start, or a restart while the last run goes on, can start two for one folder. Each writes its own
|
||||
# <out>.part-<pid>; the first to finish renames it into place, the other drops its own.
|
||||
romfs = self.romfs['v121']
|
||||
want = (self.outs['v121', 'xbox'][0] / 'manifest.tsv').read_bytes()
|
||||
base = self.base / 'publish'
|
||||
out = base / 'romfs-patches-xbox'
|
||||
base.mkdir()
|
||||
command = [sys.executable, '-I', str(TOOL), '--version', 'v121', '--romfs', str(romfs.folder), '--out', str(out),
|
||||
'--prompts', 'xbox', '--background']
|
||||
runs = [subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for _ in range(2)]
|
||||
results = [(p.returncode, *out) for p in runs for out in [p.communicate()]]
|
||||
for code, _, err in results:
|
||||
self.assertEqual(code, 0, err)
|
||||
summaries = [json.loads(o.strip().splitlines()[-1]) for _, o, _ in results]
|
||||
self.assertEqual(sum('kept' not in s for s in summaries), 1, summaries)
|
||||
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
|
||||
self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox'])
|
||||
# Once it is there a --background run leaves it as it is (a running game checks its files on every open)
|
||||
stamp = (out / 'manifest.tsv').stat().st_mtime_ns
|
||||
proc = romfs.run(out, '--prompts', 'xbox', '--background')
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertEqual(json.loads(proc.stdout.strip().splitlines()[-1]), {'status': 'pass', 'kept': str(out)})
|
||||
self.assertEqual((out / 'manifest.tsv').stat().st_mtime_ns, stamp)
|
||||
# An unfinished folder without a manifest (an older generator's) is replaced; a stopped run's .part folder an
|
||||
# hour old goes, a younger one (a run going on) stays
|
||||
shutil.rmtree(out)
|
||||
(out / 'UI').mkdir(parents=True)
|
||||
(out / 'UI' / 'half.zs').write_bytes(b'half')
|
||||
stale, young = base / 'romfs-patches-xbox.part-1', base / 'romfs-patches-xbox.part-2'
|
||||
for folder in (stale, young):
|
||||
folder.mkdir()
|
||||
(folder / 'x').write_bytes(b'x')
|
||||
hour = time.time() - 3700
|
||||
for path in (stale / 'x', stale):
|
||||
os.utime(path, (hour, hour))
|
||||
proc = romfs.run(out, '--prompts', 'xbox', '--background')
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertNotIn('kept', json.loads(proc.stdout.strip().splitlines()[-1]))
|
||||
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
|
||||
self.assertFalse((out / 'UI' / 'half.zs').exists())
|
||||
self.assertEqual(sorted(p.name for p in base.iterdir()), ['romfs-patches-xbox', young.name])
|
||||
# Without --background (the setup, a source build) the folder is replaced whole: no file of the old one stays
|
||||
(out / 'extra.zs').write_bytes(b'old')
|
||||
proc = romfs.run(out, '--prompts', 'xbox')
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertEqual((out / 'manifest.tsv').read_bytes(), want)
|
||||
self.assertFalse((out / 'extra.zs').exists())
|
||||
|
||||
def test_unrecognised_romfs_fails_before_writing(self):
|
||||
cases = {'no text fonts': (dict(text_fonts=False), icons.TEXT_FONTS[0]),
|
||||
'no size entry': (dict(restbl_skips=(REGIONAL_FONT,)), 'has no entry in the resource size table'),
|
||||
@@ -801,6 +853,7 @@ class PromptsTest(unittest.TestCase):
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
self.assertIn(message, proc.stderr)
|
||||
self.assertFalse(out.exists(), 'a failed run writes nothing')
|
||||
self.assertEqual([p.name for p in out.parent.iterdir()], ['romfs'], 'and leaves no .part folder')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
+4
-2
@@ -314,8 +314,10 @@ def copy_romfs_patches(version, out):
|
||||
manifest, the same way when it differs: romfs-patches, romfs-patches-xbox, romfs-patches-xbox-label and
|
||||
romfs-patches-pc (the same with Xbox or keyboard and mouse button prompts, make_romfs_patches.py --prompts) and
|
||||
the ones of another shape, romfs-patches-<w>x<h> and romfs-patches-<style>-<w>x<h> (make_romfs_patches.py
|
||||
--aspect), which runtime/romfs.c overlay_init needs for a game resolution that is not 16:9."""
|
||||
names = sorted(p.name for p in (ROOT / 'build' / version).glob('romfs-patches*') if p.is_dir())
|
||||
--aspect), which runtime/romfs.c overlay_init needs for a game resolution that is not 16:9. The <name>.part-<pid>
|
||||
folders a make_romfs_patches.py run writes before it renames them into place are no overlay and stay behind."""
|
||||
names = sorted(p.name for p in (ROOT / 'build' / version).glob('romfs-patches*')
|
||||
if p.is_dir() and '.part-' not in p.name)
|
||||
for name in names:
|
||||
src = ROOT / 'build' / version / name
|
||||
dst = out / name
|
||||
|
||||
+125
-23
@@ -66,15 +66,27 @@ lists them. The output is byte for byte the same as without it.
|
||||
Input: build/extracted/<version>/romfs (romfs.bin and index.json, verified by tools/extract.py). Output:
|
||||
build/<version>/romfs-patches/<path> by default (romfs-patches-<style> with --prompts, -<w>x<h> with --aspect,
|
||||
and -hud<a>x<b> with --hud), and <out>/manifest.tsv (path, size, SHA-256 of the overlay, SHA-256 of the original). Run with .venv-py314.
|
||||
|
||||
The files go into <out>.part-<pid> beside <out>, which is renamed to <out> once complete (publish), so a folder is
|
||||
never seen half written and two runs for one folder do not mix their files. Without --background an older <out> is
|
||||
removed first (the setup and a source build replace it). --background is the game's (runtime/romfs.c
|
||||
romfs_folders_prepare, for a folder the settings need and the install lacks): it runs at a lower priority, does
|
||||
nothing when <out> has a manifest at its start, and drops its own result when another run finished <out> first, so a
|
||||
folder a running game loaded (whose files totk.exe checks again on every open) is never written into. A .part folder
|
||||
left by a run that was stopped is removed by the next run for that folder once it is an hour old.
|
||||
"""
|
||||
import argparse
|
||||
import compression.zstd as zstd
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import ctypes
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
@@ -96,6 +108,8 @@ ap.add_argument('--binds', type=Path, help='--prompts pc: a totk.ini whose bind_
|
||||
ap.add_argument('--xbox-prompts', dest='prompts', action='store_const', const='xbox', help='the same as --prompts xbox')
|
||||
ap.add_argument('--reuse', type=Path,
|
||||
help='an overlay folder made before: its files with the same contents are linked, not compressed again')
|
||||
ap.add_argument('--background', action='store_true',
|
||||
help='started by the game: lower priority, and an <out> that has a manifest is kept, not replaced')
|
||||
args = ap.parse_args()
|
||||
args.romfs = args.romfs or root / 'build/extracted' / args.version / 'romfs'
|
||||
width = height = None
|
||||
@@ -116,6 +130,54 @@ if not args.out:
|
||||
if args.aspect: name += '-' + args.aspect
|
||||
if args.hud != 'full': name += '-hud' + args.hud.replace(':', 'x')
|
||||
args.out = root / 'build' / args.version / name
|
||||
PART_STALE_S = 3600 # a .part folder untouched this long belongs to a run that was stopped
|
||||
|
||||
|
||||
def finished(folder):
|
||||
return (folder / 'manifest.tsv').is_file()
|
||||
|
||||
|
||||
def newest_mtime(folder):
|
||||
times = [folder.stat().st_mtime]
|
||||
for path in folder.rglob('*'):
|
||||
try:
|
||||
times.append(path.stat().st_mtime)
|
||||
except OSError:
|
||||
pass
|
||||
return max(times)
|
||||
|
||||
|
||||
def remove_stale_parts(out):
|
||||
"""Remove the <out>.part-* folders of runs that were stopped (none of their files changed for PART_STALE_S)."""
|
||||
for part in out.parent.glob(glob.escape(out.name) + '.part-*'):
|
||||
try:
|
||||
if part.is_dir() and time.time() - newest_mtime(part) > PART_STALE_S:
|
||||
shutil.rmtree(part, ignore_errors=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def lower_priority():
|
||||
"""Below-normal priority, so the game that started this run keeps its cores."""
|
||||
try:
|
||||
if os.name == 'nt':
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
kernel32.SetPriorityClass(kernel32.GetCurrentProcess(), 0x4000) # BELOW_NORMAL_PRIORITY_CLASS
|
||||
else:
|
||||
os.nice(10)
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
if args.background:
|
||||
if finished(args.out): # another run made it since the game looked
|
||||
print(json.dumps(dict(status='pass', kept=str(args.out))))
|
||||
raise SystemExit(0)
|
||||
lower_priority()
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
remove_stale_parts(args.out)
|
||||
part = args.out.with_name(f'{args.out.name}.part-{os.getpid()}') # the files go here first (publish)
|
||||
shutil.rmtree(part, ignore_errors=True)
|
||||
index = json.loads((args.romfs / 'index.json').read_text())
|
||||
assert index['status'] == 'pass'
|
||||
files = {f['path']: f for f in index['files']}
|
||||
@@ -514,30 +576,70 @@ def reusable(path, contents, dictionary):
|
||||
return data if same else None
|
||||
|
||||
|
||||
lines, reused = [], []
|
||||
for path, contents in outputs.items():
|
||||
raw = originals[path]
|
||||
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
|
||||
target = args.out / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Unlinked first: a file linked from another folder (--reuse) is not written through, which would break the other
|
||||
# folder's manifest.
|
||||
target.unlink(missing_ok=True)
|
||||
data = reusable(path, contents, dictionary)
|
||||
if data is not None:
|
||||
try: # a hard link takes no room; a volume without them gets a copy
|
||||
os.link(args.reuse / path, target)
|
||||
except OSError:
|
||||
def publish(part, out, sizes, keep_existing):
|
||||
"""Rename the complete folder part to out. Each file the manifest lists must be there with its size first: a
|
||||
setup's clean-up that removed part's files under a running generator must not leave a manifest without them.
|
||||
keep_existing (--background): an out that has a manifest by now stays and part is dropped; an out without one is
|
||||
an unfinished folder of an older generator, which nothing loads. Without it out is replaced. Returns whether part
|
||||
became out."""
|
||||
for path, size in sizes.items():
|
||||
assert (part / path).is_file() and (part / path).stat().st_size == size, f'{part / path} is missing or changed'
|
||||
for attempt in range(3):
|
||||
if keep_existing and finished(out):
|
||||
shutil.rmtree(part, ignore_errors=True)
|
||||
return False
|
||||
if out.exists() and not keep_existing:
|
||||
shutil.rmtree(out)
|
||||
elif out.exists():
|
||||
# Moved aside, never removed in place: another run may publish between the check and this rename, and a
|
||||
# folder moved aside with a manifest goes back (a removed one could be the one a game just loaded).
|
||||
stale = out.with_name(f'{out.name}.part-stale-{os.getpid()}')
|
||||
try:
|
||||
os.rename(out, stale)
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
if finished(stale):
|
||||
os.rename(stale, out)
|
||||
continue
|
||||
shutil.rmtree(stale, ignore_errors=True)
|
||||
try:
|
||||
os.rename(part, out)
|
||||
return True
|
||||
except OSError: # another run renamed its folder into place after the check: the next pass looks again
|
||||
if attempt == 2:
|
||||
raise
|
||||
shutil.rmtree(part, ignore_errors=True)
|
||||
return False
|
||||
|
||||
|
||||
lines, reused, sizes = [], [], {}
|
||||
try:
|
||||
for path, contents in outputs.items():
|
||||
raw = originals[path]
|
||||
dictionary = dictionaries.get(zstd.get_frame_info(raw).dictionary_id)
|
||||
target = part / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = reusable(path, contents, dictionary)
|
||||
if data is not None:
|
||||
try: # a hard link takes no room; a volume without them gets a copy
|
||||
os.link(args.reuse / path, target)
|
||||
except OSError:
|
||||
target.write_bytes(data)
|
||||
reused.append(path)
|
||||
else:
|
||||
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
|
||||
assert zstd.decompress(data, zstd_dict=dictionary) == contents
|
||||
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
|
||||
target.write_bytes(data)
|
||||
reused.append(path)
|
||||
else:
|
||||
data = zstd.compress(contents, level=19, zstd_dict=dictionary)
|
||||
assert zstd.decompress(data, zstd_dict=dictionary) == contents
|
||||
assert zstd.get_frame_info(data).dictionary_id == zstd.get_frame_info(raw).dictionary_id
|
||||
target.write_bytes(data)
|
||||
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
|
||||
(args.out / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
|
||||
sizes[path] = len(data)
|
||||
lines.append('\t'.join([path, str(len(data)), hashlib.sha256(data).hexdigest(), hashlib.sha256(raw).hexdigest()]))
|
||||
(part / 'manifest.tsv').write_text('# path\tsize\tsha256\toriginal_sha256 (tools/make_romfs_patches.py)\n'
|
||||
+ ''.join(line + '\n' for line in lines))
|
||||
published = publish(part, args.out, sizes, args.background)
|
||||
except BaseException:
|
||||
shutil.rmtree(part, ignore_errors=True)
|
||||
raise
|
||||
print(json.dumps(dict(status='pass', path=ARCHIVE, members=patched, changed_bytes=len(diff),
|
||||
ultrawide_layout=dict(aspect=args.aspect, hud=args.hud, members=len(layout)) if args.aspect else None,
|
||||
prompts=prompts, reused=reused)))
|
||||
prompts=prompts, reused=reused, **({} if published else {'kept': str(args.out)}))))
|
||||
Reference in new issue
Block a user