mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-20 04:35:57 +00:00
Size the code generator's worker pool by free memory so it stops running out of RAM
gen.py started one emit worker per CPU, about 1.3 GiB each on 1.4.3, so many-core machines with modest RAM died with a MemoryError during "Translating code to C" (issues 17, 28, probably 66). The default is now min(CPU count, 80% of the free physical memory / 1.5 GiB), at least 1, capped by the memory limit of the enclosing job object if there is one. --jobs still overrides it, and the chosen count and its reason are logged. The memory helper is tools/hostmem.py, shipped with the release tools. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3b9a4e7f6
commit
a5711539e1
3 files changed
+75
-3
No files matched your search
+22
-2
@@ -39,6 +39,7 @@ sys.path.insert(0, str(ROOT / 'tools'))
|
||||
sys.path.insert(0, str(ROOT / 'recomp'))
|
||||
|
||||
import flags # noqa: E402
|
||||
import hostmem # noqa: E402
|
||||
import jumptables # noqa: E402
|
||||
import patches # noqa: E402
|
||||
import regs # noqa: E402
|
||||
@@ -587,12 +588,29 @@ def remove_stale(moddir, name, nchunks, errors):
|
||||
return removed
|
||||
|
||||
|
||||
# Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB.
|
||||
WORKER_MEM = 1536 << 20
|
||||
MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs
|
||||
|
||||
|
||||
def pick_jobs(requested):
|
||||
"""Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory."""
|
||||
if requested:
|
||||
return requested, '--jobs'
|
||||
cpus = os.cpu_count() or 4
|
||||
free = hostmem.available_ram()
|
||||
if free is None:
|
||||
return cpus, f'{cpus} CPUs, free memory unknown'
|
||||
by_mem = max(1, int(free * MEM_SHARE) // WORKER_MEM)
|
||||
return min(cpus, by_mem), f'{cpus} CPUs, {free / 2**30:.1f} GiB free memory allows {by_mem}'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--exefs', type=Path, required=True)
|
||||
ap.add_argument('--out', type=Path, required=True)
|
||||
ap.add_argument('--modules', default='main,subsdk0,sdk')
|
||||
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
|
||||
ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)')
|
||||
ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module')
|
||||
ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most')
|
||||
ap.add_argument('--chunk-bytes', type=int, default=3000000, help='estimated bytes of C per chunk')
|
||||
@@ -651,7 +669,9 @@ def main():
|
||||
done_at, errors = {}, []
|
||||
t_emit = time.perf_counter()
|
||||
ctx = mp.get_context('spawn')
|
||||
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
|
||||
jobs, why = pick_jobs(args.jobs)
|
||||
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
|
||||
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
|
||||
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
|
||||
results[r['module']].append(r)
|
||||
if r['error']:
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
|
||||
import ctypes
|
||||
import ctypes.wintypes as wt
|
||||
import sys
|
||||
|
||||
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
|
||||
LIMIT_PROCESS_MEMORY = 0x100
|
||||
LIMIT_JOB_MEMORY = 0x200
|
||||
|
||||
|
||||
class _MemStatus(ctypes.Structure):
|
||||
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
|
||||
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
|
||||
|
||||
|
||||
class _JobLimits(ctypes.Structure):
|
||||
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
|
||||
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
|
||||
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
|
||||
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD),
|
||||
('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong),
|
||||
('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong),
|
||||
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
|
||||
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
|
||||
|
||||
|
||||
def _job_headroom():
|
||||
"""Bytes left under the memory limit of the job object this process runs in, or None without such a limit."""
|
||||
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
|
||||
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
|
||||
info = _JobLimits()
|
||||
# A null handle asks about the job of the calling process.
|
||||
if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
|
||||
ctypes.sizeof(info), None):
|
||||
return None
|
||||
if info.LimitFlags & LIMIT_JOB_MEMORY:
|
||||
return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed)
|
||||
if info.LimitFlags & LIMIT_PROCESS_MEMORY:
|
||||
return info.ProcessMemoryLimit
|
||||
return None
|
||||
|
||||
|
||||
def available_ram():
|
||||
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
|
||||
if sys.platform != 'win32':
|
||||
return None
|
||||
status = _MemStatus()
|
||||
status.dwLength = ctypes.sizeof(status)
|
||||
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
||||
return None
|
||||
headroom = _job_headroom()
|
||||
return status.avail if headroom is None else min(status.avail, headroom)
|
||||
@@ -97,7 +97,7 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32'
|
||||
LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
|
||||
|
||||
# tools/ files the installer steps run (and what they import), copied from the repo.
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py',
|
||||
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py',
|
||||
'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
|
||||
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
|
||||
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon
|
||||
|
||||
Reference in new issue
Block a user