Size the code generator's worker pool by free memory so it stops running out of RAM

gen.py started one emit worker per CPU, about 1.3 GiB each on 1.4.3, so many-core machines with modest RAM died with a MemoryError during "Translating code to C" (issues 17, 28, probably 66). The default is now min(CPU count, 80% of the free physical memory / 1.5 GiB), at least 1, capped by the memory limit of the enclosing job object if there is one. --jobs still overrides it, and the chosen count and its reason are logged. The memory helper is tools/hostmem.py, shipped with the release tools.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-10-30 12:43:50 +03:00
1 parent c3b9a4e7f6
commit a5711539e1
3 files changed
+75 -3

No files matched your search

+22 -2
View File
@@ -39,6 +39,7 @@ sys.path.insert(0, str(ROOT / 'tools'))
sys.path.insert(0, str(ROOT / 'recomp'))
import flags # noqa: E402
import hostmem # noqa: E402
import jumptables # noqa: E402
import patches # noqa: E402
import regs # noqa: E402
@@ -587,12 +588,29 @@ def remove_stale(moddir, name, nchunks, errors):
return removed
# Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB.
WORKER_MEM = 1536 << 20
MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs
def pick_jobs(requested):
"""Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory."""
if requested:
return requested, '--jobs'
cpus = os.cpu_count() or 4
free = hostmem.available_ram()
if free is None:
return cpus, f'{cpus} CPUs, free memory unknown'
by_mem = max(1, int(free * MEM_SHARE) // WORKER_MEM)
return min(cpus, by_mem), f'{cpus} CPUs, {free / 2**30:.1f} GiB free memory allows {by_mem}'
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--exefs', type=Path, required=True)
ap.add_argument('--out', type=Path, required=True)
ap.add_argument('--modules', default='main,subsdk0,sdk')
ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4)
ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)')
ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module')
ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most')
ap.add_argument('--chunk-bytes', type=int, default=3000000, help='estimated bytes of C per chunk')
@@ -651,7 +669,9 @@ def main():
done_at, errors = {}, []
t_emit = time.perf_counter()
ctx = mp.get_context('spawn')
with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
jobs, why = pick_jobs(args.jobs)
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
results[r['module']].append(r)
if r['error']:
+52
View File
@@ -0,0 +1,52 @@
"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows)."""
import ctypes
import ctypes.wintypes as wt
import sys
JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9
LIMIT_PROCESS_MEMORY = 0x100
LIMIT_JOB_MEMORY = 0x200
class _MemStatus(ctypes.Structure):
_fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \
[(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')]
class _JobLimits(ctypes.Structure):
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong),
('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t),
('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD),
('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD),
('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong),
('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong),
('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t),
('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
def _job_headroom():
"""Bytes left under the memory limit of the job object this process runs in, or None without such a limit."""
k32 = ctypes.WinDLL('kernel32', use_last_error=True)
k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p]
info = _JobLimits()
# A null handle asks about the job of the calling process.
if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info),
ctypes.sizeof(info), None):
return None
if info.LimitFlags & LIMIT_JOB_MEMORY:
return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed)
if info.LimitFlags & LIMIT_PROCESS_MEMORY:
return info.ProcessMemoryLimit
return None
def available_ram():
"""Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows."""
if sys.platform != 'win32':
return None
status = _MemStatus()
status.dwLength = ctypes.sizeof(status)
if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
return None
headroom = _job_headroom()
return status.avail if headroom is None else min(status.avail, headroom)
+1 -1
View File
@@ -97,7 +97,7 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32'
LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'}
# tools/ files the installer steps run (and what they import), copied from the repo.
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py',
TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py',
'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py']
TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes
RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon