From a5711539e132612e634e353399c5ee28839ea556 Mon Sep 17 00:00:00 2001 From: Valiantsin Dzerakh Date: Fri, 30 Oct 2026 12:43:50 +0300 Subject: [PATCH] Size the code generator's worker pool by free memory so it stops running out of RAM gen.py started one emit worker per CPU, about 1.3 GiB each on 1.4.3, so many-core machines with modest RAM died with a MemoryError during "Translating code to C" (issues 17, 28, probably 66). The default is now min(CPU count, 80% of the free physical memory / 1.5 GiB), at least 1, capped by the memory limit of the enclosing job object if there is one. --jobs still overrides it, and the chosen count and its reason are logged. The memory helper is tools/hostmem.py, shipped with the release tools. Co-Authored-By: Claude Opus 5.5 --- recomp/gen.py | 24 ++++++++++++++++++-- tools/hostmem.py | 52 +++++++++++++++++++++++++++++++++++++++++++ tools/make_release.py | 2 +- 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tools/hostmem.py diff --git a/recomp/gen.py b/recomp/gen.py index e07396fa..1eaa00f2 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -39,6 +39,7 @@ sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import flags # noqa: E402 +import hostmem # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 import regs # noqa: E402 @@ -587,12 +588,29 @@ def remove_stale(moddir, name, nchunks, errors): return removed +# Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB. +WORKER_MEM = 1536 << 20 +MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs + + +def pick_jobs(requested): + """Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory.""" + if requested: + return requested, '--jobs' + cpus = os.cpu_count() or 4 + free = hostmem.available_ram() + if free is None: + return cpus, f'{cpus} CPUs, free memory unknown' + by_mem = max(1, int(free * MEM_SHARE) // WORKER_MEM) + return min(cpus, by_mem), f'{cpus} CPUs, {free / 2**30:.1f} GiB free memory allows {by_mem}' + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exefs', type=Path, required=True) ap.add_argument('--out', type=Path, required=True) ap.add_argument('--modules', default='main,subsdk0,sdk') - ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4) + ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)') ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module') ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most') ap.add_argument('--chunk-bytes', type=int, default=3000000, help='estimated bytes of C per chunk') @@ -651,7 +669,9 @@ def main(): done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') - with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: + jobs, why = pick_jobs(args.jobs) + print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True) + with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)): results[r['module']].append(r) if r['error']: diff --git a/tools/hostmem.py b/tools/hostmem.py new file mode 100644 index 00000000..7916d937 --- /dev/null +++ b/tools/hostmem.py @@ -0,0 +1,52 @@ +"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows).""" +import ctypes +import ctypes.wintypes as wt +import sys + +JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 +LIMIT_PROCESS_MEMORY = 0x100 +LIMIT_JOB_MEMORY = 0x200 + + +class _MemStatus(ctypes.Structure): + _fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \ + [(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')] + + +class _JobLimits(ctypes.Structure): + _fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong), + ('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t), + ('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD), + ('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD), + ('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong), + ('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong), + ('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t), + ('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)] + + +def _job_headroom(): + """Bytes left under the memory limit of the job object this process runs in, or None without such a limit.""" + k32 = ctypes.WinDLL('kernel32', use_last_error=True) + k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p] + info = _JobLimits() + # A null handle asks about the job of the calling process. + if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), + ctypes.sizeof(info), None): + return None + if info.LimitFlags & LIMIT_JOB_MEMORY: + return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed) + if info.LimitFlags & LIMIT_PROCESS_MEMORY: + return info.ProcessMemoryLimit + return None + + +def available_ram(): + """Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows.""" + if sys.platform != 'win32': + return None + status = _MemStatus() + status.dwLength = ctypes.sizeof(status) + if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return None + headroom = _job_headroom() + return status.avail if headroom is None else min(status.avail, headroom) diff --git a/tools/make_release.py b/tools/make_release.py index 4489c8d8..9fd9d33c 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -97,7 +97,7 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32' LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. -TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', +TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon