diff --git a/recomp/gen.py b/recomp/gen.py index e07396fa..1eaa00f2 100644 --- a/recomp/gen.py +++ b/recomp/gen.py @@ -39,6 +39,7 @@ sys.path.insert(0, str(ROOT / 'tools')) sys.path.insert(0, str(ROOT / 'recomp')) import flags # noqa: E402 +import hostmem # noqa: E402 import jumptables # noqa: E402 import patches # noqa: E402 import regs # noqa: E402 @@ -587,12 +588,29 @@ def remove_stale(moddir, name, nchunks, errors): return removed +# Peak memory of one emit worker on 1.4.3 is 1.3 GiB (1.5 GiB with margin); the parent holds about 0.7 GiB. +WORKER_MEM = 1536 << 20 +MEM_SHARE = 0.8 # of the free memory the workers may take, the rest is for the system and other programs + + +def pick_jobs(requested): + """Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory.""" + if requested: + return requested, '--jobs' + cpus = os.cpu_count() or 4 + free = hostmem.available_ram() + if free is None: + return cpus, f'{cpus} CPUs, free memory unknown' + by_mem = max(1, int(free * MEM_SHARE) // WORKER_MEM) + return min(cpus, by_mem), f'{cpus} CPUs, {free / 2**30:.1f} GiB free memory allows {by_mem}' + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--exefs', type=Path, required=True) ap.add_argument('--out', type=Path, required=True) ap.add_argument('--modules', default='main,subsdk0,sdk') - ap.add_argument('--jobs', type=int, default=os.cpu_count() or 4) + ap.add_argument('--jobs', type=int, help='emit workers (default: by free memory and CPU count)') ap.add_argument('--limit', type=int, default=0, help='emit only the first N functions of each module') ap.add_argument('--chunk-fns', type=int, default=3000, help='functions per chunk at most') ap.add_argument('--chunk-bytes', type=int, default=3000000, help='estimated bytes of C per chunk') @@ -651,7 +669,9 @@ def main(): done_at, errors = {}, [] t_emit = time.perf_counter() ctx = mp.get_context('spawn') - with ctx.Pool(max(1, args.jobs), initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: + jobs, why = pick_jobs(args.jobs) + print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True) + with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool: for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)): results[r['module']].append(r) if r['error']: diff --git a/tools/hostmem.py b/tools/hostmem.py new file mode 100644 index 00000000..7916d937 --- /dev/null +++ b/tools/hostmem.py @@ -0,0 +1,52 @@ +"""Memory the machine can still give to new processes, for sizing worker pools (stdlib only, Windows).""" +import ctypes +import ctypes.wintypes as wt +import sys + +JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 +LIMIT_PROCESS_MEMORY = 0x100 +LIMIT_JOB_MEMORY = 0x200 + + +class _MemStatus(ctypes.Structure): + _fields_ = [('dwLength', wt.DWORD), ('dwMemoryLoad', wt.DWORD)] + \ + [(n, ctypes.c_ulonglong) for n in ('total', 'avail', 'tpf', 'apf', 'tv', 'av', 'aev')] + + +class _JobLimits(ctypes.Structure): + _fields_ = [('PerProcessUserTimeLimit', ctypes.c_longlong), ('PerJobUserTimeLimit', ctypes.c_longlong), + ('LimitFlags', wt.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t), + ('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wt.DWORD), + ('Affinity', ctypes.c_size_t), ('PriorityClass', wt.DWORD), ('SchedulingClass', wt.DWORD), + ('r_ops', ctypes.c_ulonglong), ('w_ops', ctypes.c_ulonglong), ('o_ops', ctypes.c_ulonglong), + ('r_bytes', ctypes.c_ulonglong), ('w_bytes', ctypes.c_ulonglong), ('o_bytes', ctypes.c_ulonglong), + ('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t), + ('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)] + + +def _job_headroom(): + """Bytes left under the memory limit of the job object this process runs in, or None without such a limit.""" + k32 = ctypes.WinDLL('kernel32', use_last_error=True) + k32.QueryInformationJobObject.argtypes = [wt.HANDLE, ctypes.c_int, ctypes.c_void_p, wt.DWORD, ctypes.c_void_p] + info = _JobLimits() + # A null handle asks about the job of the calling process. + if not k32.QueryInformationJobObject(None, JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, ctypes.byref(info), + ctypes.sizeof(info), None): + return None + if info.LimitFlags & LIMIT_JOB_MEMORY: + return max(0, info.JobMemoryLimit - info.PeakJobMemoryUsed) + if info.LimitFlags & LIMIT_PROCESS_MEMORY: + return info.ProcessMemoryLimit + return None + + +def available_ram(): + """Physical memory not in use right now, capped by the job memory limit if there is one. None off Windows.""" + if sys.platform != 'win32': + return None + status = _MemStatus() + status.dwLength = ctypes.sizeof(status) + if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return None + headroom = _job_headroom() + return status.avail if headroom is None else min(status.avail, headroom) diff --git a/tools/make_release.py b/tools/make_release.py index 4489c8d8..9fd9d33c 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -97,7 +97,7 @@ LLVM_SKIP_DIRS = {'aarch64-w64-mingw32', 'armv7-w64-mingw32', 'i686-w64-mingw32' LLVM_SKIP_SUFFIXES = {'.idl', '.tlb'} # tools/ files the installer steps run (and what they import), copied from the repo. -TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'make_romfs_patches.py', +TOOLS_FILES = ['build_game.py', 'build_shaders.py', 'check_content.py', 'extract.py', 'hostmem.py', 'make_romfs_patches.py', 'prepare_shaders.py', 'romfs_index.py', 'shader_census.py', 'shader_list.py'] TOOLS_DIRS = ['vendor/macnativegames', 'data'] # data/: expected hashes RUNTIME_SUFFIXES = {'.h', '.inc', '.ico'} # .ico: the installer window icon