mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-21 12:55:57 +00:00
Merge task/liftretry: retry the lift with half the workers after a MemoryError and take the count from TOTK_LIFT_JOBS (#240)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
0851638064
2 files changed
+41
-11
No files matched your search
@@ -97,6 +97,8 @@ of them.
|
||||
`--cpu v2` targets x86-64-v2 (SSE4.2, no AVX2/FMA) for older CPUs; the default `v3` needs AVX2. The v2 build links
|
||||
`runtime/fma.c`, a correctly rounded software fma the FP lowering needs, and runs slower.
|
||||
|
||||
`gen.py` sizes its emit pool from the CPU count and the free memory; `--jobs N` sets it, and so does the environment variable `TOTK_LIFT_JOBS=N`, which the installer's lift step honours too (it has no `--jobs`). When a worker runs out of memory, the pool restarts with half the workers (`emit: out of memory with N workers, retrying with M`), down to one; only an out-of-memory error with one worker fails the step.
|
||||
|
||||
Lifting takes about 15 s, a clean compile about 200 s on 24 threads; later builds recompile only what changed.
|
||||
A compiler cache in front of clang shortens it further: with `SCCACHE` set to an sccache executable, `build_game.py`
|
||||
writes it into the ninja file ahead of `cc`, and every lift rewrites the generated chunks with the same contents, so
|
||||
|
||||
+39
-11
@@ -558,7 +558,8 @@ def _retry(fn, *args):
|
||||
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
|
||||
|
||||
|
||||
def _worker_init(mods, starts, table, opts):
|
||||
def _worker_init(mods, starts, table, opts, jobs):
|
||||
_W['jobs'] = jobs
|
||||
_W['mods'] = mods
|
||||
_W['starts'] = starts
|
||||
_W['patches'] = table
|
||||
@@ -581,6 +582,10 @@ def _emit_chunk(task):
|
||||
"""Worker: write one chunk. Returns its statistics."""
|
||||
name, number, path, fns = task
|
||||
t0 = time.perf_counter()
|
||||
# Test hook: a pool wider than this raises the MemoryError a busy machine does, so the retry can be exercised.
|
||||
limit = os.environ.get('TOTK_LIFT_TEST_OOM_ABOVE')
|
||||
if limit and _W['jobs'] > int(limit) and number % 2 == 1:
|
||||
raise MemoryError
|
||||
e = _emitter(name)
|
||||
e.stats.clear()
|
||||
e.unsupported.clear()
|
||||
@@ -632,6 +637,14 @@ def pick_jobs(requested):
|
||||
"""Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory."""
|
||||
if requested:
|
||||
return requested, '--jobs'
|
||||
env = os.environ.get('TOTK_LIFT_JOBS', '').strip()
|
||||
if env:
|
||||
try:
|
||||
if int(env) >= 1:
|
||||
return int(env), 'TOTK_LIFT_JOBS'
|
||||
except ValueError:
|
||||
pass
|
||||
print(f'ignoring TOTK_LIFT_JOBS={env!r}: not a positive integer', file=sys.stderr, flush=True)
|
||||
cpus = os.cpu_count() or 4
|
||||
free = hostmem.available_ram()
|
||||
if free is None:
|
||||
@@ -705,16 +718,31 @@ def main():
|
||||
t_emit = time.perf_counter()
|
||||
ctx = mp.get_context('spawn')
|
||||
jobs, why = pick_jobs(args.jobs)
|
||||
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
|
||||
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
|
||||
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
|
||||
results[r['module']].append(r)
|
||||
if r['error']:
|
||||
errors.append(r['error'])
|
||||
done_at[r['module']] = time.perf_counter() - t_emit
|
||||
if (i + 1) % 200 == 0 or i + 1 == len(tasks):
|
||||
print(f'emitted {i + 1}/{len(tasks)} chunks, {time.perf_counter() - t_emit:.0f}s', file=sys.stderr,
|
||||
flush=True)
|
||||
pending = [t for _, t in tasks]
|
||||
emitted = 0
|
||||
while True:
|
||||
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
|
||||
try:
|
||||
# Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again.
|
||||
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool:
|
||||
for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1):
|
||||
results[r['module']].append(r)
|
||||
if r['error']:
|
||||
errors.append(r['error'])
|
||||
done_at[r['module']] = time.perf_counter() - t_emit
|
||||
emitted += 1
|
||||
if emitted % 200 == 0 or emitted == len(tasks):
|
||||
print(f'emitted {emitted}/{len(tasks)} chunks, {time.perf_counter() - t_emit:.0f}s',
|
||||
file=sys.stderr, flush=True)
|
||||
break
|
||||
except MemoryError:
|
||||
if jobs == 1:
|
||||
raise
|
||||
smaller = jobs // 2
|
||||
print(f'emit: out of memory with {jobs} workers, retrying with {smaller}', file=sys.stderr, flush=True)
|
||||
jobs, why = smaller, 'after out of memory'
|
||||
done = {(r['module'], r['number']) for rs in results.values() for r in rs}
|
||||
pending = [t for t in pending if (t[0], t[1]) not in done]
|
||||
|
||||
for name, d in info.items():
|
||||
rs = sorted(results[name], key=lambda r: r['number'])
|
||||
|
||||
Reference in new issue
Block a user