Merge task/liftretry: retry the lift with half the workers after a MemoryError and take the count from TOTK_LIFT_JOBS (#240)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valiantsin DzerakhandClaude Opus 5.5 committed 2026-11-01 23:28:03 +03:00
commit 0851638064
2 files changed
+41 -11

No files matched your search

+2
View File
@@ -97,6 +97,8 @@ of them.
`--cpu v2` targets x86-64-v2 (SSE4.2, no AVX2/FMA) for older CPUs; the default `v3` needs AVX2. The v2 build links
`runtime/fma.c`, a correctly rounded software fma the FP lowering needs, and runs slower.
`gen.py` sizes its emit pool from the CPU count and the free memory; `--jobs N` sets it, and so does the environment variable `TOTK_LIFT_JOBS=N`, which the installer's lift step honours too (it has no `--jobs`). When a worker runs out of memory, the pool restarts with half the workers (`emit: out of memory with N workers, retrying with M`), down to one; only an out-of-memory error with one worker fails the step.
Lifting takes about 15 s, a clean compile about 200 s on 24 threads; later builds recompile only what changed.
A compiler cache in front of clang shortens it further: with `SCCACHE` set to an sccache executable, `build_game.py`
writes it into the ninja file ahead of `cc`, and every lift rewrites the generated chunks with the same contents, so
+39 -11
View File
@@ -558,7 +558,8 @@ def _retry(fn, *args):
_W = {} # worker state: 'mods' {name: (path, base)}, 'starts' {name: sorted array}, 'emitters' {name: Emitter}
def _worker_init(mods, starts, table, opts):
def _worker_init(mods, starts, table, opts, jobs):
_W['jobs'] = jobs
_W['mods'] = mods
_W['starts'] = starts
_W['patches'] = table
@@ -581,6 +582,10 @@ def _emit_chunk(task):
"""Worker: write one chunk. Returns its statistics."""
name, number, path, fns = task
t0 = time.perf_counter()
# Test hook: a pool wider than this raises the MemoryError a busy machine does, so the retry can be exercised.
limit = os.environ.get('TOTK_LIFT_TEST_OOM_ABOVE')
if limit and _W['jobs'] > int(limit) and number % 2 == 1:
raise MemoryError
e = _emitter(name)
e.stats.clear()
e.unsupported.clear()
@@ -632,6 +637,14 @@ def pick_jobs(requested):
"""Worker count and the reason for it: --jobs if given, else the CPU count capped by the free memory."""
if requested:
return requested, '--jobs'
env = os.environ.get('TOTK_LIFT_JOBS', '').strip()
if env:
try:
if int(env) >= 1:
return int(env), 'TOTK_LIFT_JOBS'
except ValueError:
pass
print(f'ignoring TOTK_LIFT_JOBS={env!r}: not a positive integer', file=sys.stderr, flush=True)
cpus = os.cpu_count() or 4
free = hostmem.available_ram()
if free is None:
@@ -705,16 +718,31 @@ def main():
t_emit = time.perf_counter()
ctx = mp.get_context('spawn')
jobs, why = pick_jobs(args.jobs)
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts)) as pool:
for i, r in enumerate(pool.imap_unordered(_emit_chunk, [t for _, t in tasks], chunksize=1)):
results[r['module']].append(r)
if r['error']:
errors.append(r['error'])
done_at[r['module']] = time.perf_counter() - t_emit
if (i + 1) % 200 == 0 or i + 1 == len(tasks):
print(f'emitted {i + 1}/{len(tasks)} chunks, {time.perf_counter() - t_emit:.0f}s', file=sys.stderr,
flush=True)
pending = [t for _, t in tasks]
emitted = 0
while True:
print(f'emitting with {jobs} workers ({why})', file=sys.stderr, flush=True)
try:
# Chunk writes are atomic and idempotent, so a chunk lost with a failed pool is simply emitted again.
with ctx.Pool(jobs, initializer=_worker_init, initargs=(paths, starts_abs, table, opts, jobs)) as pool:
for r in pool.imap_unordered(_emit_chunk, pending, chunksize=1):
results[r['module']].append(r)
if r['error']:
errors.append(r['error'])
done_at[r['module']] = time.perf_counter() - t_emit
emitted += 1
if emitted % 200 == 0 or emitted == len(tasks):
print(f'emitted {emitted}/{len(tasks)} chunks, {time.perf_counter() - t_emit:.0f}s',
file=sys.stderr, flush=True)
break
except MemoryError:
if jobs == 1:
raise
smaller = jobs // 2
print(f'emit: out of memory with {jobs} workers, retrying with {smaller}', file=sys.stderr, flush=True)
jobs, why = smaller, 'after out of memory'
done = {(r['module'], r['number']) for rs in results.values() for r in rs}
pending = [t for t in pending if (t[0], t[1]) not in done]
for name, d in info.items():
rs = sorted(results[name], key=lambda r: r['number'])