mirror of
https://github.com/valentderah/totk-pc-recomp.git
synced 2026-11-19 20:25:57 +00:00
Record the runtime headers a PGO profile was trained with and warn when they change
The floating-point helpers in simd.h and simd_fp.h inline into every lifted function before llvm-profdata matches the profile, so the fpcheap change left pgo-v143 with a fifth of its weight while recomp/ stayed the same and make_release saw nothing. pgo.py merge now reads the instrumented build's Ninja deps log and stores the digests of the runtime/ headers the generated objects include (cpu.h, simd.h, simd_fp.h); make_release warns when one differs or a record has none. docs/building.md says to retrain after a change to those headers, and build_game.py names the libiconv-2.dll that MSYS2's SDL3.dll needs in deps-mingw/bin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bdcfb42c07
commit
2d2c6782af
4 files changed
+85
-16
No files matched your search
+10
-5
@@ -165,9 +165,12 @@ with it too: hot paths laid out straight, hot calls inlined, code that never ran
|
||||
bench ran 12-15% faster per frame and a fight outside the training about 6%. The installer builds with
|
||||
`data/pgo-<ver>.profdata` of the release when the update has one, else at -O1.
|
||||
|
||||
A profile only fits the generated C it was trained on: functions are matched by name and control-flow hash, so a
|
||||
function whose C changes loses its counts without a warning. Retrain after a lifter or generator change, from a clean
|
||||
`recomp/`. The instrumented build needs llvm-mingw, as MSYS2's clang has no profile runtime:
|
||||
A profile only fits the code it was trained on: functions are matched by name and control-flow hash, so a function
|
||||
whose code changes loses its counts without a warning. That code is the generated C plus the runtime headers it
|
||||
includes (`cpu.h`, `simd.h`, `simd_fp.h`), whose inline helpers become part of every lifted function: a change to the
|
||||
floating-point helpers once left a profile with a fifth of its weight and 2% of its gain. Retrain after a lifter or
|
||||
generator change or a change to those headers, from a clean `recomp/`. The instrumented build needs llvm-mingw, as
|
||||
MSYS2's clang has no profile runtime:
|
||||
|
||||
```powershell
|
||||
$llvm = "toolchain\llvm-mingw-<date>\llvm-mingw-<date>-ucrt-x86_64"
|
||||
@@ -182,8 +185,10 @@ Scripted runs from saves (`tools/run_game.py`) add to the same file. Then:
|
||||
.venv\Scripts\python tools\pgo.py merge build\v143-pgogen --version v143
|
||||
```
|
||||
|
||||
writes `build\pgo\pgo-v143.profdata` and its record (`.json`: the `recomp/` tree it fits). `make_release.py` ships every
|
||||
profile of `build\pgo` in `data\` and warns when a record names another `recomp/` tree than the one released.
|
||||
writes `build\pgo\pgo-v143.profdata` and its record (`.json`: the `recomp/` tree it fits and the digests of the runtime
|
||||
headers the generated C included, read from the build's Ninja deps log). Merge from the checkout the instrumented build
|
||||
was made in. `make_release.py` ships every profile of `build\pgo` in `data\` and warns when a record names another
|
||||
`recomp/` tree than the one released or a header differs.
|
||||
|
||||
## Linux (in progress)
|
||||
|
||||
|
||||
+3
-2
@@ -46,8 +46,9 @@ of its processes (Windows job object accounting: committed private bytes).
|
||||
--ninja PATH names the Ninja to run; without MSYS2 the bin/ninja.exe of the release folder (tools/make_release.py) is used.
|
||||
--toolchain DIR builds with an llvm-mingw tree (DIR/bin/clang.exe) instead of MSYS2 UCRT64 clang. Ninja still comes
|
||||
from MSYS2. The libraries and headers MSYS2 supplies (Vulkan, SDL3, Opus, libvpx, VMA) then come from --deps DIR
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs), and the C++
|
||||
runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
|
||||
(default toolchain/deps-mingw: include/, lib/ with the import libraries and libvpx.a, bin/ with the DLLs copied next to
|
||||
totk.exe: SDL3.dll, libopus-0.dll and, when SDL3.dll comes from MSYS2, the libiconv-2.dll it imports, without which
|
||||
the exe stops at start with 0xc000007b or a missing-DLL error), and the C++ runtime is libc++ from llvm-mingw. Such a build never reuses another build's generated objects.
|
||||
|
||||
Split build. --make-runtime-lib compiles only runtime/ into <out>/libtotkrt.a (what CI prebuilds). --runtime-lib PATH
|
||||
then compiles only the generated C and the install-time files (modules.c and INSTALL_SIDE, which bake the player's
|
||||
|
||||
+14
-3
@@ -54,6 +54,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
import build_xlate # noqa: E402
|
||||
import check_content # noqa: E402
|
||||
import pgo # noqa: E402
|
||||
import shader_list # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -385,11 +386,13 @@ def lifted_version():
|
||||
|
||||
|
||||
def profiles():
|
||||
"""{version tag: .profdata} of build/pgo (tools/pgo.py merge). A profile trained on another recomp/ than the one
|
||||
shipped still works but misses the functions whose generated C changed since: warn, so the next release retrains."""
|
||||
"""{version tag: .profdata} of build/pgo (tools/pgo.py merge). A profile trained on another recomp/, or with other
|
||||
runtime headers inlined into the generated C, than the ones shipped still works but misses the functions whose
|
||||
code changed since: warn, so the next release retrains."""
|
||||
tree = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD:recomp'], capture_output=True, text=True).stdout.strip()
|
||||
dirty = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain', '--', 'recomp'], capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
retrain = 'build without a profile; retrain (docs/building.md, "Profile-guided build")'
|
||||
found = {}
|
||||
for p in sorted(PGO_DIR.glob('pgo-*.profdata')):
|
||||
rec = p.with_suffix('.json')
|
||||
@@ -397,7 +400,15 @@ def profiles():
|
||||
if record.get('recomp_tree') != tree or dirty:
|
||||
print(f'make_release: warning: {p.name} was trained on recomp tree {record.get("recomp_tree", "unknown")[:12]}, '
|
||||
f'this is {tree[:12]}{" with local changes" if dirty else ""}: functions whose generated C changed '
|
||||
'build without a profile; retrain (docs/building.md, "Profile-guided build")', file=sys.stderr)
|
||||
f'{retrain}', file=sys.stderr)
|
||||
stale = pgo.stale_headers(record, ROOT)
|
||||
if stale is None:
|
||||
print(f'make_release: warning: {p.name} has no runtime header digests (merged before they were recorded): '
|
||||
'a change to a header the generated C includes would go unnoticed; re-merge or retrain',
|
||||
file=sys.stderr)
|
||||
elif stale:
|
||||
print(f'make_release: warning: {p.name} was trained with other {", ".join(stale)}: every lifted function '
|
||||
f'that inlines them may {retrain}', file=sys.stderr)
|
||||
found[p.stem.removeprefix('pgo-')] = p
|
||||
if not found:
|
||||
print(f'make_release: no execution profile in {PGO_DIR}: installs build the game at -O1', file=sys.stderr)
|
||||
|
||||
+58
-6
@@ -6,18 +6,24 @@ An instrumented build (build_game.py --pgo gen) writes <build>/pgo/default_<id>.
|
||||
minute and at exit); successive runs of the same exe add to that file. llvm-profdata comes from the llvm-mingw toolchain
|
||||
the instrumented build used: its raw format is tied to the compiler version.
|
||||
|
||||
Beside the profile goes <out>.json, the training record: the game version and the git tree of recomp/ at the merge.
|
||||
The profile matches the generated C of that generator; tools/make_release.py ships it only while recomp/ is unchanged,
|
||||
since llvm-profdata matches functions by name and control-flow hash and a regenerated function silently loses its
|
||||
counts. Merge right after the training runs, from the commit they were built at.
|
||||
Beside the profile goes <out>.json, the training record: the game version, the git tree of recomp/ at the merge and
|
||||
the digests of the runtime/ headers the generated C includes (gen_headers; their inline helpers end up inside every
|
||||
lifted function). The profile matches the generated C of that generator and those headers; tools/make_release.py warns
|
||||
when either changed, since llvm-profdata matches functions by name and control-flow hash and a changed function
|
||||
silently loses its counts. Merge right after the training runs, from the checkout and commit they were built at.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import build_game # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXE = '.exe' if sys.platform == 'win32' else ''
|
||||
DEFAULT_DIR = ROOT / 'build' / 'pgo' # tools/make_release.py takes pgo-<version>.profdata from here
|
||||
@@ -44,6 +50,51 @@ def recomp_tree():
|
||||
text=True).stdout.strip() or None
|
||||
|
||||
|
||||
def header_digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
|
||||
|
||||
|
||||
def gen_headers(build):
|
||||
"""{runtime-relative path: digest} of the runtime/ headers the generated objects of an instrumented build include,
|
||||
from its Ninja deps log. Their inline helpers (simd_fp.h, cpu.h, ...) become part of every lifted function before
|
||||
the profile is matched, so a change to one invalidates the profile as surely as a generator change."""
|
||||
env = dict(os.environ, PATH=str(build_game.UCRT) + os.pathsep + os.environ.get('PATH', ''))
|
||||
r = subprocess.run([str(build_game.NINJA), '-f', str(build / 'build.ninja'), '-t', 'deps'], cwd=ROOT, env=env,
|
||||
capture_output=True, text=True)
|
||||
if r.returncode:
|
||||
sys.exit(f'ninja -t deps failed in {build}: {(r.stdout + r.stderr).strip()[-500:]}')
|
||||
runtime = (ROOT / 'runtime').resolve()
|
||||
found, cur = {}, False
|
||||
for line in r.stdout.splitlines():
|
||||
if not line.startswith(' '):
|
||||
target = line.split(': #deps')[0].replace('\\', '/')
|
||||
cur = '/obj/' in target and '/obj/runtime/' not in target
|
||||
continue
|
||||
if not cur:
|
||||
continue
|
||||
p = Path(line.strip())
|
||||
p = (p if p.is_absolute() else ROOT / p).resolve()
|
||||
try:
|
||||
rel = p.relative_to(runtime).as_posix()
|
||||
except ValueError:
|
||||
continue
|
||||
if rel not in found:
|
||||
found[rel] = header_digest(p)
|
||||
if not found:
|
||||
sys.exit(f'no runtime/ header in the deps log of {build}: was it built from this checkout?')
|
||||
return dict(sorted(found.items()))
|
||||
|
||||
|
||||
def stale_headers(record, root):
|
||||
"""The headers of a training record that differ in root/runtime now (missing ones included); None when the record
|
||||
predates the header list."""
|
||||
headers = record.get('gen_headers')
|
||||
if headers is None:
|
||||
return None
|
||||
rt = root / 'runtime'
|
||||
return [h for h, d in headers.items() if not (rt / h).is_file() or header_digest(rt / h) != d]
|
||||
|
||||
|
||||
def merge(build, out, version, toolchain):
|
||||
raw = sorted((build / 'pgo').glob('*.profraw'))
|
||||
if not raw:
|
||||
@@ -51,6 +102,7 @@ def merge(build, out, version, toolchain):
|
||||
tree = recomp_tree()
|
||||
if not tree:
|
||||
sys.exit('recomp/ has uncommitted changes: the record could not name the generator the profile matches')
|
||||
headers = gen_headers(build)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = out.with_suffix('.profdata.tmp')
|
||||
cmd = [str(llvm_profdata(toolchain)), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)]
|
||||
@@ -60,11 +112,11 @@ def merge(build, out, version, toolchain):
|
||||
sys.exit(f'llvm-profdata merge failed: {(p.stdout + p.stderr).strip()[-2000:]}')
|
||||
tmp.replace(out)
|
||||
record = {'version': version, 'recomp_tree': tree, 'merged': datetime.date.today().isoformat(),
|
||||
'raw_files': len(raw)}
|
||||
'raw_files': len(raw), 'gen_headers': headers}
|
||||
out.with_suffix('.json').write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8')
|
||||
size = sum(f.stat().st_size for f in raw)
|
||||
print(f'{out}: {len(raw)} raw files ({size / 2**20:.1f} MiB) -> {out.stat().st_size / 2**20:.1f} MiB; '
|
||||
f'recomp tree {tree[:12]}')
|
||||
f'recomp tree {tree[:12]}, {len(headers)} runtime headers')
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Reference in new issue
Block a user