diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index d9ea474b..feba5764 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2568,7 +2568,8 @@ static struct { PlatThread *th[LB_MAX_THREADS]; uint8_t clamp, poly, minus_w, samples; // library state (GKey fields) double t0, ms[3]; // ms: build time per kind (vertex, fragment, compute) - uint64_t built[3], have[3], fail[3], warmed[3], noshader, done, cs_draw; // warmed: driver cache only + // warmed: driver cache only; nolib: vertex and fragment variants skipped with the graphics pipeline library off + uint64_t built[3], have[3], fail[3], warmed[3], noshader, nolib, done, cs_draw; double warm_ms[3]; volatile int32_t kept; // libraries entered in libs[] so far int32_t keep_n; @@ -2734,14 +2735,21 @@ static void lb_library(const GfxShader *sh, int vertex, int keep) { lb_count(stat, !pipe ? 2 : have || got != pipe ? 1 : 0, ms); } +// A vertex or fragment variant with the graphics pipeline library off: no stage library to build ahead. +static void lb_skip_lib(void) { + plat_mutex_lock(&LB.stat); + LB.nolib++; + plat_mutex_unlock(&LB.stat); +} + static void lb_report(const char *how) { float mem_used, mem_total, mem_proc; plat_mem_info(&mem_used, &mem_total, &mem_proc); plat_mutex_lock_shared(&LB.stat); rt_log("vk_pipeline: stage library prebuild %s: %llu of %zu variants in %.0f ms; kept: vertex %llu built (avg %.1f ms), " "%llu there; fragment %llu built (avg %.1f ms), %llu there; compute %llu built (avg %.1f ms), %llu there; driver " - "cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache; " - "process working set %.0f MB", how, (unsigned long long)LB.done, LB.n, + "cache only: vertex %llu (avg %.1f ms), fragment %llu (avg %.1f ms), compute %llu (avg %.1f ms); %llu failed, %llu not in the shader cache, " + "%llu vertex/fragment skipped (graphics pipeline library off); process working set %.0f MB", how, (unsigned long long)LB.done, LB.n, now_ms() - LB.t0, (unsigned long long)LB.built[0], LB.built[0] ? LB.ms[0] / LB.built[0] : 0.0, (unsigned long long)LB.have[0], (unsigned long long)LB.built[1], LB.built[1] ? LB.ms[1] / LB.built[1] : 0.0, (unsigned long long)LB.have[1], (unsigned long long)LB.built[2], LB.built[2] ? LB.ms[2] / LB.built[2] : 0.0, @@ -2749,7 +2757,7 @@ static void lb_report(const char *how) { (unsigned long long)LB.warmed[1], LB.warmed[1] ? LB.warm_ms[1] / LB.warmed[1] : 0.0, (unsigned long long)LB.warmed[2], LB.warmed[2] ? LB.warm_ms[2] / LB.warmed[2] : 0.0, (unsigned long long)(LB.fail[0] + LB.fail[1] + LB.fail[2]), - (unsigned long long)LB.noshader, (double)mem_proc); + (unsigned long long)LB.noshader, (unsigned long long)LB.nolib, (double)mem_proc); plat_mutex_unlock_shared(&LB.stat); } @@ -2769,8 +2777,9 @@ static void lb_worker(void *arg) { plat_mutex_unlock(&LB.stat); } else if (gfx_shader_stage(sh) == GFX_STAGE_CS) { lb_compute(sh); - } else if (P.gpl && (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS)) { - lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); + } else if (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS) { + if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); + else lb_skip_lib(); } plat_mutex_lock(&LB.stat); LB.done++; @@ -2861,8 +2870,9 @@ static void lb_warm_worker(void *arg) { VkPipeline pipe = l ? compute_create(sh, l) : VK_NULL_HANDLE; if (pipe) vkDestroyPipeline(P.device, pipe, NULL); lb_count(2, pipe ? 3 : 2, now_ms() - t0); - } else if (P.gpl && (st == GFX_STAGE_VS || st == GFX_STAGE_FS)) { - lb_library(sh, st == GFX_STAGE_VS, 0); + } else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) { + if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0); + else lb_skip_lib(); } if (owned) gfx_shader_release_private(sh); plat_mutex_lock(&LB.stat); @@ -2894,7 +2904,12 @@ static void lb_warm_prepared(const char *dir) { if (!k) lb_warm_worker(NULL); for (int i = 0; i < k; i++) plat_thread_join(t[i]); lb_report("preparation done"); - printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", n, k, now_ms() - LB.t0); + // with gpl off only compute variants are built: say so, rather than count the skipped ones as built + if (LB.nolib) + printf("prepare-shaders: %zu variants on %d threads in %.0f ms, %llu vertex/fragment variants skipped (graphics " + "pipeline library off, they build at their first draw)\n", n, k, now_ms() - LB.t0, (unsigned long long)LB.nolib); + else + printf("prepare-shaders: %zu variants built into the driver cache on %d threads in %.0f ms\n", n, k, now_ms() - LB.t0); free(LB.e), LB.e = NULL, LB.n = 0, LB.next = 0, LB.done = 0; } diff --git a/tools/build_xlate.py b/tools/build_xlate.py index ac1293c5..7c8b0412 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -51,7 +51,7 @@ def mng_pin() -> str: """The MacNativeGames commit tools/fetch_refs.sh pins.""" for line in (ROOT / "tools" / "fetch_refs.sh").read_text(encoding="utf-8").splitlines(): f = line.split() - if len(f) == 4 and f[:2] == ["fetch", "MacNativeGames"]: + if len(f) >= 4 and f[:2] == ["fetch", "MacNativeGames"]: # a fifth word asks for the history return f[3] sys.exit("tools/fetch_refs.sh pins no MacNativeGames commit") diff --git a/tools/fetch_refs.sh b/tools/fetch_refs.sh index 8c2aee0b..6addaa70 100644 --- a/tools/fetch_refs.sh +++ b/tools/fetch_refs.sh @@ -6,23 +6,33 @@ set -eu cd "$(dirname "$0")/.." mkdir -p refs +# fetch [history]: a shallow checkout of the commit, or with "history" the commit and all its +# ancestors (tools/shader_list.py walks the log of MacNativeGames' shader list). origin/HEAD names the commit either +# way, so tools can read the pinned revision as origin/HEAD. fetch() { - name=$1 url=$2 commit=$3 + name=$1 url=$2 commit=$3 history=${4:-} dir=refs/$name - if [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$commit" ]; then + if [ "$(git -C "$dir" rev-parse HEAD 2>/dev/null)" = "$commit" ] && + [ "$(git -C "$dir" rev-parse -q --verify refs/remotes/origin/HEAD 2>/dev/null)" = "$commit" ] && + { [ -z "$history" ] || [ "$(git -C "$dir" rev-parse --is-shallow-repository)" = false ]; }; then echo "$name: up to date" return fi rm -rf "$dir" git init -q "$dir" git -C "$dir" remote add origin "$url" - git -C "$dir" fetch -q --depth 1 origin "$commit" + if [ -n "$history" ]; then + git -C "$dir" fetch -q origin "$commit" + else + git -C "$dir" fetch -q --depth 1 origin "$commit" + fi git -C "$dir" checkout -q FETCH_HEAD + git -C "$dir" update-ref refs/remotes/origin/HEAD "$commit" echo "$name: $commit" } fetch LASRecomp https://github.com/peiikai/LASRecomp.git da35b83a9611d046a9a681aae668d7085954d37f -fetch MacNativeGames https://github.com/ly05010419/MacNativeGames.git b75be8cfa704df3701e86130c8ced268e79d6b11 +fetch MacNativeGames https://github.com/ly05010419/MacNativeGames.git b75be8cfa704df3701e86130c8ced268e79d6b11 history fetch d2r-mobile https://github.com/jhaizhou-ops/d2r-mobile.git 53e7c5bb9ca3ea53e2160005be8fbe920cd9cb53 fetch nxnative https://github.com/seedds/nxnative.git d5b380b3a80495609c2e751bb994911a638935ad fetch SwitchRecomp https://github.com/bgyss/SwitchRecomp.git 0f99e1c5c1e78fdc8e01e58a0f8dc8622e82976b diff --git a/tools/make_release.py b/tools/make_release.py index 86d0a92d..11bf7e0f 100644 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -38,6 +38,7 @@ import subprocess import sys import tarfile import time +import urllib.error import urllib.request import zipfile from pathlib import Path @@ -126,13 +127,31 @@ def fetch(name): path = DL / f'{name}-9.0.4.tar.gz' # upstream archives have identical basenames if name == 'libvpx-src': path = DL / 'libvpx-1.17.0.tar.gz' - if not path.is_file(): - print(f'download {url}') - with urllib.request.urlopen(url) as r, open(path, 'wb') as f: - shutil.copyfileobj(r, f) + if path.is_file(): + if sha256_file(path) == want: + return path + print(f'{path}: not the pinned file (a cut download or an older pin); downloading it again') + path.unlink() + # into .part, renamed once complete: a cut connection leaves a .part that the next run resumes, never a + # short file under the final name + part = path.with_name(path.name + '.part') + have = part.stat().st_size if part.is_file() else 0 + print(f'download {url}' + (f' (resuming at {have} bytes)' if have else '')) + req = urllib.request.Request(url, headers={'Range': f'bytes={have}-'} if have else {}) + try: + with urllib.request.urlopen(req) as r: + if have and r.status != 206: # the server ignored the range: start over + have = 0 + with open(part, 'ab' if have else 'wb') as f: + shutil.copyfileobj(r, f) + except urllib.error.HTTPError as e: + if e.code != 416: # 416: the .part already holds the whole file + raise + part.replace(path) got = sha256_file(path) if got != want: - sys.exit(f'{path}: sha256 {got}, pinned {want}') + path.unlink() # a .part resumed from another upload, or the URL now serves another file: start clean next run + sys.exit(f'{path}: sha256 {got}, pinned {want}; deleted, run again to download it anew') return path diff --git a/tools/make_release_linux.py b/tools/make_release_linux.py index 248f2078..64428053 100644 --- a/tools/make_release_linux.py +++ b/tools/make_release_linux.py @@ -72,12 +72,12 @@ PINS = { 'vulkan-headers-src': ('https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v1.4.363.tar.gz', 'cbaf687d3c59b9666fe080e7f8396b6b4f4d344a768ee6b337c59852f4526b68'), 'vma-header': build_game.VMA_FETCH, - # appimagetool and the runtime are published only under the moving "continuous" tag: pinned by hash, so a new - # upload stops the build until the pins are updated on purpose - 'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage', - 'ac7fd56feade6002c2f57db39587113275d351fce911a5fbb29ea84cd0666cab'), - 'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-x86_64', - '617aa77314d57c8b8e4fae9c3f0b35af7b30d5529cb69e98f64f92510ed98a46'), + # appimagetool and the type-2 runtime from release tags, not "continuous", whose uploads change under the same + # URL and would stop the build at the hash check + 'appimagetool': ('https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-x86_64.AppImage', + 'ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0'), + 'appimage-runtime': ('https://github.com/AppImage/type2-runtime/releases/download/20251108/runtime-x86_64', + '2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d'), } mr.PINS.update(PINS) # one cache (toolchain/release-dl) and one fetch for both releases diff --git a/tools/shader_list.py b/tools/shader_list.py index ec3c6310..493f0d00 100644 --- a/tools/shader_list.py +++ b/tools/shader_list.py @@ -8,6 +8,7 @@ our play meets: at 1340bee the list alone has 134 rows of 1.4.3 shaders, its his 25,238. A row no draw asks for costs one translation. No copy of the list goes into the repository. """ import subprocess +import sys PATH = 'games/totk/assets/shader_list.tsv' @@ -18,6 +19,9 @@ def git(repo, *args): def history(repo, rev='origin/HEAD'): """rev's list, then each V row of an older revision that no newer one has, newest first.""" + if git(repo, 'rev-parse', '--is-shallow-repository').strip() == 'true': + # a shallow clone has rev's list only: preparing from it skips most variants and shows only as stutter in play + sys.exit(f'{repo} is a shallow clone without the shader list\'s history; run tools/fetch_refs.sh') lines = git(repo, 'show', f'{rev}:{PATH}').splitlines() seen = set(lines) for h in git(repo, 'log', '--format=%H', rev, '--', PATH).split()[1:]: