diff --git a/.github/ISSUE_TEMPLATE/1-crash.yml b/.github/ISSUE_TEMPLATE/1-crash.yml new file mode 100644 index 00000000..adb68726 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-crash.yml @@ -0,0 +1,65 @@ +name: Crash +description: The game closes, freezes or shows a FATAL message. +labels: [bug, crash] +body: + - type: markdown + attributes: + value: | + Check the pinned issues at the top of the issue list first: they cover the known crashes and their workarounds. + + Do not attach or link game files, keys, firmware or builds. Issues that do are closed. + - type: checkboxes + id: duplicates + attributes: + label: Existing issues + options: + - label: I read the pinned issues and [searched open and closed issues](https://github.com/valentderah/totk-pc-recomp/issues?q=is%3Aissue) for this crash. + required: true + - type: input + id: version + attributes: + label: Version + description: The title line of the settings menu (F1), for example "TotK Recomp 0.1.0, game 1.4.3". + validations: + required: true + - type: input + id: system + attributes: + label: GPU, driver and Windows version + description: Copy system info in the menu's Advanced tab puts them on the clipboard. + validations: + required: true + - type: dropdown + id: when + attributes: + label: When does it crash? + options: + - At startup, before the Nintendo logo + - Between the Nintendo logo and the title screen + - While loading a save + - In gameplay + - In a cutscene + validations: + required: true + - type: dropdown + id: blocker + attributes: + label: Can you keep playing? + options: + - "Yes, it does not happen again after a restart, or I can avoid it" + - "No, it happens every time at the same point" + validations: + required: true + - type: textarea + id: steps + attributes: + label: What you did before the crash + validations: + required: true + - type: textarea + id: log + attributes: + label: Log + description: Attach logs\totk.log from the folder of totk.exe. Copy it before you start the game again, because the next start overwrites it. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/2-graphics.yml b/.github/ISSUE_TEMPLATE/2-graphics.yml new file mode 100644 index 00000000..bf297032 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-graphics.yml @@ -0,0 +1,59 @@ +name: Graphics glitch +description: Artifacts, missing or broken textures, flicker, wrong colours or shadows. +labels: [bug, graphics] +body: + - type: markdown + attributes: + value: | + Check the pinned issues at the top of the issue list first: they cover the known glitches on each GPU. + + Do not attach or link game files, keys, firmware or builds. Issues that do are closed. + - type: checkboxes + id: duplicates + attributes: + label: Existing issues + options: + - label: I read the pinned issues and [searched open and closed issues](https://github.com/valentderah/totk-pc-recomp/issues?q=is%3Aissue) for this glitch. + required: true + - type: input + id: version + attributes: + label: Version + description: The title line of the settings menu (F1), for example "TotK Recomp 0.1.0, game 1.4.3". + validations: + required: true + - type: input + id: system + attributes: + label: GPU, driver and Windows version + description: Copy system info in the menu's Advanced tab puts them on the clipboard. + validations: + required: true + - type: textarea + id: screenshot + attributes: + label: Screenshot or video + description: Drag the file into this field. Say where in the game it was taken. + validations: + required: true + - type: textarea + id: details + attributes: + label: What looks wrong and when + description: Does it show all the time, in one place, or after some time in play? Which settings did you change from the defaults? + validations: + required: true + - type: dropdown + id: blocker + attributes: + label: Can you keep playing? + options: + - "Yes" + - "No, I cannot see enough to play" + validations: + required: true + - type: textarea + id: log + attributes: + label: Log + description: Attach logs\totk.log from the folder of totk.exe. diff --git a/.github/ISSUE_TEMPLATE/3-bug.yml b/.github/ISSUE_TEMPLATE/3-bug.yml new file mode 100644 index 00000000..20d7087f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3-bug.yml @@ -0,0 +1,65 @@ +name: Other bug +description: Sound, controls, performance, the settings menu, the installer, or game logic. +labels: [bug] +body: + - type: markdown + attributes: + value: | + Check the pinned issues at the top of the issue list first. + + Do not attach or link game files, keys, firmware or builds. Issues that do are closed. + - type: checkboxes + id: duplicates + attributes: + label: Existing issues + options: + - label: I read the pinned issues and [searched open and closed issues](https://github.com/valentderah/totk-pc-recomp/issues?q=is%3Aissue) for this bug. + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - Audio + - Controls and input + - Performance and stutter + - Settings menu and overlay + - Installer + - Game logic and physics + - Something else + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: The title line of the settings menu (F1), for example "TotK Recomp 0.1.0, game 1.4.3". + validations: + required: true + - type: input + id: system + attributes: + label: GPU, driver and Windows version + description: Copy system info in the menu's Advanced tab puts them on the clipboard. + validations: + required: true + - type: dropdown + id: blocker + attributes: + label: Can you keep playing? + options: + - "Yes" + - "No, it stops the game or a quest" + validations: + required: true + - type: textarea + id: steps + attributes: + label: What happened and how to reproduce it + validations: + required: true + - type: textarea + id: log + attributes: + label: Log + description: Attach logs\totk.log from the folder of totk.exe. diff --git a/.github/ISSUE_TEMPLATE/4-feature.yml b/.github/ISSUE_TEMPLATE/4-feature.yml new file mode 100644 index 00000000..7d2f4ffe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4-feature.yml @@ -0,0 +1,17 @@ +name: Feature request +description: A setting, option or support for something the port does not do yet. +labels: [enhancement] +body: + - type: checkboxes + id: duplicates + attributes: + label: Existing issues + options: + - label: I [searched open and closed issues](https://github.com/valentderah/totk-pc-recomp/issues?q=is%3Aissue) and the Planned section of the README for this request. + required: true + - type: textarea + id: request + attributes: + label: What you want and why + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml deleted file mode 100644 index eb22ae3b..00000000 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Bug report -description: Something breaks, crashes or looks wrong. -body: - - type: markdown - attributes: - value: | - Do not attach or link game files, keys, firmware or builds. Issues that do are closed. - - type: input - id: version - attributes: - label: Version - description: The title line of the settings menu (F1), for example "TotK Recomp 0.1.0, game 1.4.3". - validations: - required: true - - type: input - id: system - attributes: - label: GPU, driver and Windows version - description: Copy system info in the menu's Advanced tab puts them on the clipboard. - validations: - required: true - - type: textarea - id: steps - attributes: - label: What happened and how to reproduce it - validations: - required: true - - type: textarea - id: log - attributes: - label: Log - description: Attach logs\totk.log from the folder of totk.exe. After a crash, copy it before you start the game again. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3ba13e0c..7e047e82 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1 +1,5 @@ blank_issues_enabled: false +contact_links: + - name: Question or help with setup + url: https://github.com/valentderah/totk-pc-recomp/discussions + about: Installing, updating and configuring the port. Ask in Discussions. diff --git a/.gitignore b/.gitignore index d2fdab74..e365e074 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ __pycache__/ *.o *.lib *.a +*.so +*.so.* +*.AppImage # .NET build output of the shader translator (tools/build_xlate.py) /tools/xlate/bin/ diff --git a/README.md b/README.md index 089d3f2b..cd851018 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Game updates 1.2.1 and 1.4.3, with saves that carry over between them - 60 fps by default; up to 144 fps or the monitor's refresh rate (experimental) - Custom resolutions (1080p, 2k...), supersampling and AMD FSR 1 upscaling +- 16:10 (Steam Deck, handhelds), 21:9 and 32:9 (ultrawide) game resolutions, with the camera and the interface in that shape - In-game settings menu (F1) and performance overlay (F10) - Keyboard, mouse and gamepad with remapping, rumble and gyro - Xbox button prompts: a layout that keeps the Switch button positions and shows the Xbox labels @@ -28,7 +29,6 @@ Report problems as issues, following [CONTRIBUTING.md](CONTRIBUTING.md). - Linux and the Steam Deck - Mods made for emulators: unpack a mod into a `mods` folder -- Ultrawide on 1.2.1 (1.4.3 has it for 2560x1080, 3440x1440, 3840x1600 and 5120x2160; other shapes get the wide camera with a 16:9 HUD) - VRR ## Installing diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 2d52b952..86d96747 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -74,6 +74,27 @@ These tools run on the developer's or player's machine during the build. They ar | Vulkan-Headers and Vulkan-Loader 1.4.363.0 (MSYS2 packages; headers and `libvulkan-1.dll.a` only, no loader DLL) | https://github.com/KhronosGroup | Apache-2.0 OR MIT | `runtime/deps/` | | Vulkan Memory Allocator 3.4.0 (MSYS2 package), header | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator | MIT | `runtime/deps/` | | Ninja 1.13.2 (`ninja-win.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja.exe` | -| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus `tools/hactool/portable.patch`, from MacNativeGames' `build_hactool()`: `MNG_TITLE_KEYS` and relaxed BKTR layout checks; our `tools/hactool/unicode.patch`: an embedded manifest that makes UTF-8 the process code page), built by `tools/build_hactool.py` with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | +| hactool (SciresM @ `1d64a83450e025622f3468c28fc4164dad2c5ef6` plus `tools/hactool/portable.patch`, from MacNativeGames' `build_hactool()`: `MNG_TITLE_KEYS` and relaxed BKTR layout checks; our `tools/hactool/unicode.patch`: an embedded manifest that makes UTF-8 the process code page; our `tools/hactool/mbedtls-asm-clobbers.patch`: the clobbers the bundled mbedtls's x86-64 multiply asm lacks), built by `tools/build_hactool.py` with MSYS2 UCRT64 gcc; imports only KERNEL32 and the UCRT | https://github.com/SciresM/hactool | ISC | `bin/hactool.exe` | | `totk_xlate.exe` (Ryujinx shader translator, NativeAOT build of `tools/xlate/`, see the Ryujinx row above) | built by `tools/build_xlate.py` | MIT | `bin/totk_xlate.exe` | | MacNativeGames shader variant list | https://github.com/ly05010419/MacNativeGames @ `b75be8cfa704df3701e86130c8ced268e79d6b11` (`games/totk/assets/shader_list.tsv`), Copyright (c) 2026 MacNativeGames contributors | MIT | `data/shader_list.tsv` | + +## Distributed in the Linux release (AppImage) + +`tools/make_release_linux.py` assembles these into the folder of `docs/installer.md` ("Linux release") and packs it as +`TotK-Setup-x86_64.AppImage`. Versions and sha256 of every download are pinned in that script; licence texts are in the +folder's `licenses/` and in each component's own folder. + +| Component | Source | License | Where in the release | +|---|---|---|---| +| CPython 3.14.10, python-build-standalone 20261026 `install_only` build, with `tkinter` and the libraries it bundles (OpenSSL, libffi, SQLite, zlib, Tcl/Tk and others; their licences ship in the folder) | https://github.com/astral-sh/python-build-standalone/releases/tag/20261026 | PSF-2.0; bundled libraries under their own licences | `python/` | +| numpy 2.5.4 (with the libraries its manylinux wheel bundles), lz4 4.4.5, cryptography 50.0.2, cffi 2.1.1, pycparser 3.11 | PyPI binary wheels, pins in `tools/requirements-installer.txt` | as in the installer release row above | `python/lib/python3.14/site-packages/` | +| LLVM 23.1.4 official Linux X64 release: `clang`, `lld`, `llvm-ar`, `llvm-ranlib`, `llvm-objcopy`, `llvm-strip` and the clang resource headers; `llvm-LICENSE.txt` from the `llvmorg-23.1.4` source tag | https://github.com/llvm/llvm-project/releases/tag/llvmorg-23.1.4 | Apache-2.0 with LLVM exception | `llvm/` | +| Steam Runtime 3 "sniper" SDK sysroot 3.0.20261021.266025 (Debian 11 based), only the headers, libraries and objects a game build reads: glibc 2.31 (headers, `crt*.o`, `libc_nonshared.a`, link libraries), GCC 10 `libstdc++.a` and `libgcc`, SDL 3.4.18, libopus, libvpx 1.9 (static), the Vulkan loader's link library; their Debian `copyright` files go into `licenses/` | https://repo.steampowered.com/steamrt-images-sniper/snapshots/3.0.20261021.266025/ | glibc LGPL-2.1+; libstdc++ and libgcc GPL-3.0 with the GCC Runtime Library Exception; SDL zlib; libopus BSD-3-Clause; libvpx BSD-3-Clause with the WebM patent grant (`libvpx-PATENTS.txt` from the v1.17.0 source tag); Vulkan loader Apache-2.0 | `llvm/sysroot/`, `bin/libSDL3.so.0` (the sysroot's, copied next to the game by the setup), `licenses/` | +| Vulkan-Headers 1.4.363 (source tag) | https://github.com/KhronosGroup/Vulkan-Headers/tree/v1.4.363 | Apache-2.0 OR MIT | `runtime/deps/include/` | +| Vulkan Memory Allocator 3.4.0, header (the licence text is its header comment) | https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator/tree/v3.4.0 | MIT | `runtime/deps/include/vma/` | +| Zstandard 1.5.7, built from the release tarball into a static `libzstd.a` with the folder's clang and sysroot | https://github.com/facebook/zstd/releases/tag/v1.5.7 | BSD-3-Clause | `runtime/deps/` | +| Ninja 1.13.2 (`ninja-linux.zip`) | https://github.com/ninja-build/ninja/releases/tag/v1.13.2 | Apache-2.0 | `bin/ninja` | +| hactool (as in the installer release row above), built by `tools/build_hactool.py --cc` with the folder's clang and sysroot; includes its bundled mbedtls | https://github.com/SciresM/hactool | ISC; mbedtls Apache-2.0 | `bin/hactool` | +| `totk_xlate` (NativeAOT linux-x64 build of `tools/xlate/`, linked with the folder's clang, lld and sysroot) | built by `tools/build_xlate.py --llvm --sysroot` | MIT | `bin/totk_xlate` | +| AppImage type-2 runtime (static; includes libfuse and squashfuse), the AppImage's first part; appimagetool packs the folder and is not distributed | https://github.com/AppImage/type2-runtime | MIT; libfuse LGPL-2.1; squashfuse BSD-2-Clause | the AppImage header | +| MacNativeGames shader variant list | as in the installer release row above | MIT | `data/shader_list.tsv` | diff --git a/VERSION b/VERSION index 1a030947..0c62199f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.9 +0.2.1 diff --git a/docs/architecture.md b/docs/architecture.md index 29967538..12f75987 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,11 +94,12 @@ Every guest thread is a host thread. TotK assumes Horizon's scheduler: three app (MacNativeGames' sites); `resolution.c` sets them from `output_res` and `render_scale` and grows the texture heap and the application pool to match. When the window is larger than the output, `vk_fsr.c` runs AMD FSR 1 (EASU, RCAS) on the present. An `output_res` that is not 16:9 (more than 1% off, `res_is_16x9`) also switches on the - `aspect` patch set (1.4.3 only), which hands the output's ratio to the camera, and `romfs.c` `overlay_init` loads the + `aspect` patch set, which hands the output's ratio to the camera, and `romfs.c` `overlay_init` loads the `romfs-patches-x` overlay for it when it exists (`tools/make_romfs_patches.py --aspect`, laid out by - `tools/ultrawide_layout.py`; the installer builds four sizes: 2560x1080, 3440x1440, 3840x1600, 5120x2160). Other - shapes, such as 16:10 or 32:9, get the wide camera and the 16:9 HUD stretched over it. 1.2.1 has no camera patch: a - wide size there keeps the 16:9 view. The `aspect` setting (`fit`, the default; `stretch`) decides what the present does + `tools/ultrawide_layout.py`; the installer builds ten sizes: 21:9 2560x1080, 3440x1440, 3840x1600, 5120x2160, 32:9 + 3840x1080, 5120x1440 and 16:10 1280x800, 1680x1050, 1920x1200, 2560x1600). A wider layout moves the HUD to the + screen's corners, a narrower one to its top and bottom edges. Other sizes get the camera of their shape and the 16:9 + HUD stretched over it. The `aspect` setting (`fit`, the default; `stretch`) decides what the present does when the window's shape differs from the output's: `fit` letterboxes, where a 16:9 game in a wider window used to be stretched. - Occlusion queries never block the CPU; a result that is not ready counts as visible. diff --git a/docs/building.md b/docs/building.md index 44216cc0..e7d48480 100644 --- a/docs/building.md +++ b/docs/building.md @@ -37,13 +37,15 @@ PATH: ### hactool -`tools/build_hactool.py` fetches SciresM/hactool at `1d64a83` into `toolchain\src\hactool`, applies the two patches in +`tools/build_hactool.py` fetches SciresM/hactool at `1d64a83` into `toolchain\src\hactool`, applies the three patches in `tools/hactool/` and builds a static `toolchain\hactool\hactool.exe` with MSYS2 UCRT64 gcc (a few seconds): - `portable.patch`: the parts of MacNativeGames' `build_hactool()` that run on Windows, the `MNG_TITLE_KEYS` variable and the two BKTR fixes in `nca.c` for the subsection table padding and the subsection layout of newer cumulative updates; - `unicode.patch`: a manifest with `UTF-8`, so paths under a non-ASCII user profile - reach it intact. + reach it intact; +- `mbedtls-asm-clobbers.patch`: the `memory` and `cc` clobbers the bundled mbedtls 2.6.1's x86-64 multiply asm lacks. + Without them a clang build (the Linux release) fails every RSA signature check and the update merge stops. hactool reads `-k `; MSYS2 shells set `HOME`, which hides its default key location, so pass `-k` there. @@ -78,7 +80,8 @@ From the repository root, in PowerShell. Every step that depends on the game ver .venv\Scripts\python tools\build_xlate.py ``` -5. Lift the modules to C, write the RomFS half of the frame-rate patches, then compile and link `build\\totk.exe`: +5. Lift the modules to C, write the RomFS overlay (the frame-rate fix), then compile and link + `build\\totk.exe`: ```powershell .venv\Scripts\python recomp\gen.py --exefs build\extracted\\exefs --out generated\ --patches recomp\patches-.txt @@ -87,6 +90,14 @@ From the repository root, in PowerShell. Every step that depends on the game ver ``` Lifting takes about 15 s, a clean compile about 200 s on 24 threads; later builds recompile only what changed. + A compiler cache in front of clang shortens it further: with `SCCACHE` set to an sccache executable, `build_game.py` + writes it into the ninja file ahead of `cc`, and every lift rewrites the generated chunks with the same contents, so + a second build of the same version hits all of their objects: + + ```powershell + $env:SCCACHE = "path\to\sccache.exe" + .venv\Scripts\python tools\build_game.py --version + ``` `--patches` builds in the frame-rate, resolution and other patch sets, switchable at run time. The link copies `SDL3.dll`, `libiconv-2.dll` and `libopus-0.dll` next to the exe. For Xbox button prompts, run `make_romfs_patches.py` once more with `--xbox-prompts`: it writes @@ -104,7 +115,8 @@ From the repository root, in PowerShell. Every step that depends on the game ver The AMD driver keeps its pipeline cache per exe path and holds a limited amount: preparing another build folder can evict this one's. `totk.exe --warm-shaders` rebuilds it from the shader cache. -7. Run. Keep the window in the foreground; an unfocused window stalls the game. +7. Run. The game keeps running when its window loses the focus (`assume_focus = 1` in `totk.ini`, the default; the + Game tab switches it). With 0 it pauses, as it does on the Switch when Home is pressed. ```powershell build\\totk.exe @@ -120,6 +132,48 @@ your machines without MSYS2, Python or .NET. `romfs.bin` stays out unless you pa `romfs\` next to the exe or point `TOTK_ROMFS` at its folder. Run `totk.exe --prepare-shaders` once in the copied folder. The folder holds code and data derived from your game: for your own machines only. +## Linux (in progress) + +The runtime builds and runs on x86-64 Linux with a window, input and menu on SDL3 (`*_sdl.c`) in place of the Win32 +files (`*_win.c`); `runtime/platform/linux.c` implements `platform.h`. Status: the selftest and the probes pass; 1.2.1 +plays at 60 fps on RADV (RX 7800 XT) and NVIDIA (RTX 3060); 1.4.3 builds, probes and plays on RADV (RX 6600, and the +Vega 7 of a Ryzen 5 4600G at about 34 fps). Long play sessions and the Steam Deck itself are not verified yet. + +Requirements: + +- Linux 6.7 or later: the write watch uses userfaultfd write protection with `PAGEMAP_SCAN`. SteamOS 3.8 qualifies. +- A Vulkan 1.3 driver with `VK_EXT_external_memory_host` (RADV, NVIDIA). +- Packages on Arch: + + ```sh + pacman -S --needed clang lld ninja make sdl3 opus libvpx zstd vulkan-headers vulkan-icd-loader \ + vulkan-validation-layers shaderc spirv-tools dotnet-sdk uv git + ``` + + Vulkan Memory Allocator is a single header (AUR `vulkan-memory-allocator`). Without it, `build_game.py` fetches the + version the Windows release pins (3.4.0) into `toolchain/vma/` on first use and checks its hash. + +The steps are the Windows ones with POSIX paths: `.venv/bin/python` in place of `.venv\Scripts\python`, and `totk` in +place of `totk.exe`. `build_game.py` takes clang, lld and Ninja from PATH (or the ones in `/bin` with +`--toolchain `) and links against the system libraries, with `libstdc++` static and an `$ORIGIN` rpath; +`build_hactool.py` builds `hactool` with make. `tools/prepare_shaders.py` reads the shader list from +`refs/MacNativeGames` at `origin/HEAD`, which must be the pinned commit with its history (the list format changed later). + +The probes open a window, so from an SSH session they need the desktop session's display, or they stop at `gpu 0 +skipped: no graphics+compute+present queue family`: on Wayland `XDG_RUNTIME_DIR=/run/user/$(id -u) +WAYLAND_DISPLAY=wayland-1 SDL_VIDEODRIVER=wayland` (the socket name is in `$XDG_RUNTIME_DIR`), on X11 `DISPLAY=:0`. + +Memory: on Windows the GPU imports guest memory and `GetWriteWatch` tracks the CPU's writes to it. On Linux, +re-protecting imported memory invalidates the driver's userptr buffer on every submit (about 1.3 ms per 16 MiB on +amdgpu), and importing pins the pages, which drops their write protection. Each CPU-visible pool therefore gets a +host-visible copy (shadow storage, `runtime/vk_memory.c`): the pages the CPU wrote go in before each submit, and the GPU's +writes come back at its fence. In play that costs about 0.2 ms a frame and doubles the memory of those pools (about +1.1 GB). `TOTK_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as +well: the shadow copies are chosen for the measured cost above, not for a missing extension. + +The Deck's GPU (Van Gogh) has no native ASTC, so its textures go through the compute decoder (`runtime/vk_astc.c`, +`TOTK_ASTC`) as on desktop AMD GPUs without it; frame times measured there include it. + ## Checks Before a commit: @@ -142,7 +196,9 @@ change passes these, all run with `.venv`: | `tests\jumptables_check.py` | `recomp/jumptables.py` against a Unicorn oracle; must report 0 disagreements, on 1.4.3 too | yes | Runtime probes, each a mode of `totk.exe` (the list is at the top of `runtime/main.c`): `--probe-kernel`, -`--probe-vkexec`, `--probe-nvn-tex` and `--probe-shader` cover most runtime changes. +`--probe-vkexec`, `--probe-nvn-tex` and `--probe-shader` cover most runtime changes. `--probe-shader` and the draw +part of `--probe-vkexec` translate shaders of the census below: they look for `build//shaders/jobs.json` of the +exe's own version first, then of the other one. Shaders: `.venv-py314\Scripts\python tools\shader_census.py --version --translate` extracts every Maxwell shader to `build\\shaders\` and runs it through the translator and `spirv-val`; failure groups go to `failures.txt`. diff --git a/docs/controls.md b/docs/controls.md index 29f53f0d..3cd4082c 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -49,7 +49,7 @@ Without the folder (an install made by an older setup, or one where the setup co | Key | Gamepad | Action | |---|---|---| | F1 or Esc | Minus and Plus together | open or close the settings menu; the game gets no input while it is open | -| F10 | | performance overlay: off, frame rate, full panel | +| F10 | L3 + R3 (both stick clicks) | performance overlay: off, frame rate, full panel | | F11 | | borderless fullscreen on or off, saved to `totk.ini` | The Minus+Plus combo never reaches the game. In the menu the arrow keys, the D-pad or the left stick move, Enter or A selects, Q and E or L and R switch tabs, Esc or B goes back (Switch button names, as in the table above). @@ -86,6 +86,12 @@ Mouse motion (raw input) becomes right-stick deflection. Each presented frame ta The stick direction bits (`StickLLeft` to `StickRDown`) are derived from the sticks at 50% deflection. A gamepad plugged in later is picked up within one second; with none connected the log says `input: no gamepad`. +## Nintendo Switch 2 controllers + +Windows does not present the Switch 2 Pro Controller as a gamepad SDL can open (the log says `input: no gamepad`), so it needs a bridge program that wakes it and creates a virtual gamepad. The DS4Windows fork by hbashton ([github.com/hbashton/DS4Windows](https://github.com/hbashton/DS4Windows), tested with release VIIPERRC4.5.1, a pre-release) with the output set to an emulated DualSense gives buttons, rumble, gyro and accelerometer, and reads the sticks through the controller's calibration (they rest at 0 to 2%). The game reads its virtual DualSense directly; the log then shows `input: gamepad "DualSense Wireless Controller" (rumble, gyro and accelerometer, ...)` and `input: motion sensors on`. Its release notes also list Joy-Con 2 support, which is untested here. + +Bridges that output a ViGEm DualShock 4 (ProCon2Windows, connectyourpro as tested) give buttons and rumble but no motion: their virtual DualShock 4 reports a gyro and an accelerometer that never change, so motion aiming and Ultrahand rotation do nothing. Run one bridge at a time; Steam Input can also claim the controller, so close Steam if the bridge does not find it. + ## Scripted input `TOTK_INPUT_SCRIPT=` replays presses. It is ORed with real input and still applies under `TOTK_INPUT=none`. One event per line, `#` starts a comment: diff --git a/docs/installer.md b/docs/installer.md index a18373b6..44b55810 100644 --- a/docs/installer.md +++ b/docs/installer.md @@ -110,9 +110,46 @@ totk.ini.example, VERSION the default ini; the release string (the repository's - The release contains nothing game-derived; `make_release.py` runs `tools/check_content.py` over the folder before it finishes. +## Linux release + +`tools/make_release_linux.py` builds the same setup for Linux and packs it as `TotK-Setup-x86_64.AppImage` (the static +type-2 runtime: no libfuse2 needed on the host). The player makes it executable and starts it; on a Steam Deck, from +Desktop Mode. The setup builds the game into the folder they pick, as on Windows, and the game is `totk` there with +`libSDL3.so.0` beside it. The folder inside the AppImage: + +``` +AppRun starts python/bin/python3 -m installer.gui (no bytecode written into the read-only mount) +totk-setup.desktop, totk-setup.png +installer/, recomp/, tools/, data/, licenses/, THIRD_PARTY_NOTICES.md, totk.ini.example, VERSION as on Windows +python/ python-build-standalone 3.14 (with tkinter) + the installer's packages; a .pth puts the folder, + tools/ and recomp/ on sys.path +llvm/ LLVM 23 (clang, lld, llvm-ar) and llvm/sysroot: the files of the Steam Runtime 3 "sniper" SDK the + game's compile and link read (about 130 files) +runtime/ libtotkrt.a, headers, the install-side sources, totk.png; deps/ with the Vulkan headers, VMA and a + static zstd +bin/ totk_xlate, hactool, ninja, libSDL3.so.0 +``` + +- What we compile is built against the sniper sysroot (glibc 2.31): the game, hostcheck, hactool and the translator + need no newer glibc. The game links Opus, libvpx, zstd and libstdc++ statically and needs only glibc, the system's + Vulkan loader and the SDL3 beside it. The setup itself needs glibc 2.34 (Ubuntu 22.04, Debian 12, Fedora 35, + SteamOS 3): the official LLVM binaries and the `cryptography` wheel are built against it. `make_release_linux.py` + checks every ELF file in the folder against that floor (`check_glibc`), and the setup's check step stops with one + message on an older system. +- The setup's temp folder is `.setup-temp` inside the install folder (`--temp` changes it): the AppImage is mounted + read-only, and a Steam Deck's `/home` is often too full for the generated C while the game goes to the SD card. +- The game keeps Mesa's shader cache in `mesa-cache` beside `totk` unless `MESA_SHADER_CACHE_DIR` is set, for the + same reason: RADV fills its default folder in `~/.cache` up to 1 GB. +- The window follows `LANG` for English or Russian, cancels through SIGTERM to the CLI's process group (the CLI handles + it as Ctrl+Break on Windows) and opens the finished folder with `xdg-open`. +- Requirements beyond Windows': Linux 6.7 or later (the write watch, `docs/building.md`) and glibc 2.34. +- Building the release needs one game version extracted and lifted (`tools/extract.py`, `recomp/gen.py`, either + version): the runtime library includes the generated `layout.h`, whose shared part (`MOD_COUNT`, `MOD_MAIN_BASE`) + is the same for both. `tools/make_release.py` takes `generated/v121`, else `generated/v143`. + ## Constraints -- The installer steps contain no Win32 calls: Linux follows with its own compiler and the same steps. +- The installer steps contain no Win32 calls: Linux runs the same steps with its own compiler ("Linux release"). - The keys file path is passed to the tools; key values never appear in the UI, logs or files. - The window is a front end over the CLI (`python -m installer --base ... --update ... --keys ... --lang ... --dest ...`), which prints one JSON progress event per line. Tests and CI use the CLI. @@ -128,4 +165,4 @@ totk.ini.example, VERSION the default ini; the release string (the repository's ## Later - Re-warm the driver cache on start after a move, GPU or driver change. -- NSZ and XCZ input. Linux release. +- NSZ and XCZ input. diff --git a/docs/maxwell.md b/docs/maxwell.md index c8fa4886..dc71a041 100644 --- a/docs/maxwell.md +++ b/docs/maxwell.md @@ -66,7 +66,7 @@ Source shorthands: | 2.7 | **Swizzles** per texture and per view (`TextureBuilderSetSwizzle`, `TextureViewSetSwizzle`); BotW layout captures rebuild a texture on the same storage with (1, 1, 1, R) or alpha 1. Default per-format mapping R8 → (R, 0, 0, 1). | `VkComponentMapping` on the view. | NXN: font atlas white boxes, HUD dark boxes without it. Ours: mapped (`vk_exec_draw.c` `get_view`). | NXN | | 2.8 | **Sample dimension comes from the instruction**: a non-array sample of an array texture reads layer 0; 1,512 BotW shaders sample a cube as a cube array. | View type must match the declared type. | NXN: 2D array bound as 2D gave one pale colour (washed-out scene); cubes created as 6-layer arrays with cube / cube-array views. Ours: views per resource type (`get_view`). | NXN | | 2.9 | **LOD**: TSC min/max LOD clamp, LOD bias, `TWO_D_NO_MIPMAP` texture type, a min filter without mips stays on the base level; TIC carries its own max anisotropy and aniso quality fields. | `minLod`/`maxLod`, `mipLodBias` limited by `maxSamplerLodBias`. | yuzu: anisotropy heuristics for mods forcing trilinear (YZ-2306, PR 10744). Ours: non-mip filter → `maxLod = minLod + 0.25`, bias clamped (`vk_texture.c` `make_sampler`). | G80TEX | -| 2.10 | **Texture gather rounding**: Maxwell sub-texel precision differs from AMD/Intel; edge texels picked by TLD4 differ by one. | Implementation-defined `subTexelPrecisionBits`. | yuzu adds a sub-pixel offset to gathers on non-NVIDIA hosts: fixed BotW shadow artifacts at object edges ("shader_recompiler: Add subpixel offset for correct rounding at ImageGather", , title only). | — | +| 2.10 | **Texture gather rounding**: Maxwell sub-texel precision differs from AMD/Intel; edge texels picked by TLD4 differ by one. | Implementation-defined `subTexelPrecisionBits`. | yuzu adds a sub-pixel offset to gathers on non-NVIDIA hosts: fixed BotW shadow artifacts at object edges ("shader_recompiler: Add subpixel offset for correct rounding at ImageGather", , title only). Ours: the translator biases gather coordinates by the host's `subTexelPrecisionBits` on every non-NVIDIA GPU (`vk_caps.c` → job `gather_bias` → Ryujinx `TexturePass.InsertCoordGatherBias`); TotK on RADV drew dashed dark lines along wall and floor edges in the prologue caves without it. `TOTK_GATHER_BIAS=0` turns it off. | — | | 2.11 | **Gather offsets**: `TLD4.PTP` takes four per-texel offsets, possibly from registers. | `ConstOffsets` must be constant. | Ours: `QueryHostSupportsNonConstantTextureOffset() => false` makes Ryujinx emulate register offsets (`tools/xlate/Program.cs`). | RYU-SH `Translation/Optimizations` (TexturePass) | | 2.12 | **Depth compare** (`TSC` depth compare + `TIC_4_DEPTH_TEXTURE`), min/max reduction filtering. | `compareEnable`; reduction mode needs `samplerFilterMinmax` and is not allowed with compare. | Ours: reduction dropped when compare is on (`vk_texture.c` `make_sampler`). | G80TEX | | 2.13 | **ASTC / BC**: native ASTC; zero ASTC block = error colour on hosts. | AMD and NVIDIA desktop GPUs lack ASTC: decode to RGBA8. yuzu offered ASTC recompression to BC1/BC3 for low VRAM (YZ-2305, PR 10398). | Ours: native upload where the device samples ASTC, else a compute-shader decode (`vk_astc.c`, `TOTK_ASTC`), CPU decode (`tex_astc.c`) as the fallback. | `runtime/vk_astc.c` | @@ -141,6 +141,7 @@ Source shorthands: | TotK | AO broken above 1008p mods | rescale thresholds | YZ-2306 PR 10675 | | TotK | "detective mode" Link on AMD | AMD EDS3 colour-blend driver regression | YZ-2306 PR 10946 | | BotW | shadow artifacts at object edges (AMD) | gather rounding | yuzu commit 780240e (2.10) | +| TotK | dashed dark lines along wall and floor edges, changing with the camera angle (AMD, RADV) | gather rounding | gather bias (2.10) | | BotW | black textures on GCN | HTILE (depth compression) | `AMD_DEBUG=nohyperz` wiki (1.9) | | BotW | 3D scene black | deferred texture handles | NXN | | BotW | albedo scattered squares | duplicate colour attachments | NXN | diff --git a/docs/tester-guide.md b/docs/tester-guide.md index 4c9dcad4..67a9fc70 100644 --- a/docs/tester-guide.md +++ b/docs/tester-guide.md @@ -38,7 +38,7 @@ build\\totk.exe `TOTK_VK_VALIDATION=1` turns on the Vulkan validation layer (it comes with the Vulkan SDK). It makes the game slower; do one run with it and one without (`set TOTK_VK_VALIDATION=`). -Keep the window in the foreground: an unfocused window stalls the game. Shaders the preparation did not cover compile on first use, so +The game keeps running behind other windows (`assume_focus = 0` in `totk.ini` pauses it instead). Shaders the preparation did not cover compile on first use, so the first minutes of play can stutter. Play into the open field (load a save, walk around for a minute, fight something if you can), then close the window. @@ -63,6 +63,27 @@ The line `vk: using gpu N: ` says which one ran. 5. For a bug you can trigger again: a recording of your input (`set TOTK_INPUT_RECORD=input.txt`, see [controls.md](controls.md), "Scripted input") and the save you started from. +## Steam Deck + +Build or install on Linux ([installer.md](installer.md), "Linux release"), in Desktop Mode. The game then runs in either +mode; for Game Mode, add `totk` from the install folder to Steam as a non-Steam game. + +The game recognises the Deck (LCD and OLED) and uses its device profile for every setting `totk.ini` does not set: +fullscreen, the screen's own 1280x800 (16:10: the camera and interface take that shape), the game's own 3D scale, +vsync and a 40 fps limit. These are starting points, not measurements. What helps most: + +1. The performance overlay (F10, or both stick clicks L3 + R3) in full mode: the frame rate, mean and longest frame + time over a minute in Lookout Landing, in the Depths and in the Sky, with the settings named. +2. The same with `fps_limit` 30, 45 and 60 and `render_scale` 50, 67 and game, so the defaults can be picked from + measurements. Steam's own performance overlay (the ... button) adds power draw and temperatures. +3. LCD or OLED, SteamOS version (`uname -r` in Desktop Mode's terminal), and the game update. +4. Anything that looks stretched, squeezed or cut off, with a screenshot. At 16:10 the menus keep their 16:9 + proportions in the middle of the taller screen, the HUD sits at the top and bottom edges, and full-screen + backgrounds reach them; pre-rendered cutscenes play stretched. + +`device_profile = desktop` in `totk.ini` turns the profile off; `deck` turns it on elsewhere, to try the Deck's +settings on a PC. + ## Switches for the developer's questions Only set these when asked; each forces a fallback the build picks by itself on GPUs that need it. diff --git a/installer/cli.py b/installer/cli.py index a754c427..b6d4238a 100644 --- a/installer/cli.py +++ b/installer/cli.py @@ -28,6 +28,11 @@ class Parser(argparse.ArgumentParser): sys.exit(2) +# The temp folder inside the install folder when the layout names none (the Linux release); the check step does not +# count it when it requires an empty install folder +TEMP_IN_DEST = '.setup-temp' + + def temp_folder(base, dest): """One temp folder per install folder, so two installs never share generated files.""" tag = hashlib.sha256(str(dest).lower().encode()).hexdigest()[:8] @@ -46,8 +51,8 @@ def main(argv=None): ap.add_argument('--lang', help='game language, a Switch language code (en-US, en-GB, ja, de, ru, ...)') ap.add_argument('--dest', type=Path, required=True, help='install folder') ap.add_argument('--root', type=Path, help='release folder or repository (default: the folder above installer/)') - ap.add_argument('--temp', type=Path, help='temp folder (default: temp/ of the release, build/install-temp in a ' - 'repository)') + ap.add_argument('--temp', type=Path, help='temp folder (default: temp/ of the release on Windows, .setup-temp in ' + 'the install folder on Linux, build/install-temp in a repository)') ap.add_argument('--verify', action='store_true', help='hash every kept file, also those this release made') args = ap.parse_args(argv) @@ -58,7 +63,8 @@ def main(argv=None): signal.signal(getattr(signal, name), cancel) layout = layouts.find(args.root) dest = args.dest.resolve() - temp = temp_folder((args.temp or layout.temp).resolve(), dest) + base = args.temp or layout.temp + temp = temp_folder(base.resolve(), dest) if base else dest / TEMP_IN_DEST log = temp / 'logs' / time.strftime('install-%Y%m%d-%H%M%S.log') rep = Reporter(log_path=log) t0 = time.monotonic() diff --git a/installer/events.py b/installer/events.py index 121ccfe9..ae31358e 100644 --- a/installer/events.py +++ b/installer/events.py @@ -15,6 +15,7 @@ Text is never the payload: ids name a step ("step."), a phase ("phase. and installer/text/en.json holds their English text with {param} placeholders. The log renders events with it. """ import json +import os import sys import threading import time @@ -28,11 +29,14 @@ def load_catalog(path=CATALOG): return json.loads(path.read_text(encoding='utf-8')) +GAME_EXE = 'totk.exe' if os.name == 'nt' else 'totk' # {game_exe} in the catalogs + + def render(catalog, id, params=None): """The English text of id, or the id itself when the catalog lacks it.""" text = catalog.get(id, id) try: - return text.format(**(params or {})) + return text.format(**{'game_exe': GAME_EXE, **(params or {})}) except (KeyError, IndexError, ValueError): return f'{text} {params}' diff --git a/installer/gui.py b/installer/gui.py index 8a6c551e..edfca969 100644 --- a/installer/gui.py +++ b/installer/gui.py @@ -10,6 +10,7 @@ this file only draws. The window text is in installer/text/.json un import argparse import json import os +import subprocess import sys import tkinter as tk from pathlib import Path @@ -58,8 +59,12 @@ class App: root.title(self.cat('ui.title')) icon = layout.runtime / 'totk.ico' + png = layout.runtime / 'totk.png' # Tk loads no .ico off Windows; the Linux release converts it if os.name == 'nt' and icon.is_file(): root.iconbitmap(default=str(icon)) + elif png.is_file(): + self.icon = tk.PhotoImage(file=str(png)) + root.iconphoto(True, self.icon) root.protocol('WM_DELETE_WINDOW', self.on_close) style = ttk.Style(root) style.configure('Title.TLabel', font=('Segoe UI', 15, 'bold')) @@ -478,7 +483,11 @@ class App: def open_folder(self): dest = self.model.dest if dest and Path(dest).is_dir(): - os.startfile(dest) + if os.name == 'nt': + os.startfile(dest) + else: + subprocess.Popen(['xdg-open', dest], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) # ---- self test --------------------------------------------------------------------------------------------------- diff --git a/installer/guilogic.py b/installer/guilogic.py index 4725dc18..2056e86c 100644 --- a/installer/guilogic.py +++ b/installer/guilogic.py @@ -11,6 +11,7 @@ import ctypes import json import os import queue +import signal import subprocess import sys import threading @@ -76,7 +77,11 @@ def flat(value): def windows_ui_language(): - """'ru' when the Windows display language is Russian, 'pt-BR' when it is Brazilian Portuguese, else 'en'.""" + """'ru' when the display language (Windows; elsewhere LC_ALL, LC_MESSAGES or LANG) is Russian, 'pt-BR' when it is + Brazilian Portuguese (LANGID 0x0416, or a pt_BR locale), else 'en'.""" + if os.name != 'nt': + loc = (os.environ.get('LC_ALL') or os.environ.get('LC_MESSAGES') or os.environ.get('LANG') or '').lower() + return 'ru' if loc.startswith('ru') else 'pt-BR' if loc.startswith(('pt_br', 'pt-br')) else 'en' try: langid = ctypes.windll.kernel32.GetUserDefaultUILanguage() except (AttributeError, OSError): @@ -317,7 +322,14 @@ else: def send_break(proc, python=None): """Ctrl+Break to the process group of proc (started with CREATE_NEW_PROCESS_GROUP). The window itself may have no - console (pythonw), so a short helper process attaches to the child's console and sends it.""" + console (pythonw), so a short helper process attaches to the child's console and sends it. Elsewhere SIGTERM to + proc's session (Process starts it in one of its own), which installer/cli.py handles as Ctrl+Break.""" + if os.name != 'nt': + try: + os.killpg(proc.pid, signal.SIGTERM) + return True + except OSError: + return False helper = [str(python or sys.executable), '-c', _BREAK, str(proc.pid)] flags = CREATE_NO_WINDOW if os.name == 'nt' else 0 r = subprocess.run(helper, creationflags=flags, capture_output=True) @@ -333,10 +345,14 @@ class Process: self.command = command self.helper_python = helper_python self.lines = queue.Queue() - flags = (CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW) if creationflags is None else creationflags env = dict(os.environ, PYTHONIOENCODING='utf-8', PYTHONUTF8='1', PYTHONUNBUFFERED='1') + if os.name == 'nt': + flags = (CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW) if creationflags is None else creationflags + group = {'creationflags': flags} + else: + group = {'start_new_session': True} self.proc = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, creationflags=flags) + stderr=subprocess.STDOUT, **group) self.cancelled_at = None self.reader = threading.Thread(target=self.read, daemon=True) self.reader.start() diff --git a/installer/hostcheck.c b/installer/hostcheck.c index 2f86571a..6cd08332 100644 --- a/installer/hostcheck.c +++ b/installer/hostcheck.c @@ -1,6 +1,7 @@ // Host check of the installer's first step, compiled with the bundled clang at install time: does the CPU run // -march=x86-64-v3 code, and does a Vulkan 1.3 device exist. Prints one JSON object on stdout. // The Vulkan loader is opened at run time, so a machine without one gets "loader": false instead of a failed start. +#include #include #include #include @@ -24,9 +25,26 @@ static void json_string(const char *s) { putchar('"'); } +// The x86-64-v3 level from CPUID: AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT and MOVBE, with the OS saving the YMM state. +// Not __builtin_cpu_supports("x86-64-v3"): clang reads that from the C runtime's CPU model, and libgcc before GCC 12 +// (the Linux release's sysroot has GCC 10) has no level bits, so it answers false on every CPU. +static int x86_64_v3(void) { + unsigned a, b, c, d; + if (!__get_cpuid(1, &a, &b, &c, &d)) return 0; + const unsigned fma = 1u << 12, movbe = 1u << 22, osxsave = 1u << 27, avx = 1u << 28, f16c = 1u << 29; + if ((c & (fma | movbe | osxsave | avx | f16c)) != (fma | movbe | osxsave | avx | f16c)) return 0; + unsigned lo, hi; + __asm__("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + if ((lo & 6) != 6) return 0; // XMM and YMM state enabled by the OS + if (!__get_cpuid_count(7, 0, &a, &b, &c, &d)) return 0; + const unsigned bmi1 = 1u << 3, avx2 = 1u << 5, bmi2 = 1u << 8; + if ((b & (bmi1 | avx2 | bmi2)) != (bmi1 | avx2 | bmi2)) return 0; + if (!__get_cpuid(0x80000001, &a, &b, &c, &d)) return 0; + return (c & (1u << 5)) != 0; // LZCNT (ABM) +} + int main(void) { - __builtin_cpu_init(); - printf("{\"x86_64_v3\": %s, ", __builtin_cpu_supports("x86-64-v3") ? "true" : "false"); + printf("{\"x86_64_v3\": %s, ", x86_64_v3() ? "true" : "false"); void *lib = open_lib(); PFN_vkGetInstanceProcAddr gipa = lib ? (PFN_vkGetInstanceProcAddr)lib_sym(lib, "vkGetInstanceProcAddr") : NULL; if (!gipa) { diff --git a/installer/layout.py b/installer/layout.py index 3ff47139..23c7137b 100644 --- a/installer/layout.py +++ b/installer/layout.py @@ -15,19 +15,26 @@ Release (docs/installer.md "Release layout"), root = the unpacked folder: VERSION the release version, one line temp/ the default temp folder (created) +Linux release (the AppImage's folder, docs/installer.md "Linux release"): python/bin/python3 (python-build-standalone, +with tkinter), llvm/ with clang, lld and llvm-ar plus llvm/sysroot (the glibc headers and libraries the game links +against), runtime/deps/ with include/ and the static Opus, libvpx and zstd, bin/ without .exe and with libSDL3.so.0. + Dev mode, root = a repository checkout (no python/python.exe): build/py314-embed/py/python.exe (the embeddable Python the release ships, with the same packages), else this Python; toolchain/llvm-mingw-*/ and toolchain/deps-mingw; runtime/ as the sources with build/install-rtlib/libtotkrt.a (tools/build_game.py --make-runtime-lib --out build/install-rtlib); MSYS2 UCRT64 ninja.exe; toolchain/hactool; build/xlate/totk_xlate.exe; the shader list from -refs/MacNativeGames; build/install-temp. The version is VERSION with -dev and the commit: 0.1.0-dev+973c626. +refs/MacNativeGames; build/install-temp. The version is VERSION with -dev and the commit: 0.1.0-dev+973c626. On Linux, +dev mode takes clang, lld and ninja from PATH and the system's libraries (no deps folder, no SDL3 copy). """ import os +import shutil import subprocess import sys from dataclasses import dataclass, field from pathlib import Path -EXE = '.exe' if os.name == 'nt' else '' +WIN = os.name == 'nt' +EXE = '.exe' if WIN else '' UCRT = Path(r'C:\msys64\ucrt64\bin') SUPPORTED = ('1.2.1', '1.4.3') # updates the installer builds; the check identifies the player's by hash @@ -45,8 +52,8 @@ class Layout: python: Path # runs the tools tools: Path recomp: Path - llvm: Path # llvm-mingw tree - deps: Path # include/ and lib/ for the link + llvm: Path # llvm-mingw tree; on Linux the tree with bin/clang (a release's, or the system's /usr) + deps: Path | None # include/ and lib/ for the link; None: the system's (Linux dev mode) runtime: Path # runtime headers and install-side sources runtime_lib: Path # libtotkrt.a ninja: Path @@ -56,6 +63,7 @@ class Layout: shader_list: Path | None = None # None: dev, read from refs/MacNativeGames through git ini_example: Path | None = None temp: Path | None = None + sysroot: Path | None = None # Linux release: what the game links against, instead of the host's @property def clang(self): @@ -75,15 +83,21 @@ class Layout: self.tools / 'pack_shader_cache.py', self.tools / 'make_romfs_patches.py', self.tools / 'ultrawide_layout.py', self.recomp / 'gen.py', self.clang, - self.deps / 'include', self.deps / 'lib', self.runtime, self.runtime_lib, self.ninja, self.hactool, + *((self.deps / 'include', self.deps / 'lib') if self.deps else ()), self.runtime, self.runtime_lib, self.ninja, self.hactool, self.xlate, *self.dlls, *(f(v) for v in SUPPORTED for f in (self.patches, self.expected))] if self.shader_list: need.append(self.shader_list) + if self.sysroot: + need.append(self.sysroot) return [str(p) for p in need if not p.exists()] +def release_python(root): + return root / 'python' / ('python.exe' if WIN else 'bin/python3') + + def is_release(root): - return (root / 'python' / ('python' + EXE)).is_file() + return release_python(root).is_file() def release_layout(root): @@ -91,17 +105,23 @@ def release_layout(root): return Layout( root=root, dev=False, release=version.read_text(encoding='utf-8').strip() if version.is_file() else 'unknown', - python=root / 'python' / ('python' + EXE), + python=release_python(root), tools=root / 'tools', recomp=root / 'recomp', llvm=root / 'llvm', deps=root / 'runtime' / 'deps', runtime=root / 'runtime', runtime_lib=root / 'runtime' / 'libtotkrt.a', ninja=root / 'bin' / ('ninja' + EXE), hactool=root / 'bin' / ('hactool' + EXE), xlate=root / 'bin' / ('totk_xlate' + EXE), - dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll'), - shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example', temp=root / 'temp') + dlls=(root / 'bin' / 'SDL3.dll', root / 'bin' / 'libopus-0.dll') if WIN else (root / 'bin' / 'libSDL3.so.0',), + shader_list=root / 'data' / 'shader_list.tsv', ini_example=root / 'totk.ini.example', + # Linux: None, the install folder's .setup-temp (installer/cli.py): the AppImage is mounted read-only, and on + # a Steam Deck /home is often too full for the generated C while the install folder is on the SD card + temp=root / 'temp' if WIN else None, + sysroot=None if WIN else root / 'llvm' / 'sysroot') def dev_layout(root): + if not WIN: + return dev_layout_posix(root) commit = subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True).stdout.strip() embed = root / 'build' / 'py314-embed' / 'py' / ('python' + EXE) @@ -120,6 +140,24 @@ def dev_layout(root): shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp') +def dev_commit(root): + return subprocess.run(['git', '-C', str(root), 'rev-parse', '--short', 'HEAD'], capture_output=True, + text=True).stdout.strip() + + +def dev_layout_posix(root): + clang, ninja = shutil.which('clang'), shutil.which('ninja') + semver = (root / 'VERSION').read_text(encoding='utf-8').strip() + return Layout( + root=root, dev=True, release=f'{semver}-dev+{dev_commit(root) or "unknown"}', + python=Path(sys.executable), tools=root / 'tools', recomp=root / 'recomp', + llvm=Path(clang).parent.parent if clang else root / 'toolchain' / 'clang-missing', deps=None, + runtime=root / 'runtime', runtime_lib=root / 'build' / 'install-rtlib' / 'libtotkrt.a', + ninja=Path(ninja) if ninja else root / 'toolchain' / 'ninja-missing', + hactool=root / 'toolchain' / 'hactool' / 'hactool', xlate=root / 'build' / 'xlate' / 'totk_xlate', + shader_list=None, ini_example=root / 'totk.ini.example', temp=root / 'build' / 'install-temp') + + def find(root=None): """The layout of root (default: the folder above this package).""" root = Path(root or Path(__file__).resolve().parent.parent).resolve() diff --git a/installer/steps.py b/installer/steps.py index d72167ec..b03a69f1 100644 --- a/installer/steps.py +++ b/installer/steps.py @@ -16,6 +16,7 @@ import json import os import re import shutil +import signal import subprocess import sys import threading @@ -95,10 +96,36 @@ def kill_tree(proc): return if os.name == 'nt': subprocess.run(['taskkill', '/T', '/F', '/PID', str(proc.pid)], capture_output=True) - else: + return + try: # run_tool starts each tool in a session of its own: the group holds ninja, clang and the game + os.killpg(proc.pid, signal.SIGKILL) + except OSError: proc.kill() +def linux_glibc(): + """(major, minor) of the C library this Linux process runs on, None elsewhere or when it is not glibc.""" + try: + name = os.confstr('CS_GNU_LIBC_VERSION') if sys.platform.startswith('linux') else None + except (ValueError, OSError): + return None + m = re.match(r'glibc (\d+)\.(\d+)', name or '') + return (int(m.group(1)), int(m.group(2))) if m else None + + +# The bundled clang, lld and the cryptography wheel are built against glibc 2.34 (tools/make_release_linux.py +# GLIBC_FLOOR); on an older one they cannot start +GLIBC_NEEDED = (2, 34) + + +def linux_kernel(): + """(major, minor) of the running Linux kernel, None elsewhere.""" + if not sys.platform.startswith('linux'): + return None + m = re.match(r'(\d+)\.(\d+)', os.uname().release) + return (int(m.group(1)), int(m.group(2))) if m else None + + def run_tool(ctx, step, script, args, parse=None, cwd=None): """Run a tool script with the bundled Python. Its output goes to /logs/-