diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 94dbf6baf8..5ea175cbfe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,9 +16,15 @@ on: - 'LICENSE' - '.gitignore' workflow_dispatch: + inputs: + rebuild_preloads: + description: Rebuild preloads instead of reusing the approved binary snapshot + type: boolean + default: false permissions: contents: read + actions: read pull-requests: write concurrency: @@ -26,7 +32,7 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: - PRELOAD_CACHE_VERSION: v7 + PRELOAD_CACHE_VERSION: v8 jobs: xbox: @@ -257,34 +263,43 @@ jobs: app/src/main/assets/linuxfs !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-comet !app/src/main/assets/linuxfs/usr/local/lib/droiddeck-xbox - key: preloads-${{ env.PRELOAD_CACHE_VERSION }}-${{ hashFiles('app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/cpp/adrenotools/src/hook/kgsl.h', 'tools/quest/*', 'tools/linuxfs/preload/*.c', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/fex/*', 'tools/linuxfs/clipboard/*.c', 'tools/linuxfs/ssbs/*.c', 'tools/proot/fastpath/*.c', 'tools/linuxfs/overlay/usr/local/bin/*', 'tools/linuxfs/overlay/usr/bin/**', 'tools/linuxfs/desktop/*', 'tools/gamescope/release.env', 'tools/wlroots/release.env', 'tools/linuxfs/uruntime.env', 'tools/linuxfs/licenses/*', 'tools/mangoapp/*', 'tools/msitools/**', 'tools/linuxfs/debian-image.env', 'tools/pipewire/**', 'tools/ci/install-ubuntu-packages.sh', 'tools/vr/**') }} + !app/src/main/assets/linuxfs/i386/libblsession.so + !app/src/main/assets/linuxfs/i386/libfakeinput.so + !app/src/main/assets/linuxfs/i386/libfaultreport.so + !app/src/main/assets/linuxfs/libblaudit.so + !app/src/main/assets/linuxfs/libblfastpath.so + !app/src/main/assets/linuxfs/libblsession.so + !app/src/main/assets/linuxfs/libfakeinput.so + !app/src/main/assets/linuxfs/libssbs.so + !app/src/main/assets/linuxfs/usr/local/bin/droiddeck-clipboard + !app/src/main/assets/linuxfs/x86_64/libblsession.so + !app/src/main/assets/linuxfs/x86_64/libfakeinput.so + !app/src/main/assets/linuxfs/x86_64/libfaultreport.so + !app/src/main/assets/linuxfs/x86_64/libvulkan-thunk.so + !app/src/main/assets/linuxfs/x86_64/libwebkitinput.so + key: preloads-${{ env.PRELOAD_CACHE_VERSION }}-${{ hashFiles('app/src/main/cpp/adrenotools/src/hook/kgsl.h', 'tools/quest/*', 'tools/linuxfs/overlay/usr/local/bin/*', 'tools/linuxfs/overlay/usr/bin/**', 'tools/linuxfs/desktop/*', 'tools/gamescope/release.env', 'tools/wlroots/release.env', 'tools/linuxfs/uruntime.env', 'tools/linuxfs/licenses/*', 'tools/mangoapp/*', 'tools/msitools/**', 'tools/pipewire/**', 'tools/ci/install-ubuntu-packages.sh', 'tools/vr/**') }} - - name: Build libfakeinput.so and libblsession.so (glibc aarch64) + - name: Restore verified preload snapshots + id: component-preloads + env: + GH_TOKEN: ${{ github.token }} + FORCE_PRELOAD_REBUILD: ${{ inputs.rebuild_preloads || 'false' }} + COMPONENT_TRUSTED_BASE: ${{ needs.xbox.outputs.base-sha || github.sha }} + run: python3 tools/components/preloads.py restore + + - name: Build changed ARM64 preloads + if: steps.component-preloads.outputs.arm64 != 'true' + run: | + bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + bash tools/components/build-arm64-preloads.sh app/src/main/assets/linuxfs + + - name: Prepare remaining Linux runtime assets if: steps.preloads.outputs.cache-hit != 'true' run: | set -euo pipefail bash tools/ci/install-ubuntu-packages.sh gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libvulkan1 mesa-vulkan-drivers D=app/src/main/assets/linuxfs mkdir -p "$D" - aarch64-linux-gnu-g++ -shared -fPIC -O2 -Wall -Wno-attributes -Wno-nonnull-compare \ - -pthread -std=c++17 -static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL \ - -o "$D/libfakeinput.so" app/src/main/cpp/fakeinput_steam.cpp -ldl - aarch64-linux-gnu-strip --strip-unneeded "$D/libfakeinput.so" - aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ - -o "$D/libblsession.so" tools/linuxfs/preload/*.c -ldl - aarch64-linux-gnu-strip --strip-unneeded "$D/libblsession.so" - aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ - -o "$D/libblfastpath.so" tools/proot/fastpath/fastpath.c -ldl - aarch64-linux-gnu-strip --strip-unneeded "$D/libblfastpath.so" - aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -nostdlib -ffreestanding -fno-stack-protector \ - -fno-tree-loop-distribute-patterns -Wl,-z,defs -o "$D/libblaudit.so" tools/proot/fastpath/audit.c - aarch64-linux-gnu-strip --strip-unneeded "$D/libblaudit.so" - aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ - -o "$D/libssbs.so" tools/linuxfs/ssbs/ssbs_adapter.c -ldl - aarch64-linux-gnu-strip --strip-unneeded "$D/libssbs.so" - mkdir -p "$D/usr/local/bin" - aarch64-linux-gnu-gcc -O2 -Wall -Wextra -o "$D/usr/local/bin/droiddeck-clipboard" tools/linuxfs/clipboard/clipboard.c -ldl - aarch64-linux-gnu-strip --strip-unneeded "$D/usr/local/bin/droiddeck-clipboard" VK_INCLUDE=$(ls -d "${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-$ANDROID_NDK_ROOT}}"/toolchains/llvm/prebuilt/*/sysroot/usr/include | head -1) \ tools/vr/build.sh --native "$D/usr/local/lib/droiddeck-vr" for script in tools/linuxfs/overlay/usr/local/bin/droiddeck-* tools/linuxfs/overlay/usr/local/bin/steam-compatibility; do @@ -366,7 +381,7 @@ jobs: ls -lR "$D" - name: Fetch the Debian image the preloads build in - if: steps.preloads.outputs.cache-hit != 'true' + if: steps.component-preloads.outputs.x86 != 'true' env: GH_TOKEN: ${{ github.token }} run: | @@ -389,7 +404,7 @@ jobs: docker image ls "$image" - name: Build the x86 preloads FEX guests load (glibc 2.31) - if: steps.preloads.outputs.cache-hit != 'true' + if: steps.component-preloads.outputs.x86 != 'true' run: | set -euo pipefail docker run --rm -v "$PWD:/src" -w /src public.ecr.aws/docker/library/debian:bullseye bash -c ' diff --git a/tools/components/README.md b/tools/components/README.md new file mode 100644 index 0000000000..646c099ccb --- /dev/null +++ b/tools/components/README.md @@ -0,0 +1,35 @@ +# Verified preload snapshots + +`preloads.lock.json` pins a small immutable preload archive. It records each +binary's hash, its producing main CI run, the original APK artifact identity and +digest, source/header hashes, the ARM64 build recipe and x86 compiler-image inputs. +It covers core ARM64 preloads and the x86 preload libraries. VR, Android native +libraries, Comet and Xbox keep their existing preparation paths. + +Every build compares current source inputs, including transitive quoted headers, +and build recipes with this manifest. Changed groups rebuild from source. A +missing release also selects a rebuild. Corrupt archives, unexpected files and +binary hash mismatches fail the build. Existing preload checks, runtime packaging +checks and all test jobs still run. + +The trust anchor is the manifest in the pinned main base. A new or changed PR +manifest is independently checked against a successful `push` build of main: +GitHub must confirm the run/source/workflow and original APK artifact digest, and +every claimed binary must match that APK. The claimed source inputs and recipe +must match that run's Git source. A PR cannot establish trust by changing both a +binary and its claimed checksum. This is approved-build reuse, not a claim of +bit-for-bit reproducible compilation or a signed SLSA attestation. + +To refresh after source or compiler changes: + +1. Let a main build compile and test the new inputs. +2. Run `python tools/components/preloads.py snapshot --run RUN_ID`. +3. Publish `app/build/component-snapshot/preloads.tar.gz` to + DroidDeck-Components as `preloads-`, without replacing an + existing release. Include the manifest and producing run in the release notes. +4. Review and commit the updated manifest. CI independently checks its origin. + +The workflow dispatch `rebuild_preloads` option forces source compilation even +when an approved snapshot matches. Use it when refreshing a compiler or system +header revision. Runner image updates do not silently replace an approved binary; +the producing run remains the record of its compiler environment. diff --git a/tools/components/build-arm64-preloads.sh b/tools/components/build-arm64-preloads.sh new file mode 100644 index 0000000000..a38a1edaaa --- /dev/null +++ b/tools/components/build-arm64-preloads.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +D=${1:?output directory} +mkdir -p "$D" +aarch64-linux-gnu-g++ -shared -fPIC -O2 -Wall -Wno-attributes -Wno-nonnull-compare \ + -pthread -std=c++17 -static-libstdc++ -static-libgcc -Wl,--exclude-libs,ALL \ + -o "$D/libfakeinput.so" app/src/main/cpp/fakeinput_steam.cpp -ldl +aarch64-linux-gnu-strip --strip-unneeded "$D/libfakeinput.so" +aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ + -o "$D/libblsession.so" tools/linuxfs/preload/*.c -ldl +aarch64-linux-gnu-strip --strip-unneeded "$D/libblsession.so" +aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ + -o "$D/libblfastpath.so" tools/proot/fastpath/fastpath.c -ldl +aarch64-linux-gnu-strip --strip-unneeded "$D/libblfastpath.so" +aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -nostdlib -ffreestanding -fno-stack-protector \ + -fno-tree-loop-distribute-patterns -Wl,-z,defs -o "$D/libblaudit.so" tools/proot/fastpath/audit.c +aarch64-linux-gnu-strip --strip-unneeded "$D/libblaudit.so" +aarch64-linux-gnu-gcc -shared -fPIC -O2 -Wall -pthread \ + -o "$D/libssbs.so" tools/linuxfs/ssbs/ssbs_adapter.c -ldl +aarch64-linux-gnu-strip --strip-unneeded "$D/libssbs.so" +mkdir -p "$D/usr/local/bin" +aarch64-linux-gnu-gcc -O2 -Wall -Wextra -o "$D/usr/local/bin/droiddeck-clipboard" tools/linuxfs/clipboard/clipboard.c -ldl +aarch64-linux-gnu-strip --strip-unneeded "$D/usr/local/bin/droiddeck-clipboard" diff --git a/tools/components/preloads.lock.json b/tools/components/preloads.lock.json new file mode 100644 index 0000000000..4093912e32 --- /dev/null +++ b/tools/components/preloads.lock.json @@ -0,0 +1,125 @@ +{ + "archive_sha256": "cb6487a94f25a3068fd07958174eb5b5e14f709655ff4ef0a8cdbd99138513b2", + "arm64_recipe": "62c8b9cf001f1b04b52856bd74d353842a4be29960f033067f34c7d3a9a48681", + "groups": { + "arm64": { + "@runner": "84f825635599f81ed33f62c5fc4c73f66044d1468b060d65f83855eb9c8bfe77", + "app/src/main/cpp/adrenotools/src/hook/kgsl.h": "396c56761034a4ed783543ee3312c6ec64d63b2a1cf7eecf1939c23251f25adf", + "app/src/main/cpp/fakeinput_steam.cpp": "3d11e859ca58a42a9da92a3fc589d404df790e151f7e18ea6114645750e70ef5", + "app/src/main/cpp/venc/venc_proto.h": "2242ddbc8981d14a33bac70264bb3e7f701fd1c3ebb988e17bbb2b19b6b808f7", + "tools/linuxfs/clipboard/clipboard.c": "96291c2bbc040c83caabf059ab3e95a9ed90d6a768c759d7cb4c0b64706bddfa", + "tools/linuxfs/preload/binfmt.c": "0933cc49cb53f0aa1206e2eae516d38dc3337e74174be07e10619c42aa56792b", + "tools/linuxfs/preload/dircache.c": "8e7f19ea4a1198783d53a3f357326c6177da157b0cc7e0d22e1670963e1c26e7", + "tools/linuxfs/preload/drm.c": "e0eabab0d8ed9e841918e827bf87168efa7f2dfb8404169e9bed3ab5eda2836c", + "tools/linuxfs/preload/fsync.c": "795c114c2a05d3f250d07145d04c5658091699da3b96290da36348ccadfb0c09", + "tools/linuxfs/preload/inputudev.c": "2648248d3d235a582eb8459709e0a85de537c03ac10c5040fbfb50d9846c68eb", + "tools/linuxfs/preload/kgslid.c": "88a7875fd5289930204d155d92ac9d15492cd06db16ed87aa9c95af37f3e7089", + "tools/linuxfs/preload/kgslpoll.c": "f0d853d919a52487c04cbda5fb3fa22269da54b7f4ee2f7dd3b5bd9909418670", + "tools/linuxfs/preload/links.c": "b7bbcf9112393b152f358fe6f8dc9a1955f0c8532e1eddf016a971415cd13c64", + "tools/linuxfs/preload/lock.c": "f71b3fd40e6a2b74908d3cdc601e8748699ae9c7a89b542d2445b999015ffd66", + "tools/linuxfs/preload/net.c": "335853c8ea0e9a8ebbc5a5500dc4bded9c34c4310a6ab8aa46b07d5ce1c2b55e", + "tools/linuxfs/preload/netif.c": "cc26d50b8d4639d8dd4697811bbf85c9d9f31761f9504a4af42473de3cb22b3c", + "tools/linuxfs/preload/noexec.c": "1855aeaeda3d506329bd9af1107580c74d1f56787c411b2e01c0a8a9d4059c7b", + "tools/linuxfs/preload/ntsync.c": "0614c48af0c062bfcb4c30141efe1b792b340767a67d2fe92ee9877d2ba16e0d", + "tools/linuxfs/preload/opens.c": "92ae4aea16aa457a898dedf4a70eb332d80d3b1e77f3a60bace7715a4317399e", + "tools/linuxfs/preload/pathcache.c": "eb0d4b5238fe6c8891e805d90bd837e466754100c978fad2d0ff717116b9b8a3", + "tools/linuxfs/preload/perms.c": "f600619c4ca130f69001ed7ad7bd74a3dc215b14c61479c45bb770f916d244b9", + "tools/linuxfs/preload/pulseguard.c": "57674ea40283e53a27bc78362de1ba41ee972d172ea38b1f818b1d6eed0abe65", + "tools/linuxfs/preload/quest_kgsl.c": "59d9b8416be6a1c680f1d1e4c2427d2cbb7b415344261adb357d9d211ec95cfe", + "tools/linuxfs/preload/robust.c": "9eb6c0ed50e0f48a8cfb86bd92587f471354bdf2531b9177f4b3ab83bdfed39a", + "tools/linuxfs/preload/rptouch.c": "0e3f8bf2f9a41324361f38fa4da8d2d47593c9555f0cabf8228767397846c7c3", + "tools/linuxfs/preload/space.c": "118389bcc89dfa2b4230b80915e21d2d4331c4cff1f5a58265f85e6ecb5c29d2", + "tools/linuxfs/preload/storage.c": "08a8b46c3588f3160bb8608489768010ec36d5ccdb2b25a431ec9d6fcd413a6e", + "tools/linuxfs/preload/sysv.c": "49774ee5e2e39b84dcfadf63abdc741b3df236cb11783744a1280adee36965e8", + "tools/linuxfs/preload/tracefs.c": "be3f8a2f032b56e6e08a931e77a9386aea4bb93421a482271e06386ec0c469b6", + "tools/linuxfs/preload/tracer.c": "f34e5377c891b0a8d00489245d66f96bc352df4cd90f47a951dcadae980af4c1", + "tools/linuxfs/preload/udevmon.c": "2045dfb883cfb6e2016c359a3dc3af17f3b8f383f6dd0d5dfee407ef0d344852", + "tools/linuxfs/preload/usr2.c": "6ae704380b45a9b3a74c4fe2b3ce4482470dae59ced26cf0aee64f4b4dcc11f9", + "tools/linuxfs/preload/venc.c": "24c3c46fe49d28adfc8c977e52c9cc407f534b26f56d94b3425b8bb01803dba8", + "tools/linuxfs/preload/vmwrite.c": "1a3ea2ec4354e0716c17f7db537b19ab82bcd524f2aece2fa5dec89b076a2188", + "tools/linuxfs/preload/webhelper.c": "a8213a7a96b407459c83fb93365cfe58fd05964e7bb262b8061c1c0792a6aac8", + "tools/linuxfs/preload/writewatch.c": "61e395d1caa184d329dead0e36c4e92cb1069b1154ac36581f37406c1c242240", + "tools/linuxfs/preload/xmode.c": "9a09995bf52d40b07f16828bcc545b59b8fb1ba60d9cb09f14f7f31dba1e65a6", + "tools/linuxfs/preload/zram.c": "12bcf07459728d4a7ede6fa0fea8deb330f7c5d0f0dd61e610e81b9efecd4262", + "tools/linuxfs/ssbs/ssbs_adapter.c": "9e4e0fe637f7b8f91f12d8151f02e8d31bcb4294be2b2c62d3cd3ce76f200306", + "tools/proot/fastpath/audit.c": "bfe347a2ddaceef40a78d7d590e4e679724ee4c82f3f1f1e3c7017a18ad536d4", + "tools/proot/fastpath/fastpath.c": "4cd07b25ba14f6faa6bac5091ae323ded6b646dbb34b9e5a92ea6385e4d37cb5", + "tools/quest/kgsl_compat.c": "111afffda7a6df2469610f6c342ae086b1a97a559c586efc4a28015b86efd6bd", + "tools/quest/kgsl_compat.h": "2e9e0eb647530a35277954b7e61a22ee958067fe46056a608fb32f40f273f1bf" + }, + "x86": { + "@runner": "84f825635599f81ed33f62c5fc4c73f66044d1468b060d65f83855eb9c8bfe77", + "app/src/main/cpp/adrenotools/src/hook/kgsl.h": "396c56761034a4ed783543ee3312c6ec64d63b2a1cf7eecf1939c23251f25adf", + "app/src/main/cpp/fakeinput_steam.cpp": "3d11e859ca58a42a9da92a3fc589d404df790e151f7e18ea6114645750e70ef5", + "app/src/main/cpp/venc/venc_proto.h": "2242ddbc8981d14a33bac70264bb3e7f701fd1c3ebb988e17bbb2b19b6b808f7", + "tools/linuxfs/build-x86-preloads.sh": "d0820252809881db89db5a17427e74b9aa4964aae70623cb7f4b474e90c7a23b", + "tools/linuxfs/debian-image.env": "1c79dea4314c784042a5c59208765c8401771098f293185d61a157d89edf6e52", + "tools/linuxfs/fex/faultreport.c": "573bf780f32f7256dc2fbf40fe850d67e67c354509fdedcac3ff7f1a261daac3", + "tools/linuxfs/fex/vulkanpin.c": "f2693ac2be8d15f7881115e61b1c6b3fe4aeec8f603584c6fce5b0f046563d51", + "tools/linuxfs/fex/webkitinput.c": "693ffd6c8c8c97e87156444ad2255c84dbf22cd0053d135fb8d9be094b5534d1", + "tools/linuxfs/preload/binfmt.c": "0933cc49cb53f0aa1206e2eae516d38dc3337e74174be07e10619c42aa56792b", + "tools/linuxfs/preload/dircache.c": "8e7f19ea4a1198783d53a3f357326c6177da157b0cc7e0d22e1670963e1c26e7", + "tools/linuxfs/preload/drm.c": "e0eabab0d8ed9e841918e827bf87168efa7f2dfb8404169e9bed3ab5eda2836c", + "tools/linuxfs/preload/fsync.c": "795c114c2a05d3f250d07145d04c5658091699da3b96290da36348ccadfb0c09", + "tools/linuxfs/preload/inputudev.c": "2648248d3d235a582eb8459709e0a85de537c03ac10c5040fbfb50d9846c68eb", + "tools/linuxfs/preload/kgslid.c": "88a7875fd5289930204d155d92ac9d15492cd06db16ed87aa9c95af37f3e7089", + "tools/linuxfs/preload/kgslpoll.c": "f0d853d919a52487c04cbda5fb3fa22269da54b7f4ee2f7dd3b5bd9909418670", + "tools/linuxfs/preload/links.c": "b7bbcf9112393b152f358fe6f8dc9a1955f0c8532e1eddf016a971415cd13c64", + "tools/linuxfs/preload/lock.c": "f71b3fd40e6a2b74908d3cdc601e8748699ae9c7a89b542d2445b999015ffd66", + "tools/linuxfs/preload/net.c": "335853c8ea0e9a8ebbc5a5500dc4bded9c34c4310a6ab8aa46b07d5ce1c2b55e", + "tools/linuxfs/preload/netif.c": "cc26d50b8d4639d8dd4697811bbf85c9d9f31761f9504a4af42473de3cb22b3c", + "tools/linuxfs/preload/noexec.c": "1855aeaeda3d506329bd9af1107580c74d1f56787c411b2e01c0a8a9d4059c7b", + "tools/linuxfs/preload/ntsync.c": "0614c48af0c062bfcb4c30141efe1b792b340767a67d2fe92ee9877d2ba16e0d", + "tools/linuxfs/preload/opens.c": "92ae4aea16aa457a898dedf4a70eb332d80d3b1e77f3a60bace7715a4317399e", + "tools/linuxfs/preload/pathcache.c": "eb0d4b5238fe6c8891e805d90bd837e466754100c978fad2d0ff717116b9b8a3", + "tools/linuxfs/preload/perms.c": "f600619c4ca130f69001ed7ad7bd74a3dc215b14c61479c45bb770f916d244b9", + "tools/linuxfs/preload/pulseguard.c": "57674ea40283e53a27bc78362de1ba41ee972d172ea38b1f818b1d6eed0abe65", + "tools/linuxfs/preload/quest_kgsl.c": "59d9b8416be6a1c680f1d1e4c2427d2cbb7b415344261adb357d9d211ec95cfe", + "tools/linuxfs/preload/robust.c": "9eb6c0ed50e0f48a8cfb86bd92587f471354bdf2531b9177f4b3ab83bdfed39a", + "tools/linuxfs/preload/rptouch.c": "0e3f8bf2f9a41324361f38fa4da8d2d47593c9555f0cabf8228767397846c7c3", + "tools/linuxfs/preload/space.c": "118389bcc89dfa2b4230b80915e21d2d4331c4cff1f5a58265f85e6ecb5c29d2", + "tools/linuxfs/preload/storage.c": "08a8b46c3588f3160bb8608489768010ec36d5ccdb2b25a431ec9d6fcd413a6e", + "tools/linuxfs/preload/sysv.c": "49774ee5e2e39b84dcfadf63abdc741b3df236cb11783744a1280adee36965e8", + "tools/linuxfs/preload/tracefs.c": "be3f8a2f032b56e6e08a931e77a9386aea4bb93421a482271e06386ec0c469b6", + "tools/linuxfs/preload/tracer.c": "f34e5377c891b0a8d00489245d66f96bc352df4cd90f47a951dcadae980af4c1", + "tools/linuxfs/preload/udevmon.c": "2045dfb883cfb6e2016c359a3dc3af17f3b8f383f6dd0d5dfee407ef0d344852", + "tools/linuxfs/preload/usr2.c": "6ae704380b45a9b3a74c4fe2b3ce4482470dae59ced26cf0aee64f4b4dcc11f9", + "tools/linuxfs/preload/venc.c": "24c3c46fe49d28adfc8c977e52c9cc407f534b26f56d94b3425b8bb01803dba8", + "tools/linuxfs/preload/vmwrite.c": "1a3ea2ec4354e0716c17f7db537b19ab82bcd524f2aece2fa5dec89b076a2188", + "tools/linuxfs/preload/webhelper.c": "a8213a7a96b407459c83fb93365cfe58fd05964e7bb262b8061c1c0792a6aac8", + "tools/linuxfs/preload/writewatch.c": "61e395d1caa184d329dead0e36c4e92cb1069b1154ac36581f37406c1c242240", + "tools/linuxfs/preload/xmode.c": "9a09995bf52d40b07f16828bcc545b59b8fb1ba60d9cb09f14f7f31dba1e65a6", + "tools/linuxfs/preload/zram.c": "12bcf07459728d4a7ede6fa0fea8deb330f7c5d0f0dd61e610e81b9efecd4262", + "tools/quest/kgsl_compat.c": "111afffda7a6df2469610f6c342ae086b1a97a559c586efc4a28015b86efd6bd", + "tools/quest/kgsl_compat.h": "2e9e0eb647530a35277954b7e61a22ee958067fe46056a608fb32f40f273f1bf" + } + }, + "origin": { + "artifact": 12520502054, + "artifact_sha256": "b7e800a1884a516ec52b050b7f904f9881427bca4579f660b909fd18f15e49ff", + "commit": "b76f13f7acc0af0cc00369dde61fb88e40529e6d", + "run": 39992155294, + "url": "https://github.com/Droid-Deck/DroidDeck/actions/runs/39992155294" + }, + "outputs": { + "i386/libblsession.so": "5343cdb704217ddc81ee593e67d5987363dc7b5ac78bd4bd36000219e5063268", + "i386/libfakeinput.so": "188d88dfb13b6e2f2df948e7b8b4d9ec604770f55d1ea5b6b9e21ad56a74ce5e", + "i386/libfaultreport.so": "95f9ef60f04a13ee60cfbd4d2d2efb60a7063b722e646c78c9dc27162292877b", + "libblaudit.so": "6c6abe89baf5d7309babee0c3372a4b00c2b689e2a63de38c5e9a660626509b5", + "libblfastpath.so": "0e1f151b5c1c5ea9dd380ad58d09f8fe7ba3ae037d0991cb3eef6899e9d5ec83", + "libblsession.so": "29d4b5ea46dba941107e02c0ee544244b137a339cada43bff65c350f2ef2755a", + "libfakeinput.so": "5f6dc3aea05de2dc1fbe0bbfac2ce4d11e3d47f761c6b06c8badb15fe1904392", + "libssbs.so": "c288ed0da26c2481aff270d1cdafe1205061c4a300c2c46ae1b53c6e0c94f44b", + "usr/local/bin/droiddeck-clipboard": "ec50e8f67fdf76372927189b0fdb670f9b70fd3484f2fe3e833bc8f9fdc44205", + "x86_64/libblsession.so": "61d19f06aebab6a789f8c99f687af9903376bfc853e46b70c48a2ce5e497accc", + "x86_64/libfakeinput.so": "8663dd2086bf8ae53069a480dd9ae4619109d32f0947a29e2ce38f07a21ae2e6", + "x86_64/libfaultreport.so": "bb0dc9165d0d764473e552508650edd586df16d620fc5b16b8b1c5740a28ce22", + "x86_64/libvulkan-thunk.so": "74f01b652d7e8179561a096a552328b68da640cc4bad3981edd38e6afbf90243", + "x86_64/libwebkitinput.so": "9b07380a7b017397b9c56bb7bb1161e046b0c6a647fc413888904606b0def6d0" + }, + "toolchain": { + "arm64": "Ubuntu 24.04 GCC cross compiler; exact build commands pinned by arm64_recipe", + "x86": "Debian bullseye compiler image and recipe pinned by source inputs" + }, + "version": 1 +} diff --git a/tools/components/preloads.py b/tools/components/preloads.py new file mode 100644 index 0000000000..7da904080f --- /dev/null +++ b/tools/components/preloads.py @@ -0,0 +1,248 @@ +"""Reuse reviewed preload snapshots; changed snapshots must prove their main-build origin.""" +import argparse +import gzip +import hashlib +import io +import json +import os +import posixpath +import re +from pathlib import Path +import subprocess +import tarfile +import urllib.error +import urllib.request +from zipfile import ZipFile + +ROOT = Path(__file__).resolve().parents[2] +REPO = 'Droid-Deck/DroidDeck' +LOCK = 'tools/components/preloads.lock.json' +COMMON = ['app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/cpp/adrenotools/src/hook/kgsl.h', + 'app/src/main/cpp/venc/venc_proto.h', 'tools/linuxfs/preload', + 'tools/quest/kgsl_compat.c', 'tools/quest/kgsl_compat.h'] +INPUTS = { + 'arm64': COMMON + ['tools/proot/fastpath', 'tools/linuxfs/ssbs', 'tools/linuxfs/clipboard'], + 'x86': COMMON + ['tools/linuxfs/fex', 'tools/linuxfs/build-x86-preloads.sh', 'tools/linuxfs/debian-image.env'], +} +OUTPUTS = { + 'arm64': ['libfakeinput.so', 'libblsession.so', 'libblfastpath.so', 'libblaudit.so', 'libssbs.so', + 'usr/local/bin/droiddeck-clipboard'], + 'x86': [f'{arch}/{name}.so' for arch in ('x86_64', 'i386') + for name in ('libfakeinput', 'libblsession', 'libfaultreport')] + + ['x86_64/libwebkitinput.so', 'x86_64/libvulkan-thunk.so'], +} +FILES = set(sum(OUTPUTS.values(), [])) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def run(*args, root=ROOT): + return subprocess.check_output(args, cwd=root) + + +def api(path): + return json.loads(run('gh', 'api', 'repos/' + REPO + '/' + path)) + + +def source_files(group, root=ROOT, commit=None): + if commit: + paths = run('git', 'ls-tree', '-r', '--name-only', commit, '--', *INPUTS[group], root=root).decode().splitlines() + read = lambda path: run('git', 'show', f'{commit}:{path}', root=root) + else: + paths = run('git', 'ls-files', '--cached', '--others', '--exclude-standard', '--', *INPUTS[group], root=root).decode().splitlines() + read = lambda path: (root / path).read_bytes() + paths = [p for p in paths if not p.endswith('.md') and '__pycache__' not in p and not p.endswith('.pyc')] + if not paths: + raise ValueError('No preload inputs found') + result = {} + pending = list(paths) + while pending: + path = pending.pop() + if path in result: + continue + if path.startswith('../') or path.startswith('/'): + raise ValueError('Preload include escapes repository') + content = read(path).replace(b'\r\n', b'\n') + result[path] = sha(content) + for include in re.findall(rb'^\s*#\s*include\s*"([^"\n]+)"', content, re.M): + pending.append(posixpath.normpath(posixpath.join(posixpath.dirname(path), include.decode()))) + workflow = (run('git', 'show', f'{commit}:.github/workflows/build.yml', root=root).decode() if commit + else (root / '.github/workflows/build.yml').read_text()) + runner = re.search(r'(?ms)^ build:\n.*?^ runs-on: ([^\n]+)', workflow) + if runner is None: + raise ValueError('APK build runner policy is missing') + result['@runner'] = sha(runner[1].strip().encode()) + return dict(sorted(result.items())) + + +def recipe(root=ROOT, commit=None): + def read(path): + return (run('git', 'show', f'{commit}:{path}', root=root).decode() if commit + else (root / path).read_text()) + script = 'tools/components/build-arm64-preloads.sh' + if commit and run('git', 'ls-tree', commit, script, root=root) == b'': + workflow = read('.github/workflows/build.yml') + text = workflow[workflow.index(' aarch64-linux-gnu-g++ -shared'):] + text = text[:text.index(' VK_INCLUDE=')] + else: + text = read(script) + preamble = '#!/usr/bin/env bash\nset -euo pipefail\nD=${1:?output directory}\nmkdir -p "$D"\n' + if not text.startswith(preamble): + return sha(text.encode()) + text = text[len(preamble):] + return sha(' '.join(text.split()).encode()) + + +def artifact_bytes(origin, cache): + build = api(f"actions/runs/{origin['run']}") + if (build['event'] != 'push' or build['head_branch'] != 'main' or build['conclusion'] != 'success' + or build['head_sha'] != origin['commit'] or build['path'] != '.github/workflows/build.yml' + or build['repository']['full_name'] != REPO): + raise ValueError('Snapshot is not from a successful main APK build') + artifact = api(f"actions/artifacts/{origin['artifact']}") + if (artifact['name'] != 'droiddeck-apk' or artifact['workflow_run']['id'] != origin['run'] + or artifact['digest'] != 'sha256:' + origin['artifact_sha256']): + raise ValueError('Original APK artifact identity or digest mismatch') + cache.mkdir(parents=True, exist_ok=True) + path = cache / (origin['artifact_sha256'] + '.zip') + if not path.exists(): + path.write_bytes(run('gh', 'api', f"repos/{REPO}/actions/artifacts/{origin['artifact']}/zip")) + content = path.read_bytes() + if sha(content) != origin['artifact_sha256']: + raise ValueError('Original APK archive checksum mismatch') + with ZipFile(io.BytesIO(content)) as archive: + names = [name for name in archive.namelist() if name.endswith('.apk')] + if len(names) != 1: + raise ValueError('Expected one original APK') + with ZipFile(io.BytesIO(archive.read(names[0]))) as apk: + return {name: apk.read('assets/linuxfs/' + name) for name in FILES} + + +def validate_lock(data): + if data.get('version') != 1 or set(data['groups']) != set(INPUTS): + raise ValueError('Unsupported preload snapshot') + if set(data['outputs']) != FILES: + raise ValueError('Unexpected preload output set') + for digest in [data['archive_sha256'], *data['outputs'].values()]: + if len(digest) != 64 or any(c not in '0123456789abcdef' for c in digest): + raise ValueError('Invalid preload checksum') + + +def prove_origin(data, cache, root=ROOT): + content = artifact_bytes(data['origin'], cache) + if {name: sha(value) for name, value in content.items()} != data['outputs']: + raise ValueError('Snapshot binaries differ from the original main APK') + commit = data['origin']['commit'] + run('git', 'fetch', '--no-tags', 'origin', commit, root=root) + for group in INPUTS: + if source_files(group, root, commit) != data['groups'][group]: + raise ValueError('Snapshot inputs differ from the producing source') + if recipe(root, commit) != data['arm64_recipe']: + raise ValueError('Snapshot recipe differs from the producing source') + + +def unpack(content, data): + if sha(content) != data['archive_sha256']: + raise ValueError('Preload archive checksum mismatch') + result = {} + total = 0 + with tarfile.open(fileobj=io.BytesIO(content), mode='r:gz') as archive: + for entry in archive: + if entry.name not in FILES or entry.name in result or not entry.isfile() or entry.size > 32 * 1024 * 1024: + raise ValueError('Unsafe or unexpected preload archive entry') + total += entry.size + if total > 64 * 1024 * 1024: + raise ValueError('Preload archive exceeds expanded size limit') + value = archive.extractfile(entry).read() + if sha(value) != data['outputs'][entry.name]: + raise ValueError('Preload binary checksum mismatch') + result[entry.name] = value + if set(result) != FILES: + raise ValueError('Incomplete preload archive') + return result + + +def restore(root=ROOT): + data = json.loads((root / LOCK).read_text()) + validate_lock(data) + matching = {group: source_files(group, root) == data['groups'][group] for group in INPUTS} + matching['arm64'] &= recipe(root) == data['arm64_recipe'] + if os.environ.get('FORCE_PRELOAD_REBUILD') == 'true': + matching = dict.fromkeys(INPUTS, False) + if any(matching.values()): + base = os.environ.get('COMPONENT_TRUSTED_BASE') + approved = None + if base: + result = subprocess.run(['git', 'show', f'{base}:{LOCK}'], cwd=root, capture_output=True) + if result.returncode == 0: + approved = json.loads(result.stdout) + if approved != data: + prove_origin(data, root / 'app/build/component-downloads', root) + url = (f"https://github.com/Droid-Deck/DroidDeck-Components/releases/download/" + f"preloads-{data['archive_sha256']}/preloads.tar.gz") + try: + with urllib.request.urlopen(url, timeout=60) as response: + content = response.read(64 * 1024 * 1024) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + print('Preload snapshot is unavailable; rebuilding from source.') + matching = dict.fromkeys(INPUTS, False) + else: + binaries = unpack(content, data) + for group, matches in matching.items(): + if matches: + for name in OUTPUTS[group]: + path = root / 'app/src/main/assets/linuxfs' / name + if not path.resolve().is_relative_to(root.resolve()) or path.is_symlink(): + raise ValueError('Preload output escapes repository or is a symlink') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(binaries[name]) + path.chmod(0o755 if name.endswith('droiddeck-clipboard') else 0o644) + for group, matches in matching.items(): + print(f"{group}: {'verified snapshot reused' if matches else 'inputs changed or snapshot unavailable; rebuild required'}") + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'{group}={str(matches).lower()}\n') + return matching + + +def snapshot(run_id, destination, root=ROOT): + build = api(f'actions/runs/{run_id}') + candidates = api(f'actions/runs/{run_id}/artifacts?per_page=100')['artifacts'] + artifact = next(a for a in candidates if a['name'] == 'droiddeck-apk') + origin = dict(run=run_id, commit=build['head_sha'], artifact=artifact['id'], + artifact_sha256=artifact['digest'].removeprefix('sha256:'), + url=build['html_url']) + content = artifact_bytes(origin, root / 'app/build/component-downloads') + run('git', 'fetch', '--no-tags', 'origin', origin['commit'], root=root) + data = dict(version=1, origin=origin, groups={g: source_files(g, root, origin['commit']) for g in INPUTS}, + arm64_recipe=recipe(root, origin['commit']), outputs={name: sha(value) for name, value in content.items()}, + toolchain=dict(arm64='Ubuntu 24.04 GCC cross compiler; exact build commands pinned by arm64_recipe', + x86='Debian bullseye compiler image and recipe pinned by source inputs')) + stream = io.BytesIO() + with gzip.GzipFile(fileobj=stream, mode='wb', mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode='w') as archive: + for name, value in sorted(content.items()): + entry = tarfile.TarInfo(name) + entry.size, entry.mode = len(value), 0o644 + archive.addfile(entry, io.BytesIO(value)) + destination.mkdir(parents=True, exist_ok=True) + (destination / 'preloads.tar.gz').write_bytes(stream.getvalue()) + data['archive_sha256'] = sha(stream.getvalue()) + (root / LOCK).write_text(json.dumps(data, indent=2, sort_keys=True) + '\n') + print('Snapshot SHA-256: ' + data['archive_sha256']) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('operation', choices=['restore', 'snapshot']) + parser.add_argument('--run', type=int) + parser.add_argument('--out', type=Path, default=ROOT / 'app/build/component-snapshot') + args = parser.parse_args() + if args.operation == 'restore': + restore() + else: + snapshot(args.run, args.out) diff --git a/tools/release/runtime_inputs.py b/tools/release/runtime_inputs.py index a0ac10bfdd..d6cf46bc49 100644 --- a/tools/release/runtime_inputs.py +++ b/tools/release/runtime_inputs.py @@ -23,7 +23,7 @@ PACK_NAMES = ['proot', 'directaudio', 'tracking', 'xbox', 'comet'] SOURCE_PATHS = ['app/build.gradle', 'app/src/main/cpp/fakeinput_steam.cpp', 'app/src/main/assets/pulseaudio.tzst', 'tools/linuxfs', 'tools/proot', 'tools/aaudio-sink', 'tools/directaudio', 'tools/gamescope/release.env', 'tools/droiddeck-esync/release.env', - 'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/build_local.sh', + 'tools/mangoapp', 'tools/msitools', 'tools/pipewire', 'tools/comet', 'tools/xbox', 'tools/components', 'tools/build_local.sh', 'tools/release/runtime_inputs.py', 'app/src/main/assets/desktop-removal', 'tools/quest', 'tools/vr/src', 'tools/vr/driver_droiddeck', 'tools/vr/layer', 'tools/vr/build.sh', 'app/src/main/cpp/waylandcomp/src/headpose_shm.h', diff --git a/tools/tests/test_preload_snapshots.py b/tools/tests/test_preload_snapshots.py new file mode 100644 index 0000000000..5a4cfb606c --- /dev/null +++ b/tools/tests/test_preload_snapshots.py @@ -0,0 +1,146 @@ +import copy +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('preload_snapshots', Path(__file__).resolve().parents[1] / 'components/preloads.py') +snapshots = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(snapshots) + + +def archive(files): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode='w:gz') as tar: + for name, content in files.items(): + entry = tarfile.TarInfo(name) + entry.size = len(content) + tar.addfile(entry, io.BytesIO(content)) + return stream.getvalue() + + +class PreloadSnapshotTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + # Real repository inputs exercise the transitive include walker and build recipe. + for group in snapshots.INPUTS: + for name in snapshots.source_files(group): + if name.startswith('@'): + continue + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((snapshots.ROOT / name).read_bytes()) + recipe = 'tools/components/build-arm64-preloads.sh' + (self.root / recipe).parent.mkdir(parents=True, exist_ok=True) + (self.root / recipe).write_text((snapshots.ROOT / recipe).read_text()) + (self.root / '.github/workflows').mkdir(parents=True) + (self.root / '.github/workflows/build.yml').write_text((snapshots.ROOT / '.github/workflows/build.yml').read_text()) + self.git('init', '-q') + self.git('config', 'core.autocrlf', 'false') + self.git('config', 'user.name', 'Test Developer') + self.git('config', 'user.email', 'test@example.test') + self.git('add', '.') + self.binaries = {name: ('verified-' + name).encode() for name in snapshots.FILES} + self.content = archive(self.binaries) + self.data = dict(version=1, origin=dict(run=123, commit='abc', artifact=456, artifact_sha256='0' * 64), + groups={g: snapshots.source_files(g, self.root) for g in snapshots.INPUTS}, + arm64_recipe=snapshots.recipe(self.root), archive_sha256=snapshots.sha(self.content), + outputs={name: snapshots.sha(value) for name, value in self.binaries.items()}) + (self.root / snapshots.LOCK).write_text(json.dumps(self.data)) + self.git('add', '.') + self.git('commit', '-qm', 'Approved baseline') + self.base = self.git('rev-parse', 'HEAD').strip() + + def git(self, *args): + return subprocess.check_output(['git', *args], cwd=self.root, text=True) + + def restore(self, base=None): + with patch.dict(os.environ, COMPONENT_TRUSTED_BASE=base or self.base, FORCE_PRELOAD_REBUILD='false'), \ + patch.object(snapshots.urllib.request, 'urlopen', return_value=io.BytesIO(self.content)) as fetch, \ + patch.object(snapshots, 'prove_origin') as prove: + result = snapshots.restore(self.root) + return result, fetch, prove + + def test_reviewed_unchanged_binaries_reused_and_written(self): + result, fetch, prove = self.restore() + self.assertEqual(result, dict(arm64=True, x86=True)) + prove.assert_not_called() + fetch.assert_called_once() + for name, value in self.binaries.items(): + self.assertEqual((self.root / 'app/src/main/assets/linuxfs' / name).read_bytes(), value) + + def test_changed_common_source_requires_both_rebuilds(self): + path = self.root / snapshots.COMMON[0] + path.write_text(path.read_text() + '\n// Changed source\n') + result, fetch, prove = self.restore() + self.assertEqual(result, dict(arm64=False, x86=False)) + fetch.assert_not_called() + prove.assert_not_called() + + def test_new_transitive_header_is_an_input(self): + source = self.root / snapshots.COMMON[0] + source.write_text(source.read_text() + '\n#include "new-header.h"\n') + header = source.with_name('new-header.h') + header.write_text('#define VALUE 1\n') + before = snapshots.source_files('arm64', self.root) + header.write_text('#define VALUE 2\n') + self.assertNotEqual(before, snapshots.source_files('arm64', self.root)) + + def test_changed_recipe_rebuilds_arm64_only(self): + script = self.root / 'tools/components/build-arm64-preloads.sh' + script.write_text(script.read_text().replace('-O2', '-O3')) + result, _, _ = self.restore() + self.assertEqual(result, dict(arm64=False, x86=True)) + + def test_new_manifest_requires_main_build_proof(self): + result, _, prove = self.restore('HEAD~1') + self.assertTrue(all(result.values())) + prove.assert_called_once() + + def test_forged_binary_hash_cannot_prove_main_origin(self): + data = copy.deepcopy(self.data) + data['outputs']['libblsession.so'] = 'f' * 64 + with patch.object(snapshots, 'artifact_bytes', return_value=self.binaries): + with self.assertRaisesRegex(ValueError, 'original main APK'): + snapshots.prove_origin(data, self.root / 'cache', self.root) + + def test_non_main_or_unsuccessful_run_cannot_prove_origin(self): + build = dict(event='pull_request', head_branch='main', conclusion='success', head_sha='abc', + path='.github/workflows/build.yml', repository=dict(full_name=snapshots.REPO)) + for field, value in [('event', 'pull_request'), ('head_branch', 'feature'), ('conclusion', 'failure')]: + candidate = dict(build, event='push') + candidate[field] = value + with self.subTest(field=field), patch.object(snapshots, 'api', return_value=candidate): + with self.assertRaisesRegex(ValueError, 'successful main'): + snapshots.artifact_bytes(self.data['origin'], self.root / 'cache') + + def test_corrupt_archive_is_rejected(self): + with self.assertRaisesRegex(ValueError, 'checksum'): + snapshots.unpack(self.content + b'tampered', self.data) + + def test_missing_duplicate_or_traversal_entries_are_rejected(self): + for files in ({}, {'../escaped': b'bad'}, {'libblsession.so': b'wrong'}): + content = archive(files) + data = dict(self.data, archive_sha256=snapshots.sha(content)) + with self.subTest(names=list(files)), self.assertRaises(ValueError): + snapshots.unpack(content, data) + + def test_missing_release_rebuilds_without_accepting_unverified_bytes(self): + error = snapshots.urllib.error.HTTPError('https://example.test', 404, 'missing', {}, None) + with patch.dict(os.environ, COMPONENT_TRUSTED_BASE=self.base), \ + patch.object(snapshots.urllib.request, 'urlopen', side_effect=error): + self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False)) + + def test_manual_rebuild_does_not_download_snapshot(self): + with patch.dict(os.environ, FORCE_PRELOAD_REBUILD='true'), \ + patch.object(snapshots.urllib.request, 'urlopen') as fetch: + self.assertEqual(snapshots.restore(self.root), dict(arm64=False, x86=False)) + fetch.assert_not_called()