diff --git a/README.md b/README.md index 9b5dc82b8..653df9f72 100644 --- a/README.md +++ b/README.md @@ -66,3 +66,5 @@ Joy-Con VR controllers port the arm model of Google's [Google VR SDK for Unity]( Camera hand tracking runs Google's [MediaPipe](https://github.com/google-ai-edge/mediapipe) Tasks Vision library with its Pose Landmarker (Lite) model, BlazePose GHUM 3D, and its Hand Landmarker model (all Apache-2.0), on frames from AndroidX [CameraX](https://developer.android.com/jetpack/androidx/releases/camera) (Apache-2.0); MediaPipe's usage reports are switched off. The design was checked against [@F8LEFT](https://github.com/F8LEFT)'s [JoyConVr](https://github.com/F8LEFT/JoyConVr) camera app (MIT), and the short Joy-Con movements follow the accelerometer mode of the [Google VR SDK for Unity](https://github.com/googlevr/gvr-unity-sdk)'s arm model, Eric Foxlin's zero-velocity updates and Frosio, Pedersini and Borghese's accelerometer autocalibration, and the camera and Joy-Con are joined by a Kalman filter (R. E. Kalman, 1960). Bare camera hands take their gestures after [ALVR](https://github.com/alvr-org/ALVR)'s `hand_gestures.rs` (MIT) and Meta Quest's menu pinch, with distances checked on photos from the [HaGRID](https://github.com/hukenovs/hagrid) gesture set (CC BY-SA 4.0, not included), fit the palm with Horn's quaternion method (1987) and smooth it with the 1€ filter (Casiez, Roussel and Vogel, 2012); the white glove model follows Valve's OpenVR render model reference and Index controller poses, is meshed with surface nets (Gibson, 1998; Mikola Lysenko) and blends and shades it with Inigo Quilez's smooth minimum and distance-field ambient occlusion, porting his polygon distance (MIT); see [docs/third-party/handtracking](docs/third-party/handtracking/README.md). The Store installs and runs apps with [Flatpak](https://github.com/flatpak/flatpak) (LGPL-2.1). Its sandbox goes through DroidDeck's own bubblewrap shim, which follows how Flatpak hands bwrap its environment. Chromium and Electron apps from Flathub start their browser processes through [zypak](https://github.com/refi64/zypak) by [@refi64](https://github.com/refi64) (BSD-3-Clause); its source was read to see that it always spawns them through Flatpak's spawn portal, which is why DroidDeck starts them directly without Chromium's sandbox instead. + +Steam's Browse buttons open a GTK 3 file chooser through PyGObject (LGPL-2.1-or-later). It runs under Steam's own `reaper` with Steam's app ID, which [gamescope](https://github.com/ValveSoftware/gamescope) (BSD-2-Clause) reads from the process tree (`GetAppIdFromReaper` in `src/Utils/Process.cpp`) to show it over Steam. diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-compat b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-compat index 1de4bcf1c..3127ace34 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-compat +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-compat @@ -196,6 +196,43 @@ TOUR_JS = r""" return "off"; })() """ +FILE_BINDING = "droiddeckOpenFileDialog" +FILE_JS = r""" +(() => { + if (window.__droiddeckFiles) return "present"; + const self = { next: 0, pending: new Map() }; + const cancelled = () => ({ result: 2, message: "" }); + const open = function (options) { + return new Promise((resolve, reject) => { + if (typeof window.__BINDING__ !== "function") { reject(cancelled()); return; } + const id = ++self.next; + self.pending.set(id, { resolve, reject }); + try { window.__BINDING__(JSON.stringify({ id, options: options || {} })); } + catch (e) { self.pending.delete(id); reject(cancelled()); } + }); + }; + self.done = (id, path) => { + const item = self.pending.get(id); + if (!item) return false; + self.pending.delete(id); + if (typeof path === "string" && path) item.resolve(path); else item.reject(cancelled()); + return true; + }; + const patch = (w) => { + const system = w?.SteamClient?.System; + if (system && system.OpenFileDialog !== open) system.OpenFileDialog = open; + }; + patch(window); + if (SteamClient.System.OpenFileDialog !== open) throw new Error("Steam OpenFileDialog could not be redirected"); + const popups = window.g_PopupManager; + if (popups) { + for (const popup of popups.GetPopups?.() || []) patch(popup.window); + popups.AddPopupCreatedCallback?.((popup) => patch(popup.window)); + } + window.__droiddeckFiles = self; + return "installed"; +})() +""".replace("__BINDING__", FILE_BINDING) VR_BINDING = "droiddeckLaunchSteamVr" VR_HELPER = "/usr/local/bin/droiddeck-steamvr" VR_JS = r""" @@ -274,6 +311,62 @@ def launch_steamvr(): say("SteamVR library launch failed: %s" % e) +def choose_file(options): + import gi + gi.require_version("Gdk", "3.0") + gi.require_version("Gtk", "3.0") + from gi.repository import Gdk, Gtk + directory = bool(options.get("bChooseDirectory")) + save = bool(options.get("bSaveMode")) and not directory + action = (Gtk.FileChooserAction.SELECT_FOLDER if directory else + Gtk.FileChooserAction.SAVE if save else Gtk.FileChooserAction.OPEN) + dialog = Gtk.FileChooserDialog(title=str(options.get("strTitle") or ""), action=action) + dialog.add_buttons("gtk-cancel", Gtk.ResponseType.CANCEL, + "gtk-save" if save else "gtk-open", Gtk.ResponseType.ACCEPT) + dialog.set_default_response(Gtk.ResponseType.ACCEPT) + dialog.set_local_only(True) + dialog.set_do_overwrite_confirmation(True) + for folder in ("/root/Storage", "/mnt/droiddeck-sd"): + if os.path.isdir(folder): + try: + dialog.add_shortcut_folder(folder) + except Exception: + pass + if not directory: + for entry in options.get("rgFilters") or []: + patterns = [str(p) for p in entry.get("rFilePatterns") or [] if p] + if not patterns: + continue + chosen = Gtk.FileFilter() + chosen.set_name(str(entry.get("strFileTypeName") or ", ".join(patterns))) + for pattern in patterns: + pattern = "*" if pattern == "*.*" else pattern + chosen.add_pattern(pattern) + if pattern.lower() != pattern.upper(): + chosen.add_pattern(pattern.upper()) + dialog.add_filter(chosen) + if entry.get("bUseAsDefault"): + dialog.set_filter(chosen) + initial = str(options.get("strInitialFile") or "").strip('"') + if initial: + if os.path.isdir(initial): + dialog.set_current_folder(initial) + elif save: + if os.path.isdir(os.path.dirname(initial)): + dialog.set_current_folder(os.path.dirname(initial)) + dialog.set_current_name(os.path.basename(initial)) + elif os.path.exists(initial): + dialog.set_filename(initial) + elif os.path.isdir(os.path.dirname(initial)): + dialog.set_current_folder(os.path.dirname(initial)) + screen = Gdk.Screen.get_default() + if screen is not None: + dialog.set_default_size(screen.get_width(), screen.get_height()) + chosen_path = dialog.get_filename() if dialog.run() == Gtk.ResponseType.ACCEPT else None + dialog.destroy() + return chosen_path + + class Closed(Exception): pass @@ -530,6 +623,59 @@ class Helper: self.tools_asked = {} self.tools_checked = None self.vr_launch = None + self.chooser = None + + def open_chooser(self, payload): + try: + request = json.loads(payload) + ident = int(request["id"]) + options = request.get("options", {}) + if not isinstance(options, dict): + raise ValueError("options are not an object") + except (ValueError, KeyError, TypeError) as e: + say("ignored a file dialog request (%s)" % e) + return + if self.chooser is not None: + self.finish_chooser(ident, None) + return + command = [sys.executable, os.path.realpath(__file__), "--choose-file"] + reaper = os.path.join(self.steam_root, "steamrtarm64", "reaper") + if os.access(reaper, os.X_OK): + command = [reaper, "SteamLaunch", "AppId=769", "--"] + command + try: + proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, start_new_session=True) + proc.stdin.write(json.dumps(options).encode("utf-8")) + proc.stdin.close() + except OSError as e: + say("could not open a file dialog (%s)" % e) + self.finish_chooser(ident, None) + return + say("file dialog open: %s" % (options.get("strTitle") or "untitled")) + self.chooser = (ident, proc) + + def chooser_done(self): + ident, proc = self.chooser + self.chooser = None + path = proc.stdout.read().decode("utf-8", "replace").strip() + proc.stdout.close() + proc.wait() + self.finish_chooser(ident, path if proc.returncode == 0 and path else None) + + def finish_chooser(self, ident, path): + say("file dialog %s" % ("chose " + path if path else "cancelled")) + self.page.evaluate("window.__droiddeckFiles?.done(%d, %s)" % (ident, json.dumps(path))) + + def close_chooser(self): + if self.chooser is not None: + _, proc = self.chooser + self.chooser = None + try: + os.killpg(proc.pid, signal.SIGTERM) + except OSError: + pass + proc.stdout.close() + proc.wait() def attach(self): path = shared_context(self.port) @@ -542,6 +688,8 @@ class Helper: page.call("Runtime.addBinding", {"name": BINDING}) page.call("Runtime.addBinding", {"name": VR_BINDING}) page.evaluate(VR_JS) + page.call("Runtime.addBinding", {"name": FILE_BINDING}) + page.evaluate(FILE_JS) try: page.evaluate(TOUR_JS) except RuntimeError: @@ -680,6 +828,9 @@ class Helper: if vr_games and (self.vr_launch is None or self.vr_launch.poll() is not None): say("preparing SteamVR for library app(s): " + ", ".join(sorted(vr_games))) self.vr_launch = launch_steamvr() + for e in events: + if called(e, FILE_BINDING) and isinstance(e.get("params", {}).get("payload"), str): + self.open_chooser(e["params"]["payload"]) return events def serve(self, watcher): @@ -690,7 +841,11 @@ class Helper: self.ensure_tools() while True: sources = [self.page.ws] + ([watcher] if watcher.fileno() >= 0 else []) + if self.chooser is not None: + sources.append(self.chooser[1].stdout) ready = [self.page.ws] if self.page.ws.pending() else select.select(sources, [], [], 30 if watcher.fileno() >= 0 else 2)[0] + if self.chooser is not None and self.chooser[1].stdout in ready: + self.chooser_done() due = False if self.page.ws in ready: due = any(called(e, BINDING) for e in self.drain()) @@ -718,6 +873,18 @@ def detach(): COMPAT["write_json"](state_file, state) +def chooser_main(): + try: + options = json.load(sys.stdin) + except ValueError: + return 1 + path = choose_file(options if isinstance(options, dict) else {}) + if not path: + return 1 + print(path) + return 0 + + def main(): steam_root = sys.argv[1] try: @@ -739,6 +906,7 @@ def main(): last = None helper.serve(watcher) except Exception as e: + helper.close_chooser() if helper.page is not None: say("lost the client (%s); reattaching" % e) helper.page.ws.close() @@ -752,6 +920,8 @@ def main(): if __name__ == "__main__": + if sys.argv[1:2] == ["--choose-file"]: + sys.exit(chooser_main()) signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) try: sys.exit(main()) diff --git a/tools/tests/test_steam_file_dialog.py b/tools/tests/test_steam_file_dialog.py new file mode 100644 index 000000000..baff8914f --- /dev/null +++ b/tools/tests/test_steam_file_dialog.py @@ -0,0 +1,134 @@ +import io +import json +import os +import runpy +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +BIN = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin" +LIVE = runpy.run_path(str(BIN / "droiddeck-steam-compat")) + +HARNESS = r""" +const assert = require('assert'); +globalThis.window = globalThis; +const native = () => Promise.reject({ result: 2, message: '' }); +const popupCallbacks = []; +const popup = { window: { SteamClient: { System: { OpenFileDialog: native } } } }; +const later = { window: { SteamClient: { System: { OpenFileDialog: native } } } }; +globalThis.SteamClient = { System: { OpenFileDialog: native } }; +window.g_PopupManager = { GetPopups: () => [popup], AddPopupCreatedCallback: cb => popupCallbacks.push(cb) }; +const asked = []; +window.droiddeckOpenFileDialog = payload => asked.push(JSON.parse(payload)); +const script = require('fs').readFileSync(process.argv[2], 'utf8'); +(async () => { + assert.strictEqual(eval(script), 'installed'); + assert.strictEqual(eval(script), 'present'); + assert.notStrictEqual(SteamClient.System.OpenFileDialog, native); + assert.strictEqual(popup.window.SteamClient.System.OpenFileDialog, SteamClient.System.OpenFileDialog); + popupCallbacks.forEach(cb => cb(later)); + assert.strictEqual(later.window.SteamClient.System.OpenFileDialog, SteamClient.System.OpenFileDialog); + const options = { strTitle: 'Pick', bChooseDirectory: true }; + const chosen = SteamClient.System.OpenFileDialog(options); + const cancelled = later.window.SteamClient.System.OpenFileDialog({ strTitle: 'Other' }); + assert.deepStrictEqual(asked, [{ id: 1, options }, { id: 2, options: { strTitle: 'Other' } }]); + assert.strictEqual(window.__droiddeckFiles.done(1, '/root/Storage/Games'), true); + assert.strictEqual(window.__droiddeckFiles.done(1, '/again'), false); + assert.strictEqual(await chosen, '/root/Storage/Games'); + window.__droiddeckFiles.done(2, null); + await assert.rejects(cancelled, e => e.result === 2); + delete window.droiddeckOpenFileDialog; + await assert.rejects(SteamClient.System.OpenFileDialog({}), e => e.result === 2); +})().catch(e => { console.error(e); process.exit(1); }); +""" + + +def bound(payload): + return {"method": "Runtime.bindingCalled", "params": {"name": LIVE["FILE_BINDING"], "payload": payload}} + + +class FileDialogScriptTest(unittest.TestCase): + def test_every_window_asks_the_bridge_and_gets_the_path_or_a_cancel(self): + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "files.js" + script.write_text(LIVE["FILE_JS"]) + harness = Path(tmp) / "harness.js" + harness.write_text(HARNESS) + result = subprocess.run(["node", str(harness), str(script)], capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + +class FileDialogBridgeTest(unittest.TestCase): + def setUp(self): + self.globals = LIVE["Helper"].attach.__globals__ + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.helper = LIVE["Helper"](str(self.root), 1) + self.helper.page = mock.Mock() + + def answer(self): + return self.helper.page.evaluate.call_args.args[0] + + def test_the_chooser_runs_under_a_steam_reaper_with_the_options_on_stdin(self): + reaper = self.root / "steamrtarm64/reaper" + reaper.parent.mkdir() + reaper.write_text("#!/bin/sh\n") + reaper.chmod(0o755) + proc = mock.Mock(stdin=io.BytesIO()) + proc.stdin.close = lambda: None + with mock.patch.object(self.globals["subprocess"], "Popen", return_value=proc) as popen: + self.helper.page.drain.return_value = [bound(json.dumps({"id": 3, "options": {"strTitle": "Pick"}}))] + self.helper.drain() + command = popen.call_args.args[0] + self.assertEqual(command[:4], [str(reaper), "SteamLaunch", "AppId=769", "--"]) + self.assertEqual(command[-1], "--choose-file") + self.assertEqual(json.loads(proc.stdin.getvalue()), {"strTitle": "Pick"}) + self.assertEqual(self.helper.chooser, (3, proc)) + + def test_a_second_request_while_one_is_open_is_cancelled(self): + first = mock.Mock() + self.helper.chooser = (1, first) + self.helper.open_chooser(json.dumps({"id": 2, "options": {}})) + self.assertEqual(self.answer(), "window.__droiddeckFiles?.done(2, null)") + self.assertEqual(self.helper.chooser, (1, first)) + + def test_the_chosen_path_or_a_cancel_goes_back_to_the_page(self): + for code, out, expected in ((0, b"/root/Storage/My Game/game.sh\n", '"/root/Storage/My Game/game.sh"'), + (1, b"", "null"), (0, b"\n", "null")): + proc = mock.Mock(stdout=io.BytesIO(out), returncode=code) + self.helper.chooser = (7, proc) + self.helper.chooser_done() + self.assertEqual(self.answer(), "window.__droiddeckFiles?.done(7, %s)" % expected) + self.assertIsNone(self.helper.chooser) + + def test_bad_requests_are_ignored(self): + with mock.patch.object(self.globals["subprocess"], "Popen") as popen: + for payload in ("not json", json.dumps({"options": {}}), json.dumps({"id": 1, "options": []})): + self.helper.open_chooser(payload) + popen.assert_not_called() + self.helper.page.evaluate.assert_not_called() + + def test_choose_file_mode_prints_the_path_and_fails_on_cancel(self): + for chosen, code, printed in (("/root/Storage/a.exe", 0, "/root/Storage/a.exe\n"), (None, 1, "")): + with mock.patch.dict(LIVE["chooser_main"].__globals__, {"choose_file": mock.Mock(return_value=chosen)}), \ + mock.patch.object(sys, "stdin", io.StringIO('{"strTitle": "Pick"}')), \ + mock.patch.object(sys, "stdout", io.StringIO()) as out: + self.assertEqual(LIVE["chooser_main"](), code) + self.assertEqual(out.getvalue(), printed) + + def test_attach_installs_the_file_dialog_bridge(self): + page = mock.Mock() + page.evaluate.return_value = True + with mock.patch.dict(self.globals, {"shared_context": lambda port: "/devtools/page/1", + "Page": mock.Mock(return_value=page)}): + self.helper.attach() + page.call.assert_any_call("Runtime.addBinding", {"name": LIVE["FILE_BINDING"]}) + page.evaluate.assert_any_call(LIVE["FILE_JS"]) + + +if __name__ == "__main__": + unittest.main()