mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-20 05:16:01 +00:00
fix: the on-screen pad steps aside while Steam's or DroidDeck's keyboard is open (#1217)
This commit is contained in:
1 parent
b8b3ecf915
commit
862afdf0a9
6 files changed
+465
-9
No files matched your search
@@ -23,6 +23,7 @@ import json
|
||||
import math
|
||||
import re
|
||||
import os
|
||||
import select
|
||||
import selectors
|
||||
import signal
|
||||
import socket
|
||||
@@ -532,6 +533,111 @@ def cdp_eval(agent_dir, expression, target=None, await_promise=True, timeout=15)
|
||||
ws.close()
|
||||
|
||||
|
||||
KEYBOARD_BINDING = "droiddeckSteamKeyboard"
|
||||
KEYBOARD_CHECK_SECONDS = 30
|
||||
KEYBOARD_JS = r"""
|
||||
(() => {
|
||||
const self = window.__droiddeckKeyboard || (window.__droiddeckKeyboard = { watched: new WeakSet(), last: null });
|
||||
self.check = () => {
|
||||
let showing = false;
|
||||
for (const win of Array.from(globalThis.SteamUIStore?.WindowStore?.SteamUIWindows || [])) {
|
||||
const value = win?.VirtualKeyboardManager?.IsShowingVirtualKeyboard;
|
||||
if (!value) continue;
|
||||
if (!self.watched.has(value) && typeof value.Subscribe === "function") {
|
||||
self.watched.add(value);
|
||||
value.Subscribe(() => setTimeout(() => self.check()));
|
||||
}
|
||||
if (value.m_currentValue === true) showing = true;
|
||||
}
|
||||
if (showing !== self.last) {
|
||||
self.last = showing;
|
||||
if (typeof window.__BINDING__ === "function") window.__BINDING__(String(showing));
|
||||
}
|
||||
return showing;
|
||||
};
|
||||
self.timer = self.timer || setInterval(() => self.check(), 500);
|
||||
return self.check();
|
||||
})()
|
||||
""".replace("__BINDING__", KEYBOARD_BINDING)
|
||||
|
||||
|
||||
class KeyboardWatcher(threading.Thread):
|
||||
def __init__(self, agent_dir):
|
||||
super().__init__(daemon=True, name="keyboard")
|
||||
self.agent_dir = agent_dir
|
||||
self.path = agent_dir / "keyboard.json"
|
||||
self.showing = None
|
||||
self.seq = 0
|
||||
self.next_id = 0
|
||||
self.attached = False
|
||||
|
||||
def report(self, showing):
|
||||
if showing == self.showing:
|
||||
return
|
||||
self.showing = showing
|
||||
self.seq += 1
|
||||
write_json(self.path, {"steamKeyboard": showing, "seq": self.seq})
|
||||
|
||||
def run(self):
|
||||
failures, last = 0, None
|
||||
while True:
|
||||
try:
|
||||
self.watch()
|
||||
except Exception as e:
|
||||
if self.attached:
|
||||
log("Steam keyboard: lost the client (%s); reattaching" % e)
|
||||
failures = 0
|
||||
elif str(e) != last:
|
||||
log("Steam keyboard: waiting for the client (%s)" % e)
|
||||
self.attached = False
|
||||
last = str(e)
|
||||
self.report(False)
|
||||
failures += 1
|
||||
time.sleep(min(2 * failures, 10))
|
||||
|
||||
def watch(self):
|
||||
targets = [t for t in cdp_targets(self.agent_dir)
|
||||
if t.get("title") == "SharedJSContext" and t.get("webSocketDebuggerUrl")]
|
||||
if not targets:
|
||||
raise AgentError("NO_TARGET", "No SharedJSContext yet")
|
||||
ws = WebSocket(targets[0]["webSocketDebuggerUrl"], 5)
|
||||
try:
|
||||
self.call(ws, "Runtime.addBinding", {"name": KEYBOARD_BINDING})
|
||||
self.report(self.install(ws))
|
||||
self.attached = True
|
||||
while True:
|
||||
if not ws.buffer and not select.select([ws.s], [], [], KEYBOARD_CHECK_SECONDS)[0]:
|
||||
self.report(self.install(ws))
|
||||
continue
|
||||
self.event(json.loads(ws.recv()))
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
def install(self, ws):
|
||||
result = self.call(ws, "Runtime.evaluate", {"expression": KEYBOARD_JS, "returnByValue": True})
|
||||
if result.get("exceptionDetails"):
|
||||
raise AgentError("JS_EXCEPTION", json.dumps(result["exceptionDetails"])[:500])
|
||||
return result.get("result", {}).get("value") is True
|
||||
|
||||
def call(self, ws, method, params):
|
||||
self.next_id += 1
|
||||
ws.send(json.dumps({"id": self.next_id, "method": method, "params": params}))
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
message = json.loads(ws.recv())
|
||||
if message.get("id") == self.next_id:
|
||||
if "error" in message:
|
||||
raise AgentError("CDP_ERROR", json.dumps(message["error"]))
|
||||
return message.get("result", {})
|
||||
self.event(message)
|
||||
raise AgentError("TIMEOUT", "No answer to %s from DevTools" % method)
|
||||
|
||||
def event(self, message):
|
||||
params = message.get("params") or {}
|
||||
if message.get("method") == "Runtime.bindingCalled" and params.get("name") == KEYBOARD_BINDING:
|
||||
self.report(params.get("payload") == "true")
|
||||
|
||||
|
||||
def steam_idle_inhibit(agent_dir, inhibit):
|
||||
expression = """(async want => {
|
||||
const toggle = () => new Promise(done => {
|
||||
@@ -918,6 +1024,8 @@ def serve(agent_dir, focus=True, poll=0.1):
|
||||
can_focus = focus and bool(os.environ.get("DISPLAY"))
|
||||
if can_focus:
|
||||
FocusWatcher(agent_dir / "focus.json").start()
|
||||
if focus:
|
||||
KeyboardWatcher(agent_dir).start()
|
||||
write_json(agent_dir / "hello.json", {"pid": os.getpid(), "version": VERSION, "started": int(time.time() * 1000),
|
||||
"displays": displays(), "focus": can_focus})
|
||||
log("serving %s (focus %s)" % (agent_dir, "on" if can_focus else "off"))
|
||||
|
||||
@@ -6,6 +6,7 @@ import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -77,6 +78,79 @@ class FakeDevTools(threading.Thread):
|
||||
conn.close()
|
||||
|
||||
|
||||
class FakeSteamClient(threading.Thread):
|
||||
|
||||
def __init__(self, showing):
|
||||
super().__init__(daemon=True)
|
||||
self.showing = showing
|
||||
self.server = socket.socket()
|
||||
self.server.bind(("127.0.0.1", 0))
|
||||
self.server.listen(4)
|
||||
self.port = self.server.getsockname()[1]
|
||||
self.seen = []
|
||||
self.conn = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
conn, _ = self.server.accept()
|
||||
except OSError:
|
||||
return
|
||||
threading.Thread(target=self.serve, args=(conn,), daemon=True).start()
|
||||
|
||||
def send(self, conn, message):
|
||||
payload = json.dumps(message).encode()
|
||||
header = bytes([0x81, 126]) + struct.pack(">H", len(payload)) if len(payload) > 125 else bytes([0x81, len(payload)])
|
||||
with self.lock:
|
||||
conn.sendall(header + payload)
|
||||
|
||||
def push(self, payload, name=None):
|
||||
self.send(self.conn, {"method": "Runtime.bindingCalled",
|
||||
"params": {"name": name or agent.KEYBOARD_BINDING, "payload": payload}})
|
||||
|
||||
def drop(self):
|
||||
self.conn.shutdown(socket.SHUT_RDWR)
|
||||
self.conn.close()
|
||||
|
||||
def serve(self, conn):
|
||||
stream = conn.makefile("rb")
|
||||
head = b""
|
||||
while not head.endswith(b"\r\n\r\n"):
|
||||
head += stream.read(1)
|
||||
request = head.decode()
|
||||
if request.startswith("GET /json"):
|
||||
body = json.dumps([{"title": "Steam Big Picture Mode", "url": "https://steamloopback.host/routes/",
|
||||
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/2" % self.port},
|
||||
{"title": "SharedJSContext", "url": "https://steamloopback.host/",
|
||||
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/1" % self.port}]).encode()
|
||||
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body) + body)
|
||||
conn.close()
|
||||
return
|
||||
key = [l.split(":", 1)[1].strip() for l in request.split("\r\n") if l.lower().startswith("sec-websocket-key")][0]
|
||||
accept = base64.b64encode(hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()).decode()
|
||||
conn.sendall(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
|
||||
"Sec-WebSocket-Accept: %s\r\n\r\n" % accept).encode())
|
||||
self.conn = conn
|
||||
try:
|
||||
while True:
|
||||
b0, b1 = stream.read(2)
|
||||
n = b1 & 0x7F
|
||||
if n == 126:
|
||||
n = struct.unpack(">H", stream.read(2))[0]
|
||||
elif n == 127:
|
||||
n = struct.unpack(">Q", stream.read(8))[0]
|
||||
mask = stream.read(4)
|
||||
data = stream.read(n)
|
||||
message = json.loads(bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
|
||||
self.seen.append(message)
|
||||
self.push("true", name="unrelated")
|
||||
result = {"result": {"type": "boolean", "value": self.showing}} if message["method"] == "Runtime.evaluate" else {}
|
||||
self.send(conn, {"id": message["id"], "result": result})
|
||||
except (OSError, ValueError):
|
||||
conn.close()
|
||||
|
||||
|
||||
class AgentTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
@@ -409,6 +483,144 @@ class AgentTest(unittest.TestCase):
|
||||
self.assertIn(child.pid, agent.game_pids(424242))
|
||||
|
||||
|
||||
class SteamKeyboardTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.dir = Path(self.temp.name)
|
||||
|
||||
def state(self):
|
||||
try:
|
||||
return json.loads((self.dir / "keyboard.json").read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def wait_for(self, check):
|
||||
for _ in range(200):
|
||||
if check():
|
||||
return
|
||||
time.sleep(0.02)
|
||||
self.fail("timed out; keyboard.json is %s" % self.state())
|
||||
|
||||
def test_report_writes_only_changes(self):
|
||||
watcher = agent.KeyboardWatcher(self.dir)
|
||||
watcher.report(False)
|
||||
self.assertEqual({"steamKeyboard": False, "seq": 1}, self.state())
|
||||
watcher.report(False)
|
||||
watcher.report(True)
|
||||
self.assertEqual({"steamKeyboard": True, "seq": 2}, self.state())
|
||||
|
||||
def test_follows_the_clients_keyboard_and_rechecks_while_idle(self):
|
||||
client = FakeSteamClient(False)
|
||||
client.start()
|
||||
self.addCleanup(client.server.close)
|
||||
(self.dir / "cdp-port").write_text(str(client.port))
|
||||
watcher = agent.KeyboardWatcher(self.dir)
|
||||
failure = []
|
||||
with patch.object(agent, "KEYBOARD_CHECK_SECONDS", 0.3):
|
||||
thread = threading.Thread(target=lambda: failure.append(self.capture(watcher.watch)), daemon=True)
|
||||
thread.start()
|
||||
self.wait_for(lambda: self.state() == {"steamKeyboard": False, "seq": 1})
|
||||
self.assertEqual("Runtime.addBinding", client.seen[0]["method"])
|
||||
self.assertEqual(agent.KEYBOARD_BINDING, client.seen[0]["params"]["name"])
|
||||
self.assertIn("IsShowingVirtualKeyboard", client.seen[1]["params"]["expression"])
|
||||
self.assertIn(agent.KEYBOARD_BINDING, client.seen[1]["params"]["expression"])
|
||||
self.assertNotIn("__BINDING__", client.seen[1]["params"]["expression"])
|
||||
client.push("true")
|
||||
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
|
||||
client.push("false")
|
||||
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is False)
|
||||
client.showing = True
|
||||
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
|
||||
self.assertGreaterEqual(len([m for m in client.seen if m["method"] == "Runtime.evaluate"]), 2)
|
||||
client.drop()
|
||||
thread.join(5)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertIsInstance(failure[0], OSError)
|
||||
|
||||
def test_any_loss_reports_the_keyboard_closed_and_backs_off(self):
|
||||
watcher = agent.KeyboardWatcher(self.dir)
|
||||
watcher.report(True)
|
||||
|
||||
def attached_then_lost():
|
||||
watcher.attached = True
|
||||
raise OSError("DevTools closed the connection")
|
||||
|
||||
steps = iter([agent.AgentError("NO_DEVTOOLS", "no port"), agent.AgentError("NO_DEVTOOLS", "no port"),
|
||||
attached_then_lost, KeyError("SteamUIStore")])
|
||||
|
||||
def watch():
|
||||
step = next(steps, None)
|
||||
if step is None:
|
||||
raise SystemExit
|
||||
if callable(step):
|
||||
step()
|
||||
raise step
|
||||
|
||||
with patch.object(watcher, "watch", side_effect=watch), patch.object(agent.time, "sleep") as sleep, \
|
||||
patch.object(agent, "log") as log:
|
||||
with self.assertRaises(SystemExit):
|
||||
watcher.run()
|
||||
self.assertEqual({"steamKeyboard": False, "seq": 2}, self.state())
|
||||
self.assertEqual([2, 4, 2, 4], [c.args[0] for c in sleep.call_args_list])
|
||||
self.assertFalse(watcher.attached)
|
||||
self.assertEqual(3, log.call_count)
|
||||
|
||||
def test_no_shared_context_yet(self):
|
||||
with patch.object(agent, "cdp_targets", return_value=[{"title": "Steam", "webSocketDebuggerUrl": "ws://127.0.0.1:1/x"}]):
|
||||
with self.assertRaises(agent.AgentError) as caught:
|
||||
agent.KeyboardWatcher(self.dir).watch()
|
||||
self.assertEqual("NO_TARGET", caught.exception.code)
|
||||
|
||||
@unittest.skipUnless(shutil.which("node"), "node is not installed")
|
||||
def test_page_script_watches_every_window_with_or_without_subscribe(self):
|
||||
harness = r"""
|
||||
const assert = require('assert');
|
||||
globalThis.window = globalThis;
|
||||
const script = require('fs').readFileSync(process.argv[2], 'utf8');
|
||||
const calls = [];
|
||||
window.droiddeckSteamKeyboard = value => calls.push(value);
|
||||
const tick = ms => new Promise(done => setTimeout(done, ms));
|
||||
const value = subscribe => {
|
||||
const v = { m_currentValue: false, m_callbacks: [] };
|
||||
if (subscribe) v.Subscribe = cb => { v.m_callbacks.push(cb); return { Unsubscribe() {} }; };
|
||||
v.set = next => { v.m_currentValue = next; for (const cb of v.m_callbacks) cb(next); };
|
||||
return v;
|
||||
};
|
||||
(async () => {
|
||||
assert.strictEqual(eval(script), false);
|
||||
assert.deepStrictEqual(calls, ['false']);
|
||||
const main = value(true), overlay = value(false);
|
||||
globalThis.SteamUIStore = { WindowStore: { SteamUIWindows: [{ VirtualKeyboardManager: { IsShowingVirtualKeyboard: main } }] } };
|
||||
assert.strictEqual(eval(script), false);
|
||||
main.set(true);
|
||||
await tick(20);
|
||||
assert.deepStrictEqual(calls, ['false', 'true']);
|
||||
main.set(false);
|
||||
await tick(20);
|
||||
SteamUIStore.WindowStore.SteamUIWindows.push({}, { VirtualKeyboardManager: { IsShowingVirtualKeyboard: overlay } });
|
||||
overlay.m_currentValue = true;
|
||||
await tick(700);
|
||||
assert.deepStrictEqual(calls, ['false', 'true', 'false', 'true']);
|
||||
assert.strictEqual(eval(script), true);
|
||||
assert.strictEqual(main.m_callbacks.length, 1);
|
||||
clearInterval(window.__droiddeckKeyboard.timer);
|
||||
})().catch(e => { console.error(e); process.exit(1); });
|
||||
"""
|
||||
(self.dir / "keyboard.js").write_text(agent.KEYBOARD_JS)
|
||||
(self.dir / "harness.js").write_text(harness)
|
||||
result = subprocess.run(["node", str(self.dir / "harness.js"), str(self.dir / "keyboard.js")],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
|
||||
@staticmethod
|
||||
def capture(call):
|
||||
try:
|
||||
call()
|
||||
except Exception as e:
|
||||
return e
|
||||
|
||||
|
||||
class VrFocusTest(unittest.TestCase):
|
||||
def test_scene_dashboard_and_external_focus_changes(self):
|
||||
focus = agent.VrSceneFocus()
|
||||
|
||||
Reference in new issue
Block a user