fix: the on-screen pad steps aside while Steam's or DroidDeck's keyboard is open (#1217)

This commit is contained in:
MaxsTechReview authored and GitHub committed 2026-10-20 14:29:50 -04:00
1 parent b8b3ecf915
commit 862afdf0a9
6 files changed
+465 -9

No files matched your search

@@ -23,6 +23,7 @@ import json
import math
import re
import os
import select
import selectors
import signal
import socket
@@ -532,6 +533,111 @@ def cdp_eval(agent_dir, expression, target=None, await_promise=True, timeout=15)
ws.close()
KEYBOARD_BINDING = "droiddeckSteamKeyboard"
KEYBOARD_CHECK_SECONDS = 30
KEYBOARD_JS = r"""
(() => {
const self = window.__droiddeckKeyboard || (window.__droiddeckKeyboard = { watched: new WeakSet(), last: null });
self.check = () => {
let showing = false;
for (const win of Array.from(globalThis.SteamUIStore?.WindowStore?.SteamUIWindows || [])) {
const value = win?.VirtualKeyboardManager?.IsShowingVirtualKeyboard;
if (!value) continue;
if (!self.watched.has(value) && typeof value.Subscribe === "function") {
self.watched.add(value);
value.Subscribe(() => setTimeout(() => self.check()));
}
if (value.m_currentValue === true) showing = true;
}
if (showing !== self.last) {
self.last = showing;
if (typeof window.__BINDING__ === "function") window.__BINDING__(String(showing));
}
return showing;
};
self.timer = self.timer || setInterval(() => self.check(), 500);
return self.check();
})()
""".replace("__BINDING__", KEYBOARD_BINDING)
class KeyboardWatcher(threading.Thread):
def __init__(self, agent_dir):
super().__init__(daemon=True, name="keyboard")
self.agent_dir = agent_dir
self.path = agent_dir / "keyboard.json"
self.showing = None
self.seq = 0
self.next_id = 0
self.attached = False
def report(self, showing):
if showing == self.showing:
return
self.showing = showing
self.seq += 1
write_json(self.path, {"steamKeyboard": showing, "seq": self.seq})
def run(self):
failures, last = 0, None
while True:
try:
self.watch()
except Exception as e:
if self.attached:
log("Steam keyboard: lost the client (%s); reattaching" % e)
failures = 0
elif str(e) != last:
log("Steam keyboard: waiting for the client (%s)" % e)
self.attached = False
last = str(e)
self.report(False)
failures += 1
time.sleep(min(2 * failures, 10))
def watch(self):
targets = [t for t in cdp_targets(self.agent_dir)
if t.get("title") == "SharedJSContext" and t.get("webSocketDebuggerUrl")]
if not targets:
raise AgentError("NO_TARGET", "No SharedJSContext yet")
ws = WebSocket(targets[0]["webSocketDebuggerUrl"], 5)
try:
self.call(ws, "Runtime.addBinding", {"name": KEYBOARD_BINDING})
self.report(self.install(ws))
self.attached = True
while True:
if not ws.buffer and not select.select([ws.s], [], [], KEYBOARD_CHECK_SECONDS)[0]:
self.report(self.install(ws))
continue
self.event(json.loads(ws.recv()))
finally:
ws.close()
def install(self, ws):
result = self.call(ws, "Runtime.evaluate", {"expression": KEYBOARD_JS, "returnByValue": True})
if result.get("exceptionDetails"):
raise AgentError("JS_EXCEPTION", json.dumps(result["exceptionDetails"])[:500])
return result.get("result", {}).get("value") is True
def call(self, ws, method, params):
self.next_id += 1
ws.send(json.dumps({"id": self.next_id, "method": method, "params": params}))
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
message = json.loads(ws.recv())
if message.get("id") == self.next_id:
if "error" in message:
raise AgentError("CDP_ERROR", json.dumps(message["error"]))
return message.get("result", {})
self.event(message)
raise AgentError("TIMEOUT", "No answer to %s from DevTools" % method)
def event(self, message):
params = message.get("params") or {}
if message.get("method") == "Runtime.bindingCalled" and params.get("name") == KEYBOARD_BINDING:
self.report(params.get("payload") == "true")
def steam_idle_inhibit(agent_dir, inhibit):
expression = """(async want => {
const toggle = () => new Promise(done => {
@@ -918,6 +1024,8 @@ def serve(agent_dir, focus=True, poll=0.1):
can_focus = focus and bool(os.environ.get("DISPLAY"))
if can_focus:
FocusWatcher(agent_dir / "focus.json").start()
if focus:
KeyboardWatcher(agent_dir).start()
write_json(agent_dir / "hello.json", {"pid": os.getpid(), "version": VERSION, "started": int(time.time() * 1000),
"displays": displays(), "focus": can_focus})
log("serving %s (focus %s)" % (agent_dir, "on" if can_focus else "off"))
+212
View File
@@ -6,6 +6,7 @@ import importlib.util
import json
import os
from pathlib import Path
import shutil
import socket
import struct
import subprocess
@@ -77,6 +78,79 @@ class FakeDevTools(threading.Thread):
conn.close()
class FakeSteamClient(threading.Thread):
def __init__(self, showing):
super().__init__(daemon=True)
self.showing = showing
self.server = socket.socket()
self.server.bind(("127.0.0.1", 0))
self.server.listen(4)
self.port = self.server.getsockname()[1]
self.seen = []
self.conn = None
self.lock = threading.Lock()
def run(self):
while True:
try:
conn, _ = self.server.accept()
except OSError:
return
threading.Thread(target=self.serve, args=(conn,), daemon=True).start()
def send(self, conn, message):
payload = json.dumps(message).encode()
header = bytes([0x81, 126]) + struct.pack(">H", len(payload)) if len(payload) > 125 else bytes([0x81, len(payload)])
with self.lock:
conn.sendall(header + payload)
def push(self, payload, name=None):
self.send(self.conn, {"method": "Runtime.bindingCalled",
"params": {"name": name or agent.KEYBOARD_BINDING, "payload": payload}})
def drop(self):
self.conn.shutdown(socket.SHUT_RDWR)
self.conn.close()
def serve(self, conn):
stream = conn.makefile("rb")
head = b""
while not head.endswith(b"\r\n\r\n"):
head += stream.read(1)
request = head.decode()
if request.startswith("GET /json"):
body = json.dumps([{"title": "Steam Big Picture Mode", "url": "https://steamloopback.host/routes/",
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/2" % self.port},
{"title": "SharedJSContext", "url": "https://steamloopback.host/",
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/1" % self.port}]).encode()
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body) + body)
conn.close()
return
key = [l.split(":", 1)[1].strip() for l in request.split("\r\n") if l.lower().startswith("sec-websocket-key")][0]
accept = base64.b64encode(hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()).decode()
conn.sendall(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
"Sec-WebSocket-Accept: %s\r\n\r\n" % accept).encode())
self.conn = conn
try:
while True:
b0, b1 = stream.read(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", stream.read(2))[0]
elif n == 127:
n = struct.unpack(">Q", stream.read(8))[0]
mask = stream.read(4)
data = stream.read(n)
message = json.loads(bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
self.seen.append(message)
self.push("true", name="unrelated")
result = {"result": {"type": "boolean", "value": self.showing}} if message["method"] == "Runtime.evaluate" else {}
self.send(conn, {"id": message["id"], "result": result})
except (OSError, ValueError):
conn.close()
class AgentTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
@@ -409,6 +483,144 @@ class AgentTest(unittest.TestCase):
self.assertIn(child.pid, agent.game_pids(424242))
class SteamKeyboardTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.dir = Path(self.temp.name)
def state(self):
try:
return json.loads((self.dir / "keyboard.json").read_text())
except (OSError, ValueError):
return None
def wait_for(self, check):
for _ in range(200):
if check():
return
time.sleep(0.02)
self.fail("timed out; keyboard.json is %s" % self.state())
def test_report_writes_only_changes(self):
watcher = agent.KeyboardWatcher(self.dir)
watcher.report(False)
self.assertEqual({"steamKeyboard": False, "seq": 1}, self.state())
watcher.report(False)
watcher.report(True)
self.assertEqual({"steamKeyboard": True, "seq": 2}, self.state())
def test_follows_the_clients_keyboard_and_rechecks_while_idle(self):
client = FakeSteamClient(False)
client.start()
self.addCleanup(client.server.close)
(self.dir / "cdp-port").write_text(str(client.port))
watcher = agent.KeyboardWatcher(self.dir)
failure = []
with patch.object(agent, "KEYBOARD_CHECK_SECONDS", 0.3):
thread = threading.Thread(target=lambda: failure.append(self.capture(watcher.watch)), daemon=True)
thread.start()
self.wait_for(lambda: self.state() == {"steamKeyboard": False, "seq": 1})
self.assertEqual("Runtime.addBinding", client.seen[0]["method"])
self.assertEqual(agent.KEYBOARD_BINDING, client.seen[0]["params"]["name"])
self.assertIn("IsShowingVirtualKeyboard", client.seen[1]["params"]["expression"])
self.assertIn(agent.KEYBOARD_BINDING, client.seen[1]["params"]["expression"])
self.assertNotIn("__BINDING__", client.seen[1]["params"]["expression"])
client.push("true")
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
client.push("false")
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is False)
client.showing = True
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
self.assertGreaterEqual(len([m for m in client.seen if m["method"] == "Runtime.evaluate"]), 2)
client.drop()
thread.join(5)
self.assertFalse(thread.is_alive())
self.assertIsInstance(failure[0], OSError)
def test_any_loss_reports_the_keyboard_closed_and_backs_off(self):
watcher = agent.KeyboardWatcher(self.dir)
watcher.report(True)
def attached_then_lost():
watcher.attached = True
raise OSError("DevTools closed the connection")
steps = iter([agent.AgentError("NO_DEVTOOLS", "no port"), agent.AgentError("NO_DEVTOOLS", "no port"),
attached_then_lost, KeyError("SteamUIStore")])
def watch():
step = next(steps, None)
if step is None:
raise SystemExit
if callable(step):
step()
raise step
with patch.object(watcher, "watch", side_effect=watch), patch.object(agent.time, "sleep") as sleep, \
patch.object(agent, "log") as log:
with self.assertRaises(SystemExit):
watcher.run()
self.assertEqual({"steamKeyboard": False, "seq": 2}, self.state())
self.assertEqual([2, 4, 2, 4], [c.args[0] for c in sleep.call_args_list])
self.assertFalse(watcher.attached)
self.assertEqual(3, log.call_count)
def test_no_shared_context_yet(self):
with patch.object(agent, "cdp_targets", return_value=[{"title": "Steam", "webSocketDebuggerUrl": "ws://127.0.0.1:1/x"}]):
with self.assertRaises(agent.AgentError) as caught:
agent.KeyboardWatcher(self.dir).watch()
self.assertEqual("NO_TARGET", caught.exception.code)
@unittest.skipUnless(shutil.which("node"), "node is not installed")
def test_page_script_watches_every_window_with_or_without_subscribe(self):
harness = r"""
const assert = require('assert');
globalThis.window = globalThis;
const script = require('fs').readFileSync(process.argv[2], 'utf8');
const calls = [];
window.droiddeckSteamKeyboard = value => calls.push(value);
const tick = ms => new Promise(done => setTimeout(done, ms));
const value = subscribe => {
const v = { m_currentValue: false, m_callbacks: [] };
if (subscribe) v.Subscribe = cb => { v.m_callbacks.push(cb); return { Unsubscribe() {} }; };
v.set = next => { v.m_currentValue = next; for (const cb of v.m_callbacks) cb(next); };
return v;
};
(async () => {
assert.strictEqual(eval(script), false);
assert.deepStrictEqual(calls, ['false']);
const main = value(true), overlay = value(false);
globalThis.SteamUIStore = { WindowStore: { SteamUIWindows: [{ VirtualKeyboardManager: { IsShowingVirtualKeyboard: main } }] } };
assert.strictEqual(eval(script), false);
main.set(true);
await tick(20);
assert.deepStrictEqual(calls, ['false', 'true']);
main.set(false);
await tick(20);
SteamUIStore.WindowStore.SteamUIWindows.push({}, { VirtualKeyboardManager: { IsShowingVirtualKeyboard: overlay } });
overlay.m_currentValue = true;
await tick(700);
assert.deepStrictEqual(calls, ['false', 'true', 'false', 'true']);
assert.strictEqual(eval(script), true);
assert.strictEqual(main.m_callbacks.length, 1);
clearInterval(window.__droiddeckKeyboard.timer);
})().catch(e => { console.error(e); process.exit(1); });
"""
(self.dir / "keyboard.js").write_text(agent.KEYBOARD_JS)
(self.dir / "harness.js").write_text(harness)
result = subprocess.run(["node", str(self.dir / "harness.js"), str(self.dir / "keyboard.js")],
capture_output=True, text=True, timeout=20)
self.assertEqual(0, result.returncode, result.stderr)
@staticmethod
def capture(call):
try:
call()
except Exception as e:
return e
class VrFocusTest(unittest.TestCase):
def test_scene_dashboard_and_external_focus_changes(self):
focus = agent.VrSceneFocus()