fix: the on-screen pad steps aside while Steam's or DroidDeck's keyboard is open (#1217)

This commit is contained in:
MaxsTechReview authored and GitHub committed 2026-10-20 14:29:50 -04:00
1 parent b8b3ecf915
commit 862afdf0a9
6 files changed
+465 -9

No files matched your search

+212
View File
@@ -6,6 +6,7 @@ import importlib.util
import json
import os
from pathlib import Path
import shutil
import socket
import struct
import subprocess
@@ -77,6 +78,79 @@ class FakeDevTools(threading.Thread):
conn.close()
class FakeSteamClient(threading.Thread):
def __init__(self, showing):
super().__init__(daemon=True)
self.showing = showing
self.server = socket.socket()
self.server.bind(("127.0.0.1", 0))
self.server.listen(4)
self.port = self.server.getsockname()[1]
self.seen = []
self.conn = None
self.lock = threading.Lock()
def run(self):
while True:
try:
conn, _ = self.server.accept()
except OSError:
return
threading.Thread(target=self.serve, args=(conn,), daemon=True).start()
def send(self, conn, message):
payload = json.dumps(message).encode()
header = bytes([0x81, 126]) + struct.pack(">H", len(payload)) if len(payload) > 125 else bytes([0x81, len(payload)])
with self.lock:
conn.sendall(header + payload)
def push(self, payload, name=None):
self.send(self.conn, {"method": "Runtime.bindingCalled",
"params": {"name": name or agent.KEYBOARD_BINDING, "payload": payload}})
def drop(self):
self.conn.shutdown(socket.SHUT_RDWR)
self.conn.close()
def serve(self, conn):
stream = conn.makefile("rb")
head = b""
while not head.endswith(b"\r\n\r\n"):
head += stream.read(1)
request = head.decode()
if request.startswith("GET /json"):
body = json.dumps([{"title": "Steam Big Picture Mode", "url": "https://steamloopback.host/routes/",
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/2" % self.port},
{"title": "SharedJSContext", "url": "https://steamloopback.host/",
"webSocketDebuggerUrl": "ws://127.0.0.1:%d/devtools/page/1" % self.port}]).encode()
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body) + body)
conn.close()
return
key = [l.split(":", 1)[1].strip() for l in request.split("\r\n") if l.lower().startswith("sec-websocket-key")][0]
accept = base64.b64encode(hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()).decode()
conn.sendall(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
"Sec-WebSocket-Accept: %s\r\n\r\n" % accept).encode())
self.conn = conn
try:
while True:
b0, b1 = stream.read(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", stream.read(2))[0]
elif n == 127:
n = struct.unpack(">Q", stream.read(8))[0]
mask = stream.read(4)
data = stream.read(n)
message = json.loads(bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
self.seen.append(message)
self.push("true", name="unrelated")
result = {"result": {"type": "boolean", "value": self.showing}} if message["method"] == "Runtime.evaluate" else {}
self.send(conn, {"id": message["id"], "result": result})
except (OSError, ValueError):
conn.close()
class AgentTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
@@ -409,6 +483,144 @@ class AgentTest(unittest.TestCase):
self.assertIn(child.pid, agent.game_pids(424242))
class SteamKeyboardTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.dir = Path(self.temp.name)
def state(self):
try:
return json.loads((self.dir / "keyboard.json").read_text())
except (OSError, ValueError):
return None
def wait_for(self, check):
for _ in range(200):
if check():
return
time.sleep(0.02)
self.fail("timed out; keyboard.json is %s" % self.state())
def test_report_writes_only_changes(self):
watcher = agent.KeyboardWatcher(self.dir)
watcher.report(False)
self.assertEqual({"steamKeyboard": False, "seq": 1}, self.state())
watcher.report(False)
watcher.report(True)
self.assertEqual({"steamKeyboard": True, "seq": 2}, self.state())
def test_follows_the_clients_keyboard_and_rechecks_while_idle(self):
client = FakeSteamClient(False)
client.start()
self.addCleanup(client.server.close)
(self.dir / "cdp-port").write_text(str(client.port))
watcher = agent.KeyboardWatcher(self.dir)
failure = []
with patch.object(agent, "KEYBOARD_CHECK_SECONDS", 0.3):
thread = threading.Thread(target=lambda: failure.append(self.capture(watcher.watch)), daemon=True)
thread.start()
self.wait_for(lambda: self.state() == {"steamKeyboard": False, "seq": 1})
self.assertEqual("Runtime.addBinding", client.seen[0]["method"])
self.assertEqual(agent.KEYBOARD_BINDING, client.seen[0]["params"]["name"])
self.assertIn("IsShowingVirtualKeyboard", client.seen[1]["params"]["expression"])
self.assertIn(agent.KEYBOARD_BINDING, client.seen[1]["params"]["expression"])
self.assertNotIn("__BINDING__", client.seen[1]["params"]["expression"])
client.push("true")
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
client.push("false")
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is False)
client.showing = True
self.wait_for(lambda: (self.state() or {}).get("steamKeyboard") is True)
self.assertGreaterEqual(len([m for m in client.seen if m["method"] == "Runtime.evaluate"]), 2)
client.drop()
thread.join(5)
self.assertFalse(thread.is_alive())
self.assertIsInstance(failure[0], OSError)
def test_any_loss_reports_the_keyboard_closed_and_backs_off(self):
watcher = agent.KeyboardWatcher(self.dir)
watcher.report(True)
def attached_then_lost():
watcher.attached = True
raise OSError("DevTools closed the connection")
steps = iter([agent.AgentError("NO_DEVTOOLS", "no port"), agent.AgentError("NO_DEVTOOLS", "no port"),
attached_then_lost, KeyError("SteamUIStore")])
def watch():
step = next(steps, None)
if step is None:
raise SystemExit
if callable(step):
step()
raise step
with patch.object(watcher, "watch", side_effect=watch), patch.object(agent.time, "sleep") as sleep, \
patch.object(agent, "log") as log:
with self.assertRaises(SystemExit):
watcher.run()
self.assertEqual({"steamKeyboard": False, "seq": 2}, self.state())
self.assertEqual([2, 4, 2, 4], [c.args[0] for c in sleep.call_args_list])
self.assertFalse(watcher.attached)
self.assertEqual(3, log.call_count)
def test_no_shared_context_yet(self):
with patch.object(agent, "cdp_targets", return_value=[{"title": "Steam", "webSocketDebuggerUrl": "ws://127.0.0.1:1/x"}]):
with self.assertRaises(agent.AgentError) as caught:
agent.KeyboardWatcher(self.dir).watch()
self.assertEqual("NO_TARGET", caught.exception.code)
@unittest.skipUnless(shutil.which("node"), "node is not installed")
def test_page_script_watches_every_window_with_or_without_subscribe(self):
harness = r"""
const assert = require('assert');
globalThis.window = globalThis;
const script = require('fs').readFileSync(process.argv[2], 'utf8');
const calls = [];
window.droiddeckSteamKeyboard = value => calls.push(value);
const tick = ms => new Promise(done => setTimeout(done, ms));
const value = subscribe => {
const v = { m_currentValue: false, m_callbacks: [] };
if (subscribe) v.Subscribe = cb => { v.m_callbacks.push(cb); return { Unsubscribe() {} }; };
v.set = next => { v.m_currentValue = next; for (const cb of v.m_callbacks) cb(next); };
return v;
};
(async () => {
assert.strictEqual(eval(script), false);
assert.deepStrictEqual(calls, ['false']);
const main = value(true), overlay = value(false);
globalThis.SteamUIStore = { WindowStore: { SteamUIWindows: [{ VirtualKeyboardManager: { IsShowingVirtualKeyboard: main } }] } };
assert.strictEqual(eval(script), false);
main.set(true);
await tick(20);
assert.deepStrictEqual(calls, ['false', 'true']);
main.set(false);
await tick(20);
SteamUIStore.WindowStore.SteamUIWindows.push({}, { VirtualKeyboardManager: { IsShowingVirtualKeyboard: overlay } });
overlay.m_currentValue = true;
await tick(700);
assert.deepStrictEqual(calls, ['false', 'true', 'false', 'true']);
assert.strictEqual(eval(script), true);
assert.strictEqual(main.m_callbacks.length, 1);
clearInterval(window.__droiddeckKeyboard.timer);
})().catch(e => { console.error(e); process.exit(1); });
"""
(self.dir / "keyboard.js").write_text(agent.KEYBOARD_JS)
(self.dir / "harness.js").write_text(harness)
result = subprocess.run(["node", str(self.dir / "harness.js"), str(self.dir / "keyboard.js")],
capture_output=True, text=True, timeout=20)
self.assertEqual(0, result.returncode, result.stderr)
@staticmethod
def capture(call):
try:
call()
except Exception as e:
return e
class VrFocusTest(unittest.TestCase):
def test_scene_dashboard_and_external_focus_changes(self):
focus = agent.VrSceneFocus()