From de70e37f30b732c4c4f6bed7aaf685d11c96b912 Mon Sep 17 00:00:00 2001 From: alien2003 Date: Sun, 15 Nov 2026 15:37:56 +0100 Subject: [PATCH] feat(ctl): input pen draws with a virtual pen through Android's input reader (#2639) droiddeckctl input pen hover, tap, stroke and button register a virtual direct pen with `adb shell uinput -` and play the gesture through it, so Android's input reader makes the same stylus events a built-in pen does, with pressure, tilt, distance, the eraser and both side buttons. The app is not involved, which lets pen handling be tested without a pen. Positions are fractions of the screen as shown, placed inside the physical frame of the internal display viewport in dumpsys input: the frame is smaller than the panel when `wm size` sets another aspect ratio, and Android drops pen events outside it. The rotation and size come from the same viewport. The values are checked before anything runs on the device. The script waits 1 s after registering, since uinput reads it in one go and cannot wait for the device, and does not reset uinput's time base after that, which would drop the first samples. Contact sends a pressure of at least 1, because Android takes a pen at pressure 0 for a hovering one, and a lift first moves to where it lifts, because Android's UP repeats the last contact. A pen held still moves by one unit without leaving the physical frame so that it keeps reporting, since the kernel drops frames that change nothing. Co-authored-by: alien2003 <11610652+alien2003@users.noreply.github.com> --- docs/agent-control.md | 18 ++ tools/droiddeckctl | 232 +++++++++++++++++++++++- tools/tests/test_droiddeckctl.py | 13 ++ tools/tests/test_droiddeckctl_pen.py | 255 +++++++++++++++++++++++++++ 4 files changed, 515 insertions(+), 3 deletions(-) create mode 100644 tools/tests/test_droiddeckctl_pen.py diff --git a/docs/agent-control.md b/docs/agent-control.md index 4ac8706706..dc9ced7e60 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -122,6 +122,10 @@ tools/droiddeckctl input swipe 100 600 100 100 --duration 400 tools/droiddeckctl input pad a tools/droiddeckctl input stick left 0 -1 --hold 500 tools/droiddeckctl input trigger right 1.0 +tools/droiddeckctl input pen tap 0.5 0.5 --pressure 0.7 +tools/droiddeckctl input pen stroke 0.2 0.5 0.8 0.5 --pressure 0.1:0.9 --tilt-x=-30:30 +tools/droiddeckctl input pen hover 0.3 0.5 --to 0.7 0.5 --duration 600 +tools/droiddeckctl input pen button primary 0.5 0.5 tools/droiddeckctl ui dump tools/droiddeckctl ui click rail-steam @@ -196,6 +200,20 @@ virtual pad, the same one the on-screen controls drive, so games that ignore tou `launch`, `quit` and `cdp` need a Steam session; `focus` and `wait focused` need gamescope (every mode except the desktop). +`input pen` does not go through the app. It registers a virtual pen with `adb shell uinput -`, so +Android's own input reader makes the stylus events, as for a built-in pen, and it works on whatever +is on screen, without a session or Setup > Developer > Debugging tools. Positions are fractions of +the screen as shown; `dumpsys input` gives the frame, rotation and size of the display Android maps +internal touchscreens to. `hover` comes near and glides to `--to`; `tap`, `stroke` and `button` +hover for `--hover-ms` (80) before and after the touch or the button press. Each takes +`--tool pen|eraser`, `--tilt-x` and `--tilt-y` in degrees, `--distance` in device units (0 to 255) +and `--rate` (120 samples a second); `--pressure` and tilt take `FROM:TO` ranges, written +`--tilt-x=-30:30` when they start negative. A gesture lasts at most 30 s. `UINPUT_UNAVAILABLE` +means the shell has no `uinput` or cannot write `/dev/uinput`, `NO_VIEWPORT` that there is no +internal display or it is off, and `UINPUT_FAILED` that `uinput` stopped early; all three exit 3. +It needs Android 14 QPR2 or later: the `uinput` of Android 12 to 14 QPR1 takes only numeric codes +and the USB or Bluetooth bus, so there the pen ends with `UINPUT_FAILED`. + ## State (schema 2) `state` returns `schema`, `build`, `appVersion`, `runtime {installed, version}`, diff --git a/tools/droiddeckctl b/tools/droiddeckctl index acdfce5fdb..2ade313d41 100755 --- a/tools/droiddeckctl +++ b/tools/droiddeckctl @@ -67,10 +67,11 @@ def adb_path(configured: str | None) -> str: raise ControlError("adb not found; set ADB, ANDROID_HOME, or ANDROID_SDK_ROOT", "ADB_NOT_FOUND") -def run_adb(adb: str, serial: str, *args: str, timeout: float = 30) -> subprocess.CompletedProcess[bytes]: +def run_adb(adb: str, serial: str, *args: str, timeout: float = 30, + stdin: bytes | None = None) -> subprocess.CompletedProcess[bytes]: try: return subprocess.run( - [adb, "-s", serial, *args], capture_output=True, check=False, timeout=timeout + [adb, "-s", serial, *args], input=stdin, capture_output=True, check=False, timeout=timeout ) except (OSError, subprocess.TimeoutExpired) as exc: raise ControlError(str(exc), "ADB_FAILED") from exc @@ -598,6 +599,199 @@ def input_request(args: argparse.Namespace) -> dict: raise ControlError(f"Unknown input type {kind}", "INVALID_INPUT") +PEN_MAX_MS = 30_000 +UINPUT_SCALE = 16 +UINPUT_PRESSURE = 4095 +UINPUT_DISTANCE = 255 +UINPUT_REGISTER_MS = 1000 +UINPUT_TAIL_MS = 300 +UINPUT_SYNC = "droiddeckctl-pen-done" + + +def pen_span(value: str) -> tuple[float, float]: + try: + low, high = map(float, value.split(":")) if ":" in value else (float(value),) * 2 + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a number or FROM:TO") from None + return low, high + + +def pen_times(start: float, duration: float, rate: float) -> list[float]: + steps = max(1, round(duration * rate / 1000)) + return [start + duration * i / steps for i in range(steps + 1)] + + +def pen_lerp(span: tuple[float, float], fraction: float) -> float: + return span[0] + (span[1] - span[0]) * fraction + + +def pen_samples(args: argparse.Namespace) -> list[dict]: + op = args.pen_op + contact = op in ("tap", "stroke") + side = args.button if op == "button" else None + lead = 0 if op == "hover" else args.hover_ms + length = args.duration if op in ("hover", "stroke") else args.hold + total = 2 * lead + length + start = end = (args.x, args.y) + if op == "stroke": + end = (args.x2, args.y2) + elif op == "hover" and args.to: + end = tuple(args.to) + for name in ("duration", "hold", "hover_ms"): + if getattr(args, name, 0) < 0: + raise ControlError(f"--{name.replace('_', '-')} must not be negative", "INVALID_INPUT") + if not 1 <= args.rate <= 1000: + raise ControlError("--rate must be 1..1000 Hz", "INVALID_INPUT") + if total > PEN_MAX_MS: + raise ControlError(f"The gesture lasts {total} ms, more than {PEN_MAX_MS}", "INVALID_INPUT") + if not all(0 <= v <= 1 for v in start + end): + raise ControlError("Positions are fractions of the screen, 0..1", "INVALID_INPUT") + if contact and not all(0 <= p <= 1 for p in args.pressure): + raise ControlError("--pressure must be 0..1", "INVALID_INPUT") + if not all(-90 <= a <= 90 for a in args.tilt_x + args.tilt_y): + raise ControlError("--tilt-x and --tilt-y must be -90..90", "INVALID_INPUT") + if not 0 <= args.distance <= UINPUT_DISTANCE: + raise ControlError(f"--distance must be 0..{UINPUT_DISTANCE}", "INVALID_INPUT") + samples = [] + + def add(t: float, phase: str, x: float, y: float, pressure: float = 0.0, button: str | None = None) -> None: + progress = t / (total or 1) + samples.append({"t": round(t), "phase": phase, "x": x, "y": y, "pressure": pressure, "button": button, + "distance": args.distance, "tilt_x": pen_lerp(args.tilt_x, progress), + "tilt_y": pen_lerp(args.tilt_y, progress)}) + + if lead: + for t in pen_times(0, lead, args.rate)[:-1]: + add(t, "hover", *start) + middle = pen_times(lead, length, args.rate) + steps = len(middle) - 1 + for i, t in enumerate(middle): + f = i / steps + x, y = pen_lerp((start[0], end[0]), f), pen_lerp((start[1], end[1]), f) + if contact: + add(t, "down" if i == 0 else "up" if i == steps else "move", x, y, pen_lerp(args.pressure, f)) + else: + add(t, "hover", x, y, button=None if i == steps else side) + if lead: + for t in pen_times(lead + length, lead, args.rate)[1:]: + add(t, "hover", *end) + add(total, "out", *end) + return samples + + +def input_viewport(dump: str) -> dict: + blocks = re.findall(r"^[ \t]*Viewports:\n((?:[ \t]+Viewport .*\n?)+)", dump, re.M) + viewports = [] + for line in re.findall(r"Viewport INTERNAL: (.*)", blocks[-1] if blocks else dump): + fields = dict(re.findall(r"(\w+)=(\[[^\]]*\]|[^,\s]+)", line)) + size = re.fullmatch(r"\[(\d+), (\d+)\]", fields.get("deviceSize", "")) + frame = re.fullmatch(r"\[(-?\d+), (-?\d+), (-?\d+), (-?\d+)\]", fields.get("physicalFrame", "")) + if size and frame and re.fullmatch(r"-?\d+", fields.get("displayId", "")) and fields.get("orientation") in ("0", "1", "2", "3"): + viewports.append({"display": int(fields["displayId"]), "rotation": int(fields["orientation"]), + "size": [int(size.group(1)), int(size.group(2))], + "frame": [int(edge) for edge in frame.groups()], "active": fields.get("isActive") != "[0]"}) + if not viewports: + raise ControlError("`dumpsys input` shows no internal display viewport to map the pen to", "NO_VIEWPORT") + viewport = next((v for v in viewports if v["display"] == 0), viewports[0]) + if not viewport["active"]: + raise ControlError(f"Display {viewport['display']} is off, and Android ignores touchscreens on it; " + "wake the device first", "NO_VIEWPORT") + return viewport + + +def uinput_point(x: float, y: float, tilt_x: float, tilt_y: float, rotation: int, width: int, + height: int, frame: list[int]) -> tuple[int, int, int, int]: + left, top, right, bottom = frame + x = min(right - 1, round(left + x * (right - left))) + y = min(bottom - 1, round(top + y * (bottom - top))) + if rotation == 1: + return width - 1 - y, x, round(-tilt_y), round(tilt_x) + if rotation == 2: + return width - 1 - x, height - 1 - y, round(-tilt_x), round(-tilt_y) + if rotation == 3: + return y, height - 1 - x, round(tilt_y), round(-tilt_x) + return x, y, round(tilt_x), round(tilt_y) + + +def uinput_commands(samples: list[dict], tool: str, viewport: dict) -> list[dict]: + rotation = viewport["rotation"] + natural = viewport["size"][::-1] if rotation % 2 else viewport["size"] + width, height = (n * UINPUT_SCALE for n in natural) + bounds = [edge * UINPUT_SCALE for edge in viewport["frame"]] + last_x = max(uinput_point(f, f, 0, 0, rotation, width, height, bounds)[0] for f in (0, 1)) + tool_key = "BTN_TOOL_RUBBER" if tool == "eraser" else "BTN_TOOL_PEN" + + def axis(code: str, low: int, high: int) -> dict: + return {"code": code, "info": {"value": 0, "minimum": low, "maximum": high, "fuzz": 0, "flat": 0, "resolution": 0}} + + commands = [ + {"id": 1, "command": "register", "name": "DroidDeck agent pen", "bus": "virtual", "configuration": [ + {"type": "UI_SET_EVBIT", "data": ["EV_KEY", "EV_ABS"]}, + {"type": "UI_SET_KEYBIT", "data": ["BTN_TOOL_PEN", "BTN_TOOL_RUBBER", "BTN_TOUCH", "BTN_STYLUS", "BTN_STYLUS2"]}, + {"type": "UI_SET_ABSBIT", "data": ["ABS_X", "ABS_Y", "ABS_PRESSURE", "ABS_DISTANCE", "ABS_TILT_X", "ABS_TILT_Y"]}, + {"type": "UI_SET_PROPBIT", "data": ["INPUT_PROP_DIRECT"]}, + ], "abs_info": [axis("ABS_X", 0, width - 1), axis("ABS_Y", 0, height - 1), axis("ABS_PRESSURE", 0, UINPUT_PRESSURE), + axis("ABS_DISTANCE", 0, UINPUT_DISTANCE), axis("ABS_TILT_X", -90, 90), axis("ABS_TILT_Y", -90, 90)]}, + {"id": 1, "command": "delay", "duration": UINPUT_REGISTER_MS}, + ] + sent = {tool_key: 0, "BTN_TOUCH": 0, "BTN_STYLUS": 0, "BTN_STYLUS2": 0} + + def frame(sample: dict, phase: str) -> list: + near, contact = phase != "out", phase in ("down", "move") + values = {tool_key: int(near), "BTN_TOUCH": int(contact), "BTN_STYLUS": int(sample["button"] == "primary"), + "BTN_STYLUS2": int(sample["button"] == "secondary")} + if near: + x, y, tilt_x, tilt_y = uinput_point(sample["x"], sample["y"], sample["tilt_x"], sample["tilt_y"], + rotation, width, height, bounds) + values.update(ABS_X=x, ABS_Y=y, ABS_PRESSURE=max(1, round(sample["pressure"] * UINPUT_PRESSURE)) if contact else 0, + ABS_DISTANCE=0 if contact else sample["distance"], ABS_TILT_X=tilt_x, ABS_TILT_Y=tilt_y) + events = [] + for code, value in values.items(): + if sent.get(code) != value: + events += ["EV_KEY" if code.startswith("BTN_") else "EV_ABS", code, value] + sent[code] = value + return events + ["EV_SYN", "SYN_REPORT", 0] if events else [] + + now = 0 + for sample in samples: + events = frame(sample, "move") if sample["phase"] == "up" else [] + events += frame(sample, sample["phase"]) + if not events: + sent["ABS_X"] += 1 if sent["ABS_X"] < last_x else -1 + events = ["EV_ABS", "ABS_X", sent["ABS_X"], "EV_SYN", "SYN_REPORT", 0] + if sample["t"] > now: + commands.append({"id": 1, "command": "delay", "duration": sample["t"] - now}) + now = sample["t"] + commands.append({"id": 1, "command": "inject", "events": events}) + return commands + [{"id": 1, "command": "delay", "duration": UINPUT_TAIL_MS}, + {"id": 1, "command": "sync", "syncToken": UINPUT_SYNC}] + + +def pen_uinput(adb: str, serial: str, samples: list[dict], tool: str) -> dict: + probe = run_adb(adb, serial, "shell", + "command -v uinput >/dev/null 2>&1 || echo no-uinput; [ -w /dev/uinput ] || echo no-dev-uinput") + if probe.returncode != 0: + raise ControlError((probe.stdout + probe.stderr).decode(errors="replace").strip() or "adb shell failed", "ADB_FAILED") + missing = probe.stdout.decode(errors="replace").split() + if "no-uinput" in missing: + raise ControlError("This device's shell has no `uinput` command", "UINPUT_UNAVAILABLE") + if "no-dev-uinput" in missing: + raise ControlError("The ADB shell cannot write /dev/uinput on this device", "UINPUT_UNAVAILABLE") + viewport = input_viewport(run_adb(adb, serial, "shell", "dumpsys", "input").stdout.decode(errors="replace")) + script = "".join(json.dumps(command) + "\n" for command in uinput_commands(samples, tool, viewport)) + started = time.monotonic() + result = run_adb(adb, serial, "shell", "uinput", "-", stdin=script.encode(), + timeout=(UINPUT_REGISTER_MS + samples[-1]["t"] + UINPUT_TAIL_MS) / 1000 + 30) + elapsed = round((time.monotonic() - started) * 1000) + if UINPUT_SYNC not in result.stdout.decode(errors="replace"): + detail = (result.stderr or result.stdout).decode(errors="replace").strip()[-400:] + raise ControlError(f"uinput stopped before the end of the gesture (exit {result.returncode})" + + (f": {detail}" if detail else "") + "; `adb logcat -s UINPUT UinputDevice` tells why", + "UINPUT_FAILED") + return {"ok": True, "command": "input", "type": "pen", "samples": len(samples), "ms": elapsed, + "viewport": {key: viewport[key] for key in ("display", "rotation", "size")}} + + def xr_ppm_to_png(data: bytes, linear: bool = False) -> bytes: header = re.match(rb"P6\n([0-9]+) ([0-9]+)\n255\n", data) if not header: @@ -826,6 +1020,8 @@ def execute(args: argparse.Namespace, adb: str, serial: str) -> dict: "CDP_FAILED"), serial) if command == "input": + if args.input_type == "pen": + return add_serial(pen_uinput(adb, serial, pen_samples(args), args.tool), serial) return add_serial(checked(provider_call(adb, serial, "input", request=input_request(args), timeout=60), "INPUT_FAILED"), serial) @@ -1097,6 +1293,36 @@ def build_parser() -> argparse.ArgumentParser: trigger.add_argument("trigger", choices=["left", "right"]) trigger.add_argument("value", type=float, nargs="?", default=1.0) trigger.add_argument("--hold", type=int, default=200) + pen = kinds.add_parser("pen", help="a virtual pen at fractions of the screen as shown") + pen_ops = pen.add_subparsers(dest="pen_op", required=True) + pen_hover = pen_ops.add_parser("hover", help="come near at X Y, glide to --to X2 Y2 and leave") + pen_hover.add_argument("x", type=float) + pen_hover.add_argument("y", type=float) + pen_hover.add_argument("--to", type=float, nargs=2, metavar=("X2", "Y2")) + pen_hover.add_argument("--duration", type=int, default=300) + pen_tap = pen_ops.add_parser("tap", help="hover, touch down at X Y, lift and leave") + pen_tap.add_argument("x", type=float) + pen_tap.add_argument("y", type=float) + pen_tap.add_argument("--hold", type=int, default=60, help="ms in contact") + pen_stroke = pen_ops.add_parser("stroke", help="hover, draw a line from X Y to X2 Y2 and leave") + for name in ("x", "y", "x2", "y2"): + pen_stroke.add_argument(name, type=float) + pen_stroke.add_argument("--duration", type=int, default=300, help="ms in contact") + pen_button = pen_ops.add_parser("button", help="press and release a side button while hovering at X Y") + pen_button.add_argument("button", choices=["primary", "secondary"]) + pen_button.add_argument("x", type=float) + pen_button.add_argument("y", type=float) + pen_button.add_argument("--hold", type=int, default=120, help="ms the button is down") + for op in (pen_tap, pen_stroke): + op.add_argument("--pressure", type=pen_span, default=(0.5, 0.5), metavar="P[:P]") + for op in (pen_tap, pen_stroke, pen_button): + op.add_argument("--hover-ms", type=int, default=80, help="ms of hover before and after") + for op in (pen_hover, pen_tap, pen_stroke, pen_button): + op.add_argument("--tool", choices=["pen", "eraser"], default="pen") + op.add_argument("--tilt-x", type=pen_span, default=(0.0, 0.0), metavar="DEG[:DEG]") + op.add_argument("--tilt-y", type=pen_span, default=(0.0, 0.0), metavar="DEG[:DEG]") + op.add_argument("--distance", type=int, default=0, help="while hovering, in device units 0..255") + op.add_argument("--rate", type=float, default=120, help="samples per second") ui = commands.add_parser("ui", help="the app's own UI: dump nodes, click or type by test tag") ui_ops = ui.add_subparsers(dest="ui_op", required=True) @@ -1199,7 +1425,7 @@ def build_parser() -> argparse.ArgumentParser: return parser -EXIT_ADB = {"ADB_NOT_FOUND", "ADB_FAILED", "INVALID_PROVIDER_RESPONSE"} +EXIT_ADB = {"ADB_NOT_FOUND", "ADB_FAILED", "INVALID_PROVIDER_RESPONSE", "UINPUT_UNAVAILABLE", "NO_VIEWPORT", "UINPUT_FAILED"} EXIT_SESSION = { "SESSION_ACTIVE", "NO_ACTIVE_SESSION", "SESSION_NOT_SUSPENDED", "SESSION_FAILED", "GUEST_EXIT", "GUEST_START_FAILED", "RUNTIME_INSTALL_FAILED", "DESKTOP_INSTALL_FAILED", "ACTIVITY_START_FAILED", diff --git a/tools/tests/test_droiddeckctl.py b/tools/tests/test_droiddeckctl.py index 3d26a409fb..9dd7dff676 100644 --- a/tools/tests/test_droiddeckctl.py +++ b/tools/tests/test_droiddeckctl.py @@ -55,6 +55,19 @@ FAKE_ADB = textwrap.dedent('''\ print("Result: Bundle[{json=" + json.dumps(answers) + "}]") elif args[:3] == ["shell", "dumpsys", "display"]: print(config.get("dumpsys_display", "")) + elif args[:3] == ["shell", "dumpsys", "input"]: + print(config.get("dumpsys_input", "")) + elif args[:3] == ["shell", "uinput", "-"]: + entry["stdin"] = sys.stdin.read() + if "uinput_stdout" in config: + print(config["uinput_stdout"]) + else: + for line in entry["stdin"].splitlines(): + command = json.loads(line) + if command["command"] == "sync": + print(json.dumps({"reason": "sync", "id": command["id"], "syncToken": command["syncToken"]})) + elif args[:1] == ["shell"] and len(args) == 2 and "command -v uinput" in args[1]: + print(config.get("uinput_probe", "")) elif args[:3] == ["shell", "dumpsys", "power"]: print(" mWakefulness=Awake") elif args[:3] == ["shell", "dumpsys", "SurfaceFlinger"] and "-dump" in args: diff --git a/tools/tests/test_droiddeckctl_pen.py b/tools/tests/test_droiddeckctl_pen.py new file mode 100644 index 0000000000..a4dcf5cc54 --- /dev/null +++ b/tools/tests/test_droiddeckctl_pen.py @@ -0,0 +1,255 @@ +import json +import unittest +import unittest.mock + +from test_droiddeckctl import FakeAdbTest, ctl + + +def samples(*argv): + return ctl.pen_samples(ctl.build_parser().parse_args(["input", "pen", *argv])) + + +def phases(gesture): + return [s["phase"] for s in gesture] + + +def dumpsys_input(rotation=0, size=(1140, 2616), active=1, frame=None): + other = ("Viewport INTERNAL: displayId=2, uniqueId=local:147, port=147, orientation=0, " + "logicalFrame=[0, 0, 2248, 2480], physicalFrame=[0, 0, 2248, 2480], deviceSize=[2248, 2480], isActive=[0]") + shown = (f"Viewport INTERNAL: displayId=0, uniqueId=local:148, port=148, orientation={rotation}, " + f"logicalFrame=[0, 0, {size[0]}, {size[1]}], physicalFrame=[{', '.join(map(str, frame or (0, 0, *size)))}], " + f"deviceSize=[{size[0]}, {size[1]}], isActive=[{active}]") + reader = (f"Input Reader State (Nums of device: 11):\n Device 6: touchpanel_pen\n {other}\n" + f" Configuration:\n Viewports:\n {other}\n {shown}\n OplusInputReader mMergeEnable = 0\n") + return reader + "\nINPUT MANAGER (dumpsys input)\n\n" + reader + + +def frames(commands): + found, frame = [], {} + for command in commands: + if command["command"] == "inject": + events = command["events"] + for code, value in zip(events[1::3], events[2::3]): + if code == "SYN_REPORT": + found.append(frame) + frame = {} + else: + frame[code] = value + return found + + +class PenSamplesTest(unittest.TestCase): + def test_a_tap_hovers_lands_lifts_and_leaves(self): + gesture = samples("tap", "0.5", "0.5", "--hold", "20") + self.assertEqual(["hover"] * 10 + ["down", "move", "up"] + ["hover"] * 10 + ["out"], phases(gesture)) + times = [s["t"] for s in gesture] + self.assertEqual(sorted(times), times) + self.assertEqual([0, 8, 80, 100, 180, 180], [times[0], times[1], times[10], times[12], times[-2], times[-1]]) + self.assertEqual({0.5}, {s["pressure"] for s in gesture[10:13]}) + self.assertEqual({0}, {s["pressure"] for s in gesture[:10] + gesture[13:]}) + self.assertEqual({(0.5, 0.5, None)}, {(s["x"], s["y"], s["button"]) for s in gesture}) + + def test_without_hover_the_pen_lands_and_leaves_at_once(self): + gesture = samples("tap", "0.5", "0.5", "--hover-ms", "0", "--hold", "0") + self.assertEqual([("down", 0), ("up", 0), ("out", 0)], [(s["phase"], s["t"]) for s in gesture]) + + def test_a_stroke_ramps_pressure_and_tilt(self): + gesture = samples("stroke", "0.25", "0.5", "0.75", "1", "--duration", "100", "--hover-ms", "20", "--rate", "100", + "--pressure", "0.25:0.75", "--tilt-x=-30:30", "--tilt-y", "10") + self.assertEqual(["hover", "hover", "down"] + ["move"] * 9 + ["up", "hover", "hover", "out"], phases(gesture)) + contact = gesture[2:13] + self.assertEqual([(20, 0.25, 0.5, 0.25), (70, 0.5, 0.75, 0.5), (120, 0.75, 1, 0.75)], + [(s["t"], s["x"], s["y"], s["pressure"]) for s in (contact[0], contact[5], contact[-1])]) + self.assertEqual((-30, 30), (gesture[0]["tilt_x"], gesture[-1]["tilt_x"])) + self.assertAlmostEqual(-30 + 60 * 20 / 140, contact[0]["tilt_x"]) + self.assertEqual({10}, {s["tilt_y"] for s in gesture}) + + def test_hover_glides_to_the_target(self): + gesture = samples("hover", "0.25", "0.25", "--to", "0.75", "0.5", "--duration", "300", "--distance", "5") + self.assertEqual(["hover"] * 37 + ["out"], phases(gesture)) + self.assertEqual([(0.25, 0.25), (0.75, 0.5), (0.75, 0.5)], + [(s["x"], s["y"]) for s in (gesture[0], gesture[-2], gesture[-1])]) + self.assertEqual((300, {5}), (gesture[-1]["t"], {s["distance"] for s in gesture})) + + def test_a_side_button_is_held_while_hovering(self): + gesture = samples("button", "secondary", "0.5", "0.5", "--hold", "40") + self.assertEqual([80, 88, 96, 104, 112], [s["t"] for s in gesture if s["button"] == "secondary"]) + self.assertEqual(["hover"] * (len(gesture) - 1) + ["out"], phases(gesture)) + self.assertEqual((200, None), (gesture[-1]["t"], gesture[-2]["button"])) + + +class PenUinputTest(FakeAdbTest): + def setUp(self): + super().setUp() + self.configure({}, dumpsys_input=dumpsys_input()) + + def pen(self, *argv): + return self.run_ctl("input", "pen", *argv) + + def script(self): + [call] = [c for c in self.calls() if c["args"][:3] == ["shell", "uinput", "-"]] + return [json.loads(line) for line in call["stdin"].splitlines()] + + def test_a_stroke_draws_with_a_virtual_direct_pen(self): + code, out = self.pen("stroke", "0.25", "0.5", "0.75", "0.5", "--duration", "100", "--rate", "100", + "--hover-ms", "20", "--pressure", "0.2:1") + self.assertEqual(0, code) + self.assertEqual({"ok": True, "command": "input", "type": "pen", "samples": 16, "deviceSerial": "fake-1", + "viewport": {"display": 0, "rotation": 0, "size": [1140, 2616]}}, + {k: v for k, v in out.items() if k != "ms"}) + self.assertFalse(self.calls("input")) + commands = self.script() + register = commands[0] + bits = {c["type"]: c["data"] for c in register["configuration"]} + self.assertEqual(("register", "virtual", ["EV_KEY", "EV_ABS"], ["INPUT_PROP_DIRECT"]), + (register["command"], register["bus"], bits["UI_SET_EVBIT"], bits["UI_SET_PROPBIT"])) + self.assertEqual({"BTN_TOOL_PEN", "BTN_TOOL_RUBBER", "BTN_TOUCH", "BTN_STYLUS", "BTN_STYLUS2"}, + set(bits["UI_SET_KEYBIT"])) + ranges = {a["code"]: (a["info"]["minimum"], a["info"]["maximum"]) for a in register["abs_info"]} + self.assertEqual({"ABS_X": (0, 1140 * 16 - 1), "ABS_Y": (0, 2616 * 16 - 1), "ABS_PRESSURE": (0, 4095), + "ABS_DISTANCE": (0, 255), "ABS_TILT_X": (-90, 90), "ABS_TILT_Y": (-90, 90)}, ranges) + self.assertEqual(set(bits["UI_SET_ABSBIT"]), set(ranges)) + self.assertEqual({"id": 1, "command": "delay", "duration": 1000}, commands[1]) + self.assertNotIn("updateTimeBase", [c["command"] for c in commands]) + self.assertEqual(140, sum(c["duration"] for c in commands[2:-2] if c["command"] == "delay")) + self.assertEqual([{"id": 1, "command": "delay", "duration": 300}, + {"id": 1, "command": "sync", "syncToken": ctl.UINPUT_SYNC}], commands[-2:]) + drawn = frames(commands) + self.assertEqual({"BTN_TOOL_PEN": 1, "ABS_X": 4560, "ABS_Y": 20928, "ABS_PRESSURE": 0, "ABS_DISTANCE": 0, + "ABS_TILT_X": 0, "ABS_TILT_Y": 0}, drawn[0]) + self.assertEqual([{"ABS_X": 4561}, {"BTN_TOUCH": 1, "ABS_X": 4560, "ABS_PRESSURE": 819}], drawn[1:3]) + self.assertEqual([{"ABS_X": 13680, "ABS_PRESSURE": 4095}, {"BTN_TOUCH": 0, "ABS_PRESSURE": 0}, {"ABS_X": 13681}, + {"ABS_X": 13680}, {"BTN_TOOL_PEN": 0}], drawn[-5:]) + pressures = [f["ABS_PRESSURE"] for f in drawn[2:-4]] + self.assertEqual(sorted(pressures), pressures) + + def test_positions_and_tilt_follow_the_rotation(self): + cases = {0: ([1000, 2000], 4000, 24000, 30, -10), 1: ([2000, 1000], 3999, 8000, 10, 30), + 2: ([1000, 2000], 11999, 7999, -30, 10), 3: ([2000, 1000], 12000, 23999, -10, -30)} + for rotation, (size, x, y, tilt_x, tilt_y) in cases.items(): + with self.subTest(rotation=rotation): + self.configure({}, dumpsys_input=dumpsys_input(rotation, size)) + self.log.unlink(missing_ok=True) + code, out = self.pen("tap", "0.25", "0.75", "--tilt-x", "30", "--tilt-y=-10") + self.assertEqual((0, rotation), (code, out["viewport"]["rotation"])) + commands = self.script() + self.assertEqual([15999, 31999], [a["info"]["maximum"] for a in commands[0]["abs_info"][:2]]) + first = frames(commands)[0] + self.assertEqual((x, y, tilt_x, tilt_y), + tuple(first[key] for key in ("ABS_X", "ABS_Y", "ABS_TILT_X", "ABS_TILT_Y"))) + + def test_the_mapping_inverts_aosps_input_coordinates_example(self): + self.assertEqual((0, 2, 0, 0), ctl.uinput_point(2 / 4, 2 / 3, 0, 0, 1, 3, 4, [0, 0, 4, 3])) + self.assertEqual((2, 3, 0, 0), ctl.uinput_point(1, 1, 0, 0, 0, 3, 4, [0, 0, 3, 4])) + self.assertEqual((0, 0, 0, 0), ctl.uinput_point(1, 1, 0, 0, 2, 3, 4, [0, 0, 3, 4])) + + def test_positions_map_onto_the_physical_frame(self): + cases = {0: ((1000, 2000), (100, 250, 900, 1750), ("1", "0.05"), (14399, 5200)), + 1: ((2000, 1000), (250, 100, 1750, 900), ("0.05", "1"), (1600, 5200))} + for rotation, (size, frame, position, first) in cases.items(): + with self.subTest(rotation=rotation): + self.configure({}, dumpsys_input=dumpsys_input(rotation, size, frame=frame)) + self.log.unlink(missing_ok=True) + self.assertEqual(0, self.pen("tap", *position)[0]) + drawn = frames(self.script()) + self.assertEqual(first, (drawn[0]["ABS_X"], drawn[0]["ABS_Y"])) + self.assertTrue(all(1600 <= f.get("ABS_X", 1600) < 14400 for f in drawn)) + + def test_eraser_and_buttons_use_the_stylus_keys(self): + for button, side in (("primary", "BTN_STYLUS"), ("secondary", "BTN_STYLUS2")): + with self.subTest(button=button): + self.log.unlink(missing_ok=True) + code, _ = self.pen("button", button, "0.5", "0.5", "--hold", "40", "--tool", "eraser", "--distance", "12") + self.assertEqual(0, code) + drawn = frames(self.script()) + keys = [(key, value) for frame in drawn for key, value in frame.items() if key.startswith("BTN_")] + self.assertEqual([("BTN_TOOL_RUBBER", 1), (side, 1), (side, 0), ("BTN_TOOL_RUBBER", 0)], keys) + self.assertEqual(12, drawn[0]["ABS_DISTANCE"]) + + def test_a_still_pen_keeps_reporting_at_the_rate(self): + code, out = self.pen("hover", "1", "0.5", "--duration", "100", "--rate", "50") + self.assertEqual(0, code) + commands = self.script() + drawn = frames(commands) + self.assertEqual(out["samples"], len(drawn)) + self.assertEqual([18239, 18238, 18239, 18238, 18239, 18238], [f["ABS_X"] for f in drawn[:-1]]) + self.assertEqual([20, 20, 20, 20, 20, 300], [c["duration"] for c in commands[2:] if c["command"] == "delay"]) + + def test_a_contact_without_pressure_still_touches(self): + code, _ = self.pen("tap", "0.5", "0.5", "--pressure", "0", "--hover-ms", "0", "--hold", "0", "--distance", "5") + self.assertEqual(0, code) + drawn = frames(self.script()) + self.assertEqual((1, 1, 1, 0), + tuple(drawn[0][key] for key in ("BTN_TOOL_PEN", "BTN_TOUCH", "ABS_PRESSURE", "ABS_DISTANCE"))) + self.assertEqual([{"BTN_TOUCH": 0, "ABS_PRESSURE": 0, "ABS_DISTANCE": 5}, {"BTN_TOOL_PEN": 0}], drawn[1:]) + + def test_bad_values_are_refused(self): + cases = [ + (("tap", "1.5", "0.5"), "Positions are fractions of the screen"), + (("hover", "0.5", "0.5", "--to", "nan", "0.5"), "Positions are fractions of the screen"), + (("stroke", "0", "0", "1", "1", "--pressure", "0.5:1.5"), "--pressure must be 0..1"), + (("tap", "0.5", "0.5", "--tilt-x=-91"), "--tilt-x and --tilt-y must be -90..90"), + (("tap", "0.5", "0.5", "--tilt-y", "120"), "--tilt-x and --tilt-y must be -90..90"), + (("hover", "0.5", "0.5", "--distance", "256"), "--distance must be 0..255"), + (("tap", "0.5", "0.5", "--rate", "1e9"), "--rate must be 1..1000 Hz"), + (("button", "primary", "0.5", "0.5", "--hold", "-5"), "--hold must not be negative"), + (("tap", "0.5", "0.5", "--hover-ms", "-1"), "--hover-ms must not be negative"), + (("stroke", "0", "0", "1", "1", "--duration", "40000"), "lasts 40160 ms, more than 30000"), + (("hover", "0.5", "0.5", "--duration", "1000000000"), "more than 30000"), + ] + with unittest.mock.patch.object(ctl, "pen_times", side_effect=AssertionError): + for argv, message in cases: + with self.subTest(message=message): + code, out = self.pen(*argv) + self.assertEqual((2, "INVALID_INPUT"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse(self.calls()) + self.assertEqual(30_000, samples("tap", "1", "1", "--hover-ms", "14970")[-1]["t"]) + + def test_a_shell_without_uinput_is_unavailable(self): + for probe, message in (("no-uinput", "no `uinput` command"), ("no-dev-uinput", "cannot write /dev/uinput")): + with self.subTest(probe=probe): + self.configure({}, dumpsys_input=dumpsys_input(), uinput_probe=probe) + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "UINPUT_UNAVAILABLE"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse([c for c in self.calls() if c["args"][:2] == ["shell", "uinput"]]) + + def test_a_missing_or_dark_display_is_refused(self): + for dump, message in (("Input Reader State (Nums of device: 0):\n", "no internal display viewport"), + (dumpsys_input(active=0), "Display 0 is off")): + with self.subTest(message=message): + self.configure({}, dumpsys_input=dump) + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "NO_VIEWPORT"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse([c for c in self.calls() if c["args"][:2] == ["shell", "uinput"]]) + + def test_uinput_that_stops_early_fails(self): + self.configure({}, dumpsys_input=dumpsys_input(), uinput_stdout="") + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "UINPUT_FAILED"), (code, out["error"]["code"])) + self.assertIn("adb logcat -s UINPUT", out["error"]["message"]) + + def test_the_viewport_android_maps_touchscreens_to(self): + self.assertEqual({"display": 0, "rotation": 0, "size": [1140, 2616], "frame": [0, 0, 1140, 2616], "active": True}, + ctl.input_viewport(dumpsys_input())) + device = " Device 7: touchpanel\n Viewport INTERNAL: displayId=0, orientation=0, deviceSize=[1140, 2616]\n" + self.assertEqual(1, ctl.input_viewport(device + dumpsys_input(1, (2480, 2248)))["rotation"]) + newest = (" Viewports:\n Viewport INTERNAL: displayId=3, uniqueId=local:1, port=, orientation=3, " + "densityDpi=420 xDpi=409.432007, yDpi=411.891998, logicalFrame=[0, 0, 2400, 1080], " + "physicalFrame=[0, 0, 2400, 1080], deviceSize=[2400, 1080], isActive=[1]\n" + " Viewport INTERNAL: displayId=5, uniqueId=local:3, port=2, orientation=0, " + "deviceSize=[1080, 2400], isActive=[1]\n" + " Viewport EXTERNAL: displayId=0, uniqueId=local:2, port=1, orientation=0, " + "deviceSize=[1920, 1080], isActive=[1]\n") + self.assertEqual({"display": 3, "rotation": 3, "size": [2400, 1080], "frame": [0, 0, 2400, 1080], "active": True}, + ctl.input_viewport(newest)) + older = ("Viewport INTERNAL: displayId=0, orientation=2, logicalFrame=[0, 0, 720, 1280], " + "physicalFrame=[-3, 0, 717, 1280], deviceSize=[720, 1280]") + self.assertEqual({"display": 0, "rotation": 2, "size": [720, 1280], "frame": [-3, 0, 717, 1280], "active": True}, + ctl.input_viewport(older)) + + +if __name__ == "__main__": + unittest.main()