diff --git a/docs/agent-control.md b/docs/agent-control.md index 4ac8706706..dc9ced7e60 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -122,6 +122,10 @@ tools/droiddeckctl input swipe 100 600 100 100 --duration 400 tools/droiddeckctl input pad a tools/droiddeckctl input stick left 0 -1 --hold 500 tools/droiddeckctl input trigger right 1.0 +tools/droiddeckctl input pen tap 0.5 0.5 --pressure 0.7 +tools/droiddeckctl input pen stroke 0.2 0.5 0.8 0.5 --pressure 0.1:0.9 --tilt-x=-30:30 +tools/droiddeckctl input pen hover 0.3 0.5 --to 0.7 0.5 --duration 600 +tools/droiddeckctl input pen button primary 0.5 0.5 tools/droiddeckctl ui dump tools/droiddeckctl ui click rail-steam @@ -196,6 +200,20 @@ virtual pad, the same one the on-screen controls drive, so games that ignore tou `launch`, `quit` and `cdp` need a Steam session; `focus` and `wait focused` need gamescope (every mode except the desktop). +`input pen` does not go through the app. It registers a virtual pen with `adb shell uinput -`, so +Android's own input reader makes the stylus events, as for a built-in pen, and it works on whatever +is on screen, without a session or Setup > Developer > Debugging tools. Positions are fractions of +the screen as shown; `dumpsys input` gives the frame, rotation and size of the display Android maps +internal touchscreens to. `hover` comes near and glides to `--to`; `tap`, `stroke` and `button` +hover for `--hover-ms` (80) before and after the touch or the button press. Each takes +`--tool pen|eraser`, `--tilt-x` and `--tilt-y` in degrees, `--distance` in device units (0 to 255) +and `--rate` (120 samples a second); `--pressure` and tilt take `FROM:TO` ranges, written +`--tilt-x=-30:30` when they start negative. A gesture lasts at most 30 s. `UINPUT_UNAVAILABLE` +means the shell has no `uinput` or cannot write `/dev/uinput`, `NO_VIEWPORT` that there is no +internal display or it is off, and `UINPUT_FAILED` that `uinput` stopped early; all three exit 3. +It needs Android 14 QPR2 or later: the `uinput` of Android 12 to 14 QPR1 takes only numeric codes +and the USB or Bluetooth bus, so there the pen ends with `UINPUT_FAILED`. + ## State (schema 2) `state` returns `schema`, `build`, `appVersion`, `runtime {installed, version}`, diff --git a/tools/droiddeckctl b/tools/droiddeckctl index acdfce5fdb..2ade313d41 100755 --- a/tools/droiddeckctl +++ b/tools/droiddeckctl @@ -67,10 +67,11 @@ def adb_path(configured: str | None) -> str: raise ControlError("adb not found; set ADB, ANDROID_HOME, or ANDROID_SDK_ROOT", "ADB_NOT_FOUND") -def run_adb(adb: str, serial: str, *args: str, timeout: float = 30) -> subprocess.CompletedProcess[bytes]: +def run_adb(adb: str, serial: str, *args: str, timeout: float = 30, + stdin: bytes | None = None) -> subprocess.CompletedProcess[bytes]: try: return subprocess.run( - [adb, "-s", serial, *args], capture_output=True, check=False, timeout=timeout + [adb, "-s", serial, *args], input=stdin, capture_output=True, check=False, timeout=timeout ) except (OSError, subprocess.TimeoutExpired) as exc: raise ControlError(str(exc), "ADB_FAILED") from exc @@ -598,6 +599,199 @@ def input_request(args: argparse.Namespace) -> dict: raise ControlError(f"Unknown input type {kind}", "INVALID_INPUT") +PEN_MAX_MS = 30_000 +UINPUT_SCALE = 16 +UINPUT_PRESSURE = 4095 +UINPUT_DISTANCE = 255 +UINPUT_REGISTER_MS = 1000 +UINPUT_TAIL_MS = 300 +UINPUT_SYNC = "droiddeckctl-pen-done" + + +def pen_span(value: str) -> tuple[float, float]: + try: + low, high = map(float, value.split(":")) if ":" in value else (float(value),) * 2 + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a number or FROM:TO") from None + return low, high + + +def pen_times(start: float, duration: float, rate: float) -> list[float]: + steps = max(1, round(duration * rate / 1000)) + return [start + duration * i / steps for i in range(steps + 1)] + + +def pen_lerp(span: tuple[float, float], fraction: float) -> float: + return span[0] + (span[1] - span[0]) * fraction + + +def pen_samples(args: argparse.Namespace) -> list[dict]: + op = args.pen_op + contact = op in ("tap", "stroke") + side = args.button if op == "button" else None + lead = 0 if op == "hover" else args.hover_ms + length = args.duration if op in ("hover", "stroke") else args.hold + total = 2 * lead + length + start = end = (args.x, args.y) + if op == "stroke": + end = (args.x2, args.y2) + elif op == "hover" and args.to: + end = tuple(args.to) + for name in ("duration", "hold", "hover_ms"): + if getattr(args, name, 0) < 0: + raise ControlError(f"--{name.replace('_', '-')} must not be negative", "INVALID_INPUT") + if not 1 <= args.rate <= 1000: + raise ControlError("--rate must be 1..1000 Hz", "INVALID_INPUT") + if total > PEN_MAX_MS: + raise ControlError(f"The gesture lasts {total} ms, more than {PEN_MAX_MS}", "INVALID_INPUT") + if not all(0 <= v <= 1 for v in start + end): + raise ControlError("Positions are fractions of the screen, 0..1", "INVALID_INPUT") + if contact and not all(0 <= p <= 1 for p in args.pressure): + raise ControlError("--pressure must be 0..1", "INVALID_INPUT") + if not all(-90 <= a <= 90 for a in args.tilt_x + args.tilt_y): + raise ControlError("--tilt-x and --tilt-y must be -90..90", "INVALID_INPUT") + if not 0 <= args.distance <= UINPUT_DISTANCE: + raise ControlError(f"--distance must be 0..{UINPUT_DISTANCE}", "INVALID_INPUT") + samples = [] + + def add(t: float, phase: str, x: float, y: float, pressure: float = 0.0, button: str | None = None) -> None: + progress = t / (total or 1) + samples.append({"t": round(t), "phase": phase, "x": x, "y": y, "pressure": pressure, "button": button, + "distance": args.distance, "tilt_x": pen_lerp(args.tilt_x, progress), + "tilt_y": pen_lerp(args.tilt_y, progress)}) + + if lead: + for t in pen_times(0, lead, args.rate)[:-1]: + add(t, "hover", *start) + middle = pen_times(lead, length, args.rate) + steps = len(middle) - 1 + for i, t in enumerate(middle): + f = i / steps + x, y = pen_lerp((start[0], end[0]), f), pen_lerp((start[1], end[1]), f) + if contact: + add(t, "down" if i == 0 else "up" if i == steps else "move", x, y, pen_lerp(args.pressure, f)) + else: + add(t, "hover", x, y, button=None if i == steps else side) + if lead: + for t in pen_times(lead + length, lead, args.rate)[1:]: + add(t, "hover", *end) + add(total, "out", *end) + return samples + + +def input_viewport(dump: str) -> dict: + blocks = re.findall(r"^[ \t]*Viewports:\n((?:[ \t]+Viewport .*\n?)+)", dump, re.M) + viewports = [] + for line in re.findall(r"Viewport INTERNAL: (.*)", blocks[-1] if blocks else dump): + fields = dict(re.findall(r"(\w+)=(\[[^\]]*\]|[^,\s]+)", line)) + size = re.fullmatch(r"\[(\d+), (\d+)\]", fields.get("deviceSize", "")) + frame = re.fullmatch(r"\[(-?\d+), (-?\d+), (-?\d+), (-?\d+)\]", fields.get("physicalFrame", "")) + if size and frame and re.fullmatch(r"-?\d+", fields.get("displayId", "")) and fields.get("orientation") in ("0", "1", "2", "3"): + viewports.append({"display": int(fields["displayId"]), "rotation": int(fields["orientation"]), + "size": [int(size.group(1)), int(size.group(2))], + "frame": [int(edge) for edge in frame.groups()], "active": fields.get("isActive") != "[0]"}) + if not viewports: + raise ControlError("`dumpsys input` shows no internal display viewport to map the pen to", "NO_VIEWPORT") + viewport = next((v for v in viewports if v["display"] == 0), viewports[0]) + if not viewport["active"]: + raise ControlError(f"Display {viewport['display']} is off, and Android ignores touchscreens on it; " + "wake the device first", "NO_VIEWPORT") + return viewport + + +def uinput_point(x: float, y: float, tilt_x: float, tilt_y: float, rotation: int, width: int, + height: int, frame: list[int]) -> tuple[int, int, int, int]: + left, top, right, bottom = frame + x = min(right - 1, round(left + x * (right - left))) + y = min(bottom - 1, round(top + y * (bottom - top))) + if rotation == 1: + return width - 1 - y, x, round(-tilt_y), round(tilt_x) + if rotation == 2: + return width - 1 - x, height - 1 - y, round(-tilt_x), round(-tilt_y) + if rotation == 3: + return y, height - 1 - x, round(tilt_y), round(-tilt_x) + return x, y, round(tilt_x), round(tilt_y) + + +def uinput_commands(samples: list[dict], tool: str, viewport: dict) -> list[dict]: + rotation = viewport["rotation"] + natural = viewport["size"][::-1] if rotation % 2 else viewport["size"] + width, height = (n * UINPUT_SCALE for n in natural) + bounds = [edge * UINPUT_SCALE for edge in viewport["frame"]] + last_x = max(uinput_point(f, f, 0, 0, rotation, width, height, bounds)[0] for f in (0, 1)) + tool_key = "BTN_TOOL_RUBBER" if tool == "eraser" else "BTN_TOOL_PEN" + + def axis(code: str, low: int, high: int) -> dict: + return {"code": code, "info": {"value": 0, "minimum": low, "maximum": high, "fuzz": 0, "flat": 0, "resolution": 0}} + + commands = [ + {"id": 1, "command": "register", "name": "DroidDeck agent pen", "bus": "virtual", "configuration": [ + {"type": "UI_SET_EVBIT", "data": ["EV_KEY", "EV_ABS"]}, + {"type": "UI_SET_KEYBIT", "data": ["BTN_TOOL_PEN", "BTN_TOOL_RUBBER", "BTN_TOUCH", "BTN_STYLUS", "BTN_STYLUS2"]}, + {"type": "UI_SET_ABSBIT", "data": ["ABS_X", "ABS_Y", "ABS_PRESSURE", "ABS_DISTANCE", "ABS_TILT_X", "ABS_TILT_Y"]}, + {"type": "UI_SET_PROPBIT", "data": ["INPUT_PROP_DIRECT"]}, + ], "abs_info": [axis("ABS_X", 0, width - 1), axis("ABS_Y", 0, height - 1), axis("ABS_PRESSURE", 0, UINPUT_PRESSURE), + axis("ABS_DISTANCE", 0, UINPUT_DISTANCE), axis("ABS_TILT_X", -90, 90), axis("ABS_TILT_Y", -90, 90)]}, + {"id": 1, "command": "delay", "duration": UINPUT_REGISTER_MS}, + ] + sent = {tool_key: 0, "BTN_TOUCH": 0, "BTN_STYLUS": 0, "BTN_STYLUS2": 0} + + def frame(sample: dict, phase: str) -> list: + near, contact = phase != "out", phase in ("down", "move") + values = {tool_key: int(near), "BTN_TOUCH": int(contact), "BTN_STYLUS": int(sample["button"] == "primary"), + "BTN_STYLUS2": int(sample["button"] == "secondary")} + if near: + x, y, tilt_x, tilt_y = uinput_point(sample["x"], sample["y"], sample["tilt_x"], sample["tilt_y"], + rotation, width, height, bounds) + values.update(ABS_X=x, ABS_Y=y, ABS_PRESSURE=max(1, round(sample["pressure"] * UINPUT_PRESSURE)) if contact else 0, + ABS_DISTANCE=0 if contact else sample["distance"], ABS_TILT_X=tilt_x, ABS_TILT_Y=tilt_y) + events = [] + for code, value in values.items(): + if sent.get(code) != value: + events += ["EV_KEY" if code.startswith("BTN_") else "EV_ABS", code, value] + sent[code] = value + return events + ["EV_SYN", "SYN_REPORT", 0] if events else [] + + now = 0 + for sample in samples: + events = frame(sample, "move") if sample["phase"] == "up" else [] + events += frame(sample, sample["phase"]) + if not events: + sent["ABS_X"] += 1 if sent["ABS_X"] < last_x else -1 + events = ["EV_ABS", "ABS_X", sent["ABS_X"], "EV_SYN", "SYN_REPORT", 0] + if sample["t"] > now: + commands.append({"id": 1, "command": "delay", "duration": sample["t"] - now}) + now = sample["t"] + commands.append({"id": 1, "command": "inject", "events": events}) + return commands + [{"id": 1, "command": "delay", "duration": UINPUT_TAIL_MS}, + {"id": 1, "command": "sync", "syncToken": UINPUT_SYNC}] + + +def pen_uinput(adb: str, serial: str, samples: list[dict], tool: str) -> dict: + probe = run_adb(adb, serial, "shell", + "command -v uinput >/dev/null 2>&1 || echo no-uinput; [ -w /dev/uinput ] || echo no-dev-uinput") + if probe.returncode != 0: + raise ControlError((probe.stdout + probe.stderr).decode(errors="replace").strip() or "adb shell failed", "ADB_FAILED") + missing = probe.stdout.decode(errors="replace").split() + if "no-uinput" in missing: + raise ControlError("This device's shell has no `uinput` command", "UINPUT_UNAVAILABLE") + if "no-dev-uinput" in missing: + raise ControlError("The ADB shell cannot write /dev/uinput on this device", "UINPUT_UNAVAILABLE") + viewport = input_viewport(run_adb(adb, serial, "shell", "dumpsys", "input").stdout.decode(errors="replace")) + script = "".join(json.dumps(command) + "\n" for command in uinput_commands(samples, tool, viewport)) + started = time.monotonic() + result = run_adb(adb, serial, "shell", "uinput", "-", stdin=script.encode(), + timeout=(UINPUT_REGISTER_MS + samples[-1]["t"] + UINPUT_TAIL_MS) / 1000 + 30) + elapsed = round((time.monotonic() - started) * 1000) + if UINPUT_SYNC not in result.stdout.decode(errors="replace"): + detail = (result.stderr or result.stdout).decode(errors="replace").strip()[-400:] + raise ControlError(f"uinput stopped before the end of the gesture (exit {result.returncode})" + + (f": {detail}" if detail else "") + "; `adb logcat -s UINPUT UinputDevice` tells why", + "UINPUT_FAILED") + return {"ok": True, "command": "input", "type": "pen", "samples": len(samples), "ms": elapsed, + "viewport": {key: viewport[key] for key in ("display", "rotation", "size")}} + + def xr_ppm_to_png(data: bytes, linear: bool = False) -> bytes: header = re.match(rb"P6\n([0-9]+) ([0-9]+)\n255\n", data) if not header: @@ -826,6 +1020,8 @@ def execute(args: argparse.Namespace, adb: str, serial: str) -> dict: "CDP_FAILED"), serial) if command == "input": + if args.input_type == "pen": + return add_serial(pen_uinput(adb, serial, pen_samples(args), args.tool), serial) return add_serial(checked(provider_call(adb, serial, "input", request=input_request(args), timeout=60), "INPUT_FAILED"), serial) @@ -1097,6 +1293,36 @@ def build_parser() -> argparse.ArgumentParser: trigger.add_argument("trigger", choices=["left", "right"]) trigger.add_argument("value", type=float, nargs="?", default=1.0) trigger.add_argument("--hold", type=int, default=200) + pen = kinds.add_parser("pen", help="a virtual pen at fractions of the screen as shown") + pen_ops = pen.add_subparsers(dest="pen_op", required=True) + pen_hover = pen_ops.add_parser("hover", help="come near at X Y, glide to --to X2 Y2 and leave") + pen_hover.add_argument("x", type=float) + pen_hover.add_argument("y", type=float) + pen_hover.add_argument("--to", type=float, nargs=2, metavar=("X2", "Y2")) + pen_hover.add_argument("--duration", type=int, default=300) + pen_tap = pen_ops.add_parser("tap", help="hover, touch down at X Y, lift and leave") + pen_tap.add_argument("x", type=float) + pen_tap.add_argument("y", type=float) + pen_tap.add_argument("--hold", type=int, default=60, help="ms in contact") + pen_stroke = pen_ops.add_parser("stroke", help="hover, draw a line from X Y to X2 Y2 and leave") + for name in ("x", "y", "x2", "y2"): + pen_stroke.add_argument(name, type=float) + pen_stroke.add_argument("--duration", type=int, default=300, help="ms in contact") + pen_button = pen_ops.add_parser("button", help="press and release a side button while hovering at X Y") + pen_button.add_argument("button", choices=["primary", "secondary"]) + pen_button.add_argument("x", type=float) + pen_button.add_argument("y", type=float) + pen_button.add_argument("--hold", type=int, default=120, help="ms the button is down") + for op in (pen_tap, pen_stroke): + op.add_argument("--pressure", type=pen_span, default=(0.5, 0.5), metavar="P[:P]") + for op in (pen_tap, pen_stroke, pen_button): + op.add_argument("--hover-ms", type=int, default=80, help="ms of hover before and after") + for op in (pen_hover, pen_tap, pen_stroke, pen_button): + op.add_argument("--tool", choices=["pen", "eraser"], default="pen") + op.add_argument("--tilt-x", type=pen_span, default=(0.0, 0.0), metavar="DEG[:DEG]") + op.add_argument("--tilt-y", type=pen_span, default=(0.0, 0.0), metavar="DEG[:DEG]") + op.add_argument("--distance", type=int, default=0, help="while hovering, in device units 0..255") + op.add_argument("--rate", type=float, default=120, help="samples per second") ui = commands.add_parser("ui", help="the app's own UI: dump nodes, click or type by test tag") ui_ops = ui.add_subparsers(dest="ui_op", required=True) @@ -1199,7 +1425,7 @@ def build_parser() -> argparse.ArgumentParser: return parser -EXIT_ADB = {"ADB_NOT_FOUND", "ADB_FAILED", "INVALID_PROVIDER_RESPONSE"} +EXIT_ADB = {"ADB_NOT_FOUND", "ADB_FAILED", "INVALID_PROVIDER_RESPONSE", "UINPUT_UNAVAILABLE", "NO_VIEWPORT", "UINPUT_FAILED"} EXIT_SESSION = { "SESSION_ACTIVE", "NO_ACTIVE_SESSION", "SESSION_NOT_SUSPENDED", "SESSION_FAILED", "GUEST_EXIT", "GUEST_START_FAILED", "RUNTIME_INSTALL_FAILED", "DESKTOP_INSTALL_FAILED", "ACTIVITY_START_FAILED", diff --git a/tools/tests/test_droiddeckctl.py b/tools/tests/test_droiddeckctl.py index 3d26a409fb..9dd7dff676 100644 --- a/tools/tests/test_droiddeckctl.py +++ b/tools/tests/test_droiddeckctl.py @@ -55,6 +55,19 @@ FAKE_ADB = textwrap.dedent('''\ print("Result: Bundle[{json=" + json.dumps(answers) + "}]") elif args[:3] == ["shell", "dumpsys", "display"]: print(config.get("dumpsys_display", "")) + elif args[:3] == ["shell", "dumpsys", "input"]: + print(config.get("dumpsys_input", "")) + elif args[:3] == ["shell", "uinput", "-"]: + entry["stdin"] = sys.stdin.read() + if "uinput_stdout" in config: + print(config["uinput_stdout"]) + else: + for line in entry["stdin"].splitlines(): + command = json.loads(line) + if command["command"] == "sync": + print(json.dumps({"reason": "sync", "id": command["id"], "syncToken": command["syncToken"]})) + elif args[:1] == ["shell"] and len(args) == 2 and "command -v uinput" in args[1]: + print(config.get("uinput_probe", "")) elif args[:3] == ["shell", "dumpsys", "power"]: print(" mWakefulness=Awake") elif args[:3] == ["shell", "dumpsys", "SurfaceFlinger"] and "-dump" in args: diff --git a/tools/tests/test_droiddeckctl_pen.py b/tools/tests/test_droiddeckctl_pen.py new file mode 100644 index 0000000000..a4dcf5cc54 --- /dev/null +++ b/tools/tests/test_droiddeckctl_pen.py @@ -0,0 +1,255 @@ +import json +import unittest +import unittest.mock + +from test_droiddeckctl import FakeAdbTest, ctl + + +def samples(*argv): + return ctl.pen_samples(ctl.build_parser().parse_args(["input", "pen", *argv])) + + +def phases(gesture): + return [s["phase"] for s in gesture] + + +def dumpsys_input(rotation=0, size=(1140, 2616), active=1, frame=None): + other = ("Viewport INTERNAL: displayId=2, uniqueId=local:147, port=147, orientation=0, " + "logicalFrame=[0, 0, 2248, 2480], physicalFrame=[0, 0, 2248, 2480], deviceSize=[2248, 2480], isActive=[0]") + shown = (f"Viewport INTERNAL: displayId=0, uniqueId=local:148, port=148, orientation={rotation}, " + f"logicalFrame=[0, 0, {size[0]}, {size[1]}], physicalFrame=[{', '.join(map(str, frame or (0, 0, *size)))}], " + f"deviceSize=[{size[0]}, {size[1]}], isActive=[{active}]") + reader = (f"Input Reader State (Nums of device: 11):\n Device 6: touchpanel_pen\n {other}\n" + f" Configuration:\n Viewports:\n {other}\n {shown}\n OplusInputReader mMergeEnable = 0\n") + return reader + "\nINPUT MANAGER (dumpsys input)\n\n" + reader + + +def frames(commands): + found, frame = [], {} + for command in commands: + if command["command"] == "inject": + events = command["events"] + for code, value in zip(events[1::3], events[2::3]): + if code == "SYN_REPORT": + found.append(frame) + frame = {} + else: + frame[code] = value + return found + + +class PenSamplesTest(unittest.TestCase): + def test_a_tap_hovers_lands_lifts_and_leaves(self): + gesture = samples("tap", "0.5", "0.5", "--hold", "20") + self.assertEqual(["hover"] * 10 + ["down", "move", "up"] + ["hover"] * 10 + ["out"], phases(gesture)) + times = [s["t"] for s in gesture] + self.assertEqual(sorted(times), times) + self.assertEqual([0, 8, 80, 100, 180, 180], [times[0], times[1], times[10], times[12], times[-2], times[-1]]) + self.assertEqual({0.5}, {s["pressure"] for s in gesture[10:13]}) + self.assertEqual({0}, {s["pressure"] for s in gesture[:10] + gesture[13:]}) + self.assertEqual({(0.5, 0.5, None)}, {(s["x"], s["y"], s["button"]) for s in gesture}) + + def test_without_hover_the_pen_lands_and_leaves_at_once(self): + gesture = samples("tap", "0.5", "0.5", "--hover-ms", "0", "--hold", "0") + self.assertEqual([("down", 0), ("up", 0), ("out", 0)], [(s["phase"], s["t"]) for s in gesture]) + + def test_a_stroke_ramps_pressure_and_tilt(self): + gesture = samples("stroke", "0.25", "0.5", "0.75", "1", "--duration", "100", "--hover-ms", "20", "--rate", "100", + "--pressure", "0.25:0.75", "--tilt-x=-30:30", "--tilt-y", "10") + self.assertEqual(["hover", "hover", "down"] + ["move"] * 9 + ["up", "hover", "hover", "out"], phases(gesture)) + contact = gesture[2:13] + self.assertEqual([(20, 0.25, 0.5, 0.25), (70, 0.5, 0.75, 0.5), (120, 0.75, 1, 0.75)], + [(s["t"], s["x"], s["y"], s["pressure"]) for s in (contact[0], contact[5], contact[-1])]) + self.assertEqual((-30, 30), (gesture[0]["tilt_x"], gesture[-1]["tilt_x"])) + self.assertAlmostEqual(-30 + 60 * 20 / 140, contact[0]["tilt_x"]) + self.assertEqual({10}, {s["tilt_y"] for s in gesture}) + + def test_hover_glides_to_the_target(self): + gesture = samples("hover", "0.25", "0.25", "--to", "0.75", "0.5", "--duration", "300", "--distance", "5") + self.assertEqual(["hover"] * 37 + ["out"], phases(gesture)) + self.assertEqual([(0.25, 0.25), (0.75, 0.5), (0.75, 0.5)], + [(s["x"], s["y"]) for s in (gesture[0], gesture[-2], gesture[-1])]) + self.assertEqual((300, {5}), (gesture[-1]["t"], {s["distance"] for s in gesture})) + + def test_a_side_button_is_held_while_hovering(self): + gesture = samples("button", "secondary", "0.5", "0.5", "--hold", "40") + self.assertEqual([80, 88, 96, 104, 112], [s["t"] for s in gesture if s["button"] == "secondary"]) + self.assertEqual(["hover"] * (len(gesture) - 1) + ["out"], phases(gesture)) + self.assertEqual((200, None), (gesture[-1]["t"], gesture[-2]["button"])) + + +class PenUinputTest(FakeAdbTest): + def setUp(self): + super().setUp() + self.configure({}, dumpsys_input=dumpsys_input()) + + def pen(self, *argv): + return self.run_ctl("input", "pen", *argv) + + def script(self): + [call] = [c for c in self.calls() if c["args"][:3] == ["shell", "uinput", "-"]] + return [json.loads(line) for line in call["stdin"].splitlines()] + + def test_a_stroke_draws_with_a_virtual_direct_pen(self): + code, out = self.pen("stroke", "0.25", "0.5", "0.75", "0.5", "--duration", "100", "--rate", "100", + "--hover-ms", "20", "--pressure", "0.2:1") + self.assertEqual(0, code) + self.assertEqual({"ok": True, "command": "input", "type": "pen", "samples": 16, "deviceSerial": "fake-1", + "viewport": {"display": 0, "rotation": 0, "size": [1140, 2616]}}, + {k: v for k, v in out.items() if k != "ms"}) + self.assertFalse(self.calls("input")) + commands = self.script() + register = commands[0] + bits = {c["type"]: c["data"] for c in register["configuration"]} + self.assertEqual(("register", "virtual", ["EV_KEY", "EV_ABS"], ["INPUT_PROP_DIRECT"]), + (register["command"], register["bus"], bits["UI_SET_EVBIT"], bits["UI_SET_PROPBIT"])) + self.assertEqual({"BTN_TOOL_PEN", "BTN_TOOL_RUBBER", "BTN_TOUCH", "BTN_STYLUS", "BTN_STYLUS2"}, + set(bits["UI_SET_KEYBIT"])) + ranges = {a["code"]: (a["info"]["minimum"], a["info"]["maximum"]) for a in register["abs_info"]} + self.assertEqual({"ABS_X": (0, 1140 * 16 - 1), "ABS_Y": (0, 2616 * 16 - 1), "ABS_PRESSURE": (0, 4095), + "ABS_DISTANCE": (0, 255), "ABS_TILT_X": (-90, 90), "ABS_TILT_Y": (-90, 90)}, ranges) + self.assertEqual(set(bits["UI_SET_ABSBIT"]), set(ranges)) + self.assertEqual({"id": 1, "command": "delay", "duration": 1000}, commands[1]) + self.assertNotIn("updateTimeBase", [c["command"] for c in commands]) + self.assertEqual(140, sum(c["duration"] for c in commands[2:-2] if c["command"] == "delay")) + self.assertEqual([{"id": 1, "command": "delay", "duration": 300}, + {"id": 1, "command": "sync", "syncToken": ctl.UINPUT_SYNC}], commands[-2:]) + drawn = frames(commands) + self.assertEqual({"BTN_TOOL_PEN": 1, "ABS_X": 4560, "ABS_Y": 20928, "ABS_PRESSURE": 0, "ABS_DISTANCE": 0, + "ABS_TILT_X": 0, "ABS_TILT_Y": 0}, drawn[0]) + self.assertEqual([{"ABS_X": 4561}, {"BTN_TOUCH": 1, "ABS_X": 4560, "ABS_PRESSURE": 819}], drawn[1:3]) + self.assertEqual([{"ABS_X": 13680, "ABS_PRESSURE": 4095}, {"BTN_TOUCH": 0, "ABS_PRESSURE": 0}, {"ABS_X": 13681}, + {"ABS_X": 13680}, {"BTN_TOOL_PEN": 0}], drawn[-5:]) + pressures = [f["ABS_PRESSURE"] for f in drawn[2:-4]] + self.assertEqual(sorted(pressures), pressures) + + def test_positions_and_tilt_follow_the_rotation(self): + cases = {0: ([1000, 2000], 4000, 24000, 30, -10), 1: ([2000, 1000], 3999, 8000, 10, 30), + 2: ([1000, 2000], 11999, 7999, -30, 10), 3: ([2000, 1000], 12000, 23999, -10, -30)} + for rotation, (size, x, y, tilt_x, tilt_y) in cases.items(): + with self.subTest(rotation=rotation): + self.configure({}, dumpsys_input=dumpsys_input(rotation, size)) + self.log.unlink(missing_ok=True) + code, out = self.pen("tap", "0.25", "0.75", "--tilt-x", "30", "--tilt-y=-10") + self.assertEqual((0, rotation), (code, out["viewport"]["rotation"])) + commands = self.script() + self.assertEqual([15999, 31999], [a["info"]["maximum"] for a in commands[0]["abs_info"][:2]]) + first = frames(commands)[0] + self.assertEqual((x, y, tilt_x, tilt_y), + tuple(first[key] for key in ("ABS_X", "ABS_Y", "ABS_TILT_X", "ABS_TILT_Y"))) + + def test_the_mapping_inverts_aosps_input_coordinates_example(self): + self.assertEqual((0, 2, 0, 0), ctl.uinput_point(2 / 4, 2 / 3, 0, 0, 1, 3, 4, [0, 0, 4, 3])) + self.assertEqual((2, 3, 0, 0), ctl.uinput_point(1, 1, 0, 0, 0, 3, 4, [0, 0, 3, 4])) + self.assertEqual((0, 0, 0, 0), ctl.uinput_point(1, 1, 0, 0, 2, 3, 4, [0, 0, 3, 4])) + + def test_positions_map_onto_the_physical_frame(self): + cases = {0: ((1000, 2000), (100, 250, 900, 1750), ("1", "0.05"), (14399, 5200)), + 1: ((2000, 1000), (250, 100, 1750, 900), ("0.05", "1"), (1600, 5200))} + for rotation, (size, frame, position, first) in cases.items(): + with self.subTest(rotation=rotation): + self.configure({}, dumpsys_input=dumpsys_input(rotation, size, frame=frame)) + self.log.unlink(missing_ok=True) + self.assertEqual(0, self.pen("tap", *position)[0]) + drawn = frames(self.script()) + self.assertEqual(first, (drawn[0]["ABS_X"], drawn[0]["ABS_Y"])) + self.assertTrue(all(1600 <= f.get("ABS_X", 1600) < 14400 for f in drawn)) + + def test_eraser_and_buttons_use_the_stylus_keys(self): + for button, side in (("primary", "BTN_STYLUS"), ("secondary", "BTN_STYLUS2")): + with self.subTest(button=button): + self.log.unlink(missing_ok=True) + code, _ = self.pen("button", button, "0.5", "0.5", "--hold", "40", "--tool", "eraser", "--distance", "12") + self.assertEqual(0, code) + drawn = frames(self.script()) + keys = [(key, value) for frame in drawn for key, value in frame.items() if key.startswith("BTN_")] + self.assertEqual([("BTN_TOOL_RUBBER", 1), (side, 1), (side, 0), ("BTN_TOOL_RUBBER", 0)], keys) + self.assertEqual(12, drawn[0]["ABS_DISTANCE"]) + + def test_a_still_pen_keeps_reporting_at_the_rate(self): + code, out = self.pen("hover", "1", "0.5", "--duration", "100", "--rate", "50") + self.assertEqual(0, code) + commands = self.script() + drawn = frames(commands) + self.assertEqual(out["samples"], len(drawn)) + self.assertEqual([18239, 18238, 18239, 18238, 18239, 18238], [f["ABS_X"] for f in drawn[:-1]]) + self.assertEqual([20, 20, 20, 20, 20, 300], [c["duration"] for c in commands[2:] if c["command"] == "delay"]) + + def test_a_contact_without_pressure_still_touches(self): + code, _ = self.pen("tap", "0.5", "0.5", "--pressure", "0", "--hover-ms", "0", "--hold", "0", "--distance", "5") + self.assertEqual(0, code) + drawn = frames(self.script()) + self.assertEqual((1, 1, 1, 0), + tuple(drawn[0][key] for key in ("BTN_TOOL_PEN", "BTN_TOUCH", "ABS_PRESSURE", "ABS_DISTANCE"))) + self.assertEqual([{"BTN_TOUCH": 0, "ABS_PRESSURE": 0, "ABS_DISTANCE": 5}, {"BTN_TOOL_PEN": 0}], drawn[1:]) + + def test_bad_values_are_refused(self): + cases = [ + (("tap", "1.5", "0.5"), "Positions are fractions of the screen"), + (("hover", "0.5", "0.5", "--to", "nan", "0.5"), "Positions are fractions of the screen"), + (("stroke", "0", "0", "1", "1", "--pressure", "0.5:1.5"), "--pressure must be 0..1"), + (("tap", "0.5", "0.5", "--tilt-x=-91"), "--tilt-x and --tilt-y must be -90..90"), + (("tap", "0.5", "0.5", "--tilt-y", "120"), "--tilt-x and --tilt-y must be -90..90"), + (("hover", "0.5", "0.5", "--distance", "256"), "--distance must be 0..255"), + (("tap", "0.5", "0.5", "--rate", "1e9"), "--rate must be 1..1000 Hz"), + (("button", "primary", "0.5", "0.5", "--hold", "-5"), "--hold must not be negative"), + (("tap", "0.5", "0.5", "--hover-ms", "-1"), "--hover-ms must not be negative"), + (("stroke", "0", "0", "1", "1", "--duration", "40000"), "lasts 40160 ms, more than 30000"), + (("hover", "0.5", "0.5", "--duration", "1000000000"), "more than 30000"), + ] + with unittest.mock.patch.object(ctl, "pen_times", side_effect=AssertionError): + for argv, message in cases: + with self.subTest(message=message): + code, out = self.pen(*argv) + self.assertEqual((2, "INVALID_INPUT"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse(self.calls()) + self.assertEqual(30_000, samples("tap", "1", "1", "--hover-ms", "14970")[-1]["t"]) + + def test_a_shell_without_uinput_is_unavailable(self): + for probe, message in (("no-uinput", "no `uinput` command"), ("no-dev-uinput", "cannot write /dev/uinput")): + with self.subTest(probe=probe): + self.configure({}, dumpsys_input=dumpsys_input(), uinput_probe=probe) + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "UINPUT_UNAVAILABLE"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse([c for c in self.calls() if c["args"][:2] == ["shell", "uinput"]]) + + def test_a_missing_or_dark_display_is_refused(self): + for dump, message in (("Input Reader State (Nums of device: 0):\n", "no internal display viewport"), + (dumpsys_input(active=0), "Display 0 is off")): + with self.subTest(message=message): + self.configure({}, dumpsys_input=dump) + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "NO_VIEWPORT"), (code, out["error"]["code"])) + self.assertIn(message, out["error"]["message"]) + self.assertFalse([c for c in self.calls() if c["args"][:2] == ["shell", "uinput"]]) + + def test_uinput_that_stops_early_fails(self): + self.configure({}, dumpsys_input=dumpsys_input(), uinput_stdout="") + code, out = self.pen("tap", "0.5", "0.5") + self.assertEqual((3, "UINPUT_FAILED"), (code, out["error"]["code"])) + self.assertIn("adb logcat -s UINPUT", out["error"]["message"]) + + def test_the_viewport_android_maps_touchscreens_to(self): + self.assertEqual({"display": 0, "rotation": 0, "size": [1140, 2616], "frame": [0, 0, 1140, 2616], "active": True}, + ctl.input_viewport(dumpsys_input())) + device = " Device 7: touchpanel\n Viewport INTERNAL: displayId=0, orientation=0, deviceSize=[1140, 2616]\n" + self.assertEqual(1, ctl.input_viewport(device + dumpsys_input(1, (2480, 2248)))["rotation"]) + newest = (" Viewports:\n Viewport INTERNAL: displayId=3, uniqueId=local:1, port=, orientation=3, " + "densityDpi=420 xDpi=409.432007, yDpi=411.891998, logicalFrame=[0, 0, 2400, 1080], " + "physicalFrame=[0, 0, 2400, 1080], deviceSize=[2400, 1080], isActive=[1]\n" + " Viewport INTERNAL: displayId=5, uniqueId=local:3, port=2, orientation=0, " + "deviceSize=[1080, 2400], isActive=[1]\n" + " Viewport EXTERNAL: displayId=0, uniqueId=local:2, port=1, orientation=0, " + "deviceSize=[1920, 1080], isActive=[1]\n") + self.assertEqual({"display": 3, "rotation": 3, "size": [2400, 1080], "frame": [0, 0, 2400, 1080], "active": True}, + ctl.input_viewport(newest)) + older = ("Viewport INTERNAL: displayId=0, orientation=2, logicalFrame=[0, 0, 720, 1280], " + "physicalFrame=[-3, 0, 717, 1280], deviceSize=[720, 1280]") + self.assertEqual({"display": 0, "rotation": 2, "size": [720, 1280], "frame": [-3, 0, 717, 1280], "active": True}, + ctl.input_viewport(older)) + + +if __name__ == "__main__": + unittest.main()