From cdecd09e3c29c50480ee487271736c9b193c378e Mon Sep 17 00:00:00 2001 From: MaxsTechReview Date: Sun, 15 Nov 2026 10:31:28 -0500 Subject: [PATCH] fix(audio): restart host after stalled wake recovery (#2396) * fix(audio): restart host after stalled wake recovery * Serialize audio host restarts and make the stall counter race-free * fix(audio): an overtaken restart leaves the socket alone and a sink guard only acts on its own host --------- Co-authored-by: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> --- .../launcher/audio/PulseAudioComponent.java | 102 +++++++++++++---- .../launcher/audio/PulseAudioComponentTest.kt | 105 ++++++++++++++++++ .../sink/module-directaudio-native-sink.c | 20 ++++ tools/tests/test_directaudio_watchdog.py | 97 ++++++++++++++++ 4 files changed, 304 insertions(+), 20 deletions(-) create mode 100644 app/src/test/java/com/droiddeck/launcher/audio/PulseAudioComponentTest.kt create mode 100644 tools/tests/test_directaudio_watchdog.py diff --git a/app/src/main/java/com/droiddeck/launcher/audio/PulseAudioComponent.java b/app/src/main/java/com/droiddeck/launcher/audio/PulseAudioComponent.java index 08c463dfdc..d3495a353d 100644 --- a/app/src/main/java/com/droiddeck/launcher/audio/PulseAudioComponent.java +++ b/app/src/main/java/com/droiddeck/launcher/audio/PulseAudioComponent.java @@ -16,6 +16,8 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.concurrent.TimeUnit; +import java.util.function.IntConsumer; +import java.util.function.IntSupplier; public class PulseAudioComponent extends SessionPart { private static final String TAG = "PulseAudio"; @@ -29,7 +31,55 @@ public class PulseAudioComponent extends SessionPart { private final File deviceControls; private File logFile; private final String micFifoPath; - private volatile int pid = -1; + private final HostLifecycle host = new HostLifecycle(Process::killProcess); + + static final class HostLifecycle { + volatile int pid = -1; + private long generation; + private long launches; + private final IntConsumer terminate; + + HostLifecycle(IntConsumer terminate) { + this.terminate = terminate; + } + + synchronized long begin() { + stop(); + return generation; + } + + synchronized void stop() { + generation++; + if (pid > 1) terminate.accept(pid); + pid = -1; + } + + synchronized boolean launch(long expected, IntSupplier launch) { + if (expected != generation) return false; + launches++; + pid = launch.getAsInt(); + return true; + } + + synchronized boolean current(long expected) { + return expected == generation; + } + + /** The latest launch; a sink guard holds it so it never reconfigures a host started after its own. */ + synchronized long token() { + return launches; + } + + synchronized boolean live(long token) { + return token == launches && pid > 1; + } + + synchronized boolean exited(long expected) { + if (expected != generation || pid <= 1) return false; + pid = -1; + return true; + } + } public PulseAudioComponent(Context context) { this(context, null); @@ -64,7 +114,12 @@ public class PulseAudioComponent extends SessionPart { @Override public void start() { - stop(); + start(host.begin()); + } + + private synchronized void start(long generation) { + // An exit callback's restart can be overtaken by a newer start; it must not touch the newer host's socket. + if (!host.current(generation)) return; if (!workingDir.isDirectory()) { workingDir.mkdirs(); FileUtils.chmod(workingDir, 0771); @@ -118,17 +173,23 @@ public class PulseAudioComponent extends SessionPart { + " -n --file=default.pa --daemonize=false --use-pid-file=false --exit-idle-time=-1" + " --log-level=info"; final java.io.PrintWriter out = openLog(); - pid = HostProcess.start(command, env.toArray(new String[0]), workingDir, null, + if (!host.launch(generation, () -> HostProcess.start(command, env.toArray(new String[0]), workingDir, status -> { + if (host.exited(generation)) { + Log.w(TAG, "audio host exited with status " + status + "; restarting"); + start(generation); + } + }, line -> { Log.i(TAG, line); if (out != null) synchronized (out) { out.println(line); out.flush(); } - }); - Thread guard = new Thread(() -> ensureClientSink(out), "pulse-sink-guard"); + }))) return; + long token = host.token(); + Thread guard = new Thread(() -> ensureClientSink(out, token), "pulse-sink-guard"); guard.setDaemon(true); guard.start(); } - private void ensureClientSink(java.io.PrintWriter out) { + private void ensureClientSink(java.io.PrintWriter out, long token) { try { // Nothing that may block (a sink whose AAudio open hangs: a Galaxy S24 FE (US) hung the // daemon's start-up loading one from default.pa) is loaded by the daemon itself; the @@ -137,14 +198,14 @@ public class PulseAudioComponent extends SessionPart { long deadline = started + 30000L; String sinks = null; boolean socketSeen = false; - while (System.currentTimeMillis() < deadline && pid > 1) { + while (System.currentTimeMillis() < deadline && host.live(token)) { socketSeen |= socket().exists(); - if (socketSeen && (sinks = pactl("list", "short", "sinks")) != null) break; + if (socketSeen && (sinks = pactlFor(token, "list", "short", "sinks")) != null) break; try { Thread.sleep(250L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } } note(out, "client sink: daemon " + (sinks != null ? "answered" : "did not answer") + " after " + (System.currentTimeMillis() - started) + " ms (socket " + (socketSeen ? "present" : "never appeared") + ", pactl " - + (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + pid + ")"); + + (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + host.pid + ")"); String line; if (sinks == null) { line = "== client sink: the daemon did not answer in 30 s; no fallback attempted"; @@ -155,15 +216,15 @@ public class PulseAudioComponent extends SessionPart { // still switching the output route as the session starts (seen on a Galaxy S24 FE): // try DirectAudio again, then the plain sink, a few times over ten seconds. String loadedAs = null; - for (int attempt = 1; attempt <= 5 && loadedAs == null && pid > 1; attempt++) { + for (int attempt = 1; attempt <= 5 && loadedAs == null && host.live(token); attempt++) { try { Thread.sleep(2000L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } - if (pactl("load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1", + if (pactlFor(token, "load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1", "adaptive=1", "decay=0", "limiter=1", "watchdog=1", "volume=1.0", "control_dir=" + deviceControls) != null - && pactl("set-default-sink", "DirectAudio") != null) { + && pactlFor(token, "set-default-sink", "DirectAudio") != null) { loadedAs = "DirectAudio on attempt " + attempt; - } else if (pactl("load-module", "module-aaudio-sink", "sink_name=AAudioSink", + } else if (pactlFor(token, "load-module", "module-aaudio-sink", "sink_name=AAudioSink", "performance_mode=" + (attempt == 1 ? "1" : "0"), "adaptive=1", "volume=1.0", "control_dir=" + deviceControls) != null - && pactl("set-default-sink", "AAudioSink") != null) { + && pactlFor(token, "set-default-sink", "AAudioSink") != null) { loadedAs = "the plain AAudio sink (AAudioSink) on attempt " + attempt; } } @@ -186,10 +247,14 @@ public class PulseAudioComponent extends SessionPart { if (out != null) synchronized (out) { out.println(line); out.flush(); } } + private String pactlFor(long token, String... args) { + return host.live(token) ? pactl(args) : null; + } + /** Runs pactl against this daemon; its stdout, or null when it failed or timed out. */ private String pactl(String... args) { File pactl = new File(workingDir, "pactl"); - if (!pactl.isFile() || pid <= 1) return null; + if (!pactl.isFile() || host.pid <= 1) return null; java.lang.Process process = null; try { File modules = new File(workingDir, "modules/arm64"); @@ -231,10 +296,7 @@ public class PulseAudioComponent extends SessionPart { @Override public void stop() { - if (pid != -1) { - Process.killProcess(pid); - pid = -1; - } + host.stop(); } public boolean setSinkSuspended(boolean suspended) { @@ -247,7 +309,7 @@ public class PulseAudioComponent extends SessionPart { private boolean suspend(String kind, String name, boolean suspended) { File pactl = new File(workingDir, "pactl"); - if (!pactl.isFile() || pid <= 1) { + if (!pactl.isFile() || host.pid <= 1) { Log.w(TAG, "cannot change " + kind + " state: PulseAudio control client or server unavailable"); return false; } diff --git a/app/src/test/java/com/droiddeck/launcher/audio/PulseAudioComponentTest.kt b/app/src/test/java/com/droiddeck/launcher/audio/PulseAudioComponentTest.kt new file mode 100644 index 0000000000..d06cc52d03 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/audio/PulseAudioComponentTest.kt @@ -0,0 +1,105 @@ +package com.droiddeck.launcher.audio + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.Executors + +class PulseAudioComponentTest { + @Test + fun stopCancelsSetupAndAcceptedRestart() { + val killed = mutableListOf() + val host = PulseAudioComponent.HostLifecycle { killed.add(it) } + val generation = host.begin() + assertTrue(host.launch(generation) { 31463 }) + assertTrue(host.exited(generation)) + host.stop() + assertFalse(host.launch(generation) { error("cancelled restart launched") }) + val next = host.begin() + host.stop() + assertFalse(host.launch(next) { error("cancelled setup launched") }) + assertEquals(-1, host.pid) + } + + @Test + fun explicitStartStillWorksAndStaleExitCannotRestartIt() { + val killed = mutableListOf() + val host = PulseAudioComponent.HostLifecycle { killed.add(it) } + val first = host.begin() + host.launch(first) { 31463 } + val second = host.begin() + host.launch(second) { 31464 } + assertEquals(listOf(31463), killed) + assertFalse(host.exited(first)) + assertEquals(31464, host.pid) + host.stop() + assertEquals(listOf(31463, 31464), killed) + assertFalse(host.exited(second)) + } + + @Test + fun anOvertakenRestartIsNoLongerCurrent() { + val host = PulseAudioComponent.HostLifecycle { } + val first = host.begin() + host.launch(first) { 31463 } + assertTrue(host.exited(first)) + assertTrue(host.current(first)) + val second = host.begin() + assertFalse(host.current(first)) + assertTrue(host.current(second)) + } + + @Test + fun aSinkGuardOnlyActsOnItsOwnLaunch() { + val host = PulseAudioComponent.HostLifecycle { } + val generation = host.begin() + host.launch(generation) { 31463 } + val first = host.token() + assertTrue(host.live(first)) + assertTrue(host.exited(generation)) + assertFalse(host.live(first)) + host.launch(generation) { 31464 } + assertFalse(host.live(first)) + assertTrue(host.live(host.token())) + host.stop() + assertFalse(host.live(host.token())) + } + + @Test + fun failedLaunchDoesNotRestartOrTerminateAnInvalidPid() { + val host = PulseAudioComponent.HostLifecycle { error("invalid pid terminated") } + val generation = host.begin() + assertTrue(host.launch(generation) { -1 }) + assertFalse(host.exited(generation)) + host.stop() + } + + @Test + fun earlyExitWaitsForPidPublication() { + val host = PulseAudioComponent.HostLifecycle { } + val generation = host.begin() + val entered = CountDownLatch(1) + val pool = Executors.newSingleThreadExecutor() + try { + lateinit var exit: java.util.concurrent.Future + assertTrue(host.launch(generation) { + exit = pool.submit { + entered.countDown() + host.exited(generation) + } + assertTrue(entered.await(5, TimeUnit.SECONDS)) + assertFalse(exit.isDone) + 31463 + }) + assertTrue(exit.get(5, TimeUnit.SECONDS)) + assertEquals(-1, host.pid) + assertTrue(host.launch(generation) { 31464 }) + assertEquals(31464, host.pid) + } finally { + pool.shutdownNow() + } + } +} diff --git a/tools/directaudio/sink/module-directaudio-native-sink.c b/tools/directaudio/sink/module-directaudio-native-sink.c index 3311fe0ed7..a9daf783de 100644 --- a/tools/directaudio/sink/module-directaudio-native-sink.c +++ b/tools/directaudio/sink/module-directaudio-native-sink.c @@ -65,6 +65,7 @@ PA_MODULE_USAGE( #define DA_RING_CAP_MS_LOCAL 250 #define DA_FREEZE_GAP_NS 500000000ull #define DA_CB_STALL_NS 1000000000ull +#define DA_HOST_RESTART_STALLS 3 #define DA_DECAY_QUIET_NS 10000000000ull #define DA_DECAY_PUNISH_NS 5000000000ull #define DA_DECAY_MAX_BACKOFF 32 @@ -106,6 +107,7 @@ struct userdata { uint64_t last_cb_ns, last_xrun_ns, last_decay_ns, wd_last_tick_ns; int32_t base_buf_frames, decay_floor; unsigned int decay_backoff, cb_count, wd_last_cb; + uint64_t stall_state; int reopen_running, reopen_redo; int consuming, primed, starved; int suspended; @@ -154,6 +156,13 @@ static void limit(float *p, uint32_t samples) { } } +static void record_callback(struct userdata *u) { + uint64_t state = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST); + + while (!__atomic_compare_exchange_n(&u->stall_state, &state, + ((state >> 32) + 1) << 32, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {} +} + static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *audioData, int32_t numFrames) { struct userdata *u = user; struct da_ring *r = u->ring; @@ -170,6 +179,8 @@ static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void * return AAUDIO_CALLBACK_RESULT_CONTINUE; } + record_callback(u); + ridx = __atomic_load_n(&r->ridx, __ATOMIC_ACQUIRE); avail = __atomic_load_n(&r->widx, __ATOMIC_ACQUIRE) - ridx; if (!u->primed) { @@ -412,10 +423,12 @@ static void request_reopen(struct userdata *u, const char *why) { static void watchdog(struct userdata *u) { uint64_t now = now_ns(), last_cb, prev_tick; unsigned int cb; + uint64_t stalled; if (!u->watchdog_on) return; if (!__atomic_load_n(&u->aq, __ATOMIC_SEQ_CST)) return; + stalled = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST); prev_tick = u->wd_last_tick_ns; u->wd_last_tick_ns = now; cb = __atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST); @@ -431,6 +444,13 @@ static void watchdog(struct userdata *u) { if (!last_cb || now - last_cb < DA_CB_STALL_NS) return; __atomic_store_n(&u->last_cb_ns, now, __ATOMIC_SEQ_CST); + if (__atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST) != cb) return; + if (!__atomic_compare_exchange_n(&u->stall_state, &stalled, stalled + 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) return; + if ((uint32_t) (stalled + 1) >= DA_HOST_RESTART_STALLS) { + pa_log("directaudio-native-sink: no data callback after %u stream recoveries; restarting the audio host", (unsigned int) (stalled + 1)); + _exit(EXIT_FAILURE); + return; + } request_reopen(u, "data callback stalled"); } diff --git a/tools/tests/test_directaudio_watchdog.py b/tools/tests/test_directaudio_watchdog.py new file mode 100644 index 0000000000..6e566a9d1c --- /dev/null +++ b/tools/tests/test_directaudio_watchdog.py @@ -0,0 +1,97 @@ +import pathlib +import subprocess +import tempfile +import unittest + + +class DirectAudioWatchdogTest(unittest.TestCase): + def test_recovery_threshold_callback_race_and_forced_exit(self): + source = pathlib.Path('tools/directaudio/sink/module-directaudio-native-sink.c').read_text() + watchdog = source[source.index('static void watchdog('):source.index('static void close_output(')] + callback = source[source.index('static void record_callback('): + source.index('static aaudio_data_callback_result_t out_cb(')] + constants = '\n'.join(line for line in source.splitlines() + if line.startswith(('#define DA_FREEZE_GAP_NS ', '#define DA_CB_STALL_NS ', + '#define DA_HOST_RESTART_STALLS '))) + harness = r''' +#include +#include +#include +#include +#include +#include +typedef void AAudioStream; +struct userdata { + bool watchdog_on; + AAudioStream *aq; + uint64_t wd_last_tick_ns, last_cb_ns; + unsigned int cb_count, wd_last_cb; + uint64_t stall_state; +}; +static uint64_t clock_ns = 10000000000ull; +static unsigned int reopens; +static bool callback_race; +static struct userdata *current; +static uint64_t now_ns(void) { return clock_ns; } +static void request_reopen(struct userdata *u, const char *why) { reopens++; } +#define pa_log(...) ((void)0) +CALLBACK +static bool exchange(uint64_t *value, uint64_t *expected, uint64_t next, + int weak, int success, int failure) { + if (callback_race) { + struct userdata *u = current; + record_callback(u); + __atomic_add_fetch(&u->cb_count, 1, __ATOMIC_SEQ_CST); + } + return __atomic_compare_exchange_n(value, expected, next, weak, success, failure); +} +#define __atomic_compare_exchange_n exchange +WATCHDOG +static void cleanup(void) { _exit(99); } +static void tick(struct userdata *u) { + clock_ns += 1000000000ull; + watchdog(u); +} +int main(int argc, char **argv) { + struct userdata state = {.watchdog_on = true, .aq = (void *)1}; + struct userdata *u = &state; + current = u; + watchdog(u); + if (argc > 1 && strcmp(argv[1], "zero") == 0) { + callback_race = true; + tick(u); + assert(reopens == 0 && (uint32_t) u->stall_state == 0); + return 0; + } + tick(u); + assert(reopens == 1 && (uint32_t) u->stall_state == 1); + tick(u); + assert(reopens == 2 && (uint32_t) u->stall_state == 2); + if (argc > 1 && strcmp(argv[1], "exit") == 0) { + atexit(cleanup); + tick(u); + return 42; + } + callback_race = true; + tick(u); + assert(reopens == 2 && (uint32_t) u->stall_state == 0); + callback_race = false; + tick(u); + assert(reopens == 2 && (uint32_t) u->stall_state == 0); + tick(u); + assert(reopens == 3 && (uint32_t) u->stall_state == 1); + u->watchdog_on = false; + tick(u); + assert(reopens == 3 && (uint32_t) u->stall_state == 1); + return 0; +} +'''.replace('CALLBACK', callback).replace('WATCHDOG', watchdog) + with tempfile.TemporaryDirectory() as folder: + path = pathlib.Path(folder) + (path / 'watchdog.c').write_text(constants + '\n' + harness) + subprocess.run(['cc', '-std=c11', '-Werror', str(path / 'watchdog.c'), + '-o', str(path / 'watchdog')], check=True, capture_output=True) + subprocess.run([str(path / 'watchdog')], check=True, timeout=5) + subprocess.run([str(path / 'watchdog'), 'zero'], check=True, timeout=5) + result = subprocess.run([str(path / 'watchdog'), 'exit'], timeout=5) + self.assertEqual(1, result.returncode)