fix(audio): restart host after stalled wake recovery (#2396)

* fix(audio): restart host after stalled wake recovery

* Serialize audio host restarts and make the stall counter race-free

* fix(audio): an overtaken restart leaves the socket alone and a sink guard only acts on its own host

---------

Co-authored-by: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com>
This commit is contained in:
MaxsTechReviewandKurt Himebauch authored and GitHub committed 2026-11-15 10:31:28 -05:00
1 parent a8aff1c1a3
commit cdecd09e3c
4 files changed
+304 -20

No files matched your search

@@ -16,6 +16,8 @@ import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import java.util.function.IntConsumer;
import java.util.function.IntSupplier;
public class PulseAudioComponent extends SessionPart {
private static final String TAG = "PulseAudio";
@@ -29,7 +31,55 @@ public class PulseAudioComponent extends SessionPart {
private final File deviceControls;
private File logFile;
private final String micFifoPath;
private volatile int pid = -1;
private final HostLifecycle host = new HostLifecycle(Process::killProcess);
static final class HostLifecycle {
volatile int pid = -1;
private long generation;
private long launches;
private final IntConsumer terminate;
HostLifecycle(IntConsumer terminate) {
this.terminate = terminate;
}
synchronized long begin() {
stop();
return generation;
}
synchronized void stop() {
generation++;
if (pid > 1) terminate.accept(pid);
pid = -1;
}
synchronized boolean launch(long expected, IntSupplier launch) {
if (expected != generation) return false;
launches++;
pid = launch.getAsInt();
return true;
}
synchronized boolean current(long expected) {
return expected == generation;
}
/** The latest launch; a sink guard holds it so it never reconfigures a host started after its own. */
synchronized long token() {
return launches;
}
synchronized boolean live(long token) {
return token == launches && pid > 1;
}
synchronized boolean exited(long expected) {
if (expected != generation || pid <= 1) return false;
pid = -1;
return true;
}
}
public PulseAudioComponent(Context context) {
this(context, null);
@@ -64,7 +114,12 @@ public class PulseAudioComponent extends SessionPart {
@Override
public void start() {
stop();
start(host.begin());
}
private synchronized void start(long generation) {
// An exit callback's restart can be overtaken by a newer start; it must not touch the newer host's socket.
if (!host.current(generation)) return;
if (!workingDir.isDirectory()) {
workingDir.mkdirs();
FileUtils.chmod(workingDir, 0771);
@@ -118,17 +173,23 @@ public class PulseAudioComponent extends SessionPart {
+ " -n --file=default.pa --daemonize=false --use-pid-file=false --exit-idle-time=-1"
+ " --log-level=info";
final java.io.PrintWriter out = openLog();
pid = HostProcess.start(command, env.toArray(new String[0]), workingDir, null,
if (!host.launch(generation, () -> HostProcess.start(command, env.toArray(new String[0]), workingDir, status -> {
if (host.exited(generation)) {
Log.w(TAG, "audio host exited with status " + status + "; restarting");
start(generation);
}
},
line -> {
Log.i(TAG, line);
if (out != null) synchronized (out) { out.println(line); out.flush(); }
});
Thread guard = new Thread(() -> ensureClientSink(out), "pulse-sink-guard");
}))) return;
long token = host.token();
Thread guard = new Thread(() -> ensureClientSink(out, token), "pulse-sink-guard");
guard.setDaemon(true);
guard.start();
}
private void ensureClientSink(java.io.PrintWriter out) {
private void ensureClientSink(java.io.PrintWriter out, long token) {
try {
// Nothing that may block (a sink whose AAudio open hangs: a Galaxy S24 FE (US) hung the
// daemon's start-up loading one from default.pa) is loaded by the daemon itself; the
@@ -137,14 +198,14 @@ public class PulseAudioComponent extends SessionPart {
long deadline = started + 30000L;
String sinks = null;
boolean socketSeen = false;
while (System.currentTimeMillis() < deadline && pid > 1) {
while (System.currentTimeMillis() < deadline && host.live(token)) {
socketSeen |= socket().exists();
if (socketSeen && (sinks = pactl("list", "short", "sinks")) != null) break;
if (socketSeen && (sinks = pactlFor(token, "list", "short", "sinks")) != null) break;
try { Thread.sleep(250L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; }
}
note(out, "client sink: daemon " + (sinks != null ? "answered" : "did not answer") + " after "
+ (System.currentTimeMillis() - started) + " ms (socket " + (socketSeen ? "present" : "never appeared") + ", pactl "
+ (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + pid + ")");
+ (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + host.pid + ")");
String line;
if (sinks == null) {
line = "== client sink: the daemon did not answer in 30 s; no fallback attempted";
@@ -155,15 +216,15 @@ public class PulseAudioComponent extends SessionPart {
// still switching the output route as the session starts (seen on a Galaxy S24 FE):
// try DirectAudio again, then the plain sink, a few times over ten seconds.
String loadedAs = null;
for (int attempt = 1; attempt <= 5 && loadedAs == null && pid > 1; attempt++) {
for (int attempt = 1; attempt <= 5 && loadedAs == null && host.live(token); attempt++) {
try { Thread.sleep(2000L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; }
if (pactl("load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1",
if (pactlFor(token, "load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1",
"adaptive=1", "decay=0", "limiter=1", "watchdog=1", "volume=1.0", "control_dir=" + deviceControls) != null
&& pactl("set-default-sink", "DirectAudio") != null) {
&& pactlFor(token, "set-default-sink", "DirectAudio") != null) {
loadedAs = "DirectAudio on attempt " + attempt;
} else if (pactl("load-module", "module-aaudio-sink", "sink_name=AAudioSink",
} else if (pactlFor(token, "load-module", "module-aaudio-sink", "sink_name=AAudioSink",
"performance_mode=" + (attempt == 1 ? "1" : "0"), "adaptive=1", "volume=1.0", "control_dir=" + deviceControls) != null
&& pactl("set-default-sink", "AAudioSink") != null) {
&& pactlFor(token, "set-default-sink", "AAudioSink") != null) {
loadedAs = "the plain AAudio sink (AAudioSink) on attempt " + attempt;
}
}
@@ -186,10 +247,14 @@ public class PulseAudioComponent extends SessionPart {
if (out != null) synchronized (out) { out.println(line); out.flush(); }
}
private String pactlFor(long token, String... args) {
return host.live(token) ? pactl(args) : null;
}
/** Runs pactl against this daemon; its stdout, or null when it failed or timed out. */
private String pactl(String... args) {
File pactl = new File(workingDir, "pactl");
if (!pactl.isFile() || pid <= 1) return null;
if (!pactl.isFile() || host.pid <= 1) return null;
java.lang.Process process = null;
try {
File modules = new File(workingDir, "modules/arm64");
@@ -231,10 +296,7 @@ public class PulseAudioComponent extends SessionPart {
@Override
public void stop() {
if (pid != -1) {
Process.killProcess(pid);
pid = -1;
}
host.stop();
}
public boolean setSinkSuspended(boolean suspended) {
@@ -247,7 +309,7 @@ public class PulseAudioComponent extends SessionPart {
private boolean suspend(String kind, String name, boolean suspended) {
File pactl = new File(workingDir, "pactl");
if (!pactl.isFile() || pid <= 1) {
if (!pactl.isFile() || host.pid <= 1) {
Log.w(TAG, "cannot change " + kind + " state: PulseAudio control client or server unavailable");
return false;
}
@@ -0,0 +1,105 @@
package com.droiddeck.launcher.audio
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.Executors
class PulseAudioComponentTest {
@Test
fun stopCancelsSetupAndAcceptedRestart() {
val killed = mutableListOf<Int>()
val host = PulseAudioComponent.HostLifecycle { killed.add(it) }
val generation = host.begin()
assertTrue(host.launch(generation) { 31463 })
assertTrue(host.exited(generation))
host.stop()
assertFalse(host.launch(generation) { error("cancelled restart launched") })
val next = host.begin()
host.stop()
assertFalse(host.launch(next) { error("cancelled setup launched") })
assertEquals(-1, host.pid)
}
@Test
fun explicitStartStillWorksAndStaleExitCannotRestartIt() {
val killed = mutableListOf<Int>()
val host = PulseAudioComponent.HostLifecycle { killed.add(it) }
val first = host.begin()
host.launch(first) { 31463 }
val second = host.begin()
host.launch(second) { 31464 }
assertEquals(listOf(31463), killed)
assertFalse(host.exited(first))
assertEquals(31464, host.pid)
host.stop()
assertEquals(listOf(31463, 31464), killed)
assertFalse(host.exited(second))
}
@Test
fun anOvertakenRestartIsNoLongerCurrent() {
val host = PulseAudioComponent.HostLifecycle { }
val first = host.begin()
host.launch(first) { 31463 }
assertTrue(host.exited(first))
assertTrue(host.current(first))
val second = host.begin()
assertFalse(host.current(first))
assertTrue(host.current(second))
}
@Test
fun aSinkGuardOnlyActsOnItsOwnLaunch() {
val host = PulseAudioComponent.HostLifecycle { }
val generation = host.begin()
host.launch(generation) { 31463 }
val first = host.token()
assertTrue(host.live(first))
assertTrue(host.exited(generation))
assertFalse(host.live(first))
host.launch(generation) { 31464 }
assertFalse(host.live(first))
assertTrue(host.live(host.token()))
host.stop()
assertFalse(host.live(host.token()))
}
@Test
fun failedLaunchDoesNotRestartOrTerminateAnInvalidPid() {
val host = PulseAudioComponent.HostLifecycle { error("invalid pid terminated") }
val generation = host.begin()
assertTrue(host.launch(generation) { -1 })
assertFalse(host.exited(generation))
host.stop()
}
@Test
fun earlyExitWaitsForPidPublication() {
val host = PulseAudioComponent.HostLifecycle { }
val generation = host.begin()
val entered = CountDownLatch(1)
val pool = Executors.newSingleThreadExecutor()
try {
lateinit var exit: java.util.concurrent.Future<Boolean>
assertTrue(host.launch(generation) {
exit = pool.submit<Boolean> {
entered.countDown()
host.exited(generation)
}
assertTrue(entered.await(5, TimeUnit.SECONDS))
assertFalse(exit.isDone)
31463
})
assertTrue(exit.get(5, TimeUnit.SECONDS))
assertEquals(-1, host.pid)
assertTrue(host.launch(generation) { 31464 })
assertEquals(31464, host.pid)
} finally {
pool.shutdownNow()
}
}
}
@@ -65,6 +65,7 @@ PA_MODULE_USAGE(
#define DA_RING_CAP_MS_LOCAL 250
#define DA_FREEZE_GAP_NS 500000000ull
#define DA_CB_STALL_NS 1000000000ull
#define DA_HOST_RESTART_STALLS 3
#define DA_DECAY_QUIET_NS 10000000000ull
#define DA_DECAY_PUNISH_NS 5000000000ull
#define DA_DECAY_MAX_BACKOFF 32
@@ -106,6 +107,7 @@ struct userdata {
uint64_t last_cb_ns, last_xrun_ns, last_decay_ns, wd_last_tick_ns;
int32_t base_buf_frames, decay_floor;
unsigned int decay_backoff, cb_count, wd_last_cb;
uint64_t stall_state;
int reopen_running, reopen_redo;
int consuming, primed, starved;
int suspended;
@@ -154,6 +156,13 @@ static void limit(float *p, uint32_t samples) {
}
}
static void record_callback(struct userdata *u) {
uint64_t state = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
while (!__atomic_compare_exchange_n(&u->stall_state, &state,
((state >> 32) + 1) << 32, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {}
}
static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *audioData, int32_t numFrames) {
struct userdata *u = user;
struct da_ring *r = u->ring;
@@ -170,6 +179,8 @@ static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *
return AAUDIO_CALLBACK_RESULT_CONTINUE;
}
record_callback(u);
ridx = __atomic_load_n(&r->ridx, __ATOMIC_ACQUIRE);
avail = __atomic_load_n(&r->widx, __ATOMIC_ACQUIRE) - ridx;
if (!u->primed) {
@@ -412,10 +423,12 @@ static void request_reopen(struct userdata *u, const char *why) {
static void watchdog(struct userdata *u) {
uint64_t now = now_ns(), last_cb, prev_tick;
unsigned int cb;
uint64_t stalled;
if (!u->watchdog_on) return;
if (!__atomic_load_n(&u->aq, __ATOMIC_SEQ_CST)) return;
stalled = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
prev_tick = u->wd_last_tick_ns;
u->wd_last_tick_ns = now;
cb = __atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST);
@@ -431,6 +444,13 @@ static void watchdog(struct userdata *u) {
if (!last_cb || now - last_cb < DA_CB_STALL_NS) return;
__atomic_store_n(&u->last_cb_ns, now, __ATOMIC_SEQ_CST);
if (__atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST) != cb) return;
if (!__atomic_compare_exchange_n(&u->stall_state, &stalled, stalled + 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) return;
if ((uint32_t) (stalled + 1) >= DA_HOST_RESTART_STALLS) {
pa_log("directaudio-native-sink: no data callback after %u stream recoveries; restarting the audio host", (unsigned int) (stalled + 1));
_exit(EXIT_FAILURE);
return;
}
request_reopen(u, "data callback stalled");
}
+97
View File
@@ -0,0 +1,97 @@
import pathlib
import subprocess
import tempfile
import unittest
class DirectAudioWatchdogTest(unittest.TestCase):
def test_recovery_threshold_callback_race_and_forced_exit(self):
source = pathlib.Path('tools/directaudio/sink/module-directaudio-native-sink.c').read_text()
watchdog = source[source.index('static void watchdog('):source.index('static void close_output(')]
callback = source[source.index('static void record_callback('):
source.index('static aaudio_data_callback_result_t out_cb(')]
constants = '\n'.join(line for line in source.splitlines()
if line.startswith(('#define DA_FREEZE_GAP_NS ', '#define DA_CB_STALL_NS ',
'#define DA_HOST_RESTART_STALLS ')))
harness = r'''
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
typedef void AAudioStream;
struct userdata {
bool watchdog_on;
AAudioStream *aq;
uint64_t wd_last_tick_ns, last_cb_ns;
unsigned int cb_count, wd_last_cb;
uint64_t stall_state;
};
static uint64_t clock_ns = 10000000000ull;
static unsigned int reopens;
static bool callback_race;
static struct userdata *current;
static uint64_t now_ns(void) { return clock_ns; }
static void request_reopen(struct userdata *u, const char *why) { reopens++; }
#define pa_log(...) ((void)0)
CALLBACK
static bool exchange(uint64_t *value, uint64_t *expected, uint64_t next,
int weak, int success, int failure) {
if (callback_race) {
struct userdata *u = current;
record_callback(u);
__atomic_add_fetch(&u->cb_count, 1, __ATOMIC_SEQ_CST);
}
return __atomic_compare_exchange_n(value, expected, next, weak, success, failure);
}
#define __atomic_compare_exchange_n exchange
WATCHDOG
static void cleanup(void) { _exit(99); }
static void tick(struct userdata *u) {
clock_ns += 1000000000ull;
watchdog(u);
}
int main(int argc, char **argv) {
struct userdata state = {.watchdog_on = true, .aq = (void *)1};
struct userdata *u = &state;
current = u;
watchdog(u);
if (argc > 1 && strcmp(argv[1], "zero") == 0) {
callback_race = true;
tick(u);
assert(reopens == 0 && (uint32_t) u->stall_state == 0);
return 0;
}
tick(u);
assert(reopens == 1 && (uint32_t) u->stall_state == 1);
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 2);
if (argc > 1 && strcmp(argv[1], "exit") == 0) {
atexit(cleanup);
tick(u);
return 42;
}
callback_race = true;
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
callback_race = false;
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
tick(u);
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
u->watchdog_on = false;
tick(u);
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
return 0;
}
'''.replace('CALLBACK', callback).replace('WATCHDOG', watchdog)
with tempfile.TemporaryDirectory() as folder:
path = pathlib.Path(folder)
(path / 'watchdog.c').write_text(constants + '\n' + harness)
subprocess.run(['cc', '-std=c11', '-Werror', str(path / 'watchdog.c'),
'-o', str(path / 'watchdog')], check=True, capture_output=True)
subprocess.run([str(path / 'watchdog')], check=True, timeout=5)
subprocess.run([str(path / 'watchdog'), 'zero'], check=True, timeout=5)
result = subprocess.run([str(path / 'watchdog'), 'exit'], timeout=5)
self.assertEqual(1, result.returncode)