mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-18 12:28:40 +00:00
fix(audio): restart host after stalled wake recovery (#2396)
* fix(audio): restart host after stalled wake recovery * Serialize audio host restarts and make the stall counter race-free * fix(audio): an overtaken restart leaves the socket alone and a sink guard only acts on its own host --------- Co-authored-by: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com>
This commit is contained in:
1 parent
a8aff1c1a3
commit
cdecd09e3c
4 files changed
+304
-20
No files matched your search
@@ -16,6 +16,8 @@ import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.IntConsumer;
|
||||
import java.util.function.IntSupplier;
|
||||
|
||||
public class PulseAudioComponent extends SessionPart {
|
||||
private static final String TAG = "PulseAudio";
|
||||
@@ -29,7 +31,55 @@ public class PulseAudioComponent extends SessionPart {
|
||||
private final File deviceControls;
|
||||
private File logFile;
|
||||
private final String micFifoPath;
|
||||
private volatile int pid = -1;
|
||||
private final HostLifecycle host = new HostLifecycle(Process::killProcess);
|
||||
|
||||
static final class HostLifecycle {
|
||||
volatile int pid = -1;
|
||||
private long generation;
|
||||
private long launches;
|
||||
private final IntConsumer terminate;
|
||||
|
||||
HostLifecycle(IntConsumer terminate) {
|
||||
this.terminate = terminate;
|
||||
}
|
||||
|
||||
synchronized long begin() {
|
||||
stop();
|
||||
return generation;
|
||||
}
|
||||
|
||||
synchronized void stop() {
|
||||
generation++;
|
||||
if (pid > 1) terminate.accept(pid);
|
||||
pid = -1;
|
||||
}
|
||||
|
||||
synchronized boolean launch(long expected, IntSupplier launch) {
|
||||
if (expected != generation) return false;
|
||||
launches++;
|
||||
pid = launch.getAsInt();
|
||||
return true;
|
||||
}
|
||||
|
||||
synchronized boolean current(long expected) {
|
||||
return expected == generation;
|
||||
}
|
||||
|
||||
/** The latest launch; a sink guard holds it so it never reconfigures a host started after its own. */
|
||||
synchronized long token() {
|
||||
return launches;
|
||||
}
|
||||
|
||||
synchronized boolean live(long token) {
|
||||
return token == launches && pid > 1;
|
||||
}
|
||||
|
||||
synchronized boolean exited(long expected) {
|
||||
if (expected != generation || pid <= 1) return false;
|
||||
pid = -1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public PulseAudioComponent(Context context) {
|
||||
this(context, null);
|
||||
@@ -64,7 +114,12 @@ public class PulseAudioComponent extends SessionPart {
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
stop();
|
||||
start(host.begin());
|
||||
}
|
||||
|
||||
private synchronized void start(long generation) {
|
||||
// An exit callback's restart can be overtaken by a newer start; it must not touch the newer host's socket.
|
||||
if (!host.current(generation)) return;
|
||||
if (!workingDir.isDirectory()) {
|
||||
workingDir.mkdirs();
|
||||
FileUtils.chmod(workingDir, 0771);
|
||||
@@ -118,17 +173,23 @@ public class PulseAudioComponent extends SessionPart {
|
||||
+ " -n --file=default.pa --daemonize=false --use-pid-file=false --exit-idle-time=-1"
|
||||
+ " --log-level=info";
|
||||
final java.io.PrintWriter out = openLog();
|
||||
pid = HostProcess.start(command, env.toArray(new String[0]), workingDir, null,
|
||||
if (!host.launch(generation, () -> HostProcess.start(command, env.toArray(new String[0]), workingDir, status -> {
|
||||
if (host.exited(generation)) {
|
||||
Log.w(TAG, "audio host exited with status " + status + "; restarting");
|
||||
start(generation);
|
||||
}
|
||||
},
|
||||
line -> {
|
||||
Log.i(TAG, line);
|
||||
if (out != null) synchronized (out) { out.println(line); out.flush(); }
|
||||
});
|
||||
Thread guard = new Thread(() -> ensureClientSink(out), "pulse-sink-guard");
|
||||
}))) return;
|
||||
long token = host.token();
|
||||
Thread guard = new Thread(() -> ensureClientSink(out, token), "pulse-sink-guard");
|
||||
guard.setDaemon(true);
|
||||
guard.start();
|
||||
}
|
||||
|
||||
private void ensureClientSink(java.io.PrintWriter out) {
|
||||
private void ensureClientSink(java.io.PrintWriter out, long token) {
|
||||
try {
|
||||
// Nothing that may block (a sink whose AAudio open hangs: a Galaxy S24 FE (US) hung the
|
||||
// daemon's start-up loading one from default.pa) is loaded by the daemon itself; the
|
||||
@@ -137,14 +198,14 @@ public class PulseAudioComponent extends SessionPart {
|
||||
long deadline = started + 30000L;
|
||||
String sinks = null;
|
||||
boolean socketSeen = false;
|
||||
while (System.currentTimeMillis() < deadline && pid > 1) {
|
||||
while (System.currentTimeMillis() < deadline && host.live(token)) {
|
||||
socketSeen |= socket().exists();
|
||||
if (socketSeen && (sinks = pactl("list", "short", "sinks")) != null) break;
|
||||
if (socketSeen && (sinks = pactlFor(token, "list", "short", "sinks")) != null) break;
|
||||
try { Thread.sleep(250L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; }
|
||||
}
|
||||
note(out, "client sink: daemon " + (sinks != null ? "answered" : "did not answer") + " after "
|
||||
+ (System.currentTimeMillis() - started) + " ms (socket " + (socketSeen ? "present" : "never appeared") + ", pactl "
|
||||
+ (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + pid + ")");
|
||||
+ (new File(workingDir, "pactl").isFile() ? "present" : "MISSING") + ", daemon pid " + host.pid + ")");
|
||||
String line;
|
||||
if (sinks == null) {
|
||||
line = "== client sink: the daemon did not answer in 30 s; no fallback attempted";
|
||||
@@ -155,15 +216,15 @@ public class PulseAudioComponent extends SessionPart {
|
||||
// still switching the output route as the session starts (seen on a Galaxy S24 FE):
|
||||
// try DirectAudio again, then the plain sink, a few times over ten seconds.
|
||||
String loadedAs = null;
|
||||
for (int attempt = 1; attempt <= 5 && loadedAs == null && pid > 1; attempt++) {
|
||||
for (int attempt = 1; attempt <= 5 && loadedAs == null && host.live(token); attempt++) {
|
||||
try { Thread.sleep(2000L); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; }
|
||||
if (pactl("load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1",
|
||||
if (pactlFor(token, "load-module", "module-directaudio-native-sink", "sink_name=DirectAudio", "performance_mode=1",
|
||||
"adaptive=1", "decay=0", "limiter=1", "watchdog=1", "volume=1.0", "control_dir=" + deviceControls) != null
|
||||
&& pactl("set-default-sink", "DirectAudio") != null) {
|
||||
&& pactlFor(token, "set-default-sink", "DirectAudio") != null) {
|
||||
loadedAs = "DirectAudio on attempt " + attempt;
|
||||
} else if (pactl("load-module", "module-aaudio-sink", "sink_name=AAudioSink",
|
||||
} else if (pactlFor(token, "load-module", "module-aaudio-sink", "sink_name=AAudioSink",
|
||||
"performance_mode=" + (attempt == 1 ? "1" : "0"), "adaptive=1", "volume=1.0", "control_dir=" + deviceControls) != null
|
||||
&& pactl("set-default-sink", "AAudioSink") != null) {
|
||||
&& pactlFor(token, "set-default-sink", "AAudioSink") != null) {
|
||||
loadedAs = "the plain AAudio sink (AAudioSink) on attempt " + attempt;
|
||||
}
|
||||
}
|
||||
@@ -186,10 +247,14 @@ public class PulseAudioComponent extends SessionPart {
|
||||
if (out != null) synchronized (out) { out.println(line); out.flush(); }
|
||||
}
|
||||
|
||||
private String pactlFor(long token, String... args) {
|
||||
return host.live(token) ? pactl(args) : null;
|
||||
}
|
||||
|
||||
/** Runs pactl against this daemon; its stdout, or null when it failed or timed out. */
|
||||
private String pactl(String... args) {
|
||||
File pactl = new File(workingDir, "pactl");
|
||||
if (!pactl.isFile() || pid <= 1) return null;
|
||||
if (!pactl.isFile() || host.pid <= 1) return null;
|
||||
java.lang.Process process = null;
|
||||
try {
|
||||
File modules = new File(workingDir, "modules/arm64");
|
||||
@@ -231,10 +296,7 @@ public class PulseAudioComponent extends SessionPart {
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
if (pid != -1) {
|
||||
Process.killProcess(pid);
|
||||
pid = -1;
|
||||
}
|
||||
host.stop();
|
||||
}
|
||||
|
||||
public boolean setSinkSuspended(boolean suspended) {
|
||||
@@ -247,7 +309,7 @@ public class PulseAudioComponent extends SessionPart {
|
||||
|
||||
private boolean suspend(String kind, String name, boolean suspended) {
|
||||
File pactl = new File(workingDir, "pactl");
|
||||
if (!pactl.isFile() || pid <= 1) {
|
||||
if (!pactl.isFile() || host.pid <= 1) {
|
||||
Log.w(TAG, "cannot change " + kind + " state: PulseAudio control client or server unavailable");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.droiddeck.launcher.audio
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class PulseAudioComponentTest {
|
||||
@Test
|
||||
fun stopCancelsSetupAndAcceptedRestart() {
|
||||
val killed = mutableListOf<Int>()
|
||||
val host = PulseAudioComponent.HostLifecycle { killed.add(it) }
|
||||
val generation = host.begin()
|
||||
assertTrue(host.launch(generation) { 31463 })
|
||||
assertTrue(host.exited(generation))
|
||||
host.stop()
|
||||
assertFalse(host.launch(generation) { error("cancelled restart launched") })
|
||||
val next = host.begin()
|
||||
host.stop()
|
||||
assertFalse(host.launch(next) { error("cancelled setup launched") })
|
||||
assertEquals(-1, host.pid)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun explicitStartStillWorksAndStaleExitCannotRestartIt() {
|
||||
val killed = mutableListOf<Int>()
|
||||
val host = PulseAudioComponent.HostLifecycle { killed.add(it) }
|
||||
val first = host.begin()
|
||||
host.launch(first) { 31463 }
|
||||
val second = host.begin()
|
||||
host.launch(second) { 31464 }
|
||||
assertEquals(listOf(31463), killed)
|
||||
assertFalse(host.exited(first))
|
||||
assertEquals(31464, host.pid)
|
||||
host.stop()
|
||||
assertEquals(listOf(31463, 31464), killed)
|
||||
assertFalse(host.exited(second))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun anOvertakenRestartIsNoLongerCurrent() {
|
||||
val host = PulseAudioComponent.HostLifecycle { }
|
||||
val first = host.begin()
|
||||
host.launch(first) { 31463 }
|
||||
assertTrue(host.exited(first))
|
||||
assertTrue(host.current(first))
|
||||
val second = host.begin()
|
||||
assertFalse(host.current(first))
|
||||
assertTrue(host.current(second))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aSinkGuardOnlyActsOnItsOwnLaunch() {
|
||||
val host = PulseAudioComponent.HostLifecycle { }
|
||||
val generation = host.begin()
|
||||
host.launch(generation) { 31463 }
|
||||
val first = host.token()
|
||||
assertTrue(host.live(first))
|
||||
assertTrue(host.exited(generation))
|
||||
assertFalse(host.live(first))
|
||||
host.launch(generation) { 31464 }
|
||||
assertFalse(host.live(first))
|
||||
assertTrue(host.live(host.token()))
|
||||
host.stop()
|
||||
assertFalse(host.live(host.token()))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun failedLaunchDoesNotRestartOrTerminateAnInvalidPid() {
|
||||
val host = PulseAudioComponent.HostLifecycle { error("invalid pid terminated") }
|
||||
val generation = host.begin()
|
||||
assertTrue(host.launch(generation) { -1 })
|
||||
assertFalse(host.exited(generation))
|
||||
host.stop()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun earlyExitWaitsForPidPublication() {
|
||||
val host = PulseAudioComponent.HostLifecycle { }
|
||||
val generation = host.begin()
|
||||
val entered = CountDownLatch(1)
|
||||
val pool = Executors.newSingleThreadExecutor()
|
||||
try {
|
||||
lateinit var exit: java.util.concurrent.Future<Boolean>
|
||||
assertTrue(host.launch(generation) {
|
||||
exit = pool.submit<Boolean> {
|
||||
entered.countDown()
|
||||
host.exited(generation)
|
||||
}
|
||||
assertTrue(entered.await(5, TimeUnit.SECONDS))
|
||||
assertFalse(exit.isDone)
|
||||
31463
|
||||
})
|
||||
assertTrue(exit.get(5, TimeUnit.SECONDS))
|
||||
assertEquals(-1, host.pid)
|
||||
assertTrue(host.launch(generation) { 31464 })
|
||||
assertEquals(31464, host.pid)
|
||||
} finally {
|
||||
pool.shutdownNow()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,7 @@ PA_MODULE_USAGE(
|
||||
#define DA_RING_CAP_MS_LOCAL 250
|
||||
#define DA_FREEZE_GAP_NS 500000000ull
|
||||
#define DA_CB_STALL_NS 1000000000ull
|
||||
#define DA_HOST_RESTART_STALLS 3
|
||||
#define DA_DECAY_QUIET_NS 10000000000ull
|
||||
#define DA_DECAY_PUNISH_NS 5000000000ull
|
||||
#define DA_DECAY_MAX_BACKOFF 32
|
||||
@@ -106,6 +107,7 @@ struct userdata {
|
||||
uint64_t last_cb_ns, last_xrun_ns, last_decay_ns, wd_last_tick_ns;
|
||||
int32_t base_buf_frames, decay_floor;
|
||||
unsigned int decay_backoff, cb_count, wd_last_cb;
|
||||
uint64_t stall_state;
|
||||
int reopen_running, reopen_redo;
|
||||
int consuming, primed, starved;
|
||||
int suspended;
|
||||
@@ -154,6 +156,13 @@ static void limit(float *p, uint32_t samples) {
|
||||
}
|
||||
}
|
||||
|
||||
static void record_callback(struct userdata *u) {
|
||||
uint64_t state = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
|
||||
|
||||
while (!__atomic_compare_exchange_n(&u->stall_state, &state,
|
||||
((state >> 32) + 1) << 32, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {}
|
||||
}
|
||||
|
||||
static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *audioData, int32_t numFrames) {
|
||||
struct userdata *u = user;
|
||||
struct da_ring *r = u->ring;
|
||||
@@ -170,6 +179,8 @@ static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *
|
||||
return AAUDIO_CALLBACK_RESULT_CONTINUE;
|
||||
}
|
||||
|
||||
record_callback(u);
|
||||
|
||||
ridx = __atomic_load_n(&r->ridx, __ATOMIC_ACQUIRE);
|
||||
avail = __atomic_load_n(&r->widx, __ATOMIC_ACQUIRE) - ridx;
|
||||
if (!u->primed) {
|
||||
@@ -412,10 +423,12 @@ static void request_reopen(struct userdata *u, const char *why) {
|
||||
static void watchdog(struct userdata *u) {
|
||||
uint64_t now = now_ns(), last_cb, prev_tick;
|
||||
unsigned int cb;
|
||||
uint64_t stalled;
|
||||
|
||||
if (!u->watchdog_on) return;
|
||||
if (!__atomic_load_n(&u->aq, __ATOMIC_SEQ_CST)) return;
|
||||
|
||||
stalled = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
|
||||
prev_tick = u->wd_last_tick_ns;
|
||||
u->wd_last_tick_ns = now;
|
||||
cb = __atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST);
|
||||
@@ -431,6 +444,13 @@ static void watchdog(struct userdata *u) {
|
||||
if (!last_cb || now - last_cb < DA_CB_STALL_NS) return;
|
||||
|
||||
__atomic_store_n(&u->last_cb_ns, now, __ATOMIC_SEQ_CST);
|
||||
if (__atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST) != cb) return;
|
||||
if (!__atomic_compare_exchange_n(&u->stall_state, &stalled, stalled + 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) return;
|
||||
if ((uint32_t) (stalled + 1) >= DA_HOST_RESTART_STALLS) {
|
||||
pa_log("directaudio-native-sink: no data callback after %u stream recoveries; restarting the audio host", (unsigned int) (stalled + 1));
|
||||
_exit(EXIT_FAILURE);
|
||||
return;
|
||||
}
|
||||
request_reopen(u, "data callback stalled");
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
class DirectAudioWatchdogTest(unittest.TestCase):
|
||||
def test_recovery_threshold_callback_race_and_forced_exit(self):
|
||||
source = pathlib.Path('tools/directaudio/sink/module-directaudio-native-sink.c').read_text()
|
||||
watchdog = source[source.index('static void watchdog('):source.index('static void close_output(')]
|
||||
callback = source[source.index('static void record_callback('):
|
||||
source.index('static aaudio_data_callback_result_t out_cb(')]
|
||||
constants = '\n'.join(line for line in source.splitlines()
|
||||
if line.startswith(('#define DA_FREEZE_GAP_NS ', '#define DA_CB_STALL_NS ',
|
||||
'#define DA_HOST_RESTART_STALLS ')))
|
||||
harness = r'''
|
||||
#include <assert.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
typedef void AAudioStream;
|
||||
struct userdata {
|
||||
bool watchdog_on;
|
||||
AAudioStream *aq;
|
||||
uint64_t wd_last_tick_ns, last_cb_ns;
|
||||
unsigned int cb_count, wd_last_cb;
|
||||
uint64_t stall_state;
|
||||
};
|
||||
static uint64_t clock_ns = 10000000000ull;
|
||||
static unsigned int reopens;
|
||||
static bool callback_race;
|
||||
static struct userdata *current;
|
||||
static uint64_t now_ns(void) { return clock_ns; }
|
||||
static void request_reopen(struct userdata *u, const char *why) { reopens++; }
|
||||
#define pa_log(...) ((void)0)
|
||||
CALLBACK
|
||||
static bool exchange(uint64_t *value, uint64_t *expected, uint64_t next,
|
||||
int weak, int success, int failure) {
|
||||
if (callback_race) {
|
||||
struct userdata *u = current;
|
||||
record_callback(u);
|
||||
__atomic_add_fetch(&u->cb_count, 1, __ATOMIC_SEQ_CST);
|
||||
}
|
||||
return __atomic_compare_exchange_n(value, expected, next, weak, success, failure);
|
||||
}
|
||||
#define __atomic_compare_exchange_n exchange
|
||||
WATCHDOG
|
||||
static void cleanup(void) { _exit(99); }
|
||||
static void tick(struct userdata *u) {
|
||||
clock_ns += 1000000000ull;
|
||||
watchdog(u);
|
||||
}
|
||||
int main(int argc, char **argv) {
|
||||
struct userdata state = {.watchdog_on = true, .aq = (void *)1};
|
||||
struct userdata *u = &state;
|
||||
current = u;
|
||||
watchdog(u);
|
||||
if (argc > 1 && strcmp(argv[1], "zero") == 0) {
|
||||
callback_race = true;
|
||||
tick(u);
|
||||
assert(reopens == 0 && (uint32_t) u->stall_state == 0);
|
||||
return 0;
|
||||
}
|
||||
tick(u);
|
||||
assert(reopens == 1 && (uint32_t) u->stall_state == 1);
|
||||
tick(u);
|
||||
assert(reopens == 2 && (uint32_t) u->stall_state == 2);
|
||||
if (argc > 1 && strcmp(argv[1], "exit") == 0) {
|
||||
atexit(cleanup);
|
||||
tick(u);
|
||||
return 42;
|
||||
}
|
||||
callback_race = true;
|
||||
tick(u);
|
||||
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
|
||||
callback_race = false;
|
||||
tick(u);
|
||||
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
|
||||
tick(u);
|
||||
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
|
||||
u->watchdog_on = false;
|
||||
tick(u);
|
||||
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
|
||||
return 0;
|
||||
}
|
||||
'''.replace('CALLBACK', callback).replace('WATCHDOG', watchdog)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = pathlib.Path(folder)
|
||||
(path / 'watchdog.c').write_text(constants + '\n' + harness)
|
||||
subprocess.run(['cc', '-std=c11', '-Werror', str(path / 'watchdog.c'),
|
||||
'-o', str(path / 'watchdog')], check=True, capture_output=True)
|
||||
subprocess.run([str(path / 'watchdog')], check=True, timeout=5)
|
||||
subprocess.run([str(path / 'watchdog'), 'zero'], check=True, timeout=5)
|
||||
result = subprocess.run([str(path / 'watchdog'), 'exit'], timeout=5)
|
||||
self.assertEqual(1, result.returncode)
|
||||
Reference in new issue
Block a user