fix(audio): restart host after stalled wake recovery (#2396)

* fix(audio): restart host after stalled wake recovery

* Serialize audio host restarts and make the stall counter race-free

* fix(audio): an overtaken restart leaves the socket alone and a sink guard only acts on its own host

---------

Co-authored-by: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com>
This commit is contained in:
MaxsTechReviewandKurt Himebauch authored and GitHub committed 2026-11-15 10:31:28 -05:00
1 parent a8aff1c1a3
commit cdecd09e3c
4 files changed
+304 -20

No files matched your search

@@ -65,6 +65,7 @@ PA_MODULE_USAGE(
#define DA_RING_CAP_MS_LOCAL 250
#define DA_FREEZE_GAP_NS 500000000ull
#define DA_CB_STALL_NS 1000000000ull
#define DA_HOST_RESTART_STALLS 3
#define DA_DECAY_QUIET_NS 10000000000ull
#define DA_DECAY_PUNISH_NS 5000000000ull
#define DA_DECAY_MAX_BACKOFF 32
@@ -106,6 +107,7 @@ struct userdata {
uint64_t last_cb_ns, last_xrun_ns, last_decay_ns, wd_last_tick_ns;
int32_t base_buf_frames, decay_floor;
unsigned int decay_backoff, cb_count, wd_last_cb;
uint64_t stall_state;
int reopen_running, reopen_redo;
int consuming, primed, starved;
int suspended;
@@ -154,6 +156,13 @@ static void limit(float *p, uint32_t samples) {
}
}
static void record_callback(struct userdata *u) {
uint64_t state = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
while (!__atomic_compare_exchange_n(&u->stall_state, &state,
((state >> 32) + 1) << 32, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {}
}
static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *audioData, int32_t numFrames) {
struct userdata *u = user;
struct da_ring *r = u->ring;
@@ -170,6 +179,8 @@ static aaudio_data_callback_result_t out_cb(AAudioStream *aq, void *user, void *
return AAUDIO_CALLBACK_RESULT_CONTINUE;
}
record_callback(u);
ridx = __atomic_load_n(&r->ridx, __ATOMIC_ACQUIRE);
avail = __atomic_load_n(&r->widx, __ATOMIC_ACQUIRE) - ridx;
if (!u->primed) {
@@ -412,10 +423,12 @@ static void request_reopen(struct userdata *u, const char *why) {
static void watchdog(struct userdata *u) {
uint64_t now = now_ns(), last_cb, prev_tick;
unsigned int cb;
uint64_t stalled;
if (!u->watchdog_on) return;
if (!__atomic_load_n(&u->aq, __ATOMIC_SEQ_CST)) return;
stalled = __atomic_load_n(&u->stall_state, __ATOMIC_SEQ_CST);
prev_tick = u->wd_last_tick_ns;
u->wd_last_tick_ns = now;
cb = __atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST);
@@ -431,6 +444,13 @@ static void watchdog(struct userdata *u) {
if (!last_cb || now - last_cb < DA_CB_STALL_NS) return;
__atomic_store_n(&u->last_cb_ns, now, __ATOMIC_SEQ_CST);
if (__atomic_load_n(&u->cb_count, __ATOMIC_SEQ_CST) != cb) return;
if (!__atomic_compare_exchange_n(&u->stall_state, &stalled, stalled + 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) return;
if ((uint32_t) (stalled + 1) >= DA_HOST_RESTART_STALLS) {
pa_log("directaudio-native-sink: no data callback after %u stream recoveries; restarting the audio host", (unsigned int) (stalled + 1));
_exit(EXIT_FAILURE);
return;
}
request_reopen(u, "data callback stalled");
}
+97
View File
@@ -0,0 +1,97 @@
import pathlib
import subprocess
import tempfile
import unittest
class DirectAudioWatchdogTest(unittest.TestCase):
def test_recovery_threshold_callback_race_and_forced_exit(self):
source = pathlib.Path('tools/directaudio/sink/module-directaudio-native-sink.c').read_text()
watchdog = source[source.index('static void watchdog('):source.index('static void close_output(')]
callback = source[source.index('static void record_callback('):
source.index('static aaudio_data_callback_result_t out_cb(')]
constants = '\n'.join(line for line in source.splitlines()
if line.startswith(('#define DA_FREEZE_GAP_NS ', '#define DA_CB_STALL_NS ',
'#define DA_HOST_RESTART_STALLS ')))
harness = r'''
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
typedef void AAudioStream;
struct userdata {
bool watchdog_on;
AAudioStream *aq;
uint64_t wd_last_tick_ns, last_cb_ns;
unsigned int cb_count, wd_last_cb;
uint64_t stall_state;
};
static uint64_t clock_ns = 10000000000ull;
static unsigned int reopens;
static bool callback_race;
static struct userdata *current;
static uint64_t now_ns(void) { return clock_ns; }
static void request_reopen(struct userdata *u, const char *why) { reopens++; }
#define pa_log(...) ((void)0)
CALLBACK
static bool exchange(uint64_t *value, uint64_t *expected, uint64_t next,
int weak, int success, int failure) {
if (callback_race) {
struct userdata *u = current;
record_callback(u);
__atomic_add_fetch(&u->cb_count, 1, __ATOMIC_SEQ_CST);
}
return __atomic_compare_exchange_n(value, expected, next, weak, success, failure);
}
#define __atomic_compare_exchange_n exchange
WATCHDOG
static void cleanup(void) { _exit(99); }
static void tick(struct userdata *u) {
clock_ns += 1000000000ull;
watchdog(u);
}
int main(int argc, char **argv) {
struct userdata state = {.watchdog_on = true, .aq = (void *)1};
struct userdata *u = &state;
current = u;
watchdog(u);
if (argc > 1 && strcmp(argv[1], "zero") == 0) {
callback_race = true;
tick(u);
assert(reopens == 0 && (uint32_t) u->stall_state == 0);
return 0;
}
tick(u);
assert(reopens == 1 && (uint32_t) u->stall_state == 1);
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 2);
if (argc > 1 && strcmp(argv[1], "exit") == 0) {
atexit(cleanup);
tick(u);
return 42;
}
callback_race = true;
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
callback_race = false;
tick(u);
assert(reopens == 2 && (uint32_t) u->stall_state == 0);
tick(u);
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
u->watchdog_on = false;
tick(u);
assert(reopens == 3 && (uint32_t) u->stall_state == 1);
return 0;
}
'''.replace('CALLBACK', callback).replace('WATCHDOG', watchdog)
with tempfile.TemporaryDirectory() as folder:
path = pathlib.Path(folder)
(path / 'watchdog.c').write_text(constants + '\n' + harness)
subprocess.run(['cc', '-std=c11', '-Werror', str(path / 'watchdog.c'),
'-o', str(path / 'watchdog')], check=True, capture_output=True)
subprocess.run([str(path / 'watchdog')], check=True, timeout=5)
subprocess.run([str(path / 'watchdog'), 'zero'], check=True, timeout=5)
result = subprocess.run([str(path / 'watchdog'), 'exit'], timeout=5)
self.assertEqual(1, result.returncode)