Fix game diagnostics recovery races

This commit is contained in:
wallentx committed 2026-10-18 03:36:30 -05:00
1 parent 536bb11df8
commit 66f5d39bab
11 files changed
+374 -55

No files matched your search

@@ -918,13 +918,19 @@ class MainActivity : ComponentActivity() {
.putExtra(com.droiddeck.launcher.files.FileManagerActivity.EXTRA_START_DIR, dir.absolutePath))
},
onLogs = {
SessionPrefs.setLogsEnabled(this, !SessionPrefs.logsEnabled(this))
val changed = SessionPrefs.setLogsEnabled(this, !SessionPrefs.logsEnabled(this))
if (!changed) android.widget.Toast.makeText(
this, R.string.setup_logging_update_failed, android.widget.Toast.LENGTH_LONG,
).show()
logsEnabled = SessionPrefs.logsEnabled(this)
gameDebugLogsEnabled = SessionPrefs.gameDebugLogsEnabled(this)
gameDebugLogsNeedsRestart = gameDebugLogsNeedsRestart()
},
onGameDebugLogs = {
SessionPrefs.setGameDebugLogsEnabled(this, !SessionPrefs.gameDebugLogsEnabled(this))
val changed = SessionPrefs.setGameDebugLogsEnabled(this, !SessionPrefs.gameDebugLogsEnabled(this))
if (!changed) android.widget.Toast.makeText(
this, R.string.setup_logging_update_failed, android.widget.Toast.LENGTH_LONG,
).show()
gameDebugLogsEnabled = SessionPrefs.gameDebugLogsEnabled(this)
gameDebugLogsNeedsRestart = gameDebugLogsNeedsRestart()
},
@@ -223,9 +223,11 @@ object LogRedactor {
"(?<![\\d.])(?:10\\.\\d{1,3}|192\\.168|172\\.(?:1[6-9]|2\\d|3[01])|169\\.254)\\.\\d{1,3}\\.\\d{1,3}(?!\\.?\\d)"
)
fun scrubTo(src: java.io.File, out: java.io.Writer) {
scrubLines(src, out, ::redact)
}
fun scrubTo(
src: java.io.File,
out: java.io.Writer,
shouldStop: () -> Boolean = { false },
): Boolean = scrubLines(src, out, ::redact, shouldStop)
fun redactForShare(line: String): String {
if (line.isEmpty()) return line
@@ -238,11 +240,18 @@ object LogRedactor {
}
}
fun scrubForShare(src: java.io.File, out: java.io.Writer) {
scrubLines(src, out, ::redactForShare)
}
fun scrubForShare(
src: java.io.File,
out: java.io.Writer,
shouldStop: () -> Boolean = { false },
): Boolean = scrubLines(src, out, ::redactForShare, shouldStop)
private fun scrubLines(src: java.io.File, out: java.io.Writer, scrub: (String) -> String) {
private fun scrubLines(
src: java.io.File,
out: java.io.Writer,
scrub: (String) -> String,
shouldStop: () -> Boolean,
): Boolean {
src.bufferedReader().use { input ->
val chunk = CharArray(8192)
val line = StringBuilder(4096)
@@ -259,8 +268,10 @@ object LogRedactor {
}
while (true) {
if (shouldStop()) return false
val count = input.read(chunk)
if (count < 0) break
if (shouldStop()) return false
for (i in 0 until count) {
val char = chunk[i]
if (pendingCarriageReturn) {
@@ -284,8 +295,12 @@ object LogRedactor {
}
}
}
if (pendingCarriageReturn || pendingContent) emit()
if (pendingCarriageReturn || pendingContent) {
if (shouldStop()) return false
emit()
}
}
return true
}
private fun quoted(value: String): String {
@@ -80,7 +80,11 @@ object GameEnvironmentStore {
}
@Synchronized
fun publish(context: Context, config: GameEnvironment.Config = read(context)) {
fun publish(
context: Context,
config: GameEnvironment.Config = read(context),
gameDebugLogsOverride: Boolean? = null,
) {
val resolved = config.copy(shared = GameEnvironment.defaults(SessionPrefs.fexPreset(context), SessionPrefs.forceSsbs(context)) + config.shared)
val json = encode(resolved)
val dxvk = TextureFiltering.dxvkOptions(
@@ -105,8 +109,8 @@ object GameEnvironmentStore {
if (gameDxvk.length() > 0) json.put(DXVK_CONFIG_GAMES, gameDxvk)
json.put(
GAME_DEBUG_LOGS,
SessionPrefs.logsEnabled(context) && SessionPrefs.gameDebugLogsEnabled(context) &&
SessionPaths.currentSessionLogsRetained(context),
gameDebugLogsOverride ?: (SessionPrefs.logsEnabled(context) &&
SessionPrefs.gameDebugLogsEnabled(context) && SessionPaths.currentSessionLogsRetained(context)),
)
write(File(LinuxRuntime.rootDir(context), GUEST_FILE), json.toString())
}
@@ -16,6 +16,9 @@ object SessionArtifacts {
const val COMPLETE_MARKER = ".complete"
private const val RECOVERY_RUNTIME_CAPTURED_MARKER = ".recovery-runtime-captured"
private const val RECOVERY_RUNTIME_EXPIRED_MARKER = ".recovery-runtime-expired"
private val SCRUBBED_MARKER = ".scrubbed-r${LogRedactor.RULES_VERSION}"
private val OLD_SCRUBBED_MARKER = Regex("""\.scrubbed-(r?\d+)""")
@@ -112,46 +115,79 @@ object SessionArtifacts {
var deferred = false
}
private fun scrubFolder(dir: File, record: Record, stopWhenSessionStarts: Boolean): Boolean {
private sealed class ScrubResult {
class Done(val scrubbed: Scrubbed) : ScrubResult()
object Failed : ScrubResult()
object Deferred : ScrubResult()
}
private fun scrubFolder(dir: File, record: Record, shouldStop: () -> Boolean): Boolean {
val files = dir.listFiles { f -> f.isFile && !f.name.startsWith(".") && f !in record.files } ?: return true
for (f in files) {
if (stopWhenSessionStarts && SessionState.running) return false
if (shouldStop()) return false
if (!LogRedactor.isText(f)) continue
val done = scrubFile(dir, f)
if (done == null) record.failed = true else record.files[f] = done
when (val result = scrubFile(dir, f, shouldStop)) {
is ScrubResult.Done -> record.files[f] = result.scrubbed
ScrubResult.Failed -> record.failed = true
ScrubResult.Deferred -> {
record.failed = true
record.deferred = true
return false
}
}
}
return true
}
private fun scrubFile(dir: File, f: File): Scrubbed? {
private fun scrubFile(dir: File, f: File, shouldStop: () -> Boolean): ScrubResult {
val tmp = File(dir, ".${f.name}.scrub")
return try {
val before = Scrubbed(f.length(), f.lastModified())
tmp.bufferedWriter().use { w -> LogRedactor.scrubTo(f, w) }
val scrubbedLength = tmp.length()
if (sameContents(tmp, f)) {
val scrubbed = tmp.bufferedWriter().use { w -> LogRedactor.scrubTo(f, w, shouldStop) }
if (!scrubbed) {
tmp.delete()
before.takeIf { f.length() == it.length && f.lastModified() == it.modified }
} else {
if (!tmp.renameTo(f)) {
tmp.inputStream().use { input -> f.outputStream().use { output -> input.copyTo(output, 64 * 1024) } }
return ScrubResult.Deferred
}
val scrubbedLength = tmp.length()
when (sameContents(tmp, f, shouldStop)) {
null -> {
tmp.delete()
ScrubResult.Deferred
}
true -> {
tmp.delete()
before.takeIf { f.length() == it.length && f.lastModified() == it.modified }
?.let { ScrubResult.Done(it) } ?: ScrubResult.Failed
}
false -> {
if (shouldStop()) {
tmp.delete()
return ScrubResult.Deferred
}
if (!tmp.renameTo(f)) {
tmp.delete()
return ScrubResult.Failed
}
Scrubbed(scrubbedLength, f.lastModified()).takeIf { f.length() == it.length }
?.let { ScrubResult.Done(it) } ?: ScrubResult.Failed
}
Scrubbed(scrubbedLength, f.lastModified()).takeIf { f.length() == it.length }
}
} catch (e: Exception) {
Log.w(TAG, "could not scrub ${f.name}", e)
tmp.delete()
null
ScrubResult.Failed
}
}
private fun sameContents(a: File, b: File): Boolean {
internal fun sameContents(a: File, b: File, shouldStop: () -> Boolean = { false }): Boolean? {
if (shouldStop()) return null
if (a.length() != b.length()) return false
fun readChunk(input: java.io.InputStream, buffer: ByteArray): Int {
fun readChunk(input: java.io.InputStream, buffer: ByteArray): Int? {
var count = 0
while (count < buffer.size) {
if (shouldStop()) return null
val read = input.read(buffer, count, buffer.size - count)
if (shouldStop()) return null
if (read < 0) break
if (read == 0) continue
count += read
@@ -163,10 +199,11 @@ object SessionArtifacts {
val leftBuffer = ByteArray(64 * 1024)
val rightBuffer = ByteArray(64 * 1024)
while (true) {
val leftCount = readChunk(left, leftBuffer)
val rightCount = readChunk(right, rightBuffer)
val leftCount = readChunk(left, leftBuffer) ?: return null
val rightCount = readChunk(right, rightBuffer) ?: return null
if (leftCount != rightCount) return false
if (leftCount == 0) break
if (shouldStop()) return null
for (i in 0 until leftCount) if (leftBuffer[i] != rightBuffer[i]) return false
}
true
@@ -211,24 +248,29 @@ object SessionArtifacts {
}.toSet()
}
internal fun scrubAndMark(dir: File, stopWhenSessionStarts: Boolean = false): Boolean {
val record = scrubTree(Record(dir), stopWhenSessionStarts)
if (stopWhenSessionStarts && SessionState.running) {
internal fun scrubAndMark(
dir: File,
stopWhenSessionStarts: Boolean = false,
sessionStarted: () -> Boolean = { SessionState.running },
): Boolean {
val shouldStop = { stopWhenSessionStarts && sessionStarted() }
val record = scrubTree(Record(dir), shouldStop)
if (shouldStop()) {
record.failed = true
record.deferred = true
}
return markScrubbed(record)
}
private fun scrubTree(record: Record, stopWhenSessionStarts: Boolean = false): Record {
private fun scrubTree(record: Record, shouldStop: () -> Boolean = { false }): Record {
for (dir in record.dir.walkTopDown().filter { it.isDirectory }) {
if (stopWhenSessionStarts && SessionState.running || !scrubFolder(dir, record, stopWhenSessionStarts)) {
if (shouldStop() || !scrubFolder(dir, record, shouldStop)) {
record.failed = true
record.deferred = true
break
}
}
if (stopWhenSessionStarts && SessionState.running) {
if (shouldStop()) {
record.failed = true
record.deferred = true
}
@@ -255,7 +297,11 @@ object SessionArtifacts {
val logs = File(steamArch?.let { SessionPrefs.steamRoot(context, it) } ?: SessionPrefs.activeSteamRoot(context), "logs")
if (!logs.isDirectory) return
val out = File(dir, "steam").apply { mkdirs() }
val sources = logs.listFiles { f -> f.isFile && f.length() < 8L * 1024 * 1024 } ?: return
val sources = logs.listFiles { f -> f.isFile && f.length() < 8L * 1024 * 1024 }
if (sources == null) {
record.failed = true
return
}
for (src in sources) {
if (stopWhenSessionStarts && SessionState.running) {
record.failed = true
@@ -264,10 +310,19 @@ object SessionArtifacts {
}
try {
val dst = File(out, src.name)
dst.bufferedWriter().use { w -> LogRedactor.scrubTo(src, w) }
val copied = dst.bufferedWriter().use { w ->
LogRedactor.scrubTo(src, w) { stopWhenSessionStarts && SessionState.running }
}
if (!copied) {
dst.delete()
record.failed = true
record.deferred = true
break
}
record.files[dst] = Scrubbed(dst.length(), dst.lastModified())
} catch (e: Exception) {
Log.w(TAG, "could not scrub ${src.name}", e)
record.failed = true
}
}
Log.i(TAG, "collected ${out.listFiles()?.size ?: 0} Steam log(s), scrubbed, into $out")
@@ -282,15 +337,19 @@ object SessionArtifacts {
}?.sortedWith(SessionPaths.chronological) ?: return
if (abandoned.isEmpty()) return
val current = SessionPaths.current()
val latestSession = SessionPaths.sessionFolders(context).lastOrNull()
val latestPreviousSession = SessionPaths.sessionFolders(context).lastOrNull { it != current }
LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
for (dir in abandoned) {
if (stopWhenSessionStarts && SessionState.running) break
if (dir == current) continue
val newest = dir == latestSession
val newest = dir == latestPreviousSession
try {
val recoveryNote = File(dir, "ended-without-teardown.txt")
val recoveryAttempted = recoveryNote.exists()
val runtimeCaptured = File(dir, RECOVERY_RUNTIME_CAPTURED_MARKER)
val runtimeExpired = File(dir, RECOVERY_RUNTIME_EXPIRED_MARKER)
val recoveryPreviouslyAttempted = recoveryNote.exists()
val recoveryAttempted = recoveryPreviouslyAttempted &&
(runtimeCaptured.exists() || runtimeExpired.exists() || !beforeGuestStart)
if (!recoveryAttempted) {
recoveryNote.writeText(
"This session's process ended without running its own teardown - killed by\n" +
@@ -303,6 +362,17 @@ object SessionArtifacts {
"session died of a crash, the entry is in there unless the device rebooted.\n"
)
}
if (recoveryPreviouslyAttempted && !runtimeCaptured.exists() &&
(runtimeExpired.exists() || !beforeGuestStart) &&
!recoveryNote.readText().contains("runtime capture did not finish")) {
recoveryNote.appendText(
"Steam's logs and VAC evidence are not included: runtime capture did not finish " +
"before its evidence expired.\n"
)
}
if (recoveryPreviouslyAttempted && !runtimeCaptured.exists() && !beforeGuestStart) {
runtimeExpired.writeText("runtime evidence no longer belongs to this session, at ${now()}\n")
}
val record = Record(dir)
if (newest && !recoveryAttempted) {
copySteamLogs(
@@ -313,7 +383,22 @@ object SessionArtifacts {
stopWhenSessionStarts = stopWhenSessionStarts,
)
if (!record.deferred && (!stopWhenSessionStarts || !SessionState.running)) {
VacDiagnostics.collect(context, dir)
if (!record.failed && VacDiagnostics.collect(context, dir)) {
try {
runtimeCaptured.writeText("collected at ${now()}\n")
} catch (e: Exception) {
record.failed = true
if (beforeGuestStart) {
runtimeExpired.writeText("runtime capture marker failed, at ${now()}\n")
}
Log.w(TAG, "could not mark runtime capture for $dir", e)
}
} else {
record.failed = true
if (beforeGuestStart) {
runtimeExpired.writeText("runtime capture failed, at ${now()}\n")
}
}
}
}
if (record.deferred || stopWhenSessionStarts && SessionState.running) {
@@ -330,9 +415,10 @@ object SessionArtifacts {
break
}
SessionEvents.record("session.artifacts_recovered", mapOf("newest" to newest), dir)
val scrubbed = scrubTree(record, stopWhenSessionStarts = stopWhenSessionStarts)
markScrubbed(scrubbed)
val scrubbed = scrubTree(record) { stopWhenSessionStarts && SessionState.running }
val marked = markScrubbed(scrubbed)
if (scrubbed.deferred) break
if (!marked) continue
File(dir, COMPLETE_MARKER).writeText("collected: late, at next app start, ${now()}\n")
Log.i(TAG, "finished the abandoned session folder $dir")
} catch (e: Exception) {
@@ -441,18 +441,65 @@ object SessionPrefs {
fun logsEnabled(context: Context): Boolean =
prefs(context).getBoolean("logs", true)
fun setLogsEnabled(context: Context, on: Boolean) {
prefs(context).edit().putBoolean("logs", on).apply()
publishGameEnvironment(context)
}
fun setLogsEnabled(context: Context, on: Boolean): Boolean = setLogPreference(
context,
key = "logs",
oldValue = logsEnabled(context),
newValue = on,
newEffective = on && gameDebugLogsEnabled(context) && SessionPaths.currentSessionLogsRetained(context),
)
/** Extra Proton, Wine and graphics diagnostics for the next game launch; off unless chosen. */
fun gameDebugLogsEnabled(context: Context): Boolean =
prefs(context).getBoolean("gameDebugLogsEnabled", false)
fun setGameDebugLogsEnabled(context: Context, on: Boolean) {
prefs(context).edit().putBoolean("gameDebugLogsEnabled", on).apply()
publishGameEnvironment(context)
fun setGameDebugLogsEnabled(context: Context, on: Boolean): Boolean = setLogPreference(
context,
key = "gameDebugLogsEnabled",
oldValue = gameDebugLogsEnabled(context),
newValue = on,
newEffective = logsEnabled(context) && on && SessionPaths.currentSessionLogsRetained(context),
)
private fun setLogPreference(
context: Context,
key: String,
oldValue: Boolean,
newValue: Boolean,
newEffective: Boolean,
): Boolean {
if (oldValue == newValue) return true
val saved = prefs(context)
val oldEffective = logsEnabled(context) && gameDebugLogsEnabled(context) &&
SessionPaths.currentSessionLogsRetained(context)
fun publish(value: Boolean) = GameEnvironmentStore.publish(context, gameDebugLogsOverride = value)
if (oldEffective && !newEffective) {
try {
publish(false)
} catch (e: Exception) {
android.util.Log.e("GameEnvironment", "Could not disable game diagnostics", e)
return false
}
if (saved.edit().putBoolean(key, newValue).commit()) return true
saved.edit().putBoolean(key, oldValue).commit()
runCatching { publish(oldEffective) }
.onFailure { android.util.Log.e("GameEnvironment", "Could not restore game diagnostics", it) }
return false
}
if (!saved.edit().putBoolean(key, newValue).commit()) {
saved.edit().putBoolean(key, oldValue).commit()
return false
}
return try {
publish(newEffective)
true
} catch (e: Exception) {
saved.edit().putBoolean(key, oldValue).commit()
runCatching { publish(oldEffective) }
.onFailure { android.util.Log.e("GameEnvironment", "Could not restore game diagnostics", it) }
android.util.Log.e("GameEnvironment", "Could not update game diagnostics", e)
false
}
}
fun storageDiagnosticsEnabled(context: Context): Boolean =
@@ -329,6 +329,7 @@ class SessionService : Service() {
com.droiddeck.launcher.stores.StoreLaunchRequests.start(this)
val sessionDir = openSessionFolder()
GameEnvironmentStore.publish(this)
val sessionLog = File(sessionDir, "session.log")
Log.i(TAG, GpuClockPin.start(this))
@@ -68,13 +68,13 @@ object VacDiagnostics {
}
}
fun collect(context: Context, dir: File) {
fun collect(context: Context, dir: File): Boolean {
val root = LinuxRuntime.rootDir(context)
val kept = File(dir, SNAPSHOT_MARKER).takeIf { it.isFile }
val marker = kept ?: File(root, MARKER)
val launchList = if (kept != null) File(dir, SNAPSHOT_LAUNCHES) else File(root, LAUNCHES)
if (!marker.isFile) return
try {
if (!marker.isFile) return true
return try {
val armed = marker.readLines().map { it.trim() }
val since = (armed.getOrNull(0)?.toLongOrNull() ?: 0L) * 1000 - 5_000
val arch = armed.getOrNull(1)?.takeIf { it.isNotEmpty() } ?: SessionPrefs.steamClientArch(context)
@@ -125,8 +125,10 @@ object VacDiagnostics {
File(dir, SNAPSHOT_MARKER).delete()
File(dir, SNAPSHOT_LAUNCHES).delete()
Log.i(TAG, "VAC diagnostics: ${consoles.size} console log(s), ${steamHits.size} client VAC line(s), into $dir")
true
} catch (e: Exception) {
Log.w(TAG, "collecting VAC diagnostics", e)
false
}
}