Logs: every log in app-private storage, shared as one scrubbed zip (#484)

Session folders, tool and installer logs and the stores' log now all live under the
app's private files/logs instead of Download/DroidDeck, and every existing "Share
logs" action bundles them into one zip that goes through a final scrub.

- Where: LinuxRuntime.logDir() = files/logs; sessions keep their dated folders and
  30-session pruning, tools/ and stores/ beside them. The "Logs" setting keeps its
  meaning. Game save backups stay in Download/DroidDeck/Saves.
- Migration: at first start the old session folders, tools/ and loose logs move from
  Download/DroidDeck into files/logs (Saves untouched), and every moved folder is
  scrubbed under the current rules.
- Scrubbing: one rule set (SecretScrub) for the stores' log and the zip. The scrub
  marker carries the rules version, so a rules change re-scrubs every folder at the
  next start (in the background, a few at a time). Steam login account names and
  emails, account-name fields, OAuth codes, tokens in URLs and private LAN
  addresses are removed; Windows exception codes and server addresses are kept.
- Share: one zip of the current session folder plus the last 7 days of stores/ and
  tools/, written to cache/share and handed to the share sheet.
This commit is contained in:
The412Banner authored and GitHub committed 2026-10-09 13:28:03 -04:00
1 parent 06e108a726
commit 71e2ae8683
24 files changed
+598 -121

No files matched your search

@@ -0,0 +1,73 @@
package com.droiddeck.launcher.session
import android.content.Context
import android.util.Log
import com.droiddeck.launcher.core.FileUtils
import com.droiddeck.launcher.runtime.LinuxRuntime
import java.io.File
/**
* Once, on the first start of a build with private logs: the session folders, tools/ and the loose
* tool logs that earlier builds left in `Download/DroidDeck/` move into `files/logs/`. Best effort:
* an item that cannot be moved stays where it is. Moved session folders lose their old scrub markers
* ([SessionArtifacts.unmarkMoved]), so [SessionArtifacts.scrubOlder] scrubs them under the current rules. Nothing else in that folder is touched - game
* save backups (Saves/) stay public - and after this pass the app never reads it again.
*/
object LogMigration {
private const val TAG = "LogMigration"
private const val MARKER = ".moved-from-downloads"
@Synchronized
fun run(context: Context) {
val target = LinuxRuntime.logDir(context).apply { mkdirs() }
val marker = File(target, MARKER)
if (marker.exists()) return
val legacy = LinuxRuntime.legacyLogDir()
val items = if (legacy.isDirectory) legacy.listFiles() else emptyArray()
// Unreadable (no storage access yet): try again next start rather than give up on them.
if (items == null) { Log.w(TAG, "$legacy cannot be listed; logs there stay until it can"); return }
var moved = 0
val movedFolders = ArrayList<File>()
for (f in items) {
val isLog = SessionPaths.isSessionFolder(f) ||
(f.isDirectory && f.name == SessionPaths.TOOLS_DIR) ||
(f.isFile && f.name.endsWith(".log"))
if (!isLog) continue
val dest = File(if (f.isFile) File(target, SessionPaths.TOOLS_DIR).apply { mkdirs() } else target, f.name)
if (dest.exists() && f.name != SessionPaths.TOOLS_DIR) continue
if (move(f, dest)) {
moved++
if (SessionPaths.isSessionFolder(dest)) movedFolders.add(dest)
} else Log.w(TAG, "could not move ${f.name}; left in place")
}
// Earlier builds scrubbed these under older rules (or not at all) and marked them done:
// without their markers, the pass over older folders that follows at the same start puts
// them through the current redactor.
SessionArtifacts.unmarkMoved(movedFolders)
try { marker.writeText("moved $moved item(s)\n") } catch (e: Exception) { Log.w(TAG, "could not mark the move: ${e.message}") }
if (moved > 0) Log.i(TAG, "moved $moved log item(s) from $legacy into $target")
}
/** A rename where it works (same volume), else a copy then the source's removal; false = left as it was. */
private fun move(src: File, dest: File): Boolean {
if (src.renameTo(dest)) return true
return try {
copy(src, dest)
FileUtils.delete(src)
true
} catch (e: Exception) {
Log.w(TAG, "copy of ${src.name} failed: ${e.message}")
false
}
}
private fun copy(src: File, dest: File) {
if (src.isDirectory) {
dest.mkdirs()
src.listFiles()?.forEach { copy(it, File(dest, it.name)) }
} else {
dest.parentFile?.mkdirs()
src.inputStream().use { input -> dest.outputStream().use { input.copyTo(it) } }
}
}
}
@@ -31,16 +31,20 @@ object SessionArtifacts {
/** Written last; a folder without it did not get its ending. */
const val COMPLETE_MARKER = ".complete"
/** Every file in the folder has been through the redactor, own addresses and accounts included (-2: accounts added). */
private const val SCRUBBED_MARKER = ".scrubbed-2"
/**
* Every file in the folder has been through the redactor under its current rules. Named after
* [LogRedactor.RULES_VERSION], so a folder scrubbed under older rules is scrubbed again.
*/
private val SCRUBBED_MARKER = ".scrubbed-r${LogRedactor.RULES_VERSION}"
private val OLD_SCRUBBED_MARKER = Regex("""\.scrubbed-(r?\d+)""")
/**
* Written by a session's own ending, and only when every file in the folder, subfolders
* included (whatever the session script copied into steam/ and droiddeck-esync/ as well as what
* the app wrote), went through the redactor without a failure. It lists each of those files
* with its size and modification time, under the redactor's rules version: a share copies a
* file as it is only when all three still match ([scrubbedFiles]), and scrubs anything else -
* a file added or changed since, a hidden one, or every file of a folder without it.
* with its size and modification time, under the redactor's rules version ([scrubbedFiles]).
* A share no longer relies on it: every text file goes through the share pass on the way into
* the zip regardless.
*/
const val SCRUBBED_TREE_MARKER = ".scrubbed-tree"
@@ -67,24 +71,48 @@ object SessionArtifacts {
}
/**
* Once, for session folders written before every file was scrubbed and before the device's own
* addresses were (network.txt listed them; the client's IPv6 check logs "external address"
* into steam/connection_log.txt): the whole folder, steam/ included, through the redactor.
* A marker records it. Runs at app start with [finishAbandoned].
* Every session folder not yet scrubbed under the redactor's current rules - written before
* every file was scrubbed, before the device's own addresses or the Steam account were, or
* moved in from Download/ ([LogMigration] strips their markers first, [unmarkMoved]): the
* whole folder, steam/ and every other subfolder included, through the redactor, and the
* current marker written in place of the old ones. Runs at app start with [finishAbandoned].
*
* A folder holds some 30 MB of Steam logs, about half a minute's work, so folders go through
* a few at a time; one the process does not get to keeps its old marker and is taken up at
* the next start.
*/
@Synchronized
fun scrubOlder(context: Context) {
val current = SessionPaths.current()
val dirs = LinuxRuntime.debugLogDir().listFiles { f ->
val dirs = LinuxRuntime.logDir(context).listFiles { f ->
SessionPaths.isSessionFolder(f) && f != current && !File(f, SCRUBBED_MARKER).exists()
} ?: return
}?.toList() ?: return
if (dirs.isEmpty()) return
LogRedactor.learnFromRuntime(LinuxRuntime.rootDir(context))
dirs.forEach { dir ->
Record(dir).let { r -> scrubFolder(dir, r); File(dir, "steam").takeIf { it.isDirectory }?.let { scrubFolder(it, r) } }
try { File(dir, SCRUBBED_MARKER).writeText("scrubbed ${now()}\n") } catch (e: Exception) {}
val workers = (Runtime.getRuntime().availableProcessors() / 2).coerceIn(1, 4)
val pool = java.util.concurrent.Executors.newFixedThreadPool(workers) { r ->
Thread({ android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_BACKGROUND); r.run() }, "scrub-logs")
}
val done = try {
pool.invokeAll(dirs.map { dir -> java.util.concurrent.Callable { scrubAndMark(dir) } })
.count { f -> try { f.get() } catch (e: Exception) { false } }
} finally {
pool.shutdown()
}
val failed = dirs.size - done
Log.i(TAG, "logs: scrubbed $done folders under r${LogRedactor.RULES_VERSION}" +
if (failed > 0) ", $failed not fully (again at the next start)" else "")
}
/**
* Folders [LogMigration] just moved in lose every scrub marker an earlier build left, so
* [scrubOlder], which runs right after it, puts them through the current rules whatever those
* markers claimed.
*/
fun unmarkMoved(dirs: List<File>) {
dirs.filter { SessionPaths.isSessionFolder(it) }.forEach { dir ->
dir.listFiles { f -> OLD_SCRUBBED_MARKER.matches(f.name) || f.name == SCRUBBED_TREE_MARKER }?.forEach { it.delete() }
}
Log.i(TAG, "scrubbed ${dirs.size} older session folder(s)")
}
/**
@@ -143,15 +171,17 @@ object SessionArtifacts {
* Records [record]'s folder as scrubbed - every file went through without a failure - or
* leaves it unmarked, so the next share and the pass over older folders scrub it again.
*/
private fun markScrubbed(record: Record) {
private fun markScrubbed(record: Record): Boolean {
val dir = record.dir
val tree = File(dir, SCRUBBED_TREE_MARKER)
// A marker from older rules vouches for nothing now, whether or not this pass succeeded.
dir.listFiles { f -> OLD_SCRUBBED_MARKER.matches(f.name) }?.forEach { it.delete() }
if (record.failed) {
tree.delete()
Log.w(TAG, "$dir is not fully scrubbed; it is scrubbed again when shared")
return
return false
}
try {
return try {
File(dir, SCRUBBED_MARKER).writeText("scrubbed ${now()}\n")
val lines = StringBuilder("rules ${LogRedactor.RULES_VERSION}\n")
record.files.forEach { (f, s) ->
@@ -159,8 +189,11 @@ object SessionArtifacts {
.append(f.relativeTo(dir).path).append('\n')
}
tree.writeText(lines.toString())
true
} catch (e: Exception) {
tree.delete()
File(dir, SCRUBBED_MARKER).delete()
false
}
}
@@ -181,9 +214,8 @@ object SessionArtifacts {
}.toSet()
}
/** The end-of-session scrub of [dir] on its own (steam/ already in place), for tests. */
@androidx.annotation.VisibleForTesting
internal fun scrubAndMark(dir: File) = markScrubbed(scrubTree(Record(dir)))
/** The end-of-session scrub of [dir] on its own (steam/ already in place); also the pass over older and moved folders. */
internal fun scrubAndMark(dir: File): Boolean = markScrubbed(scrubTree(Record(dir)))
/**
* [record]'s folder and every folder under it (steam/, droiddeck-esync/): the session script
@@ -223,7 +255,7 @@ object SessionArtifacts {
*/
@Synchronized
fun finishAbandoned(context: Context) {
val parent = LinuxRuntime.debugLogDir()
val parent = LinuxRuntime.logDir(context)
val abandoned = parent.listFiles { f ->
SessionPaths.isSessionFolder(f) && !File(f, COMPLETE_MARKER).exists()
}?.sortedWith(SessionPaths.chronological) ?: return
@@ -282,9 +314,7 @@ object SessionArtifacts {
val current = SessionPaths.current()
val gone = SessionPaths.sessionFolders(context).filter { it != current }
gone.forEach { com.droiddeck.launcher.core.FileUtils.delete(it) }
com.droiddeck.launcher.core.FileUtils.delete(File(LinuxRuntime.debugLogDir(), SessionPaths.TOOLS_DIR))
// Before tools/, those logs sat loose beside the session folders.
LinuxRuntime.debugLogDir().listFiles { f -> f.isFile && f.name.endsWith(".log") }?.forEach { it.delete() }
com.droiddeck.launcher.core.FileUtils.delete(File(LinuxRuntime.logDir(context), SessionPaths.TOOLS_DIR))
Log.i(TAG, "cleared ${gone.size} session folder(s)")
return gone.size
}
@@ -361,25 +361,9 @@ object SessionFiles {
).joinToString(newline)
/**
* Where the session writes its log. Downloads is the point - a failed run is handed over as a
* folder rather than dug out of app-private storage - but the session script redirects its own
* output there with `exec`, and a redirection a non-interactive shell cannot open ends that
* shell. So a public directory is used only once it is proven writable; otherwise the app's
* own files directory, which is bound into the session anyway, stands in.
* Where the session writes its log: app-private `files/logs`, bound into the session at the same
* path, so the session script's own redirection always opens. A folder leaves the device only
* through Share logs, which scrubs every file again on the way into the zip.
*/
fun logDirectory(context: Context): File {
val public = LinuxRuntime.debugLogDir()
if (public.isDirectory || public.mkdirs()) {
val probe = File(public, ".writable")
try {
if (probe.createNewFile() || probe.isFile) {
probe.delete()
return public
}
} catch (ignored: Exception) {
}
}
Log.w(TAG, "$public is not writable (storage permission?); logging to files/logs")
return File(context.filesDir, "logs").apply { mkdirs() }
}
fun logDirectory(context: Context): File = LinuxRuntime.logDir(context).apply { mkdirs() }
}
@@ -18,9 +18,11 @@ import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
/**
* Zips the most recent session's log folder and hands it to Android's share sheet. Works where
* the folder could not be written to Downloads (no storage permission, or a device that refuses
* it) and landed in the app's private files instead, which a user cannot otherwise reach.
* Share logs: one zip, made when asked, of a session folder (the newest by default) with the last
* seven days of the stores' log and the one-off command logs beside it, handed to Android's share
* sheet. Logs live in app-private `files/logs`; this is how they leave the device. Every text file
* goes through [LogRedactor.redactForShare] on the way in, so even a file written before a rule
* existed is clean in the zip. The zip sits in `cache/share/` until the next share or app start.
*/
object SessionLogShare {
private val main = Handler(Looper.getMainLooper())
@@ -62,7 +64,7 @@ object SessionLogShare {
val files = folder.walkTopDown().filter { it.isFile }
.filterNot { it.parentFile?.name == "steam" && it.length() > STEAM_LOG_MAX_BYTES }.toList()
if (files.isEmpty()) return null
val out = File(context.cacheDir, "shared-logs").apply { deleteRecursively(); mkdirs() }
val out = shareDir(context).apply { deleteRecursively(); mkdirs() }
val zip = File(out, "DroidDeck-${folder.name}.zip")
// Scrubbed on the way into the zip: a session shared while it runs has not had its end-of-
// session pass yet, and the redactor changes nothing in a line that is already clean.
@@ -71,11 +73,11 @@ object SessionLogShare {
// since, goes in as it is; anything else (added or changed since, hidden, or any file of a
// session still running) is scrubbed again. Scrubbing tens of MB of the client's logs on
// every share kept the share sheet from appearing for ten seconds or more.
val scrubbed = SessionArtifacts.scrubbedFiles(folder)
val live = liveSteamLogs(context, folder)
// The stores' own log (last 7 days, app-private, already redacted), under stores/.
val storeLogs = com.droiddeck.launcher.stores.StoreLogFiles.files(context)
val total = (files + live + storeLogs).sumOf { it.length() }.coerceAtLeast(1L)
val logs = LinuxRuntime.logDir(context)
val stores = recent(File(logs, STORES_DIR))
val tools = recent(File(logs, SessionPaths.TOOLS_DIR))
val total = (files + live + stores + tools).sumOf { it.length() }.coerceAtLeast(1L)
var done = 0L
var shown = -1
fun advance(f: File) {
@@ -84,18 +86,20 @@ object SessionLogShare {
if (percent != shown) { shown = percent; onProgress?.invoke(percent / 100f) }
}
ZipOutputStream(zip.outputStream().buffered()).use { z ->
files.forEach { f -> addEntry(z, folder.name + "/" + f.relativeTo(folder).path, f, f.relativeTo(folder).path in scrubbed); advance(f) }
live.forEach { f -> addEntry(z, folder.name + "/steam/" + f.name, f, false); advance(f) }
storeLogs.forEach { f -> addEntry(z, folder.name + "/stores/" + f.name, f, false); advance(f) }
files.forEach { f -> addEntry(z, folder.name + "/" + f.relativeTo(folder).path, f); advance(f) }
live.forEach { f -> addEntry(z, folder.name + "/steam/" + f.name, f); advance(f) }
stores.forEach { f -> addEntry(z, "$STORES_DIR/" + f.name, f); advance(f) }
tools.forEach { f -> addEntry(z, SessionPaths.TOOLS_DIR + "/" + f.name, f); advance(f) }
}
return zip
}
private fun addEntry(z: ZipOutputStream, name: String, f: File, scrubbed: Boolean) {
private fun addEntry(z: ZipOutputStream, name: String, f: File) {
z.putNextEntry(ZipEntry(name))
if (!scrubbed && LogRedactor.isText(f)) {
// Every text file, every time: the folder's own scrub may predate a rule.
if (LogRedactor.isText(f)) {
val w = z.bufferedWriter()
LogRedactor.scrubTo(f, w)
LogRedactor.scrubForShare(f, w)
w.flush()
} else {
f.inputStream().use { it.copyTo(z) }
@@ -119,6 +123,25 @@ object SessionLogShare {
/** The client's content and bootstrap logs grow large over months and say nothing about a session. */
private const val STEAM_LOG_MAX_BYTES = 8L * 1024 * 1024
/** The stores' engine log, one file a day, under files/logs. */
const val STORES_DIR = "stores"
private const val RECENT_DAYS = 7
private fun shareDir(context: Context): File = File(context.cacheDir, "share")
/** The files in [dir] changed in the last [RECENT_DAYS] days. */
private fun recent(dir: File): List<File> {
val since = System.currentTimeMillis() - RECENT_DAYS * 86_400_000L
return dir.listFiles { f -> f.isFile && f.lastModified() >= since }?.sortedBy { it.name }.orEmpty()
}
/** Deletes the last share's zip; at app start, so a zip never outlives the share it was made for. */
fun clear(context: Context) {
shareDir(context).deleteRecursively()
// Where builds before this one put it.
File(context.cacheDir, "shared-logs").deleteRecursively()
}
fun shareIntent(context: Context, zip: File): Intent {
val uri = FileProvider.getUriForFile(context, context.packageName + ".logs", zip)
val send = Intent(Intent.ACTION_SEND)
@@ -8,19 +8,20 @@ import java.util.Date
import java.util.Locale
/**
* One folder per session under `Download/DroidDeck/`, holding everything that session recorded,
* One folder per session under app-private `files/logs/`, holding everything that session recorded,
* named for the day, that day's session number and what was run, so the one a report is about can
* be picked out without opening any:
*
* ```
* Download/DroidDeck/2026-09-30-03-steam/
* files/logs/2026-09-30-03-steam/
* device.txt what this device is, and every setting the session ran with
* session.log the guest session: proot, gamescope, the client's stdout
* wayland.log the app's compositor
* steam.log the Steam client's own log, scrubbed (Steam mode)
* steam/ the rest of the client's logs, scrubbed (Steam mode)
* desktop.log KWin, Plasma and the programs on it (desktop mode)
* Download/DroidDeck/tools/ one-off commands: Flatpak installs, AppImage imports
* files/logs/tools/ one-off commands: Flatpak installs, AppImage imports
* files/logs/stores/ the stores' engine log, one file a day
* ```
*
* Folders from before this naming (`session-20260930-180642`) are still recognised, shared and
@@ -137,8 +138,7 @@ object SessionPaths {
/** Every session folder in the places logs are written, oldest first. */
fun sessionFolders(context: Context): List<File> =
listOf(com.droiddeck.launcher.runtime.LinuxRuntime.debugLogDir(), File(context.filesDir, "logs"))
.flatMap { parent -> parent.listFiles { f -> isSessionFolder(f) }?.toList() ?: emptyList() }
(com.droiddeck.launcher.runtime.LinuxRuntime.logDir(context).listFiles { f -> isSessionFolder(f) }?.toList() ?: emptyList())
.sortedWith(chronological)
internal fun slug(text: String): String =
@@ -493,7 +493,7 @@ object SessionPrefs {
}
/**
* Whether a session writes its folder under Download/DroidDeck. Off, the same logs are kept in
* Whether a session keeps its folder under files/logs. Off, the same logs are kept in
* the app's cache for the session's lifetime (the scripts need somewhere to write) and thrown
* away at the end, so nothing accumulates in Downloads.
*/
@@ -398,7 +398,7 @@ object WinComponents {
* Downloads the component's package (.msi) or installer (.exe) into the runtime and has
* droiddeck-msi-install lay it out in [staging]. Returns the installer's result (product,
* version, counts, notes), or null and why. Everything the installer prints also goes to
* Download/DroidDeck/tools.
* files/logs/tools.
*/
private fun installPackage(context: Context, c: Component, step: Step, staging: File, onProgress: (Progress) -> Unit): Pair<JSONObject?, String> {
val root = LinuxRuntime.rootDir(context)