diff --git a/app/src/main/java/com/droiddeck/launcher/stores/itch/ItchStaging.kt b/app/src/main/java/com/droiddeck/launcher/stores/itch/ItchStaging.kt index 6c2968b23..e19ec2ae5 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/itch/ItchStaging.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/itch/ItchStaging.kt @@ -4,6 +4,7 @@ import org.json.JSONObject import java.io.File import java.io.FileInputStream import java.io.FileOutputStream +import java.io.RandomAccessFile import java.nio.file.Files import java.nio.file.StandardCopyOption import java.security.MessageDigest @@ -12,6 +13,7 @@ internal object ItchStaging { const val DIR = ".droiddeck-update" const val RECORD = ".droiddeck-store.json" private const val JOURNAL = "itch-commit.json" + private const val PROGRESS = "itch-commit-progress.jsonl" private const val INVENTORY = "itch-inventory.json" private const val BACKUPS = "itch-backups" private const val DIRECTORY = "" @@ -112,10 +114,12 @@ internal object ItchStaging { onCommitted: ((Map) -> Unit)? = null, ): Map { require(!Files.isSymbolicLink(to.toPath())) { "The itch.io install folder is a symbolic link" } - val metadata = File(to, ".droiddeck-maintenance").apply { mkdirs() } + val root = to.canonicalFile + val metadata = File(root, ".droiddeck-maintenance").apply { mkdirs() } require(!Files.isSymbolicLink(metadata.toPath())) { "Maintenance metadata directory is a symbolic link" } val backups = File(metadata, BACKUPS) val journal = File(metadata, JOURNAL) + val progress = File(metadata, PROGRESS) val existing = readJournal(journal) val discovered = inventory(from) val existingNext = existing?.getJSONObject("next") @@ -123,7 +127,7 @@ internal object ItchStaging { linkedMapOf().apply { saved.keys().forEach { relative -> val hash = saved.optString(relative) - require(validInventoryEntry(to.canonicalFile, relative, hash)) { "The interrupted itch.io inventory is invalid" } + require(validInventoryEntry(root, relative, hash)) { "The interrupted itch.io inventory is invalid" } put(relative, hash) } for ((relative, hash) in discovered) { @@ -137,26 +141,37 @@ internal object ItchStaging { } ?: discovered val files = JSONObject().apply { next.forEach { (path, hash) -> put(path, hash) } } val obsolete = previous.keys.filter { it !in next }.sorted() - val root = to.canonicalFile val stage = from.canonicalFile val paths = obsolete.sortedWith(compareByDescending { it.count { ch -> ch == '/' } }.thenBy { it }) + next.keys.sortedWith(compareBy { it.count { ch -> ch == '/' } }.thenBy { it }) - val state = existing ?: JSONObject().put("schema", 1).put("next", files).put("obsolete", JSONObject().apply { obsolete.forEach { put(it, previous[it]) } }).put("done", JSONObject()) - check(state.optInt("schema") == 1 && existingNext != null && state.optJSONObject("done") != null && state.optJSONObject("obsolete") != null || existing == null) { + val obsoletePlan = JSONObject().apply { obsolete.forEach { put(it, previous[it]) } } + val legacyDone = existing?.optJSONObject("done") + check(existing == null || existing.optInt("schema") == 1 && existingNext != null && existing.optJSONObject("obsolete") != null) { "An interrupted itch.io replacement has an unsupported journal" } + check(existing == null || !existing.has("done") || legacyDone != null) { + "An interrupted itch.io progress journal is invalid" + } + require(!Files.isSymbolicLink(progress.toPath())) { "The itch.io progress log is a symbolic link" } + val allowedPaths = next.keys + obsoletePlan.keys().asSequence().toSet() + val completed = if (existing != null) readProgress(progress, root, allowedPaths) else linkedSetOf() + if (existing != null && legacyDone != null) { + validateDone(root, legacyDone, allowedPaths) + for (relative in legacyDone.keys().asSequence().filter { legacyDone.optBoolean(it) }) { + if (completed.add(relative)) appendProgress(progress, relative) + } + } require(!Files.isSymbolicLink(backups.toPath())) { "The itch.io backup directory is a symbolic link" } backups.mkdirs() require(!Files.isSymbolicLink(backups.toPath())) { "The itch.io backup directory is a symbolic link" } check(backups.isDirectory) { "The itch.io backup path is not a directory" } - val journalDone = state.optJSONObject("done") for (relative in paths) { val expected = next[relative] if (expected == null) continue val path = relative.removeSuffix("/") val dest = safe(root, path) ?: throw PreflightConflict("Unsafe itch.io archive path: $relative") safe(backups, path) ?: throw PreflightConflict("Unsafe itch.io backup path: $relative") - val wasJournaledDone = journalDone?.optBoolean(relative) == true + val wasJournaledDone = relative in completed if (expected == DIRECTORY) { if (dest.exists() && !dest.isDirectory) { val oldFileHash = previous[path] @@ -180,19 +195,21 @@ internal object ItchStaging { require(safe(stage, path)?.isFile == true) { "Staged itch.io file is missing: $relative" } } } - writeJournal(journal, state) + val plan = JSONObject().put("schema", 1).put("next", files).put("obsolete", obsoletePlan) + if (existing == null || legacyDone != null) { + if (existing == null && progress.exists()) check(progress.delete()) { "Could not clear stale itch.io progress log" } + writeJournal(journal, plan) + } var count = 0 for (relative in paths) { val directory = next[relative] == DIRECTORY val path = relative.removeSuffix("/") val expected = next[relative] val oldHash = previous[relative] - val done = state.optJSONObject("done") ?: JSONObject().also { state.put("done", it) } - val wasJournaledDone = done.optBoolean(relative) + val wasJournaledDone = relative in completed val dest = safe(root, path) if (dest == null && expected == null) { - done.put(relative, true) - writeJournal(journal, state) + if (completed.add(relative)) appendProgress(progress, relative) count++ afterEntry?.invoke(count) continue @@ -201,8 +218,7 @@ internal object ItchStaging { val staged = safe(stage, path) val backup = safe(backups, path) if (backup == null && expected == null) { - done.put(relative, true) - writeJournal(journal, state) + if (completed.add(relative)) appendProgress(progress, relative) count++ afterEntry?.invoke(count) continue @@ -213,8 +229,6 @@ internal object ItchStaging { if (expectedHash != null && actualDest.isFile && sha256(actualDest) == expectedHash) { count++; continue } if (expected == DIRECTORY && actualDest.isDirectory && !Files.isSymbolicLink(actualDest.toPath())) { count++; continue } if (expectedHash == null && expected != DIRECTORY && (!actualDest.isFile || oldHash == null || sha256(actualDest) != oldHash)) { count++; continue } - done.put(relative, false) - writeJournal(journal, state) } if (expected == null) { if (oldHash == DIRECTORY) { @@ -223,8 +237,6 @@ internal object ItchStaging { actualBackup.parentFile?.mkdirs() move(actualDest, actualBackup) } - done.put(relative, true) - writeJournal(journal, state) } else if (directory) { require(!Files.isSymbolicLink(actualDest.toPath())) { "itch.io archive directory collides with a symbolic link: $path" } if (actualDest.exists() && !actualDest.isDirectory) { @@ -234,8 +246,6 @@ internal object ItchStaging { move(actualDest, actualBackup) } check(actualDest.isDirectory || actualDest.mkdirs()) { "Could not create itch.io archive directory: $path" } - done.put(relative, true) - writeJournal(journal, state) } else { if (protectedUserData(path) && actualDest.exists()) { require((oldHash != null && actualDest.isFile && sha256(actualDest) == oldHash) || wasJournaledDone) { @@ -260,41 +270,42 @@ internal object ItchStaging { move(stagedFile, actualDest) check(sha256(actualDest) == expected) { "itch.io replacement failed verification: $relative" } } - done.put(relative, true) - writeJournal(journal, state) } + if (completed.add(relative)) appendProgress(progress, relative) count++ afterEntry?.invoke(count) } onCommitted?.invoke(next) StoreInstallsSafeDelete.delete(from) StoreInstallsSafeDelete.delete(backups) + if (progress.exists()) check(progress.delete()) { "Could not remove itch.io progress log" } journal.delete() return next } fun recover(to: File): Boolean { - val file = File(to, ".droiddeck-maintenance/$JOURNAL") + require(!Files.isSymbolicLink(to.toPath())) { "The itch.io install folder is a symbolic link" } + val root = to.canonicalFile + val file = File(root, ".droiddeck-maintenance/$JOURNAL") val state = readJournal(file) ?: return false require(state.optInt("schema") == 1) { "The interrupted itch.io replacement has an unsupported journal" } val next = state.optJSONObject("next") ?: throw IllegalStateException("The interrupted itch.io inventory is missing") - val done = state.optJSONObject("done") ?: throw IllegalStateException("The interrupted itch.io progress is missing") val obsolete = state.optJSONObject("obsolete") ?: throw IllegalStateException("The interrupted itch.io cleanup list is missing") for (relative in next.keys()) { - require(validInventoryEntry(to.canonicalFile, relative, next.optString(relative))) { + require(validInventoryEntry(root, relative, next.optString(relative))) { "The interrupted itch.io inventory is invalid" } } - for (relative in done.keys()) { - require(safe(to.canonicalFile, relative.removeSuffix("/")) != null && done.get(relative) is Boolean) { - "The interrupted itch.io progress is invalid" - } - } for (relative in obsolete.keys()) { - require(validInventoryEntry(to.canonicalFile, relative, obsolete.optString(relative))) { + require(validInventoryEntry(root, relative, obsolete.optString(relative))) { "The interrupted itch.io cleanup list is invalid" } } + val metadata = file.parentFile + val progress = File(metadata, PROGRESS) + val allowedPaths = next.keys().asSequence().toSet() + obsolete.keys().asSequence().toSet() + readProgress(progress, root, allowedPaths) + state.optJSONObject("done")?.let { validateDone(root, it, allowedPaths) } return true } @@ -306,6 +317,9 @@ internal object ItchStaging { merge(child, dest, false) } else { if (Files.isSymbolicLink(dest.toPath())) dest.delete() + require(!dest.exists() || child.isDirectory == dest.isDirectory) { + "itch.io archive path collides with an existing path: ${child.name}" + } require(!dest.exists() || !dest.isDirectory) { "itch.io archive path collides with a directory: ${child.name}" } dest.parentFile?.mkdirs() move(child, dest) @@ -331,6 +345,54 @@ internal object ItchStaging { safe(root, relative.removeSuffix("/")) != null && (if (value == DIRECTORY) relative.endsWith('/') else value.matches(HASH) && !relative.endsWith('/')) + private fun validateDone(root: File, done: JSONObject, allowed: Set) { + for (relative in done.keys()) { + require(relative in allowed && safe(root, relative.removeSuffix("/")) != null && done.get(relative) is Boolean) { + "The interrupted itch.io progress is invalid" + } + } + } + + private fun readProgress(file: File, root: File, allowed: Set): MutableSet { + require(!Files.isSymbolicLink(file.toPath())) { "The itch.io progress log is a symbolic link" } + if (!file.exists()) return linkedSetOf() + require(file.isFile) { "The itch.io progress log is not a regular file" } + val bytes = file.readBytes() + val text = bytes.toString(Charsets.UTF_8) + val completed = linkedSetOf() + var start = 0 + while (true) { + val end = text.indexOf('\n', start) + if (end < 0) break + val line = text.substring(start, end) + require(line.isNotBlank()) { "The itch.io progress log contains an empty record" } + val relative = runCatching { JSONObject(line).getString("path") } + .getOrElse { throw IllegalStateException("The itch.io progress log is corrupt", it) } + require(relative in allowed && safe(root, relative.removeSuffix("/")) != null) { + "The itch.io progress log contains an invalid path" + } + completed += relative + start = end + 1 + } + val lastNewline = bytes.indexOfLast { it == '\n'.code.toByte() } + if (lastNewline + 1 < bytes.size) { + RandomAccessFile(file, "rw").use { journal -> + journal.setLength((lastNewline + 1).toLong()) + journal.fd.sync() + } + } + return completed + } + + private fun appendProgress(file: File, relative: String) { + require(!Files.isSymbolicLink(file.toPath())) { "The itch.io progress log is a symbolic link" } + val record = JSONObject().put("path", relative).toString() + "\n" + FileOutputStream(file, true).use { output -> + output.write(record.toByteArray(Charsets.UTF_8)) + output.fd.sync() + } + } + private fun sha256(file: File): String { val digest = MessageDigest.getInstance("SHA-256") FileInputStream(file).use { input -> diff --git a/app/src/test/java/com/droiddeck/launcher/stores/itch/ItchStagingTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/itch/ItchStagingTest.kt index 0b1a8a124..0ad1dad7a 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/itch/ItchStagingTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/itch/ItchStagingTest.kt @@ -136,6 +136,87 @@ class ItchStagingTest { assertFalse(ItchStaging.recover(game)) } + @Test fun progressIsAppendedAndTornTrailingRecordIsDiscardedBeforeResume() { + val game = tmp.newFolder("progress-game") + val staged = tmp.newFolder("progress-stage") + file(staged, "a.txt", "a") + file(staged, "b.txt", "b") + assertThrows(IllegalStateException::class.java) { + ItchStaging.commit(staged, game, emptyMap(), afterEntry = { if (it == 1) error("simulated interruption") }) + } + val metadata = File(game, ".droiddeck-maintenance") + val plan = File(metadata, "itch-commit.json") + val progress = File(metadata, "itch-commit-progress.jsonl") + val planBefore = plan.readText() + assertFalse(JSONObject(planBefore).has("done")) + assertTrue(progress.readText().contains("\"path\"")) + progress.appendText("{\"path\":\"torn") + + assertTrue(ItchStaging.recover(game)) + assertFalse(progress.readText().contains("torn")) + ItchStaging.commit(staged, game, emptyMap()) + assertEquals("a", File(game, "a.txt").readText()) + assertEquals("b", File(game, "b.txt").readText()) + assertFalse(plan.exists()) + assertFalse(progress.exists()) + } + + @Test fun completeMalformedProgressRecordFailsClosed() { + val game = tmp.newFolder("bad-progress-game") + val staged = tmp.newFolder("bad-progress-stage") + file(staged, "a.txt", "a") + assertThrows(IllegalStateException::class.java) { + ItchStaging.commit(staged, game, emptyMap(), afterEntry = { error("simulated interruption") }) + } + val progress = File(game, ".droiddeck-maintenance/itch-commit-progress.jsonl") + progress.appendText("{bad}\n") + assertThrows(IllegalStateException::class.java) { ItchStaging.recover(game) } + } + + @Test fun legacyDoneJournalMigratesToAppendOnlyProgress() { + val game = tmp.newFolder("legacy-journal-game") + val staged = tmp.newFolder("legacy-journal-stage") + file(staged, "a.txt", "a") + file(staged, "b.txt", "b") + assertThrows(IllegalStateException::class.java) { + ItchStaging.commit(staged, game, emptyMap(), afterEntry = { if (it == 1) error("simulated interruption") }) + } + val metadata = File(game, ".droiddeck-maintenance") + val journal = File(metadata, "itch-commit.json") + val progress = File(metadata, "itch-commit-progress.jsonl") + val old = JSONObject(journal.readText()) + val done = JSONObject().put("a.txt", true) + journal.writeText(JSONObject().put("schema", 1).put("next", old.getJSONObject("next")) + .put("obsolete", old.getJSONObject("obsolete")).put("done", done).toString()) + progress.delete() + + ItchStaging.commit(staged, game, emptyMap()) + assertEquals("a", File(game, "a.txt").readText()) + assertEquals("b", File(game, "b.txt").readText()) + assertFalse(journal.exists()) + assertFalse(progress.exists()) + } + + @Test fun backupMetadataUsesCanonicalInstallWhenParentIsSymlinked() { + val actualParent = tmp.newFolder("actual-install-parent") + val aliasParent = File(tmp.root, "linked-install-parent") + java.nio.file.Files.createSymbolicLink(aliasParent.toPath(), actualParent.toPath()) + val game = File(aliasParent, "game").also { assertTrue(it.mkdirs()) } + val staged = tmp.newFolder("canonical-stage") + file(game, "bin/game.dat", "old") + file(staged, "bin/game.dat", "new") + val previous = ItchStaging.inventory(game) + + assertThrows(IllegalStateException::class.java) { + ItchStaging.commit(staged, game, previous, afterEntry = { error("simulated interruption") }) + } + val canonicalGame = File(actualParent, "game") + assertTrue(File(canonicalGame, ".droiddeck-maintenance/itch-backups/bin/game.dat").isFile) + assertTrue(ItchStaging.recover(game)) + ItchStaging.commit(staged, game, previous) + assertEquals("new", File(canonicalGame, "bin/game.dat").readText()) + } + @Test fun failedInventoryPublicationKeepsCommitJournalUntilResume() { val game = tmp.newFolder("inventory-publication-game") val staged = tmp.newFolder("inventory-publication-stage")