diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt index e81d594e04..9aaa8e98a1 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt @@ -22,6 +22,7 @@ object CloudSaves { private const val TAG = "CloudSaves" private const val BACKUPS = 3 private const val PARALLEL = 6 + private const val CLOUD_LAYOUT = 2 class Result(val result: String, val files: Int, val bytes: Long, val reason: String, val conflicts: Int = 0) { val ok: Boolean get() = result != "failed" @@ -69,8 +70,10 @@ object CloudSaves { GogPrefs.get(context).getString("client_id_$id", null)?.ifEmpty { null } ?: runCatching { JSONObject(File(folder, "goggame-$id.info").readText()).optString("clientId").ifEmpty { null } }.getOrNull() - private fun gogTemplate(context: Context, id: String, folder: File): String? { - val clientId = gogClientId(context, id, folder) ?: return "" + private class GogConfig(val name: String, val location: String) + + private fun gogConfig(context: Context, id: String, folder: File): GogConfig? { + val clientId = gogClientId(context, id, folder) ?: return GogConfig("", "") return try { val c = URL("https://remote-config.gog.com/components/galaxy_client/clients/$clientId?component_version=2.0.45").openConnection() as HttpURLConnection c.connectTimeout = 15_000; c.readTimeout = 15_000 @@ -78,16 +81,42 @@ object CloudSaves { if (c.responseCode !in 200..299) { c.disconnect(); return null } val cloud = JSONObject(c.inputStream.bufferedReader().readText()).optJSONObject("content")?.optJSONObject("Windows")?.optJSONObject("cloudStorage") c.disconnect() - if (cloud?.optBoolean("enabled", false) != true) return "" - val locations = cloud.optJSONArray("locations") ?: return "" + if (cloud?.optBoolean("enabled", false) != true) return GogConfig("", "") + val locations = cloud.optJSONArray("locations") ?: return GogConfig("", "") val all = (0 until locations.length()).mapNotNull { locations.optJSONObject(it) } - (all.firstOrNull { it.optString("name").equals("saves", true) } ?: all.firstOrNull())?.optString("location").orEmpty() + val chosen = all.firstOrNull { it.optString("name").equals("saves", true) } ?: all.firstOrNull() + val location = chosen?.optString("location").orEmpty() + val name = chosen?.optString("name").orEmpty() + if (location.isEmpty()) GogConfig("", "") else GogConfig(name, location) } catch (e: Exception) { Log.w(TAG, "gog remote-config: ${e.javaClass.simpleName}") null } } + private fun gogTemplate(context: Context, id: String, folder: File): String? { + val p = GogPrefs.get(context) + if (p.contains("cloud_template_$id")) return p.getString("cloud_template_$id", "") + val config = gogConfig(context, id, folder) ?: return null + p.edit().putString("cloud_template_$id", config.location).apply { + if (config.name.isNotEmpty()) putString("cloud_location_$id", config.name) + }.apply() + return config.location + } + + /** + * The selected cloud location's name: GOG prefixes every cloud file with it, and the local folder + * is the location itself. Older builds had no name cached; fetch it once for those. + */ + private fun gogLocationName(context: Context, id: String, folder: File): String? { + val p = GogPrefs.get(context) + p.getString("cloud_location_$id", null)?.ifEmpty { null }?.let { return it } + if (p.getString("cloud_template_$id", null)?.isEmpty() == true) return null + val config = gogConfig(context, id, folder) ?: return null + if (config.name.isNotEmpty()) p.edit().putString("cloud_location_$id", config.name).apply() + return config.name.ifEmpty { null } + } + private fun saveFolder(context: Context, store: Store, id: String): Pair { val (folder, sidecar) = game(context, store, id) ?: return null to "not-installed" val template = template(context, store, id, folder, sidecar) ?: return null to "store-unreachable" @@ -114,7 +143,7 @@ object CloudSaves { val galaxy = GogAuth.validToken(context) val clientId = gogClientId(context, id, folder) if (userId.isEmpty() || galaxy == null || clientId == null) null - else GogCloud(userId, clientId, gogScopedToken(context, id, clientId) ?: galaxy) + else GogCloud(userId, clientId, gogScopedToken(context, id, clientId) ?: galaxy, gogLocationName(context, id, folder)) } Store.EPIC -> { val token = EpicCredentialStore.getValidAccessToken(context) @@ -136,6 +165,83 @@ object CloudSaves { private fun saveBaseline(context: Context, store: Store, id: String, b: Map) = prefs(context, store).edit().putString("cloud_baseline_$id", CloudPlan.toJson(b)).apply() + internal class Migration(val baseline: Map, val moved: Int, val cleaned: Int, val failed: Int) + + /** + * Moves only files whose `"/"` key proves the old mapping wrote them; anything + * the game itself created under that name is left where it is. Identical duplicates are removed + * only when they prove the same. The name must be one path segment: it comes from remote config. + */ + internal fun migrateNested(dir: File, name: String, base: Map): Migration? { + if (name.isEmpty() || name == "." || name == ".." || name.contains('/') || name.contains('\\')) return null + val nested = File(dir, name) + if (!nested.isDirectory) return null + val root = runCatching { dir.canonicalFile }.getOrNull() ?: return null + if (runCatching { nested.canonicalFile }.getOrNull()?.parentFile != root) return null + val prefix = "$name/" + val next = HashMap(base) + var moved = 0 + var cleaned = 0 + var failed = 0 + for (src in nested.walkTopDown().filter { it.isFile }.toList()) { + if (runCatching { java.nio.file.Files.isSymbolicLink(src.toPath()) }.getOrDefault(true)) continue + val srcCanon = runCatching { src.canonicalFile }.getOrNull() ?: continue + if (!srcCanon.path.startsWith(root.path + File.separator)) continue + val rel = src.relativeTo(nested).path.replace(File.separatorChar, '/') + val key = prefix + rel + val entry = base[key] ?: continue + val dest = File(dir, rel) + val destCanon = runCatching { dest.canonicalFile }.getOrNull() ?: continue + if (!destCanon.path.startsWith(root.path + File.separator)) continue + if (dest.isFile && runCatching { md5(dest) == md5(src) }.getOrDefault(false)) { + if (!src.delete()) { failed++; continue } + cleaned++ + next[rel] = entry + next.remove(key) + continue + } + if (dest.exists()) continue + val copied = runCatching { + dest.parentFile?.mkdirs() + val tmp = File.createTempFile(dest.name + ".", ".migrate", dest.parentFile) + try { + src.copyTo(tmp, overwrite = true) + if (md5(tmp) != md5(src)) false + else { + java.nio.file.Files.move(tmp.toPath(), dest.toPath(), java.nio.file.StandardCopyOption.ATOMIC_MOVE) + true + } + } finally { + if (tmp.exists()) tmp.delete() + } + }.getOrDefault(false) + if (!copied) { failed++; continue } + if (!src.delete()) { failed++; continue } + moved++ + next[rel] = entry + next.remove(key) + } + if (nested.walkTopDown().none { it.isFile }) nested.deleteRecursively() + return Migration(next, moved, cleaned, failed) + } + + private fun migrateGogNested(context: Context, id: String, folder: File, dir: File) { + val p = GogPrefs.get(context) + // Run once per game: after this the flat layout is authoritative, even if the game itself + // keeps a folder named like the cloud location. + if (p.getInt("cloud_layout_$id", 0) >= CLOUD_LAYOUT) return + val name = gogLocationName(context, id, folder) ?: return + val base = baseline(context, Store.GOG, id).orEmpty() + val result = migrateNested(dir, name, base) + if (result != null) { + if (result.baseline != base) saveBaseline(context, Store.GOG, id, result.baseline) + if (result.moved + result.cleaned > 0) { + StoresState.logLine("cloud gog $id migrate result=ok moved=${result.moved} cleaned=${result.cleaned} from=$name") + } + } + if (result == null || result.failed == 0) p.edit().putInt("cloud_layout_$id", CLOUD_LAYOUT).apply() + } + fun conflicts(context: Context, store: Store, id: String): List = prefs(context, store).getString("cloud_conflicts_$id", "").orEmpty().split('\n').filter { it.isNotEmpty() } @@ -180,6 +286,7 @@ object CloudSaves { !g.second.cloud -> Result("skipped", 0, 0, "off") else -> { val (dir, reason) = saveFolder(context, store, id) + if (dir != null && store == Store.GOG) migrateGogNested(context, id, g.first, dir) val cloud = if (dir != null) transport(context, store, id, g.first) else null when { dir == null -> Result("skipped", 0, 0, reason) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt index 33c34196ff..2b2be1bc48 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt @@ -51,12 +51,43 @@ internal fun isoDate(s: String?): Long { class NoCloudSaves : Exception("no cloud saves") -class GogCloud(private val userId: String, private val clientId: String, private val token: String) : CloudTransport { +/** GOG cloud names are `/`; the name is not part of the folder. */ +internal fun splitGogName(name: String): Pair { + val slash = name.indexOf('/') + return if (slash <= 0 || slash == name.length - 1) null to name + else name.substring(0, slash) to name.substring(slash + 1) +} + +/** Maps cloud names to paths under the selected location's folder; the chosen location name is returned. */ +internal fun gogRelativeNames(files: List, location: String?): Pair> { + val locations = files.mapNotNull { splitGogName(it.name).first }.distinct() + // Only infer a location when every name is prefixed; a mixed listing keeps its old shape. + val allPrefixed = files.isNotEmpty() && files.all { splitGogName(it.name).first != null } + val chosen = when { + location != null && (location in locations || allPrefixed) -> location + location == null && allPrefixed -> locations.singleOrNull() + else -> null + } ?: return null to files + return chosen to files.mapNotNull { file -> + val (dir, rel) = splitGogName(file.name) + if (dir == chosen) CloudFile(rel, file.modifiedMs, file.md5, file.readLink) else null + } +} + +class GogCloud( + private val userId: String, + private val clientId: String, + private val token: String, + private val location: String? = null, +) : CloudTransport { private val base = "https://cloudstorage.gog.com/v1/$userId/$clientId" private val ua = "GOG Galaxy" + private var prefix: String? = location?.let { "$it/" } private fun key(name: String) = URLEncoder.encode(name, "UTF-8").replace("+", "%20") + private fun cloudKey(name: String) = key((prefix ?: "") + name) + override fun list(): List { val c = open(base, "GET", token, ua) val code = c.responseCode @@ -70,20 +101,32 @@ class GogCloud(private val userId: String, private val clientId: String, private val text = c.body().decodeToString().trim() c.disconnect() if (text.isEmpty() || text == "[]") return emptyList() - if (text.startsWith("[")) { + val raw = if (text.startsWith("[")) { val a = JSONArray(text) - return (0 until a.length()).mapNotNull { a.optJSONObject(it) }.mapNotNull { o -> + (0 until a.length()).mapNotNull { a.optJSONObject(it) }.mapNotNull { o -> val n = o.optString("name").ifEmpty { return@mapNotNull null } val lm = o.optLong("last_modified", 0L) CloudFile(n, if (lm > 1_000_000_000_000L) lm else if (lm > 0) lm * 1000 else -1L) } + } else { + text.lines().map { it.trim() }.filter { it.isNotEmpty() }.map { CloudFile(it, -1L) } } - return text.lines().map { it.trim() }.filter { it.isNotEmpty() }.map { CloudFile(it, -1L) } + return relative(raw) + } + + /** + * The API names every file `/`. The location's folder is the sync root, so the + * first segment is dropped. Older builds kept it, writing saves one folder too deep. + */ + private fun relative(raw: List): List { + val (chosen, files) = gogRelativeNames(raw, location) + prefix = chosen?.let { "$it/" } + return files } override fun details(file: CloudFile) { if (file.modifiedMs > 0 && file.md5 != null) return - val c = open("$base/${key(file.name)}", "HEAD", token, ua) + val c = open("$base/${cloudKey(file.name)}", "HEAD", token, ua) if (c.responseCode in 200..299) { httpDate(c.getHeaderField("Last-Modified")).takeIf { it > 0 }?.let { file.modifiedMs = it } c.getHeaderField("ETag")?.replace("\"", "")?.trim()?.ifEmpty { null }?.let { file.md5 = it } @@ -92,14 +135,14 @@ class GogCloud(private val userId: String, private val clientId: String, private } override fun get(file: CloudFile): Pair { - val c = open("$base/${key(file.name)}", "GET", token, ua) + val c = open("$base/${cloudKey(file.name)}", "GET", token, ua) if (c.responseCode !in 200..299) { val code = c.responseCode; c.disconnect(); throw java.io.IOException("GOG cloud get HTTP $code") } val modified = httpDate(c.getHeaderField("Last-Modified")) return c.body().also { c.disconnect() } to modified } override fun put(files: Map): Map = files.mapValues { (name, data) -> - val c = open("$base/${key(name)}", "PUT", token, ua) + val c = open("$base/${cloudKey(name)}", "PUT", token, ua) c.doOutput = true c.setRequestProperty("Content-Type", "application/octet-stream") c.setFixedLengthStreamingMode(data.size) diff --git a/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesMigrationTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesMigrationTest.kt new file mode 100644 index 0000000000..ea4e953e0d --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesMigrationTest.kt @@ -0,0 +1,76 @@ +package com.droiddeck.launcher.stores + +import java.io.File +import java.nio.file.Files +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class CloudSavesMigrationTest { + private val dir = Files.createTempDirectory("cloud-migrate").toFile() + private fun entry(bytes: ByteArray) = CloudPlan.Entry(CloudSaves.md5(bytes), null, 1000L) + + private fun nested(name: String, path: String, text: String): File = + File(dir, "$name/$path").apply { parentFile?.mkdirs(); writeText(text) } + + @Test fun movesOnlyFilesTheOldBaselineOwns() { + val a = "old save".toByteArray() + val game = "game wrote this".toByteArray() + nested("saves", "a.dat", String(a)) + nested("saves", "game.dat", String(game)) + val base = mapOf("saves/a.dat" to entry(a)) + val result = CloudSaves.migrateNested(dir, "saves", base)!! + assertEquals(1, result.moved) + assertEquals(0, result.cleaned) + assertEquals(String(a), File(dir, "a.dat").readText()) + assertEquals(String(game), File(dir, "saves/game.dat").readText()) + assertTrue("saves/a.dat" !in result.baseline) + assertEquals(entry(a), result.baseline["a.dat"]) + } + + @Test fun anIdenticalLeftoverIsRemovedAndItsBaselineKeyRewritten() { + val a = "old save".toByteArray() + File(dir, "a.dat").writeText(String(a)) + nested("saves", "a.dat", String(a)) + val result = CloudSaves.migrateNested(dir, "saves", mapOf("saves/a.dat" to entry(a)))!! + assertEquals(0, result.moved) + assertEquals(1, result.cleaned) + assertFalse(File(dir, "saves").exists()) + assertEquals(entry(a), result.baseline["a.dat"]) + assertTrue("saves/a.dat" !in result.baseline) + } + + @Test fun aDifferentDestinationKeepsTheGameFileAndTheBaseline() { + val a = "old save".toByteArray() + File(dir, "a.dat").writeText("newer game save") + nested("saves", "a.dat", String(a)) + val base = mapOf("saves/a.dat" to entry(a)) + val result = CloudSaves.migrateNested(dir, "saves", base)!! + assertEquals(0, result.moved) + assertEquals(0, result.cleaned) + assertEquals("newer game save", File(dir, "a.dat").readText()) + assertEquals(String(a), File(dir, "saves/a.dat").readText()) + assertEquals(base, result.baseline) + } + + @Test fun aSymlinkedAncestorIsNotMigrated() { + val outside = Files.createTempDirectory("cloud-outside").toFile() + File(outside, "a.dat").writeText("outside") + File(dir, "saves").mkdirs() + Files.createSymbolicLink(File(dir, "saves/link").toPath(), outside.toPath()) + val result = CloudSaves.migrateNested(dir, "saves", + mapOf("saves/link/a.dat" to entry("outside".toByteArray())))!! + assertEquals(0, result.moved) + assertEquals(0, result.cleaned) + assertTrue(File(outside, "a.dat").isFile) + } + + @Test fun aLocationNameThatCanLeaveTheFolderIsRefused() { + nested("saves", "a.dat", "x") + assertNull(CloudSaves.migrateNested(dir, "../saves", emptyMap())) + assertNull(CloudSaves.migrateNested(dir, "..", emptyMap())) + assertNull(CloudSaves.migrateNested(dir, "saves/..", emptyMap())) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/GogCloudNamesTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/GogCloudNamesTest.kt new file mode 100644 index 0000000000..33959c7314 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/GogCloudNamesTest.kt @@ -0,0 +1,52 @@ +package com.droiddeck.launcher.stores + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class GogCloudNamesTest { + private fun files(vararg names: String) = names.map { CloudFile(it, 1000L) } + + @Test fun silksongLocationIsDroppedFromThePath() { + val (location, mapped) = gogRelativeNames( + files("saves/user1.dat", "saves/shared.dat", "saves/Restore_Points1/restoreData2.dat"), "saves") + assertEquals("saves", location) + assertEquals(listOf("user1.dat", "shared.dat", "Restore_Points1/restoreData2.dat"), mapped.map { it.name }) + } + + @Test fun anotherLocationIsFilteredOut() { + val (_, mapped) = gogRelativeNames(files("saves/a.dat", "__default/settings.ini"), "saves") + assertEquals(listOf("a.dat"), mapped.map { it.name }) + } + + @Test fun theOnlyLocationIsInferredWhenItsNameWasNotCached() { + val (location, mapped) = gogRelativeNames(files("saves/a.dat", "saves/b.dat"), null) + assertEquals("saves", location) + assertEquals(listOf("a.dat", "b.dat"), mapped.map { it.name }) + } + + @Test fun namesWithoutALocationPrefixAreLeftAlone() { + val (location, mapped) = gogRelativeNames(files("a.dat", "b.dat"), null) + assertNull(location) + assertEquals(listOf("a.dat", "b.dat"), mapped.map { it.name }) + } + + @Test fun aConfiguredLocationDoesNotYieldToAnotherLocation() { + val (location, mapped) = gogRelativeNames(files("__default/settings.ini"), "saves") + assertEquals("saves", location) + assertTrue(mapped.isEmpty()) + } + + @Test fun aMixedListingWithOneSubfolderIsLeftAlone() { + val (location, mapped) = gogRelativeNames(files("a.dat", "sub/b.dat"), "saves") + assertNull(location) + assertEquals(listOf("a.dat", "sub/b.dat"), mapped.map { it.name }) + } + + @Test fun aMissingCachedLocationDoesNotHideUnprefixedNames() { + val (location, mapped) = gogRelativeNames(files("a.dat"), "saves") + assertNull(location) + assertEquals(listOf("a.dat"), mapped.map { it.name }) + } +} diff --git a/docs/development/stores.md b/docs/development/stores.md index 0d35745b6d..97b119238b 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -269,6 +269,14 @@ the prefix's user folder appears, within a minute. Every path logs game's prefix (`compatdata//pfx/drive_c/users/steamuser`) or its install folder, case-insensitively, and refuses a path that leaves its boundary. A game whose store gives none has no cloud saves. +- **GOG names are location-qualified:** the cloud API lists `saves/user1.dat`, where `saves` is the + `name` of the selected `cloudStorage.locations` entry and the folder is that entry's `location` + (`.../Hollow Knight Silksong/default`). `GogCloud` strips the location segment for the local path + and puts it back for `HEAD`/`GET`/`PUT`; files from other locations are ignored. Builds before + this wrote `/...` one folder too deep (`default/saves/user1.dat`); `migrateGogNested` + moves those copies up, deletes byte-identical leftovers and rewrites their baseline keys. The + migration runs once per game (`cloud_layout_`), so a game that legitimately keeps a folder + named like the location is not relocated again; a non-prefixed listing is left unchanged. - **The services:** GOG `cloudstorage.gog.com/v1//` with a token issued to the game's own client (the client secret is kept at install; the Galaxy token is the fallback); Epic's savesync data storage with signed read and write links (`EpicCloud` → `EpicSaveSync` → the Rust