From cd452e2b608c4dbb9c42c0b7b84c8e7864aa93cd Mon Sep 17 00:00:00 2001 From: Moroni Granja Date: Thu, 5 Nov 2026 20:56:21 -0300 Subject: [PATCH] fix: upload GOG cloud saves in Galaxy's gzip + metadata format (#1952) fix: upload GOG cloud saves in Galaxy's gzip format --- .../droiddeck/launcher/stores/CloudPlan.kt | 13 ++ .../droiddeck/launcher/stores/CloudSaves.kt | 27 +++- .../launcher/stores/CloudTransport.kt | 123 ++++++++++++++++-- .../launcher/stores/CloudPlanTest.kt | 16 +++ .../launcher/stores/CloudSavesTest.kt | 2 +- .../launcher/stores/GogCloudKeysTest.kt | 75 +++++++++++ docs/development/stores.md | 10 ++ 7 files changed, 244 insertions(+), 22 deletions(-) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudPlan.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudPlan.kt index 6368f558a..d3c63b203 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudPlan.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudPlan.kt @@ -52,6 +52,19 @@ object CloudPlan { return Plan(transfer, conflicts, same) } + /** + * Files to send. GOG objects uploaded by older builds are raw while their cloud hash is still the + * local raw MD5, so they appear in [Plan.same]; resending them stores the gzip form Galaxy reads. + * A forced GOG upload is the manual repair path and sends every local file, including unchanged + * ones the service gives no hash for; once rewritten the cloud hash is the compressed MD5. + */ + fun upSend(plan: Plan, force: Boolean, repairRawGog: Boolean, allLocal: Collection = emptyList()): List { + if (force) { + return (plan.transfer + plan.conflicts + if (repairRawGog) allLocal else emptyList()).distinct() + } + return (plan.transfer + if (repairRawGog) plan.same else emptyList()).distinct() + } + fun toJson(baseline: Map): String = JSONObject().apply { for ((k, e) in baseline) put(k, JSONObject().put("l", e.localMd5).put("c", e.cloudMd5 ?: "").put("t", e.cloudModified)) }.toString() diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt index 2af7a9bff..48244c489 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt @@ -374,18 +374,31 @@ object CloudSaves { val remote = remote(cloud) { it in files } val local = files.mapValues { CloudPlan.Local(md5(it.value)) } val plan = CloudPlan.up(local, remote.mapValues { CloudPlan.Remote(it.value.md5, it.value.modifiedMs) }, base) - val send = if (force) plan.transfer + plan.conflicts else plan.transfer + val send = CloudPlan.upSend(plan, force, cloud is GogCloud, files.keys) val next = HashMap(base) for (name in plan.same) remote[name]?.let { r -> next[name] = CloudPlan.Entry(local.getValue(name).md5, r.md5, r.modifiedMs) } if (send.isNotEmpty()) { backupCloud(context, store, id, cloud, send.mapNotNull { remote[it] }) - val data = send.associateWith { files.getValue(it).readBytes() } - val stamped = cloud.put(data) - for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t) - val after = remote(cloud) { it in data } - for (name in data.keys) { + val stamped = HashMap() + val sentMd5 = HashMap() + if (cloud is GogCloud) { + // Stream one save at a time: gzip needs a full-size buffer and uploads can be large. + val filesToSend = send.associateWith { files.getValue(it) } + val results = cloud.putFiles(filesToSend, send.associateWith { files.getValue(it).lastModified() }) + for ((name, result) in results) { + if (result.modifiedMs > 0) stamped[name] = result.modifiedMs + result.md5?.let { sentMd5[name] = it } + } + } else { + val data = send.associateWith { files.getValue(it).readBytes() } + for ((name, bytes) in data) sentMd5[name] = md5(bytes) + stamped += cloud.put(data, send.associateWith { files.getValue(it).lastModified() }) + for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t) + } + val after = remote(cloud) { it in send } + for (name in send) { val r = after[name] - next[name] = CloudPlan.Entry(md5(data.getValue(name)), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L) + next[name] = CloudPlan.Entry(sentMd5.getValue(name), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L) } } saveBaseline(context, store, id, next) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt index 3d46a28da..ecf318068 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt @@ -1,24 +1,43 @@ package com.droiddeck.launcher.stores import java.io.ByteArrayOutputStream +import java.io.File +import java.io.FileOutputStream +import java.io.InputStream +import java.io.OutputStream import java.net.HttpURLConnection import java.net.URL import java.net.URLEncoder +import java.security.DigestOutputStream import java.text.SimpleDateFormat +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit import java.util.Locale import java.util.TimeZone import java.util.zip.CRC32 import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream import org.json.JSONArray import org.json.JSONObject class CloudFile(val name: String, var modifiedMs: Long, var md5: String? = null, val readLink: String? = null) +class CloudPut(val modifiedMs: Long, val md5: String?) + interface CloudTransport { fun list(): List fun details(file: CloudFile) {} fun get(file: CloudFile): Pair - fun put(files: Map): Map + fun put(files: Map, modifiedMs: Map = emptyMap()): Map + + /** Uploads from files so a transport can stream instead of holding every save in memory. */ + fun putFiles(files: Map, modifiedMs: Map = emptyMap()): Map { + val data = files.mapValues { it.value.readBytes() } + val digests = data.mapValues { md5Hex(it.value) } + return put(data, modifiedMs).mapValues { (name, modified) -> CloudPut(modified, digests.getValue(name)) } + } } private const val TIMEOUT = 30_000 @@ -47,6 +66,9 @@ internal fun gunzip(data: ByteArray): ByteArray? = try { java.util.zip.GZIPInputStream(data.inputStream()).use { it.readBytes() } } catch (e: Exception) { null } +/** Same gzip GOG uses on PUT: deflate level 6, mtime 0, OS Unix. */ +private val GOG_GZIP_HEADER = byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3) + internal fun gzipGogPut(data: ByteArray): ByteArray { val deflater = Deflater(6, true) val raw = ByteArrayOutputStream() @@ -63,14 +85,14 @@ internal fun gzipGogPut(data: ByteArray): ByteArray { } val crc = CRC32().also { it.update(data) } return ByteArrayOutputStream(18 + raw.size()).apply { - write(byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3)) + write(GOG_GZIP_HEADER) write(raw.toByteArray()) writeIntLe(crc.value.toInt()) writeIntLe(data.size) }.toByteArray() } -private fun ByteArrayOutputStream.writeIntLe(v: Int) { +private fun OutputStream.writeIntLe(v: Int) { write(v and 0xff) write(v ushr 8 and 0xff) write(v ushr 16 and 0xff) @@ -117,13 +139,64 @@ internal fun gogRelativeNames(files: List, location: String?): Pair + val buf = ByteArray(64 * 1024) + while (true) { + val n = input.read(buf) + if (n < 0) break + raw.update(buf, 0, n) + crc.update(buf, 0, n) + size += n + deflate.write(buf, 0, n) + } + } + deflate.finish() + } finally { + deflater.end() + } + out.writeIntLe(crc.value.toInt()) + out.writeIntLe(size.toInt()) + out.flush() + return GzipFile(dst.length(), raw.digest().toHex(), compressed.digest().toHex()) + } finally { + out.close() + } +} + +/** GOG Galaxy parses `X-Object-Meta-LocalLastModified`; objects without it fail as "invalid data". */ +private val GOG_STAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssxxx", Locale.US) + +private fun gogStamp(ms: Long): String = + Instant.ofEpochMilli(ms).atOffset(ZoneOffset.UTC).truncatedTo(ChronoUnit.SECONDS).format(GOG_STAMP) + class GogCloud( private val userId: String, private val clientId: String, private val token: String, private val location: String? = null, + baseUrl: String? = null, ) : CloudTransport { - private val base = "https://cloudstorage.gog.com/v1/$userId/$clientId" + private val base = baseUrl ?: "https://cloudstorage.gog.com/v1/$userId/$clientId" private val ua = "GOG Galaxy" private var prefix: String? = location?.let { "$it/" } @@ -190,16 +263,38 @@ class GogCloud( return gogDecodeCloudBody(data, encoding) to modified } - override fun put(files: Map): Map = files.mapValues { (name, data) -> + override fun put(files: Map, modifiedMs: Map): Map = files.mapValues { (name, data) -> + val body = gzipGogPut(data) + putBody(name, body.inputStream(), body.size.toLong(), md5Hex(body), modifiedMs[name] ?: System.currentTimeMillis()) + } + + override fun putFiles(files: Map, modifiedMs: Map): Map = files.mapValues { (name, file) -> + val compressed = File.createTempFile("gog-cloud-", ".gz") + try { + val gz = gzipFile(file, compressed) + val modified = putBody(name, compressed.inputStream(), gz.length, gz.etag, modifiedMs[name] ?: file.lastModified()) + CloudPut(modified, gz.rawMd5) + } finally { + compressed.delete() + } + } + + private fun putBody(name: String, source: InputStream, length: Long, etag: String, modified: Long): Long { val c = open("$base/${cloudKey(name)}", "PUT", token, ua) - c.doOutput = true - c.setRequestProperty("Content-Type", "application/octet-stream") - c.setFixedLengthStreamingMode(data.size) - c.outputStream.use { it.write(data) } - val code = c.responseCode - c.disconnect() - if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code") - CloudFile(name, -1L).also { details(it) }.modifiedMs + try { + c.doOutput = true + c.setRequestProperty("Content-Type", "application/octet-stream") + c.setRequestProperty("Content-Encoding", "gzip") + c.setRequestProperty("Etag", etag) + c.setRequestProperty("X-Object-Meta-LocalLastModified", gogStamp(modified)) + c.setFixedLengthStreamingMode(length) + c.outputStream.use { out -> source.use { it.copyTo(out, 64 * 1024) } } + val code = c.responseCode + if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code") + return CloudFile(name, -1L).also { details(it) }.modifiedMs + } finally { + c.disconnect() + } } } @@ -208,6 +303,6 @@ class EpicCloud(private val accountId: String, private val appName: String, priv override fun get(file: CloudFile): Pair = com.droiddeck.launcher.stores.epic.EpicSaveSync.read(file) - override fun put(files: Map): Map = + override fun put(files: Map, modifiedMs: Map): Map = com.droiddeck.launcher.stores.epic.EpicSaveSync.write(token, accountId, appName, files) } diff --git a/app/src/test/java/com/droiddeck/launcher/stores/CloudPlanTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/CloudPlanTest.kt index 5338fd343..1979bece2 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/CloudPlanTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/CloudPlanTest.kt @@ -54,6 +54,22 @@ class CloudPlanTest { assertTrue(CloudPlan.down(mapOf("s" to Local("a")), mapOf("s" to Remote(null, 2500)), base).transfer.isEmpty()) } + @Test fun gogRawObjectsThatStillMatchTheLocalBytesAreResentOnce() { + val base = mapOf("s" to Entry("a", "a", 1000)) + val plan = CloudPlan.up(mapOf("s" to Local("a")), mapOf("s" to Remote("a", 1000)), base) + assertEquals(listOf("s"), plan.same) + assertEquals(listOf("s"), CloudPlan.upSend(plan, force = false, repairRawGog = true)) + assertTrue(CloudPlan.upSend(plan, force = false, repairRawGog = false).isEmpty()) + } + + @Test fun aForcedGogUploadSendsUnchangedFilesEvenWithoutACloudHash() { + val base = mapOf("s" to Entry("a", "a", 1000)) + val plan = CloudPlan.up(mapOf("s" to Local("a")), mapOf("s" to Remote(null, 1000)), base) + assertTrue(plan.transfer.isEmpty() && plan.conflicts.isEmpty() && plan.same.isEmpty()) + assertEquals(listOf("s"), CloudPlan.upSend(plan, force = true, repairRawGog = true, allLocal = listOf("s"))) + assertTrue(CloudPlan.upSend(plan, force = true, repairRawGog = false, allLocal = listOf("s")).isEmpty()) + } + @Test fun theBaselineSurvivesItsJson() { val b = mapOf("a/b.sav" to Entry("l", "c", 5), "x" to Entry("l2", null, -1)) assertEquals(b, CloudPlan.fromJson(CloudPlan.toJson(b))) diff --git a/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesTest.kt index 0b8003262..9f1dcd2b0 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/CloudSavesTest.kt @@ -22,7 +22,7 @@ class CloudSavesTest { val puts = ArrayList() override fun list() = files.map { (k, v) -> CloudFile(k, v.second, CloudSaves.md5(v.first)) } override fun get(file: CloudFile) = files.getValue(file.name) - override fun put(files: Map): Map = + override fun put(files: Map, modifiedMs: Map): Map = files.mapValues { (k, v) -> puts += k; this.files[k] = v to 99_000L; 99_000L } } diff --git a/app/src/test/java/com/droiddeck/launcher/stores/GogCloudKeysTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/GogCloudKeysTest.kt index df7ba8f2b..025dbce9b 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/GogCloudKeysTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/GogCloudKeysTest.kt @@ -56,6 +56,81 @@ class GogCloudKeysTest { assertFalse(gzipGogPut(raw).contentEquals(gameGz)) assertArrayEquals(gameGz, gogDecodeCloudBody(gameGz, null)) } + + @Test fun putUploadsTheGogGzipBodyWithEncodingAndCompressedHash() { + val raw = "savegame0".toByteArray() + val server = java.net.ServerSocket(0, 4, java.net.InetAddress.getByName("127.0.0.1")) + val requests = java.util.concurrent.CopyOnWriteArrayList() + val worker = Thread { + repeat(2) { + server.accept().use { client -> + client.soTimeout = 5_000 + val input = client.getInputStream() + val line = readLine(input) + val parts = line.split(' ') + val headers = LinkedHashMap() + while (true) { + val h = readLine(input) + if (h.isEmpty()) break + val colon = h.indexOf(':') + headers[h.substring(0, colon).trim().lowercase()] = h.substring(colon + 1).trim() + } + val length = headers["content-length"]?.toIntOrNull() ?: 0 + val body = ByteArray(length) + var offset = 0 + while (offset < length) { + val n = input.read(body, offset, length - offset) + if (n < 0) break + offset += n + } + requests += Request(parts[0], parts[1], headers, body) + val response = if (parts[0] == "PUT") { + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n" + + "Etag: \"${CloudSaves.md5(body)}\"\r\n" + + "Last-Modified: Tue, 20 Oct 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n" + } + client.getOutputStream().apply { write(response.toByteArray()); flush() } + } + } + }.apply { isDaemon = true; start() } + try { + val cloud = GogCloud("user", "client", "token", "saves", "http://127.0.0.1:${server.localPort}/v1/user/client") + val source = java.io.File.createTempFile("save", ".sav").apply { writeBytes(raw) } + val result = try { + cloud.putFiles(mapOf("save.sav" to source), mapOf("save.sav" to 1_793_490_825_000L)).getValue("save.sav") + } finally { + source.delete() + } + assertTrue(result.modifiedMs > 0) + assertEquals(CloudSaves.md5(raw), result.md5) + val put = requests[0] + assertEquals("PUT", put.method) + assertEquals("/v1/user/client/saves%2Fsave.sav", put.path) + assertEquals("gzip", put.headers["content-encoding"]) + assertEquals("2026-10-31T23:53:45+00:00", put.headers["x-object-meta-locallastmodified"]) + assertArrayEquals(gzipGogPut(raw), put.body) + assertEquals(CloudSaves.md5(put.body), put.headers["etag"]) + assertArrayEquals(raw, gogDecodeCloudBody(put.body, put.headers["content-encoding"])) + assertEquals("HEAD", requests[1].method) + } finally { + server.close() + worker.join(2_000) + } + } +} + +private class Request(val method: String, val path: String, val headers: Map, val body: ByteArray) + +private fun readLine(input: java.io.InputStream): String { + val out = ByteArrayOutputStream() + while (true) { + val b = input.read() + if (b < 0 || b == '\n'.code) break + if (b != '\r'.code) out.write(b) + } + return out.toString("UTF-8") } private fun String.hex() = chunked(2).map { it.toInt(16).toByte() }.toByteArray() diff --git a/docs/development/stores.md b/docs/development/stores.md index 5d11e48b6..78855ea74 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -300,6 +300,16 @@ the prefix's user folder appears, within a minute. Every path logs `%LocalAppData%/GOG.com/Galaxy/Applications//Storage/Shared/Files`. It goes through the same location-qualified handling below. GET peels gzip when `Content-Encoding` is gzip, or when the object round-trips as GOG's PUT gzip (deflate 6, mtime 0); a gzip-format save is left intact. + PUT mirrors Galaxy's `CloudStorageCompressingClient`: each object is gzipped (deflate 6, mtime 0) + and sent with `Content-Encoding: gzip`, an `Etag` of the compressed bytes and + `X-Object-Meta-LocalLastModified` (`yyyy-MM-dd'T'HH:mm:ss+00:00`, from the local file mtime). + Galaxy's `CloudStorageRemoteClient` rejects an object without that metadata as `invalid data` + before it ever decompresses it; a live probe showed gzip + metadata downloads, while gzip alone, + or `User-Agent` metadata alone, fails. Objects older builds uploaded raw lack it, so + `CloudPlan.upSend` resends those unchanged files once. Automatic repair does not resend + conflicts; ordinary local changes still upload through `plan.transfer`. + An explicit forced upload sends every local GOG file, including unchanged files the service gives + no hash for, so a manual repair rewrites the whole folder. - **GOG names are location-qualified:** the cloud API lists `saves/user1.dat`, where `saves` is the `name` of the selected `cloudStorage.locations` entry and the folder is that entry's `location` (`.../Hollow Knight Silksong/default`). `GogCloud` strips the location segment for the local path