fix: upload GOG cloud saves in Galaxy's gzip + metadata format (#1952)

fix: upload GOG cloud saves in Galaxy's gzip format

app/src/main/java/com/droiddeck/launcher/stores/CloudPlan.kt

@@ -52,6 +52,19 @@ object CloudPlan {
         return Plan(transfer, conflicts, same)
     }
 
+    /**
+     * Files to send. GOG objects uploaded by older builds are raw while their cloud hash is still the
+     * local raw MD5, so they appear in [Plan.same]; resending them stores the gzip form Galaxy reads.
+     * A forced GOG upload is the manual repair path and sends every local file, including unchanged
+     * ones the service gives no hash for; once rewritten the cloud hash is the compressed MD5.
+     */
+    fun upSend(plan: Plan, force: Boolean, repairRawGog: Boolean, allLocal: Collection<String> = emptyList()): List<String> {
+        if (force) {
+            return (plan.transfer + plan.conflicts + if (repairRawGog) allLocal else emptyList()).distinct()
+        }
+        return (plan.transfer + if (repairRawGog) plan.same else emptyList()).distinct()
+    }
+
     fun toJson(baseline: Map<String, Entry>): String = JSONObject().apply {
         for ((k, e) in baseline) put(k, JSONObject().put("l", e.localMd5).put("c", e.cloudMd5 ?: "").put("t", e.cloudModified))
     }.toString()

app/src/main/java/com/droiddeck/launcher/stores/CloudSaves.kt

@@ -374,18 +374,31 @@ object CloudSaves {
         val remote = remote(cloud) { it in files }
         val local = files.mapValues { CloudPlan.Local(md5(it.value)) }
         val plan = CloudPlan.up(local, remote.mapValues { CloudPlan.Remote(it.value.md5, it.value.modifiedMs) }, base)
-        val send = if (force) plan.transfer + plan.conflicts else plan.transfer
+        val send = CloudPlan.upSend(plan, force, cloud is GogCloud, files.keys)
         val next = HashMap(base)
         for (name in plan.same) remote[name]?.let { r -> next[name] = CloudPlan.Entry(local.getValue(name).md5, r.md5, r.modifiedMs) }
         if (send.isNotEmpty()) {
             backupCloud(context, store, id, cloud, send.mapNotNull { remote[it] })
-            val data = send.associateWith { files.getValue(it).readBytes() }
-            val stamped = cloud.put(data)
-            for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t)
-            val after = remote(cloud) { it in data }
-            for (name in data.keys) {
+            val stamped = HashMap<String, Long>()
+            val sentMd5 = HashMap<String, String>()
+            if (cloud is GogCloud) {
+                // Stream one save at a time: gzip needs a full-size buffer and uploads can be large.
+                val filesToSend = send.associateWith { files.getValue(it) }
+                val results = cloud.putFiles(filesToSend, send.associateWith { files.getValue(it).lastModified() })
+                for ((name, result) in results) {
+                    if (result.modifiedMs > 0) stamped[name] = result.modifiedMs
+                    result.md5?.let { sentMd5[name] = it }
+                }
+            } else {
+                val data = send.associateWith { files.getValue(it).readBytes() }
+                for ((name, bytes) in data) sentMd5[name] = md5(bytes)
+                stamped += cloud.put(data, send.associateWith { files.getValue(it).lastModified() })
+                for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t)
+            }
+            val after = remote(cloud) { it in send }
+            for (name in send) {
                 val r = after[name]
-                next[name] = CloudPlan.Entry(md5(data.getValue(name)), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L)
+                next[name] = CloudPlan.Entry(sentMd5.getValue(name), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L)
             }
         }
         saveBaseline(context, store, id, next)

app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt

@@ -1,24 +1,43 @@
 package com.droiddeck.launcher.stores
 
 import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.OutputStream
 import java.net.HttpURLConnection
 import java.net.URL
 import java.net.URLEncoder
+import java.security.DigestOutputStream
 import java.text.SimpleDateFormat
+import java.time.Instant
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 import java.util.Locale
 import java.util.TimeZone
 import java.util.zip.CRC32
 import java.util.zip.Deflater
+import java.util.zip.DeflaterOutputStream
 import org.json.JSONArray
 import org.json.JSONObject
 
 class CloudFile(val name: String, var modifiedMs: Long, var md5: String? = null, val readLink: String? = null)
 
+class CloudPut(val modifiedMs: Long, val md5: String?)
+
 interface CloudTransport {
     fun list(): List<CloudFile>
     fun details(file: CloudFile) {}
     fun get(file: CloudFile): Pair<ByteArray, Long>
-    fun put(files: Map<String, ByteArray>): Map<String, Long>
+    fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long> = emptyMap()): Map<String, Long>
+
+    /** Uploads from files so a transport can stream instead of holding every save in memory. */
+    fun putFiles(files: Map<String, File>, modifiedMs: Map<String, Long> = emptyMap()): Map<String, CloudPut> {
+        val data = files.mapValues { it.value.readBytes() }
+        val digests = data.mapValues { md5Hex(it.value) }
+        return put(data, modifiedMs).mapValues { (name, modified) -> CloudPut(modified, digests.getValue(name)) }
+    }
 }
 
 private const val TIMEOUT = 30_000
@@ -47,6 +66,9 @@ internal fun gunzip(data: ByteArray): ByteArray? = try {
     java.util.zip.GZIPInputStream(data.inputStream()).use { it.readBytes() }
 } catch (e: Exception) { null }
 
+/** Same gzip GOG uses on PUT: deflate level 6, mtime 0, OS Unix. */
+private val GOG_GZIP_HEADER = byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3)
+
 internal fun gzipGogPut(data: ByteArray): ByteArray {
     val deflater = Deflater(6, true)
     val raw = ByteArrayOutputStream()
@@ -63,14 +85,14 @@ internal fun gzipGogPut(data: ByteArray): ByteArray {
     }
     val crc = CRC32().also { it.update(data) }
     return ByteArrayOutputStream(18 + raw.size()).apply {
-        write(byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3))
+        write(GOG_GZIP_HEADER)
         write(raw.toByteArray())
         writeIntLe(crc.value.toInt())
         writeIntLe(data.size)
     }.toByteArray()
 }
 
-private fun ByteArrayOutputStream.writeIntLe(v: Int) {
+private fun OutputStream.writeIntLe(v: Int) {
     write(v and 0xff)
     write(v ushr 8 and 0xff)
     write(v ushr 16 and 0xff)
@@ -117,13 +139,64 @@ internal fun gogRelativeNames(files: List<CloudFile>, location: String?): Pair<S
     }
 }
 
+private fun md5Hex(data: ByteArray): String =
+    java.security.MessageDigest.getInstance("MD5").digest(data).joinToString("") { "%02x".format(it) }
+
+private fun ByteArray.toHex(): String = joinToString("") { "%02x".format(it) }
+
+private class GzipFile(val length: Long, val rawMd5: String, val etag: String)
+
+/** Streams the same gzip framing as [gzipGogPut] into [dst] without reading the save into memory. */
+private fun gzipFile(src: File, dst: File): GzipFile {
+    val raw = java.security.MessageDigest.getInstance("MD5")
+    val compressed = java.security.MessageDigest.getInstance("MD5")
+    val crc = CRC32()
+    var size = 0L
+    val file = FileOutputStream(dst)
+    val out = DigestOutputStream(file, compressed)
+    try {
+        out.write(GOG_GZIP_HEADER)
+        val deflater = Deflater(6, true)
+        try {
+            val deflate = DeflaterOutputStream(out, deflater, 64 * 1024)
+            src.inputStream().use { input ->
+                val buf = ByteArray(64 * 1024)
+                while (true) {
+                    val n = input.read(buf)
+                    if (n < 0) break
+                    raw.update(buf, 0, n)
+                    crc.update(buf, 0, n)
+                    size += n
+                    deflate.write(buf, 0, n)
+                }
+            }
+            deflate.finish()
+        } finally {
+            deflater.end()
+        }
+        out.writeIntLe(crc.value.toInt())
+        out.writeIntLe(size.toInt())
+        out.flush()
+        return GzipFile(dst.length(), raw.digest().toHex(), compressed.digest().toHex())
+    } finally {
+        out.close()
+    }
+}
+
+/** GOG Galaxy parses `X-Object-Meta-LocalLastModified`; objects without it fail as "invalid data". */
+private val GOG_STAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssxxx", Locale.US)
+
+private fun gogStamp(ms: Long): String =
+    Instant.ofEpochMilli(ms).atOffset(ZoneOffset.UTC).truncatedTo(ChronoUnit.SECONDS).format(GOG_STAMP)
+
 class GogCloud(
     private val userId: String,
     private val clientId: String,
     private val token: String,
     private val location: String? = null,
+    baseUrl: String? = null,
 ) : CloudTransport {
-    private val base = "https://cloudstorage.gog.com/v1/$userId/$clientId"
+    private val base = baseUrl ?: "https://cloudstorage.gog.com/v1/$userId/$clientId"
     private val ua = "GOG Galaxy"
     private var prefix: String? = location?.let { "$it/" }
 
@@ -190,16 +263,38 @@ class GogCloud(
         return gogDecodeCloudBody(data, encoding) to modified
     }
 
-    override fun put(files: Map<String, ByteArray>): Map<String, Long> = files.mapValues { (name, data) ->
+    override fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long>): Map<String, Long> = files.mapValues { (name, data) ->
+        val body = gzipGogPut(data)
+        putBody(name, body.inputStream(), body.size.toLong(), md5Hex(body), modifiedMs[name] ?: System.currentTimeMillis())
+    }
+
+    override fun putFiles(files: Map<String, File>, modifiedMs: Map<String, Long>): Map<String, CloudPut> = files.mapValues { (name, file) ->
+        val compressed = File.createTempFile("gog-cloud-", ".gz")
+        try {
+            val gz = gzipFile(file, compressed)
+            val modified = putBody(name, compressed.inputStream(), gz.length, gz.etag, modifiedMs[name] ?: file.lastModified())
+            CloudPut(modified, gz.rawMd5)
+        } finally {
+            compressed.delete()
+        }
+    }
+
+    private fun putBody(name: String, source: InputStream, length: Long, etag: String, modified: Long): Long {
         val c = open("$base/${cloudKey(name)}", "PUT", token, ua)
-        c.doOutput = true
-        c.setRequestProperty("Content-Type", "application/octet-stream")
-        c.setFixedLengthStreamingMode(data.size)
-        c.outputStream.use { it.write(data) }
-        val code = c.responseCode
-        c.disconnect()
-        if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code")
-        CloudFile(name, -1L).also { details(it) }.modifiedMs
+        try {
+            c.doOutput = true
+            c.setRequestProperty("Content-Type", "application/octet-stream")
+            c.setRequestProperty("Content-Encoding", "gzip")
+            c.setRequestProperty("Etag", etag)
+            c.setRequestProperty("X-Object-Meta-LocalLastModified", gogStamp(modified))
+            c.setFixedLengthStreamingMode(length)
+            c.outputStream.use { out -> source.use { it.copyTo(out, 64 * 1024) } }
+            val code = c.responseCode
+            if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code")
+            return CloudFile(name, -1L).also { details(it) }.modifiedMs
+        } finally {
+            c.disconnect()
+        }
     }
 }
 
@@ -208,6 +303,6 @@ class EpicCloud(private val accountId: String, private val appName: String, priv
 
     override fun get(file: CloudFile): Pair<ByteArray, Long> = com.droiddeck.launcher.stores.epic.EpicSaveSync.read(file)
 
-    override fun put(files: Map<String, ByteArray>): Map<String, Long> =
+    override fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long>): Map<String, Long> =
         com.droiddeck.launcher.stores.epic.EpicSaveSync.write(token, accountId, appName, files)
 }

app/src/test/java/com/droiddeck/launcher/stores/CloudPlanTest.kt

@@ -54,6 +54,22 @@ class CloudPlanTest {
         assertTrue(CloudPlan.down(mapOf("s" to Local("a")), mapOf("s" to Remote(null, 2500)), base).transfer.isEmpty())
     }
 
+    @Test fun gogRawObjectsThatStillMatchTheLocalBytesAreResentOnce() {
+        val base = mapOf("s" to Entry("a", "a", 1000))
+        val plan = CloudPlan.up(mapOf("s" to Local("a")), mapOf("s" to Remote("a", 1000)), base)
+        assertEquals(listOf("s"), plan.same)
+        assertEquals(listOf("s"), CloudPlan.upSend(plan, force = false, repairRawGog = true))
+        assertTrue(CloudPlan.upSend(plan, force = false, repairRawGog = false).isEmpty())
+    }
+
+    @Test fun aForcedGogUploadSendsUnchangedFilesEvenWithoutACloudHash() {
+        val base = mapOf("s" to Entry("a", "a", 1000))
+        val plan = CloudPlan.up(mapOf("s" to Local("a")), mapOf("s" to Remote(null, 1000)), base)
+        assertTrue(plan.transfer.isEmpty() && plan.conflicts.isEmpty() && plan.same.isEmpty())
+        assertEquals(listOf("s"), CloudPlan.upSend(plan, force = true, repairRawGog = true, allLocal = listOf("s")))
+        assertTrue(CloudPlan.upSend(plan, force = true, repairRawGog = false, allLocal = listOf("s")).isEmpty())
+    }
+
     @Test fun theBaselineSurvivesItsJson() {
         val b = mapOf("a/b.sav" to Entry("l", "c", 5), "x" to Entry("l2", null, -1))
         assertEquals(b, CloudPlan.fromJson(CloudPlan.toJson(b)))

app/src/test/java/com/droiddeck/launcher/stores/CloudSavesTest.kt

@@ -22,7 +22,7 @@ class CloudSavesTest {
         val puts = ArrayList<String>()
         override fun list() = files.map { (k, v) -> CloudFile(k, v.second, CloudSaves.md5(v.first)) }
         override fun get(file: CloudFile) = files.getValue(file.name)
-        override fun put(files: Map<String, ByteArray>): Map<String, Long> =
+        override fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long>): Map<String, Long> =
             files.mapValues { (k, v) -> puts += k; this.files[k] = v to 99_000L; 99_000L }
     }
 

app/src/test/java/com/droiddeck/launcher/stores/GogCloudKeysTest.kt

@@ -56,6 +56,81 @@ class GogCloudKeysTest {
         assertFalse(gzipGogPut(raw).contentEquals(gameGz))
         assertArrayEquals(gameGz, gogDecodeCloudBody(gameGz, null))
     }
+
+    @Test fun putUploadsTheGogGzipBodyWithEncodingAndCompressedHash() {
+        val raw = "savegame0".toByteArray()
+        val server = java.net.ServerSocket(0, 4, java.net.InetAddress.getByName("127.0.0.1"))
+        val requests = java.util.concurrent.CopyOnWriteArrayList<Request>()
+        val worker = Thread {
+            repeat(2) {
+                server.accept().use { client ->
+                    client.soTimeout = 5_000
+                    val input = client.getInputStream()
+                    val line = readLine(input)
+                    val parts = line.split(' ')
+                    val headers = LinkedHashMap<String, String>()
+                    while (true) {
+                        val h = readLine(input)
+                        if (h.isEmpty()) break
+                        val colon = h.indexOf(':')
+                        headers[h.substring(0, colon).trim().lowercase()] = h.substring(colon + 1).trim()
+                    }
+                    val length = headers["content-length"]?.toIntOrNull() ?: 0
+                    val body = ByteArray(length)
+                    var offset = 0
+                    while (offset < length) {
+                        val n = input.read(body, offset, length - offset)
+                        if (n < 0) break
+                        offset += n
+                    }
+                    requests += Request(parts[0], parts[1], headers, body)
+                    val response = if (parts[0] == "PUT") {
+                        "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+                    } else {
+                        "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n" +
+                            "Etag: \"${CloudSaves.md5(body)}\"\r\n" +
+                            "Last-Modified: Tue, 20 Oct 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n"
+                    }
+                    client.getOutputStream().apply { write(response.toByteArray()); flush() }
+                }
+            }
+        }.apply { isDaemon = true; start() }
+        try {
+            val cloud = GogCloud("user", "client", "token", "saves", "http://127.0.0.1:${server.localPort}/v1/user/client")
+            val source = java.io.File.createTempFile("save", ".sav").apply { writeBytes(raw) }
+            val result = try {
+                cloud.putFiles(mapOf("save.sav" to source), mapOf("save.sav" to 1_793_490_825_000L)).getValue("save.sav")
+            } finally {
+                source.delete()
+            }
+            assertTrue(result.modifiedMs > 0)
+            assertEquals(CloudSaves.md5(raw), result.md5)
+            val put = requests[0]
+            assertEquals("PUT", put.method)
+            assertEquals("/v1/user/client/saves%2Fsave.sav", put.path)
+            assertEquals("gzip", put.headers["content-encoding"])
+            assertEquals("2026-10-31T23:53:45+00:00", put.headers["x-object-meta-locallastmodified"])
+            assertArrayEquals(gzipGogPut(raw), put.body)
+            assertEquals(CloudSaves.md5(put.body), put.headers["etag"])
+            assertArrayEquals(raw, gogDecodeCloudBody(put.body, put.headers["content-encoding"]))
+            assertEquals("HEAD", requests[1].method)
+        } finally {
+            server.close()
+            worker.join(2_000)
+        }
+    }
+}
+
+private class Request(val method: String, val path: String, val headers: Map<String, String>, val body: ByteArray)
+
+private fun readLine(input: java.io.InputStream): String {
+    val out = ByteArrayOutputStream()
+    while (true) {
+        val b = input.read()
+        if (b < 0 || b == '\n'.code) break
+        if (b != '\r'.code) out.write(b)
+    }
+    return out.toString("UTF-8")
 }
 
 private fun String.hex() = chunked(2).map { it.toInt(16).toByte() }.toByteArray()

docs/development/stores.md

@@ -300,6 +300,16 @@ the prefix's user folder appears, within a minute. Every path logs
   `%LocalAppData%/GOG.com/Galaxy/Applications/<clientId>/Storage/Shared/Files`. It goes through the
   same location-qualified handling below. GET peels gzip when `Content-Encoding` is gzip, or when
   the object round-trips as GOG's PUT gzip (deflate 6, mtime 0); a gzip-format save is left intact.
+  PUT mirrors Galaxy's `CloudStorageCompressingClient`: each object is gzipped (deflate 6, mtime 0)
+  and sent with `Content-Encoding: gzip`, an `Etag` of the compressed bytes and
+  `X-Object-Meta-LocalLastModified` (`yyyy-MM-dd'T'HH:mm:ss+00:00`, from the local file mtime).
+  Galaxy's `CloudStorageRemoteClient` rejects an object without that metadata as `invalid data`
+  before it ever decompresses it; a live probe showed gzip + metadata downloads, while gzip alone,
+  or `User-Agent` metadata alone, fails. Objects older builds uploaded raw lack it, so
+  `CloudPlan.upSend` resends those unchanged files once. Automatic repair does not resend
+  conflicts; ordinary local changes still upload through `plan.transfer`.
+  An explicit forced upload sends every local GOG file, including unchanged files the service gives
+  no hash for, so a manual repair rewrites the whole folder.
 - **GOG names are location-qualified:** the cloud API lists `saves/user1.dat`, where `saves` is the
   `name` of the selected `cloudStorage.locations` entry and the folder is that entry's `location`
   (`.../Hollow Knight Silksong/default`). `GogCloud` strips the location segment for the local path