mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-18 12:28:40 +00:00
fix: upload GOG cloud saves in Galaxy's gzip + metadata format (#1952)
fix: upload GOG cloud saves in Galaxy's gzip format
This commit is contained in:
1 parent
5971b91ed4
commit
cd452e2b60
7 files changed
+244
-22
No files matched your search
@@ -52,6 +52,19 @@ object CloudPlan {
|
||||
return Plan(transfer, conflicts, same)
|
||||
}
|
||||
|
||||
/**
|
||||
* Files to send. GOG objects uploaded by older builds are raw while their cloud hash is still the
|
||||
* local raw MD5, so they appear in [Plan.same]; resending them stores the gzip form Galaxy reads.
|
||||
* A forced GOG upload is the manual repair path and sends every local file, including unchanged
|
||||
* ones the service gives no hash for; once rewritten the cloud hash is the compressed MD5.
|
||||
*/
|
||||
fun upSend(plan: Plan, force: Boolean, repairRawGog: Boolean, allLocal: Collection<String> = emptyList()): List<String> {
|
||||
if (force) {
|
||||
return (plan.transfer + plan.conflicts + if (repairRawGog) allLocal else emptyList()).distinct()
|
||||
}
|
||||
return (plan.transfer + if (repairRawGog) plan.same else emptyList()).distinct()
|
||||
}
|
||||
|
||||
fun toJson(baseline: Map<String, Entry>): String = JSONObject().apply {
|
||||
for ((k, e) in baseline) put(k, JSONObject().put("l", e.localMd5).put("c", e.cloudMd5 ?: "").put("t", e.cloudModified))
|
||||
}.toString()
|
||||
|
||||
@@ -374,18 +374,31 @@ object CloudSaves {
|
||||
val remote = remote(cloud) { it in files }
|
||||
val local = files.mapValues { CloudPlan.Local(md5(it.value)) }
|
||||
val plan = CloudPlan.up(local, remote.mapValues { CloudPlan.Remote(it.value.md5, it.value.modifiedMs) }, base)
|
||||
val send = if (force) plan.transfer + plan.conflicts else plan.transfer
|
||||
val send = CloudPlan.upSend(plan, force, cloud is GogCloud, files.keys)
|
||||
val next = HashMap(base)
|
||||
for (name in plan.same) remote[name]?.let { r -> next[name] = CloudPlan.Entry(local.getValue(name).md5, r.md5, r.modifiedMs) }
|
||||
if (send.isNotEmpty()) {
|
||||
backupCloud(context, store, id, cloud, send.mapNotNull { remote[it] })
|
||||
val data = send.associateWith { files.getValue(it).readBytes() }
|
||||
val stamped = cloud.put(data)
|
||||
for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t)
|
||||
val after = remote(cloud) { it in data }
|
||||
for (name in data.keys) {
|
||||
val stamped = HashMap<String, Long>()
|
||||
val sentMd5 = HashMap<String, String>()
|
||||
if (cloud is GogCloud) {
|
||||
// Stream one save at a time: gzip needs a full-size buffer and uploads can be large.
|
||||
val filesToSend = send.associateWith { files.getValue(it) }
|
||||
val results = cloud.putFiles(filesToSend, send.associateWith { files.getValue(it).lastModified() })
|
||||
for ((name, result) in results) {
|
||||
if (result.modifiedMs > 0) stamped[name] = result.modifiedMs
|
||||
result.md5?.let { sentMd5[name] = it }
|
||||
}
|
||||
} else {
|
||||
val data = send.associateWith { files.getValue(it).readBytes() }
|
||||
for ((name, bytes) in data) sentMd5[name] = md5(bytes)
|
||||
stamped += cloud.put(data, send.associateWith { files.getValue(it).lastModified() })
|
||||
for ((name, t) in stamped) if (t > 0) files[name]?.setLastModified(t)
|
||||
}
|
||||
val after = remote(cloud) { it in send }
|
||||
for (name in send) {
|
||||
val r = after[name]
|
||||
next[name] = CloudPlan.Entry(md5(data.getValue(name)), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L)
|
||||
next[name] = CloudPlan.Entry(sentMd5.getValue(name), r?.md5, r?.modifiedMs ?: stamped[name] ?: -1L)
|
||||
}
|
||||
}
|
||||
saveBaseline(context, store, id, next)
|
||||
|
||||
@@ -1,24 +1,43 @@
|
||||
package com.droiddeck.launcher.stores
|
||||
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.net.URLEncoder
|
||||
import java.security.DigestOutputStream
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.time.temporal.ChronoUnit
|
||||
import java.util.Locale
|
||||
import java.util.TimeZone
|
||||
import java.util.zip.CRC32
|
||||
import java.util.zip.Deflater
|
||||
import java.util.zip.DeflaterOutputStream
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
|
||||
class CloudFile(val name: String, var modifiedMs: Long, var md5: String? = null, val readLink: String? = null)
|
||||
|
||||
class CloudPut(val modifiedMs: Long, val md5: String?)
|
||||
|
||||
interface CloudTransport {
|
||||
fun list(): List<CloudFile>
|
||||
fun details(file: CloudFile) {}
|
||||
fun get(file: CloudFile): Pair<ByteArray, Long>
|
||||
fun put(files: Map<String, ByteArray>): Map<String, Long>
|
||||
fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long> = emptyMap()): Map<String, Long>
|
||||
|
||||
/** Uploads from files so a transport can stream instead of holding every save in memory. */
|
||||
fun putFiles(files: Map<String, File>, modifiedMs: Map<String, Long> = emptyMap()): Map<String, CloudPut> {
|
||||
val data = files.mapValues { it.value.readBytes() }
|
||||
val digests = data.mapValues { md5Hex(it.value) }
|
||||
return put(data, modifiedMs).mapValues { (name, modified) -> CloudPut(modified, digests.getValue(name)) }
|
||||
}
|
||||
}
|
||||
|
||||
private const val TIMEOUT = 30_000
|
||||
@@ -47,6 +66,9 @@ internal fun gunzip(data: ByteArray): ByteArray? = try {
|
||||
java.util.zip.GZIPInputStream(data.inputStream()).use { it.readBytes() }
|
||||
} catch (e: Exception) { null }
|
||||
|
||||
/** Same gzip GOG uses on PUT: deflate level 6, mtime 0, OS Unix. */
|
||||
private val GOG_GZIP_HEADER = byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3)
|
||||
|
||||
internal fun gzipGogPut(data: ByteArray): ByteArray {
|
||||
val deflater = Deflater(6, true)
|
||||
val raw = ByteArrayOutputStream()
|
||||
@@ -63,14 +85,14 @@ internal fun gzipGogPut(data: ByteArray): ByteArray {
|
||||
}
|
||||
val crc = CRC32().also { it.update(data) }
|
||||
return ByteArrayOutputStream(18 + raw.size()).apply {
|
||||
write(byteArrayOf(0x1f, 0x8b.toByte(), 8, 0, 0, 0, 0, 0, 0, 3))
|
||||
write(GOG_GZIP_HEADER)
|
||||
write(raw.toByteArray())
|
||||
writeIntLe(crc.value.toInt())
|
||||
writeIntLe(data.size)
|
||||
}.toByteArray()
|
||||
}
|
||||
|
||||
private fun ByteArrayOutputStream.writeIntLe(v: Int) {
|
||||
private fun OutputStream.writeIntLe(v: Int) {
|
||||
write(v and 0xff)
|
||||
write(v ushr 8 and 0xff)
|
||||
write(v ushr 16 and 0xff)
|
||||
@@ -117,13 +139,64 @@ internal fun gogRelativeNames(files: List<CloudFile>, location: String?): Pair<S
|
||||
}
|
||||
}
|
||||
|
||||
private fun md5Hex(data: ByteArray): String =
|
||||
java.security.MessageDigest.getInstance("MD5").digest(data).joinToString("") { "%02x".format(it) }
|
||||
|
||||
private fun ByteArray.toHex(): String = joinToString("") { "%02x".format(it) }
|
||||
|
||||
private class GzipFile(val length: Long, val rawMd5: String, val etag: String)
|
||||
|
||||
/** Streams the same gzip framing as [gzipGogPut] into [dst] without reading the save into memory. */
|
||||
private fun gzipFile(src: File, dst: File): GzipFile {
|
||||
val raw = java.security.MessageDigest.getInstance("MD5")
|
||||
val compressed = java.security.MessageDigest.getInstance("MD5")
|
||||
val crc = CRC32()
|
||||
var size = 0L
|
||||
val file = FileOutputStream(dst)
|
||||
val out = DigestOutputStream(file, compressed)
|
||||
try {
|
||||
out.write(GOG_GZIP_HEADER)
|
||||
val deflater = Deflater(6, true)
|
||||
try {
|
||||
val deflate = DeflaterOutputStream(out, deflater, 64 * 1024)
|
||||
src.inputStream().use { input ->
|
||||
val buf = ByteArray(64 * 1024)
|
||||
while (true) {
|
||||
val n = input.read(buf)
|
||||
if (n < 0) break
|
||||
raw.update(buf, 0, n)
|
||||
crc.update(buf, 0, n)
|
||||
size += n
|
||||
deflate.write(buf, 0, n)
|
||||
}
|
||||
}
|
||||
deflate.finish()
|
||||
} finally {
|
||||
deflater.end()
|
||||
}
|
||||
out.writeIntLe(crc.value.toInt())
|
||||
out.writeIntLe(size.toInt())
|
||||
out.flush()
|
||||
return GzipFile(dst.length(), raw.digest().toHex(), compressed.digest().toHex())
|
||||
} finally {
|
||||
out.close()
|
||||
}
|
||||
}
|
||||
|
||||
/** GOG Galaxy parses `X-Object-Meta-LocalLastModified`; objects without it fail as "invalid data". */
|
||||
private val GOG_STAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssxxx", Locale.US)
|
||||
|
||||
private fun gogStamp(ms: Long): String =
|
||||
Instant.ofEpochMilli(ms).atOffset(ZoneOffset.UTC).truncatedTo(ChronoUnit.SECONDS).format(GOG_STAMP)
|
||||
|
||||
class GogCloud(
|
||||
private val userId: String,
|
||||
private val clientId: String,
|
||||
private val token: String,
|
||||
private val location: String? = null,
|
||||
baseUrl: String? = null,
|
||||
) : CloudTransport {
|
||||
private val base = "https://cloudstorage.gog.com/v1/$userId/$clientId"
|
||||
private val base = baseUrl ?: "https://cloudstorage.gog.com/v1/$userId/$clientId"
|
||||
private val ua = "GOG Galaxy"
|
||||
private var prefix: String? = location?.let { "$it/" }
|
||||
|
||||
@@ -190,16 +263,38 @@ class GogCloud(
|
||||
return gogDecodeCloudBody(data, encoding) to modified
|
||||
}
|
||||
|
||||
override fun put(files: Map<String, ByteArray>): Map<String, Long> = files.mapValues { (name, data) ->
|
||||
override fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long>): Map<String, Long> = files.mapValues { (name, data) ->
|
||||
val body = gzipGogPut(data)
|
||||
putBody(name, body.inputStream(), body.size.toLong(), md5Hex(body), modifiedMs[name] ?: System.currentTimeMillis())
|
||||
}
|
||||
|
||||
override fun putFiles(files: Map<String, File>, modifiedMs: Map<String, Long>): Map<String, CloudPut> = files.mapValues { (name, file) ->
|
||||
val compressed = File.createTempFile("gog-cloud-", ".gz")
|
||||
try {
|
||||
val gz = gzipFile(file, compressed)
|
||||
val modified = putBody(name, compressed.inputStream(), gz.length, gz.etag, modifiedMs[name] ?: file.lastModified())
|
||||
CloudPut(modified, gz.rawMd5)
|
||||
} finally {
|
||||
compressed.delete()
|
||||
}
|
||||
}
|
||||
|
||||
private fun putBody(name: String, source: InputStream, length: Long, etag: String, modified: Long): Long {
|
||||
val c = open("$base/${cloudKey(name)}", "PUT", token, ua)
|
||||
c.doOutput = true
|
||||
c.setRequestProperty("Content-Type", "application/octet-stream")
|
||||
c.setFixedLengthStreamingMode(data.size)
|
||||
c.outputStream.use { it.write(data) }
|
||||
val code = c.responseCode
|
||||
c.disconnect()
|
||||
if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code")
|
||||
CloudFile(name, -1L).also { details(it) }.modifiedMs
|
||||
try {
|
||||
c.doOutput = true
|
||||
c.setRequestProperty("Content-Type", "application/octet-stream")
|
||||
c.setRequestProperty("Content-Encoding", "gzip")
|
||||
c.setRequestProperty("Etag", etag)
|
||||
c.setRequestProperty("X-Object-Meta-LocalLastModified", gogStamp(modified))
|
||||
c.setFixedLengthStreamingMode(length)
|
||||
c.outputStream.use { out -> source.use { it.copyTo(out, 64 * 1024) } }
|
||||
val code = c.responseCode
|
||||
if (code !in 200..299) throw java.io.IOException("GOG cloud put HTTP $code")
|
||||
return CloudFile(name, -1L).also { details(it) }.modifiedMs
|
||||
} finally {
|
||||
c.disconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,6 +303,6 @@ class EpicCloud(private val accountId: String, private val appName: String, priv
|
||||
|
||||
override fun get(file: CloudFile): Pair<ByteArray, Long> = com.droiddeck.launcher.stores.epic.EpicSaveSync.read(file)
|
||||
|
||||
override fun put(files: Map<String, ByteArray>): Map<String, Long> =
|
||||
override fun put(files: Map<String, ByteArray>, modifiedMs: Map<String, Long>): Map<String, Long> =
|
||||
com.droiddeck.launcher.stores.epic.EpicSaveSync.write(token, accountId, appName, files)
|
||||
}
|
||||
Reference in new issue
Block a user