From 198f667bf09a8938b050158987bd478d12e7c16c Mon Sep 17 00:00:00 2001 From: Kurt Himebauch <136133082+xXJSONDeruloXx@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:10:17 -0400 Subject: [PATCH] feat: add GOG maintenance --- app/build.gradle | 9 + .../launcher/session/SessionFiles.kt | 1 + .../launcher/stores/StoreInstallRoot.kt | 1 + .../droiddeck/launcher/stores/StoresState.kt | 3 +- .../stores/download/DownloadModels.kt | 3 + .../launcher/stores/download/DownloadQueue.kt | 20 +- .../launcher/stores/download/SavedQueue.kt | 9 +- .../droiddeck/launcher/stores/gog/GogAuth.kt | 1 + .../launcher/stores/gog/GogBackend.kt | 7 +- .../launcher/stores/gog/GogContent.kt | 187 + .../stores/gog/GogDownloadManager.java | 26 +- .../launcher/stores/gog/GogEstimate.kt | 58 + .../launcher/stores/gog/GogFallback.kt | 115 + .../launcher/stores/gog/GogInstallRecord.kt | 176 + .../launcher/stores/gog/GogMaintenance.kt | 415 ++ .../launcher/stores/gog/GogNative.kt | 4 +- .../launcher/stores/gog/GogOffline.kt | 200 + .../com/droiddeck/launcher/ui/GameDetail.kt | 3 +- .../droiddeck/launcher/ui/GogMaintenanceUi.kt | 188 + .../com/droiddeck/launcher/ui/LibraryPage.kt | 7 +- .../droiddeck/launcher/ui/StoresDownloads.kt | 6 +- .../com/droiddeck/launcher/ui/StoresPage.kt | 3 + app/src/main/res/values/strings.xml | 21 + app/src/main/rust/stores/Cargo.lock | 3 + app/src/main/rust/stores/Cargo.toml | 3 + app/src/main/rust/stores/JNI.md | 13 +- app/src/main/rust/stores/build.rs | 21 + app/src/main/rust/stores/src/gog.rs | 1 + app/src/main/rust/stores/src/gog/delta.c | 78 + app/src/main/rust/stores/src/gog/engine.rs | 7 +- .../stores/src/gog/fixtures/update.vcdiff | Bin 0 -> 66 bytes app/src/main/rust/stores/src/gog/jni.rs | 11 +- .../main/rust/stores/src/gog/maintenance.rs | 523 ++ .../main/rust/stores/vendor/xdelta3/LICENSE | 176 + .../stores/vendor/xdelta3/README.droiddeck.md | 7 + .../rust/stores/vendor/xdelta3/xdelta3-cfgs.h | 171 + .../stores/vendor/xdelta3/xdelta3-decode.h | 1218 +++++ .../rust/stores/vendor/xdelta3/xdelta3-hash.h | 167 + .../stores/vendor/xdelta3/xdelta3-internal.h | 370 ++ .../rust/stores/vendor/xdelta3/xdelta3-list.h | 127 + .../main/rust/stores/vendor/xdelta3/xdelta3.c | 4835 +++++++++++++++++ .../main/rust/stores/vendor/xdelta3/xdelta3.h | 1396 +++++ .../launcher/stores/gog/GogMaintenanceTest.kt | 402 ++ app/src/test/resources/gog/README.md | 8 + .../gog/rayman-gen1-english-excerpt.json | 29 + .../resources/gog/rayman-gen1-neutral.json | 16 + .../resources/gog/rayman-gen1-repository.json | 65 + docs/development/gog-maintenance.md | 114 + .../usr/local/bin/droiddeck-gog-maintenance | 344 ++ .../usr/local/bin/droiddeck-store-launch | 39 +- tools/release/runtime_inputs.py | 1 + tools/tests/test_game_environment.py | 18 + tools/tests/test_gog_maintenance.py | 216 + tools/tests/test_store_launch.py | 24 + 54 files changed, 11840 insertions(+), 26 deletions(-) create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogContent.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogEstimate.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogFallback.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogInstallRecord.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogMaintenance.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/stores/gog/GogOffline.kt create mode 100644 app/src/main/java/com/droiddeck/launcher/ui/GogMaintenanceUi.kt create mode 100644 app/src/main/rust/stores/build.rs create mode 100644 app/src/main/rust/stores/src/gog/delta.c create mode 100644 app/src/main/rust/stores/src/gog/fixtures/update.vcdiff create mode 100644 app/src/main/rust/stores/src/gog/maintenance.rs create mode 100644 app/src/main/rust/stores/vendor/xdelta3/LICENSE create mode 100644 app/src/main/rust/stores/vendor/xdelta3/README.droiddeck.md create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3-cfgs.h create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3-decode.h create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3-hash.h create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3-internal.h create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3-list.h create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3.c create mode 100644 app/src/main/rust/stores/vendor/xdelta3/xdelta3.h create mode 100644 app/src/test/java/com/droiddeck/launcher/stores/gog/GogMaintenanceTest.kt create mode 100644 app/src/test/resources/gog/README.md create mode 100644 app/src/test/resources/gog/rayman-gen1-english-excerpt.json create mode 100644 app/src/test/resources/gog/rayman-gen1-neutral.json create mode 100644 app/src/test/resources/gog/rayman-gen1-repository.json create mode 100644 docs/development/gog-maintenance.md create mode 100644 tools/linuxfs/overlay/usr/local/bin/droiddeck-gog-maintenance create mode 100644 tools/tests/test_gog_maintenance.py diff --git a/app/build.gradle b/app/build.gradle index 1c9b9bad8d..b97dcb2617 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -286,6 +286,13 @@ if (uvcDir) { } tasks.named('preBuild').configure { dependsOn(stageRuntimeAssets) } +def stageGogNotices = tasks.register('stageGogNotices', Copy) { + from(layout.projectDirectory.dir('src/main/rust/stores/vendor/xdelta3')) { include 'LICENSE', 'README.droiddeck.md' } + into(layout.buildDirectory.dir('gogNotices/licenses/xdelta3')) +} +android.sourceSets.main.assets.srcDir(layout.buildDirectory.dir('gogNotices')) +tasks.named('preBuild').configure { dependsOn(stageGogNotices) } + def verifyRuntimeInputs = tasks.register('verifyRuntimeInputs', Exec) { mustRunAfter(stageSessionScripts, stageRuntimeAssets) workingDir rootProject.projectDir @@ -303,6 +310,8 @@ def buildRustStores = tasks.register('buildRustStores', Exec) { description = 'Cross-compiles the store download engines (libdroiddeckstores.so) with cargo-ndk.' onlyIf { !skipRust } inputs.dir(rustCrateDir.dir('src')) + inputs.dir(rustCrateDir.dir('vendor')) + inputs.file(rustCrateDir.file('build.rs')) inputs.files(rustCrateDir.file('Cargo.toml'), rustCrateDir.file('Cargo.lock')) inputs.property('ndkVersion', android.ndkVersion) if (!skipRust) inputs.property('rustToolchain', providers.exec { diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt index d9c59b1b88..1f82f0a9fd 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt @@ -74,6 +74,7 @@ object SessionFiles { "usr/local/bin/droiddeck-steam-shortcuts" to "usr/local/bin/droiddeck-steam-shortcuts", "usr/local/bin/droiddeck-xbox" to "usr/local/bin/droiddeck-xbox", "usr/local/bin/droiddeck-store-launch" to "usr/local/bin/droiddeck-store-launch", + "usr/local/bin/droiddeck-gog-maintenance" to "usr/local/bin/droiddeck-gog-maintenance", "usr/local/bin/droiddeck-battlenet" to "usr/local/bin/droiddeck-battlenet", "usr/local/bin/droiddeck-epic" to "usr/local/bin/droiddeck-epic", "usr/local/bin/droiddeck-epic-steam" to "usr/local/bin/droiddeck-epic-steam", diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt index c4faac54e5..fa20167da9 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt @@ -254,6 +254,7 @@ object StoreInstallRoot { val sidecar = StoreGameSidecar.read(folder) when { sidecar == null -> Unfinished(store, null, folder) + store == Store.GOG && sidecar.state == StoreGameSidecar.STATE_UPDATING -> null !sidecar.isInstalled -> Unfinished(store, sidecar.id, folder) else -> null } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt index 686cfd13ba..ff44ca7638 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt @@ -81,7 +81,7 @@ object StoresState { runCatching { com.droiddeck.launcher.stores.battlenet.BattleNetClient.syncAccount(app) } val found = Store.entries.mapNotNull { store -> StoreAccounts.displayName(app, store)?.let { store to it } } val onDisk = StoreInstallRoot.gameFolders(app).mapNotNull { folder -> - StoreGameSidecar.read(folder)?.takeIf { it.isInstalled }?.let { InstalledStoreGame(it, folder) } + StoreGameSidecar.read(folder)?.takeIf { it.isInstalled || it.store == Store.GOG && it.state == StoreGameSidecar.STATE_UPDATING }?.let { InstalledStoreGame(it, folder) } } + runCatching { com.droiddeck.launcher.stores.battlenet.BattleNetBackend.installedGames(app) }.getOrDefault(emptyList()) val partial = StoreInstallRoot.unfinished(app) val version = if (StoresNative.available) StoresNative.version ?: "" else "" @@ -95,6 +95,7 @@ object StoresState { engine = version accountsLoaded = true } + com.droiddeck.launcher.stores.gog.GogMaintenance.checkInstalled(app, onDisk) } } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadModels.kt b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadModels.kt index 8683efa065..57f07742cf 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadModels.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadModels.kt @@ -31,6 +31,9 @@ data class DownloadEntry( val stageItemsTotal: Int = 0, val diskBytes: Long = 0L, val stagesPassed: Int = 0, + val interruptible: Boolean = true, + val reusedBytes: Long = 0L, + val appliedBytes: Long = 0L, ) { fun passed(s: DownloadStage): Boolean = state == DownloadState.INSTALLED || (stagesPassed and (1 shl s.ordinal)) != 0 diff --git a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt index cfb1324f0b..baae64b3cd 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt @@ -21,6 +21,11 @@ object DownloadQueue { fun diskSize(bytes: Long) = update(key) { if (bytes > 0) it.copy(diskBytes = bytes) else it } fun progress(bytesDone: Long, bytesTotal: Long, detail: String? = null, @Suppress("UNUSED_PARAMETER") speedBps: Long = -1L) = measure(key, bytesDone, bytesTotal, detail?.let(::stripSpeed)) fun log(line: String) = StoresState.logLine(line) + fun interruptible(value: Boolean) = update(key) { + check(value || !cancelled.get()) { "The operation was paused before installer application" } + it.copy(interruptible = value) + } + fun maintenance(reusedBytes: Long, appliedBytes: Long) = update(key) { it.copy(reusedBytes = reusedBytes, appliedBytes = appliedBytes) } } private fun stripSpeed(s: String): String = s.replace(Regex("\\s+\\S+ [KMG]B/s\\s*$"), "").trimEnd() @@ -166,12 +171,13 @@ object DownloadQueue { fun pause(key: String) { val item = synchronized(lock) { items[key] } ?: return synchronized(lock) { + if (!item.entry.interruptible) return if (item.entry.state == DownloadState.QUEUED) { item.entry = item.entry.copy(state = DownloadState.PAUSED, queuePosition = 0); renumber(); publishLocked(); return } if (item.entry.state != DownloadState.RUNNING) return item.pauseRequested = true + item.cancelled.set(true) + item.job?.cancel(deleteFiles = false) } - item.cancelled.set(true) - item.job?.cancel(deleteFiles = false) StoresState.logLine("paused \"${item.entry.name}\"") } @@ -204,17 +210,19 @@ object DownloadQueue { val wasRunning: Boolean synchronized(lock) { item = items[key] ?: return + if (!item.entry.interruptible) return wasRunning = item.entry.state == DownloadState.RUNNING + if (wasRunning) { + item.cancelled.set(true) + item.job?.cancel(deleteFiles = true) + } if (!wasRunning) { item.entry = item.entry.copy(state = DownloadState.CANCELLED, queuePosition = 0, finishedAt = System.currentTimeMillis()) renumber() publishLocked() } } - if (wasRunning) { - item.cancelled.set(true) - item.job?.cancel(deleteFiles = true) - } else { + if (!wasRunning) { Thread({ runCatching { item.factory?.invoke()?.cancel(deleteFiles = true) } }, "store-dl-cancel").start() } StoresState.logLine("cancelled \"${item.entry.name}\"") diff --git a/app/src/main/java/com/droiddeck/launcher/stores/download/SavedQueue.kt b/app/src/main/java/com/droiddeck/launcher/stores/download/SavedQueue.kt index e9a0acbcfa..99d08d6b8c 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/download/SavedQueue.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/download/SavedQueue.kt @@ -45,7 +45,7 @@ object SavedQueue { val f = File(context.filesDir, FILE) val tmp = File(context.filesDir, "$FILE.tmp") tmp.writeText(out.toString()) - tmp.renameTo(f) + java.nio.file.Files.move(tmp.toPath(), f.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING) }.onFailure { Log.w(TAG, "could not save the queue", it) } } } @@ -68,7 +68,12 @@ object SavedQueue { for ((item, root, paused) in back) { if (DownloadQueue.entry(key(item)) != null) continue if (paused) DownloadQueue.holdPaused(key(item)) - StoresState.install(app, item, root) + if (item.store == Store.GOG && item.extra.containsKey(com.droiddeck.launcher.stores.gog.GogMaintenance.OPERATION)) { + Thread({ + runCatching { com.droiddeck.launcher.stores.gog.GogMaintenance.restore(app, item) } + .onFailure { DownloadQueue.recordFailed(app, DownloadEntry(item.store, item.id, item.title), it.message ?: "GOG recovery failed") } + }, "gog-restore").start() + } else StoresState.install(app, item, root) } } }, "store-queue-restore").start() diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogAuth.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogAuth.kt index 8e58fab033..9332bb986f 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogAuth.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogAuth.kt @@ -33,6 +33,7 @@ object GogAuth { @Synchronized private fun saveAccount(context: Context, accessToken: String, refreshToken: String?, userId: String?) { + GogMaintenance.reset() var username = "" runCatching { val body = StoreNet.get("https://embed.gog.com/userData.json", bearer = accessToken, userAgent = GALAXY_UA) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogBackend.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogBackend.kt index bf62da5b59..59a370775a 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogBackend.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogBackend.kt @@ -28,6 +28,7 @@ object GogBackend : StoreBackend { } override fun signOut(context: Context) { + GogMaintenance.reset() GogPrefs.get(context).edit().remove("library_cache").remove("library_cache_v2").remove("library_synced_at").apply() } @@ -53,7 +54,11 @@ object GogBackend : StoreBackend { StoresState.post { StoresState.status.remove(store) when (result) { - is GogLibrary.SyncResult.Ok -> { StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], result.games.map { GogLibrary.toCatalogItem(app, it) }); StoresState.problems.remove(store) } + is GogLibrary.SyncResult.Ok -> { + StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], result.games.map { GogLibrary.toCatalogItem(app, it) }) + StoresState.problems.remove(store) + GogMaintenance.checkInstalled(app, StoresState.installed) + } is GogLibrary.SyncResult.Failed -> StoresState.problems[store] = result.message GogLibrary.SyncResult.NotLoggedIn -> StoresState.problems[store] = StoresState.notSignedInLine(store) GogLibrary.SyncResult.Busy -> {} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogContent.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogContent.kt new file mode 100644 index 0000000000..3d86fd5854 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogContent.kt @@ -0,0 +1,187 @@ +package com.droiddeck.launcher.stores.gog + +import android.content.Context +import com.droiddeck.launcher.stores.StoreLanguage +import com.droiddeck.launcher.stores.StoreNet +import org.json.JSONArray +import org.json.JSONObject +import java.net.HttpURLConnection +import java.net.URL +import java.util.zip.GZIPInputStream +import java.util.zip.InflaterInputStream + +internal object GogContent { + const val SYSTEM = "https://content-system.gog.com" + const val CDN = "https://gog-cdn-fastly.gog.com/content-system" + + fun token(context: Context) = GogAuth.validToken(context) ?: error("Sign in to GOG and retry") + + fun builds(id: String, token: String): JSONObject { + val result = json("$SYSTEM/products/$id/os/windows/builds?generation=2", token) + if (result.optJSONArray("items")?.length() != 0) return result + return json("$SYSTEM/products/$id/os/windows/builds?generation=1", token) + } + + fun product(id: String, token: String): JSONObject = json("https://api.gog.com/products/$id?expand=downloads", token) + + fun downloadFilename(url: String): String = java.net.URLDecoder.decode(URL(url).path.substringAfterLast('/').replace("+", "%2B"), "UTF-8") + + fun resolveDownload(download: GogOfflineDownload, token: String): GogOfflineDownload = download.copy(parts = download.parts.map { part -> + val resolved = GogDownloadManager.resolveRedirect(part.url, token) ?: error("The GOG download link is unavailable") + val connection = URL(resolved).openConnection() as HttpURLConnection + try { + connection.requestMethod = "HEAD" + connection.connectTimeout = 20_000; connection.readTimeout = 30_000 + check(connection.responseCode in 200..299) { "GOG download metadata failed (HTTP ${connection.responseCode})" } + val disposition = connection.getHeaderField("Content-Disposition").orEmpty() + val supplied = Regex("filename=\"?([^\";]+)", RegexOption.IGNORE_CASE).find(disposition)?.groupValues?.get(1).orEmpty() + val remote = downloadFilename(resolved).takeIf { it.endsWith(".exe", true) || it.endsWith(".bin", true) }.orEmpty() + val name = part.name.ifBlank { supplied.ifBlank { remote } } + check(name.isNotBlank()) { "GOG did not provide the installer part filename" } + part.copy(name = name, size = connection.contentLengthLong.coerceAtLeast(0)) + } finally { connection.disconnect() } + }) + + fun json(url: String, token: String? = null): JSONObject { + require(URL(url).protocol == "https") { "GOG metadata requires HTTPS" } + val connection = URL(url).openConnection() as HttpURLConnection + try { + connection.connectTimeout = 20_000; connection.readTimeout = 30_000 + connection.setRequestProperty("User-Agent", GogAuth.GALAXY_UA) + if (token != null && URL(url).host.let { it == "gog.com" || it.endsWith(".gog.com") }) connection.setRequestProperty("Authorization", "Bearer $token") + check(connection.responseCode in 200..299) { "GOG metadata request failed (HTTP ${connection.responseCode})" } + val raw = connection.inputStream.readBytes() + val text = when { + raw.size > 1 && raw[0] == 0x78.toByte() -> InflaterInputStream(raw.inputStream()).bufferedReader().use { it.readText() } + raw.size > 1 && raw[0] == 0x1f.toByte() && raw[1] == 0x8b.toByte() -> GZIPInputStream(raw.inputStream()).bufferedReader().use { it.readText() } + else -> raw.toString(Charsets.UTF_8) + } + return JSONObject(text) + } finally { connection.disconnect() } + } + + fun snapshot(old: GogInstallRecord, build: GogBuild, token: String, + metadata: (String, String?) -> JSONObject = ::json): GogInstallRecord { + check(build.link.isNotBlank()) { "GOG build manifest is unavailable" } + val manifest = metadata(build.link, token) + val files = JSONArray() + val depots = if (build.generation == 1) manifest.optJSONObject("product")?.optJSONArray("depots") ?: manifest.optJSONArray("depot") else manifest.optJSONArray("depots") + check(depots != null) { "GOG build has no depot list" } + val resolvedLanguage = StoreLanguage.pick(StoreLanguage.codesFromDepots(depots), old.language) + val selectedBitness = if (depots.objects().any { it.optJSONArray("osBitness")?.strings()?.contains("64") == true }) "64" else "32" + val owned = if (old.products.any { it != old.id }) StoreNet.get("https://embed.gog.com/user/data/games", bearer = token)?.let { JSONObject(it).optJSONArray("owned")?.strings()?.toSet() } else emptySet() + val products = old.products.filter { it == old.id || owned?.contains(it) == true } + check(old.products.all { it == old.id || owned?.contains(it) == true }) { "Could not confirm ownership of installed GOG DLC" } + for (depot in depots.objects()) { + if (depot.has("redist")) continue + val ids = depot.optJSONArray("gameIDs")?.strings() ?: listOf(depot.optString("productId", old.id)) + val product = ids.firstOrNull { it in products } ?: continue + val languages = depot.optJSONArray("languages")?.strings().orEmpty() + if (!languageSelected(languages, resolvedLanguage)) continue + val bitness = depot.optJSONArray("osBitness")?.strings().orEmpty() + if (bitness.isNotEmpty() && selectedBitness !in bitness) continue + val hash = depot.optString("manifest") + val content = when { + hash.isNotEmpty() && build.generation == 2 -> metadata("$CDN/v2/meta/${cdnPath(hash)}", null) + hash.isNotEmpty() -> { + val timestamp = manifest.getJSONObject("product").getString("timestamp") + metadata("$CDN/v1/manifests/$product/windows/$timestamp/$hash", token) + } + depot.has("files") -> JSONObject().put("depot", depot) + else -> error("GOG depot manifest is missing") + } + val entries = content.getJSONObject("depot").optJSONArray(if (build.generation == 2) "items" else "files") ?: error("GOG depot files are missing") + for (entry in entries.objects()) { + val type = entry.optString("type") + check(type != "DepotLink") { "This GOG build contains unsupported file links" } + val support = depot.optBoolean("support") || entry.optBoolean("support") || entry.optJSONArray("flags")?.strings()?.contains("support") == true + val relative = entry.getString("path").replace('\\', '/').trimEnd('/').let { if (build.generation == 1) it.trimStart('/') else it } + val path = if (support) (if (build.generation == 1) "support/$relative" else "support/$product/$relative") else relative + val directory = type == "DepotDirectory" || entry.has("directory") + val chunks = entry.optJSONArray("chunks") ?: JSONArray() + val size = if (build.generation == 2) chunks.objects().sumOf { it.optLong("size") } else entry.optLong("size") + files.put(JSONObject().put("path", path).put("size", size).put("directory", directory) + .put("md5", entry.optString("md5").ifBlank { entry.optString("hash") }.takeIf { it.matches(Regex("[a-fA-F0-9]{32}")) }.orEmpty()) + .put("sha256", entry.optString("sha256")).put("chunks", chunks).put("product", product) + .put("offset", entry.optLong("offset")).put("remotePath", entry.optString("url").substringBefore('?'))) + } + } + check(files.length() > 0) { "GOG build contains no matching files" } + return old.copy(source = "depot", generation = build.generation, build = build.id, version = build.version, + branch = build.branch, language = resolvedLanguage, manifest = sanitize(manifest), files = files, products = products, verified = true) + } + + fun languageSelected(list: List, selected: String): Boolean = list.any { it.equals("Neutral", true) } || StoreLanguage.depotMatchesResolved(list.toTypedArray(), selected) + + fun cdnPath(hash: String): String { + require(hash.matches(Regex("[a-fA-F0-9]{32,64}"))) { "Invalid GOG content hash" } + return "${hash.take(2)}/${hash.substring(2, 4)}/$hash" + } + + fun links(record: GogInstallRecord, token: String, patches: Boolean = false): JSONObject { + val out = JSONObject() + for (id in record.products) { + val path = if (record.generation == 1) "/windows/${record.manifest.optJSONObject("product")?.optString("timestamp")}/" else "/" + val query = "_version=2&" + (if (record.generation == 1) "type=depot" else "generation=2") + "&path=${java.net.URLEncoder.encode(path, "UTF-8")}" + if (patches) "&root=%2Fpatches%2Fstore" else "" + val response = json("$SYSTEM/products/$id/secure_link?$query", token) + val bases = response.optJSONArray("urls")?.objects().orEmpty().mapNotNull { item -> + val params = item.optJSONObject("parameters") + var url = item.optString("url_format") + params?.keys()?.forEach { key -> url = url.replace("{$key}", params.getString(key)) } + url.takeIf { it.startsWith("https://") } + } + check(bases.isNotEmpty()) { "GOG refused a download link" } + out.put(id, JSONArray(bases)) + } + return out + } + + fun patches(old: GogInstallRecord, target: GogInstallRecord, token: String): JSONArray { + if (old.generation != 2 || target.generation != 2 || old.build.isBlank() || old.build == target.build) return JSONArray() + return runCatching { + val index = json("$SYSTEM/products/${old.id}/patches?_version=4&from_build_id=${old.build}&to_build_id=${target.build}", token) + val meta = json(index.getString("link"), token) + if (meta.optString("algorithm") != "xdelta3") return@runCatching JSONArray() + val result = JSONArray() + for (depot in meta.getJSONArray("depots").objects()) { + if (depot.optString("productId") !in target.products || !languageSelected(depot.optJSONArray("languages")?.strings().orEmpty(), target.language)) continue + val data = json("$CDN/v2/patches/meta/${cdnPath(depot.getString("manifest"))}") + for (item in data.getJSONObject("depot").getJSONArray("items").objects()) if (item.optString("type") == "DepotDiff") + result.put(JSONObject(item.toString()).put("product", depot.getString("productId"))) + } + result + }.getOrElse { JSONArray() } + } + + fun setupFiles(record: GogInstallRecord, token: String): Pair? { + if (!record.manifest.optBoolean("scriptInterpreter")) return null + if (record.files.objects().none { file -> record.products.any { file.optString("path").substringAfterLast('/') == "goggame-$it.script" } }) return null + val index = json("$SYSTEM/dependencies/repository?generation=2") + val repository = json(index.getString("repository_manifest")) + val depot = repository.getJSONArray("depots").objects().firstOrNull { it.optString("dependencyId") == "ISI" } + ?: error("GOG setup interpreter is unavailable") + val content = json("$CDN/v2/dependencies/meta/${cdnPath(depot.getString("manifest"))}") + val files = JSONArray() + for (entry in content.getJSONObject("depot").getJSONArray("items").objects()) { + if (entry.optString("type") != "DepotFile") continue + val chunks = entry.getJSONArray("chunks") + files.put(JSONObject().put("path", "support/" + entry.getString("path").replace('\\', '/')) + .put("size", chunks.objects().sumOf { it.getLong("size") }).put("md5", entry.optString("md5")) + .put("chunks", chunks).put("product", "redist")) + } + val response = json("$SYSTEM/open_link?generation=2&_version=2&path=%2Fdependencies%2Fstore%2F", token) + val bases = GogDownloadManager.parseCdnUrls(response.toString()) + check(bases.isNotEmpty()) { "GOG setup download links are unavailable" } + return files to JSONObject().put("redist", JSONArray(bases)) + } + + fun sanitize(value: JSONObject): JSONObject { + val result = JSONObject() + for (key in value.keys()) { + if (key.lowercase() in setOf("clientsecret", "access_token", "refresh_token", "link", "urls", "url")) continue + val v = value.get(key) + result.put(key, when (v) { is JSONObject -> sanitize(v); is JSONArray -> JSONArray((0 until v.length()).map { i -> v.get(i).let { if (it is JSONObject) sanitize(it) else it } }); else -> v }) + } + return result + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java index dae63e5a29..9b66b95de9 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java @@ -106,7 +106,7 @@ public final class GogDownloadManager { String builds1Json = httpGet(builds1Url, null); if (builds1Json == null) builds1Json = httpGet(builds1Url, token); if (builds1Json == null) throw new InstallException("No builds available for this game"); - Gen1Outcome one = runGen1(ctx, game, token, builds1Json, installPath, cb, cancelled); + Gen1Outcome one = runGen1(ctx, game, token, builds1Json, installPath, cb, cancelled, lang); if (one.result != null || cancelled.get()) return one.result; if (!"NO_CS_BUILDS".equals(one.error)) throw new InstallException("Download failed: " + one.error); cb.onProgress("No Galaxy builds; fetching the installer…", 12); @@ -250,6 +250,11 @@ public final class GogDownloadManager { if (anyFailed.get()) { out.error = "one or more chunks failed to download"; return out; } cb.onProgress("Finishing…", 96); + try { + GogMaintenance.captureDepot(game.gameId, installPath, gm.buildId, buildsJson, manifest, new JSONArray(depotJsons), resolved); + } catch (Exception e) { + cb.onLog("gog: maintenance inventory not recorded: " + e.getMessage()); + } writeFile(new File(installPath, MARKER), ("{\"gameId\":\"" + game.gameId + "\",\"buildId\":\"" + gm.buildId + "\"}").getBytes("UTF-8")); deleteCounted(new File(installPath, ".gog_chunks"), cb); String clientId = manifest.optString("clientId", null); @@ -279,7 +284,7 @@ public final class GogDownloadManager { String buildId = null, manifestUrl = null; for (int i = 0; i < items.length(); i++) { JSONObject item = items.getJSONObject(i); - if ("windows".equals(item.optString("os"))) { + if ("windows".equals(item.optString("os")) && (item.isNull("branch") || item.optString("branch").isEmpty())) { buildId = item.optString("build_id"); manifestUrl = item.optString("link"); if (manifestUrl == null || manifestUrl.isEmpty()) manifestUrl = item.optString("meta_url"); @@ -328,7 +333,7 @@ public final class GogDownloadManager { private static final class Gen1Outcome { Result result; String error; } - private static Gen1Outcome runGen1(Context ctx, GogGame game, String token, String buildsJson, File installPath, Callback cb, AtomicBoolean cancelled) { + private static Gen1Outcome runGen1(Context ctx, GogGame game, String token, String buildsJson, File installPath, Callback cb, AtomicBoolean cancelled, String language) { Gen1Outcome out = new Gen1Outcome(); try { JSONObject builds = new JSONObject(buildsJson); @@ -346,6 +351,15 @@ public final class GogDownloadManager { String manifestStr = decompressBytes(raw); if (manifestStr == null) { out.error = "manifest decompress failed"; return out; } JSONObject manifest = new JSONObject(manifestStr); + if (manifest.has("product")) { + try { + out.result = GogMaintenance.installContent(ctx, game, installPath, builds, language, cb, cancelled); + if (out.result == null && !cancelled.get()) out.error = "no usable public Windows build"; + } catch (Exception e) { + out.error = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + } + return out; + } JSONArray depots = manifest.optJSONArray("depot"); if (depots == null) { out.error = "no depot array"; return out; } List files = new ArrayList<>(); @@ -472,6 +486,7 @@ public final class GogDownloadManager { String exe = pickExe(installPath, null, game.title); if (exe.isEmpty()) throw new InstallException("The installer was extracted but no game executable was found"); cb.onLog("gog: installer extracted natively; game exe " + exe); + GogMaintenance.captureOffline(game.gameId, installPath, downloads); return new Result(exe, "", total); } catch (InstallerException e) { throw e; @@ -576,13 +591,13 @@ public final class GogDownloadManager { return cleaned.isEmpty() ? "game" : cleaned; } - private static String resolveRedirect(String url, String token) { + static String resolveRedirect(String url, String token) { try { if (url.startsWith("/")) url = "https://www.gog.com" + url; for (int hop = 0; hop < 5; hop++) { HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setConnectTimeout(TIMEOUT); conn.setReadTimeout(TIMEOUT); conn.setInstanceFollowRedirects(false); - if (token != null) conn.setRequestProperty("Authorization", "Bearer " + token); + if (token != null && (new URL(url).getHost().equals("gog.com") || new URL(url).getHost().endsWith(".gog.com"))) conn.setRequestProperty("Authorization", "Bearer " + token); int code = conn.getResponseCode(); String location = conn.getHeaderField("Location"); if (code == 301 || code == 302 || code == 303 || code == 307 || code == 308) { @@ -686,6 +701,7 @@ public final class GogDownloadManager { cb.onBytes(shown, planned, speedBps.get()); } @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onMaintenance(long networkBytes, long reusedBytes, long appliedBytes) {} @Override public void onComplete(boolean success, boolean wasCancelled, boolean linkExpiry, String error, long bytesWritten, int filesDone) { okRef.set(success); linkExpiryRef.set(linkExpiry); errRef.set(error == null ? "" : error); latch.countDown(); } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogEstimate.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogEstimate.kt new file mode 100644 index 0000000000..2374ff8c58 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogEstimate.kt @@ -0,0 +1,58 @@ +package com.droiddeck.launcher.stores.gog + +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.io.RandomAccessFile +import java.security.MessageDigest + +internal object GogEstimate { + data class Size(val download: Long, val required: Long) + private data class Chunk(val file: File, val offset: Long) + + fun range(file: File, offset: Long, size: Long, md5: String): Boolean { + if (!file.isFile || offset < 0 || size < 0 || file.length() < offset + size || !md5.matches(Regex("[a-fA-F0-9]{32}"))) return false + return runCatching { + RandomAccessFile(file, "r").use { input -> + input.seek(offset) + val digest = MessageDigest.getInstance("MD5"); val buffer = ByteArray(64 * 1024); var left = size + while (left > 0) { val n = input.read(buffer, 0, minOf(left, buffer.size.toLong()).toInt()); check(n > 0); digest.update(buffer, 0, n); left -= n } + digest.digest().joinToString("") { "%02x".format(it) }.equals(md5, true) + } + }.getOrDefault(false) + } + + fun calculate(folder: File, old: GogInstallRecord, target: GogInstallRecord, patches: JSONArray, reuseChunks: Boolean = true): Size { + val reusable = HashMap, MutableList>() + for (file in old.files.objects()) { + val source = GogFiles.inside(folder, file.getString("path")); var offset = 0L + for (chunk in file.optJSONArray("chunks")?.objects().orEmpty()) { + reusable.getOrPut(chunk.optString("md5").lowercase() to chunk.optLong("size")) { ArrayList() }.add(Chunk(source, offset)) + offset += chunk.optLong("size") + } + } + var download = 0L; var required = 0L + for (file in target.files.objects()) { + if (file.optBoolean("directory") || GogFiles.matches(folder, file)) continue + val chunks = file.optJSONArray("chunks")?.objects().orEmpty() + val stamp = MessageDigest.getInstance("MD5").digest(file.getString("path").toByteArray()).joinToString("") { "%02x".format(it) } + val stage = File(GogFiles.dir(folder), "stage/$stamp") + var offset = 0L + var cost = if (chunks.isEmpty()) file.getLong("size") else chunks.sumOf { chunk -> + val size = chunk.getLong("size"); val hash = chunk.getString("md5") + val exists = reuseChunks && (range(stage, offset, size, hash) || reusable[hash.lowercase() to size].orEmpty().any { range(it.file, it.offset, size, hash) }) + offset += size + if (exists) 0L else chunk.optLong("compressedSize") + } + val patch = patches.objects().firstOrNull { it.optString("path_target").replace('\\', '/') == file.getString("path") } + if (reuseChunks && patch != null) { + val source = GogFiles.inside(folder, patch.getString("path_source").replace('\\', '/')) + val delta = patch.getJSONArray("chunks").objects().sumOf { it.optLong("compressedSize") } + if (delta < cost && source.isFile && GogFiles.hash(source).equals(patch.getString("md5_source"), true)) cost = delta + } + download = Math.addExact(download, cost) + required = Math.addExact(required, maxOf(0, file.getLong("size") - stage.length()) + cost) + } + return Size(download, required) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogFallback.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogFallback.kt new file mode 100644 index 0000000000..0f2d5b9d6f --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogFallback.kt @@ -0,0 +1,115 @@ +package com.droiddeck.launcher.stores.gog + +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadStage +import org.json.JSONObject +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.atomic.AtomicBoolean +import java.util.zip.InflaterInputStream + +internal object GogFallback { + fun transfer(folder: File, plan: JSONObject, handle: DownloadQueue.JobHandle?, stop: AtomicBoolean, + callback: GogDownloadManager.Callback? = null) { + val stageRoot = File(GogFiles.dir(folder), "stage").apply { mkdirs() } + val files = plan.getJSONArray("files") + GogFiles.validate(files, folder) + var wire = 0L + fun progress(n: Int) { wire += n; handle?.progress(wire, plan.optLong("downloadBytes").coerceAtLeast(wire)); callback?.onBytes(wire, plan.optLong("downloadBytes").coerceAtLeast(wire), 0) } + for ((index, item) in files.objects().withIndex()) { + if (stop.get()) return + val destination = GogFiles.inside(folder, item.getString("path")) + if (item.optBoolean("directory")) { java.nio.file.Files.createDirectories(destination.toPath()); continue } + if (GogFiles.matches(folder, item)) continue + val stage = File(stageRoot, java.security.MessageDigest.getInstance("MD5").digest(item.getString("path").toByteArray()).joinToString("") { "%02x".format(it) }) + val chunks = item.optJSONArray("chunks")?.objects().orEmpty() + FileOutputStream(stage).use { output -> + if (chunks.isNotEmpty()) { + for (chunk in chunks) { + if (stop.get()) return + val packed = File(stageRoot, "${stage.name}.packed") + val urls = endpoints(plan.getJSONObject("links"), item.getString("product"), GogContent.cdnPath(chunk.getString("compressedMd5"))) + download(urls, packed, chunk.getLong("compressedSize"), stop, ::progress) + check(GogFiles.hash(packed).equals(chunk.getString("compressedMd5"), true)) { "GOG compressed chunk checksum mismatch" } + val digest = java.security.MessageDigest.getInstance("MD5") + var count = 0L + packed.inputStream().buffered().use { raw -> + raw.mark(2); val first = raw.read(); raw.reset() + val stored = chunk.getLong("size") == chunk.getLong("compressedSize") && chunk.getString("md5").equals(chunk.getString("compressedMd5"), true) + val input = if (!stored && first == 0x78) InflaterInputStream(raw) else raw + val buffer = ByteArray(64 * 1024) + while (true) { + if (stop.get()) return + val n = input.read(buffer); if (n < 0) break + count += n; check(count <= chunk.getLong("size")) { "GOG chunk exceeds expected size" } + digest.update(buffer, 0, n); output.write(buffer, 0, n) + } + } + check(count == chunk.getLong("size") && digest.digest().joinToString("") { "%02x".format(it) }.equals(chunk.getString("md5"), true)) { "GOG chunk checksum mismatch" } + packed.delete() + } + } else if (item.getLong("size") > 0) { + val remote = item.getString("remotePath") + val urls = if (remote.startsWith("https://")) listOf(remote) else endpoints(plan.getJSONObject("links"), item.getString("product"), remote.trimStart('/')) + val range = item.getLong("offset") + val raw = File(stageRoot, "${stage.name}.range") + download(urls, raw, item.getLong("size"), stop, ::progress, range) + raw.inputStream().use { it.copyTo(output) }; raw.delete() + } + output.fd.sync() + } + if (stop.get()) return + check(stage.length() == item.getLong("size")) { "GOG file size mismatch" } + if (item.optString("md5").isNotBlank()) check(GogFiles.hash(stage).equals(item.getString("md5"), true)) { "GOG file checksum mismatch" } + if (item.optString("sha256").isNotBlank()) check(GogFiles.hash(stage, "SHA-256").equals(item.getString("sha256"), true)) { "GOG file checksum mismatch" } + destination.parentFile!!.mkdirs() + GogFiles.inside(folder, item.getString("path")) + GogFiles.replace(stage, destination) + handle?.stage(DownloadStage.INSTALL); handle?.stageProgress((index + 1).toLong(), files.length().toLong()) + callback?.onProgress("Downloading: ${destination.name}", ((index + 1) * 100 / files.length())) + } + val targetPaths = files.objects().mapTo(HashSet()) { it.getString("path").lowercase(java.util.Locale.ROOT) } + for (old in plan.optJSONArray("oldFiles")?.objects().orEmpty()) { + if (stop.get()) return + if (old.getString("path").lowercase(java.util.Locale.ROOT) !in targetPaths && GogFiles.matches(folder, old)) + check(GogFiles.inside(folder, old.getString("path")).delete()) { "Could not remove obsolete GOG file" } + } + } + + fun endpoints(links: JSONObject, product: String, path: String): List = links.getJSONArray(product).strings().map { base -> + if (base.contains("{GALAXY_PATH}")) base.replace("{GALAXY_PATH}", path) + else base.substringBefore('?').trimEnd('/') + "/" + path + if ('?' in base) "?" + base.substringAfter('?') else "" + } + + fun download(urls: List, destination: File, size: Long, stop: AtomicBoolean, progress: (Int) -> Unit, range: Long? = null) { + var failure: Throwable? = null + for (attempt in 0..2) for (url in urls) { + if (stop.get()) return + val connection = URL(url).openConnection() as HttpURLConnection + try { + check(URL(url).protocol == "https" || URL(url).host in setOf("127.0.0.1", "localhost")) + connection.connectTimeout = 20_000; connection.readTimeout = 30_000 + if (range != null) connection.setRequestProperty("Range", "bytes=$range-${range + size - 1}") + check(connection.responseCode == if (range == null) 200 else 206) { "GOG download failed (HTTP ${connection.responseCode})" } + FileOutputStream(destination).use { output -> + connection.inputStream.use { input -> + val buffer = ByteArray(64 * 1024); var count = 0L + while (true) { + if (stop.get()) return + val n = input.read(buffer); if (n < 0) break + count += n; check(size <= 0 || count <= size) { "GOG response exceeds expected size" } + output.write(buffer, 0, n); progress(n) + } + check(size <= 0 || count == size) { "Incomplete GOG download" } + } + output.fd.sync() + } + return + } catch (e: Exception) { failure = e } + finally { connection.disconnect() } + } + throw failure ?: IllegalStateException("No GOG download URL") + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogInstallRecord.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogInstallRecord.kt new file mode 100644 index 0000000000..df07991023 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogInstallRecord.kt @@ -0,0 +1,176 @@ +package com.droiddeck.launcher.stores.gog + +import com.droiddeck.launcher.stores.StoreGameSidecar +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.StandardCopyOption +import java.security.MessageDigest +import java.util.Locale + +internal enum class GogOperation { UPDATE, REPAIR, PATCH, REFRESH } + +internal data class GogInstallRecord( + val id: String, + val source: String, + val generation: Int = 0, + val build: String = "", + val version: String = "", + val branch: String = "", + val language: String = "en-US", + val products: List = listOf(id), + val manifest: JSONObject = JSONObject(), + val files: JSONArray = JSONArray(), + val verified: Boolean = true, +) { + fun json() = JSONObject().put("schema", 1).put("id", id).put("source", source) + .put("generation", generation).put("build", build).put("version", version).put("branch", branch) + .put("language", language).put("products", JSONArray(products)).put("manifest", manifest) + .put("files", files).put("verified", verified) + + fun write(folder: File) = GogFiles.write(File(GogFiles.dir(folder), "installed.json"), json()) + + companion object { + fun parse(o: JSONObject): GogInstallRecord { + require(o.optInt("schema") == 1) { "Unsupported GOG installation record" } + return GogInstallRecord(o.getString("id"), o.getString("source"), o.optInt("generation"), + o.optString("build"), o.optString("version"), o.optString("branch"), o.optString("language", "en-US"), + o.optJSONArray("products")?.strings().orEmpty().ifEmpty { listOf(o.getString("id")) }, + o.optJSONObject("manifest") ?: JSONObject(), o.optJSONArray("files") ?: JSONArray(), o.optBoolean("verified", true)) + } + + fun read(folder: File): GogInstallRecord? = GogFiles.read(File(GogFiles.dir(folder), "installed.json"))?.let(::parse) + + fun authenticBuild(folder: File, id: String): String = GogFiles.read(File(folder, "goggame-$id.id"))?.optString("buildId").orEmpty() + .ifBlank { GogFiles.read(File(folder, "goggame-$id.info"))?.optString("buildId").orEmpty() } + + fun migrate(folder: File, sidecar: StoreGameSidecar): GogInstallRecord { + read(folder)?.let { require(it.id == sidecar.id); return it } + val legacy = GogFiles.read(File(folder, "_gog_manifest.json")) + val info = GogFiles.read(File(folder, "goggame-${sidecar.id}.info")) + val build = authenticBuild(folder, sidecar.id).ifBlank { legacy?.optString("buildId").orEmpty().ifBlank { sidecar.installVersion } } + val ids = folder.listFiles().orEmpty().mapNotNull { Regex("goggame-([0-9]+)\\.info").matchEntire(it.name)?.groupValues?.get(1) } + return GogInstallRecord(sidecar.id, if (legacy != null || sidecar.installVersion.isNotBlank()) "depot" else "offline", + legacy?.optInt("generation", if (build.isNotBlank()) 2 else 0) ?: if (build.isNotBlank()) 2 else 0, + build, info?.optString("versionName").orEmpty(), language = sidecar.extra[com.droiddeck.launcher.stores.StoreLanguage.EXTRA] ?: "en-US", + products = (listOf(sidecar.id) + ids).distinct()).also { it.write(folder) } + } + } +} + +internal object GogFiles { + fun dir(folder: File): File = inside(folder, ".droiddeck-gog").apply { mkdirs() } + fun journal(folder: File): File = File(dir(folder), "operation.json") + + fun inside(root: File, raw: String): File { + val path = raw.replace('\\', '/') + require(path.isNotBlank() && !path.startsWith('/') && ':' !in path && '\u0000' !in path && + path.split('/').none { it.isEmpty() || it == "." || it == ".." }) { "Invalid GOG path: $raw" } + val base = root.canonicalFile + var out = base + for (part in path.split('/')) { + out = File(out, part) + require(!Files.isSymbolicLink(out.toPath())) { "GOG path is a symbolic link: $raw" } + } + require(out.canonicalFile.toPath().startsWith(base.toPath())) { "GOG path leaves installation" } + return out + } + + fun read(file: File): JSONObject? { + require(!Files.isSymbolicLink(file.toPath())) { "GOG metadata is a symbolic link" } + return if (!file.exists()) null else JSONObject(file.readText()) + } + + fun write(file: File, value: JSONObject) { + require(!Files.isSymbolicLink(file.toPath())) + file.parentFile!!.mkdirs() + val tmp = File.createTempFile(".gog-", ".stage", file.parentFile) + try { + FileOutputStream(tmp).use { it.write(value.toString().toByteArray(Charsets.UTF_8)); it.fd.sync() } + replace(tmp, file) + } finally { tmp.delete() } + } + + fun replace(from: File, to: File) { + require(!Files.isSymbolicLink(to.toPath())) + try { Files.move(from.toPath(), to.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) } + catch (_: java.nio.file.AtomicMoveNotSupportedException) { Files.move(from.toPath(), to.toPath(), StandardCopyOption.REPLACE_EXISTING) } + } + + fun hash(file: File, algorithm: String = "MD5"): String { + val digest = MessageDigest.getInstance(algorithm) + file.inputStream().use { input -> val buffer = ByteArray(128 * 1024); while (true) { val n = input.read(buffer); if (n < 0) break; digest.update(buffer, 0, n) } } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + fun matches(folder: File, file: JSONObject): Boolean { + val f = inside(folder, file.getString("path")) + if (!Files.isRegularFile(f.toPath(), LinkOption.NOFOLLOW_LINKS) || f.length() != file.getLong("size")) return false + val md5 = file.optString("md5") + if (md5.matches(Regex("[a-fA-F0-9]{32}"))) return hash(f).equals(md5, true) + val sha = file.optString("sha256") + if (sha.isNotBlank()) return hash(f, "SHA-256").equals(sha, true) + val chunks = file.optJSONArray("chunks")?.objects().orEmpty() + var offset = 0L + return chunks.isNotEmpty() && chunks.all { chunk -> + val result = GogEstimate.range(f, offset, chunk.getLong("size"), chunk.getString("md5")) + offset += chunk.getLong("size"); result + } || f.length() == 0L + } + + fun validate(files: JSONArray, folder: File) { + val paths = HashSet() + val components = HashMap() + for (file in files.objects()) { + val path = file.getString("path").replace('\\', '/') + inside(folder, path) + require(!path.startsWith(".droiddeck-", true) && path != StoreGameSidecar.FILE_NAME) { "GOG content overlaps launcher metadata" } + require(paths.add(path.lowercase(Locale.ROOT))) { "GOG paths differ only by case: $path" } + var parent = folder + var relative = "" + for (part in path.split('/')) { + relative = if (relative.isEmpty()) part else "$relative/$part" + val previous = components.putIfAbsent(relative.lowercase(Locale.ROOT), relative) + require(previous == null || previous == relative) { "GOG directory paths differ only by case: $path" } + require(parent.list()?.none { it != part && it.equals(part, true) } != false) { "An existing path differs only by case: $path" } + parent = File(parent, part) + } + } + } +} + +internal fun JSONArray.objects(): List = (0 until length()).map { getJSONObject(it) } +internal fun JSONArray.strings(): List = (0 until length()).map { optString(it) } + +internal data class GogBuild(val id: String, val version: String, val generation: Int, val branch: String, val link: String) { + companion object { + fun select(builds: JSONObject, pinned: String = "", branch: String = ""): GogBuild? { + val items = builds.optJSONArray("items")?.objects().orEmpty().filter { it.optString("os", "windows") == "windows" } + val item = if (pinned.isNotBlank()) items.firstOrNull { it.optString("build_id") == pinned || it.optString("legacy_build_id") == pinned } + else items.firstOrNull { it.optString("branch").let { b -> (if (b == "null") "" else b) == branch } } + return item?.let { GogBuild(it.optString("build_id"), it.optString("version_name"), it.optInt("generation", 2), + it.optString("branch").let { b -> if (b == "null") "" else b }, + it.optString("link").ifBlank { it.optString("meta_url").ifBlank { it.optJSONArray("urls")?.optJSONObject(0)?.optString("url").orEmpty() } }) } + } + } +} + +internal data class GogOfflinePart(val id: String, val name: String, val url: String, val size: Long, val md5: String) +internal data class GogOfflineDownload(val id: String, val name: String, val version: String, val language: String, val parts: List) { + val size: Long get() = parts.sumOf { it.size } + companion object { + fun parse(downloads: JSONObject, kind: String): List = downloads.optJSONArray(kind)?.objects().orEmpty() + .filter { it.optString("os") == "windows" }.mapNotNull { entry -> + val files = entry.optJSONArray("files")?.objects().orEmpty() + val parts = files.mapNotNull { f -> + val url = f.optString("downlink").ifBlank { f.optString("manualUrl") } + if (url.isBlank()) null else GogOfflinePart(f.optString("id"), f.optString("filename"), url, f.optLong("size"), f.optString("md5")) + } + if (parts.isEmpty()) null else GogOfflineDownload(entry.optString("id").ifBlank { parts.first().id }, + entry.optString("name", "GOG download"), entry.optString("version"), entry.optString("language"), parts) + } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogMaintenance.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogMaintenance.kt new file mode 100644 index 0000000000..796e0dbcb1 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogMaintenance.kt @@ -0,0 +1,415 @@ +package com.droiddeck.launcher.stores.gog + +import android.content.Context +import androidx.compose.runtime.mutableStateMapOf +import com.droiddeck.launcher.agent.AgentGuest +import com.droiddeck.launcher.frontend.AddedGames +import com.droiddeck.launcher.runtime.GuestCommand +import com.droiddeck.launcher.session.SessionPrefs +import com.droiddeck.launcher.session.SessionState +import com.droiddeck.launcher.stores.* +import com.droiddeck.launcher.stores.download.* +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.io.RandomAccessFile +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +internal object GogMaintenance { + enum class State { UNKNOWN, CHECKING, CURRENT, AVAILABLE, FAILED } + data class Check(val state: State, val version: String = "", val reason: String = "") + data class Proposal(val folder: File, val before: GogInstallRecord, val target: GogInstallRecord, + val operation: GogOperation, val downloadBytes: Long, val diskBytes: Long, val patch: GogOfflineDownload? = null, val deltas: JSONArray = JSONArray(), val fullFiles: Boolean = false) + + val checks = mutableStateMapOf() + private val workers = Executors.newFixedThreadPool(2) + private val checkedAt = java.util.concurrent.ConcurrentHashMap() + private val inFlight = java.util.concurrent.ConcurrentHashMap.newKeySet() + @Volatile private var account = "" + private val revision = java.util.concurrent.atomic.AtomicLong() + private const val HELPER = "/usr/local/bin/droiddeck-gog-maintenance" + const val OPERATION = "gogOperation" + const val FOLDER = "gogFolder" + + fun reset() { + revision.incrementAndGet() + account = ""; checkedAt.clear() + StoresState.post { checks.clear() } + } + + fun checkInstalled(context: Context, games: List) { + val user = StoreAccounts.displayName(context, Store.GOG).orEmpty() + if (user != account) { reset(); account = user } + if (user.isBlank()) return + games.filter { it.sidecar.store == Store.GOG && it.sidecar.isInstalled }.forEach { check(context, it) } + } + + fun check(context: Context, game: InstalledStoreGame, force: Boolean = false) { + val key = game.folder.absolutePath + if (!force && System.currentTimeMillis() - (checkedAt[key] ?: 0) < 30 * 60_000) return + if (!inFlight.add(key)) return + val owner = revision.get() + StoresState.post { checks[game.sidecar.id] = Check(State.CHECKING) } + workers.execute { + val result = runCatching { + var old = GogInstallRecord.migrate(game.folder, game.sidecar) + if (!old.verified || GogFiles.journal(game.folder).exists()) Check(State.UNKNOWN, reason = "Finish or recover the GOG maintenance operation first") + else { + val token = GogContent.token(context) + if (old.source == "depot") { + val builds = GogContent.builds(old.id, token) + if (old.files.length() == 0 && old.build.isNotBlank()) GogBuild.select(builds, pinned = old.build)?.let { old = old.copy(branch = it.branch, generation = it.generation) } + val latest = GogBuild.select(builds, branch = old.branch) + when { latest == null -> Check(State.UNKNOWN, reason = "No build is available on the installed branch") + old.build.isBlank() -> Check(State.UNKNOWN, latest.version, "The installed build is unknown") + latest.id == old.build -> Check(State.CURRENT, old.version) + else -> Check(State.AVAILABLE, latest.version) } + } else { + val downloads = GogContent.product(old.id, token).getJSONObject("downloads") + val installer = offlineInstaller(downloads, old.language) + if (old.version.isBlank()) Check(State.UNKNOWN, installer.version, "The installed version is unknown") + else Check(if (old.version == installer.version) State.CURRENT else State.AVAILABLE, installer.version) + } + } + }.getOrElse { Check(State.FAILED, reason = it.message ?: "GOG update check failed") } + if (revision.get() == owner && (result.state == State.CURRENT || result.state == State.AVAILABLE)) checkedAt[key] = System.currentTimeMillis() + inFlight.remove(key) + StoresState.post { if (revision.get() == owner) checks[game.sidecar.id] = result } + } + } + + fun offlinePatches(context: Context, game: InstalledStoreGame): List = + GogOfflineDownload.parse(GogContent.product(game.sidecar.id, GogContent.token(context)).getJSONObject("downloads"), "patches") + + internal fun offlineInstaller(downloads: JSONObject, language: String): GogOfflineDownload { + val installers = GogOfflineDownload.parse(downloads, "installers") + return installers.firstOrNull { it.language.equals(language, true) } + ?: installers.firstOrNull { !it.language.equals("Neutral", true) && StoreLanguage.depotAccepted(arrayOf(it.language), language) } + ?: installers.firstOrNull { it.language.equals("Neutral", true) } + ?: error("No Windows offline installer is available for the installed language ($language)") + } + + fun prepare(context: Context, game: InstalledStoreGame, operation: GogOperation, patch: GogOfflineDownload? = null, resume: Boolean = false): Proposal { + val folder = game.folder.canonicalFile + var old = GogInstallRecord.migrate(folder, game.sidecar) + val journal = GogFiles.read(GogFiles.journal(folder)) + require(journal == null || journal.optString("folder") == folder.path) { "GOG recovery folder does not match" } + if (journal != null && journal.optString("phase") in setOf("patch-starting", "patch-applying", "patch-uncertain") && operation != GogOperation.REFRESH) error("The patch was interrupted. Inspect the installer result or use a full offline refresh; it will not be rerun automatically.") + require(journal == null || operation == GogOperation.REFRESH || journal.optString("operation") == operation.name) { "Resume the existing GOG operation or select a full offline refresh" } + val token = GogContent.token(context) + if (operation == GogOperation.PATCH) { + if (old.source == "depot" && old.manifest.length() == 0 && old.build.isNotBlank()) { + val exact = GogBuild.select(GogContent.builds(old.id, token), pinned = old.build) + ?: error("The installed GOG setup manifest is unavailable; use a full offline refresh") + old = GogContent.snapshot(old, exact, token) + } + check(patch != null) { "Select a GOG patch" } + if (resume) require(journal?.optString("downloadVersion") == patch.version) { "The selected patch changed; select it again in Offline patches" } + val selected = GogContent.resolveDownload(patch, token) + return Proposal(folder, old, old.copy(source = "offline", version = selected.version, verified = false), operation, selected.size, + selected.size + folder.walkTopDown().onEnter { !java.nio.file.Files.isSymbolicLink(it.toPath()) && it.name != ".droiddeck-gog" }.filter { it.isFile }.sumOf { it.length() }, selected) + } + if (operation == GogOperation.REFRESH || old.source != "depot") { + require(operation != GogOperation.REPAIR) { "This offline installation has no depot manifest. Use a full offline refresh to restore its files." } + val download = GogContent.resolveDownload(offlineInstaller(GogContent.product(old.id, token).getJSONObject("downloads"), old.language), token) + if (resume) require(journal?.optString("downloadId") == download.id && journal.optString("downloadVersion") == download.version) { "The previous offline installer is no longer available; select a new full offline refresh" } + if (old.products.any { it != old.id }) { + val pinned = journal?.optJSONObject("target")?.optString("build").orEmpty() + val build = GogBuild.select(GogContent.builds(old.id, token), pinned = pinned, branch = old.branch) + ?: error("No depot build is available to preserve installed DLC during this refresh. Use Update instead.") + val target = GogContent.snapshot(old, build, token) + GogFiles.validate(target.files, folder) + val native = GogNative.isAvailable() + val remaining = GogEstimate.calculate(folder, old, target, JSONArray(), reuseChunks = native) + return Proposal(folder, old, target, GogOperation.REFRESH, Math.addExact(download.size, remaining.download), + Math.addExact(Math.multiplyExact(download.size, 3), remaining.required), download, fullFiles = !native) + } + val target = old.copy(source = "offline", build = "", generation = 0, version = download.version, manifest = JSONObject(), files = JSONArray(), verified = true) + return Proposal(folder, old, target, GogOperation.REFRESH, download.size, download.size * 3, download) + } + require(old.verified || operation != GogOperation.REPAIR) { "The patched installation must be reconciled before repair" } + val pinned = journal?.optJSONObject("target")?.let(GogInstallRecord::parse) + val builds = GogContent.builds(old.id, token) + if (old.files.length() == 0 && old.build.isNotBlank()) GogBuild.select(builds, pinned = old.build)?.let { old = old.copy(branch = it.branch, generation = it.generation) } + val build = GogBuild.select(builds, + pinned = pinned?.build ?: if (operation == GogOperation.REPAIR) old.build.ifBlank { error("The installed GOG build is unknown; update it first") } else "", + branch = pinned?.branch ?: old.branch) ?: error("The requested GOG build is no longer available") + val target = GogContent.snapshot(old, build, token) + GogFiles.validate(target.files, folder) + var before = journal?.optJSONObject("before")?.let(GogInstallRecord::parse) ?: old + if (before.files.length() == 0 && before.build.isNotBlank()) { + val historical = GogBuild.select(builds, pinned = before.build) + if (historical != null) before = runCatching { GogContent.snapshot(before, historical, token) }.getOrDefault(before) + } + val deltas = GogContent.patches(before, target, token) + val native = GogNative.isAvailable() + val size = GogEstimate.calculate(folder, before, target, deltas, reuseChunks = native) + return Proposal(folder, before, target, operation, size.download, size.required, deltas = deltas, fullFiles = !native) + } + + fun resume(context: Context, game: InstalledStoreGame): Proposal { + val journal = GogFiles.read(GogFiles.journal(game.folder)) ?: error("GOG operation journal is missing") + val operation = GogOperation.valueOf(journal.getString("operation")) + val patch = if (operation == GogOperation.PATCH) offlinePatches(context, game) + .firstOrNull { it.id == journal.optString("downloadId") } ?: error("The selected GOG patch is no longer available") else null + return prepare(context, game, operation, patch, resume = true) + } + + fun recoverPatch(context: Context, game: InstalledStoreGame) { + val folder = game.folder.canonicalFile + check(!DownloadQueue.isActive(game.key)) { "Wait for the GOG operation to finish" } + val journal = GogFiles.read(GogFiles.journal(folder)) ?: error("GOG patch journal is missing") + require(journal.optString("folder") == folder.path && journal.optString("operation") == GogOperation.PATCH.name) + require(journal.optString("phase") in setOf("patch-starting", "patch-applying", "patch-uncertain")) + RandomAccessFile(GogFiles.inside(folder, ".droiddeck-gog/lease"), "rw").use { lease -> + val lock = runCatching { lease.channel.tryLock() }.getOrNull() ?: error("Close the patch installer before verifying its result") + lock.use { + ensureIdle(context, folder) + val before = GogInstallRecord.parse(journal.getJSONObject("before")) + val buildId = GogInstallRecord.authenticBuild(folder, before.id).ifBlank { before.build } + check(buildId.isNotBlank()) { "The resulting build is unknown. Use a full offline refresh." } + val token = GogContent.token(context) + val build = GogBuild.select(GogContent.builds(before.id, token), pinned = buildId) + ?: error("The resulting build manifest is unavailable. Use a full offline refresh.") + val target = GogContent.snapshot(before, build, token) + GogFiles.validate(target.files, folder) + check(target.files.objects().all { it.optBoolean("directory") || GogFiles.matches(folder, it) }) { + "The files do not match the reported build. Use a full offline refresh to recover." + } + val sidecar = StoreGameSidecar.parse(journal.getJSONObject("sidecar").toString()) ?: error("The original GOG settings are missing") + target.write(folder) + StoreInstalls.complete(context, folder, sidecar.copy(installVersion = target.build, state = StoreGameSidecar.STATE_INSTALLED)) + check(GogFiles.journal(folder).delete()) { "Could not finish the GOG recovery record" } + } + } + checkedAt.remove(folder.path) + DownloadQueue.dismiss(game.key) + StoresState.notifyLibraryChanged(context) + StoresState.post { checks.remove(game.sidecar.id) } + } + + fun enqueue(context: Context, game: InstalledStoreGame, proposal: Proposal) { + val app = context.applicationContext + require(game.folder.canonicalFile == proposal.folder) + check(!DownloadQueue.isActive("gog:${game.sidecar.id}")) { "GOG maintenance is already queued" } + check(!StoreInstallRoot.insufficientSpace(proposal.diskBytes, StoreInstallRoot.availableBytes(proposal.folder))) { "Not enough free space for GOG maintenance" } + val journal = JSONObject().put("schema", 1).put("folder", proposal.folder.path).put("operation", proposal.operation.name) + .put("before", proposal.before.json()).put("target", proposal.target.json()).put("sidecar", game.sidecar.toJson()).put("phase", "download") + proposal.patch?.let { journal.put("downloadId", it.id).put("downloadVersion", it.version) } + GogFiles.write(GogFiles.journal(proposal.folder), journal) + val item = CatalogItem(Store.GOG, game.sidecar.id, game.sidecar.title, game.sidecar.hero, tallImageUrl = game.sidecar.cover, + extra = mapOf(OPERATION to proposal.operation.name, FOLDER to proposal.folder.path)) + SavedQueue.remember(app, item, null) + enqueueJob(app, item, proposal) + } + + private fun enqueueJob(context: Context, item: CatalogItem, proposal: Proposal) { + DownloadQueue.enqueue(context, DownloadEntry(Store.GOG, item.id, item.title, cover = item.tallImageUrl, + bytesTotal = proposal.downloadBytes, location = StoreInstallRoot.labelFor(context, proposal.folder))) { Job(context, proposal) } + } + + fun restore(context: Context, item: CatalogItem) { + val folder = File(item.extra[FOLDER] ?: error("Missing GOG recovery folder")).canonicalFile + require(StoreInstallRoot.gameFolders(context).any { it.canonicalFile == folder }) { "GOG recovery folder is unavailable" } + val sidecar = StoreGameSidecar.read(folder) ?: error("GOG installation record is missing") + require(sidecar.store == Store.GOG && sidecar.id == item.id) + val journal = GogFiles.read(GogFiles.journal(folder)) ?: error("GOG operation journal is missing") + val operation = GogOperation.valueOf(journal.getString("operation")) + if (journal.optString("phase") in setOf("patch-starting", "patch-applying", "patch-uncertain")) { + DownloadQueue.recordFailed(context, DownloadEntry(Store.GOG, item.id, item.title), "The patch result needs recovery; open Offline patches") + return + } + val game = InstalledStoreGame(sidecar, folder) + SavedQueue.remember(context, item, null) + enqueueJob(context, item, resume(context, game)) + } + + private class Job(val app: Context, val proposal: Proposal) : DownloadQueue.DownloadJob { + val cancelled = AtomicBoolean(false) + @Volatile var applyingPatch = false + @Volatile private var running = false + @Volatile private var deleteCaches = false + override fun cancel(deleteFiles: Boolean) { + if (applyingPatch) return + deleteCaches = deleteCaches || deleteFiles + cancelled.set(true) + if (!running && deleteCaches) cleanCaches() + } + + override fun run(handle: DownloadQueue.JobHandle): String? { + running = true + try { return perform(handle) } + finally { + running = false; applyingPatch = false; handle.interruptible(true) + if (deleteCaches) cleanCaches() + } + } + + private fun cleanCaches() { + for (name in listOf("stage", "downloads", "extracted")) { + val root = GogFiles.inside(proposal.folder, ".droiddeck-gog/$name") + if (root.exists()) java.nio.file.Files.walk(root.toPath()).use { paths -> + paths.sorted(java.util.Comparator.reverseOrder()).forEach { java.nio.file.Files.deleteIfExists(it) } + } + } + } + + private fun perform(handle: DownloadQueue.JobHandle): String? { + val folder = proposal.folder + val journal = GogFiles.read(GogFiles.journal(folder)) ?: error("GOG operation journal is missing") + val sidecar = StoreGameSidecar.parse(journal.getJSONObject("sidecar").toString()) ?: error("GOG previous installation is missing") + if (proposal.operation == GogOperation.PATCH) { + val result = GogOffline.apply(app, proposal, handle, cancelled, onApplying = { applyingPatch = true; handle.interruptible(false) }) + applyingPatch = false; handle.interruptible(true) + if (!result || cancelled.get()) return null + } else { + RandomAccessFile(GogFiles.inside(folder, ".droiddeck-gog/lease"), "rw").use { lease -> + val lock = runCatching { lease.channel.tryLock() }.getOrNull() ?: error("Close the GOG game before updating it") + lock.use { + ensureIdle(app, folder) + sidecar.copy(state = StoreGameSidecar.STATE_UPDATING).write(folder) + GogFiles.write(GogFiles.journal(folder), journal.put("phase", "applying")) + if (proposal.operation == GogOperation.REFRESH) { + GogOffline.refresh(app, proposal, handle, cancelled) + if (!cancelled.get() && proposal.target.source == "depot") transfer(app, proposal, handle, cancelled) + } else transfer(app, proposal, handle, cancelled) + if (cancelled.get()) return null + } + } + } + if (cancelled.get()) return null + val target = GogInstallRecord.read(folder)?.takeIf { proposal.operation == GogOperation.PATCH || proposal.operation == GogOperation.REFRESH && proposal.target.source != "depot" } ?: proposal.target + check(target.verified) { "The patch completed, but its resulting version could not be verified. Inspect the game or use a full offline refresh." } + target.write(folder) + StoreInstalls.complete(app, folder, sidecar.copy(installVersion = target.build.ifBlank { target.version }, state = StoreGameSidecar.STATE_INSTALLED)) + check(GogFiles.journal(folder).delete()) { "Could not finish the GOG recovery record" } + runCatching { cleanCaches() }.onFailure { handle.log("gog: completed; could not remove expendable cache: ${it.message}") } + checkedAt.remove(folder.path) + StoresState.post { checks.remove(sidecar.id) } + return folder.path + } + } + + private fun ensureIdle(context: Context, folder: File) { + val game = AddedGames.single(context, folder.path) ?: return + check(SessionState.gameId != (game.appId and 0xffffffffL).toString()) { "Close the GOG game before maintenance" } + if (SessionState.running) { + val id = (game.steamAppId?.toLong() ?: game.appId) and 0xffffffffL + val guest = AddedGames.guestPath(context, folder) ?: error("The GOG folder is unavailable to Linux") + if (SessionState.suspended || AgentGuest.hello(context) == null) { + val root = AddedGames.guestPath(context, SessionPrefs.activeSteamRoot(context)) ?: error("The Steam folder is unavailable to Linux") + val rc = GuestCommand.run(context, listOf("/usr/bin/env", "STEAM_COMPAT_CLIENT_INSTALL_PATH=$root", HELPER, "idle", guest, id.toString(), game.guestExe), + extraBinds = StoreInstallRoot.externalBindSpecs(context), logName = "gog-idle", onLine = StoresState::logLine) + check(rc == 0) { "Close the GOG game before maintenance" } + return + } + val request = JSONObject().put("kind", "exec").put("argv", JSONArray(listOf(HELPER, "idle", guest, id.toString(), game.guestExe))).put("timeout", 15) + val result = AgentGuest.call(context, request, 20_000, trusted = true) + check(result.optBoolean("ok") && result.optInt("exitCode", -1) == 0) { "Close the GOG game before maintenance" } + } + } + + private fun transfer(context: Context, proposal: Proposal, handle: DownloadQueue.JobHandle?, stop: AtomicBoolean, callback: GogDownloadManager.Callback? = null) { + val token = GogContent.token(context) + var patches = proposal.deltas + var previousWire = 0L + for (attempt in 0..4) { + if (stop.get()) return + val links = GogContent.links(proposal.target, GogContent.token(context)) + val patchLinks = if (patches.length() > 0) runCatching { GogContent.links(proposal.target, token, patches = true) }.getOrElse { patches = JSONArray(); JSONObject() } else JSONObject() + val plan = JSONObject().put("files", proposal.target.files).put("oldFiles", proposal.before.files) + .put("patches", patches).put("links", links).put("patchLinks", patchLinks).put("downloadBytes", proposal.downloadBytes) + val optimized = GogNative.isAvailable() + val remaining = GogEstimate.calculate(proposal.folder, proposal.before, proposal.target, patches, reuseChunks = optimized).required + check(!StoreInstallRoot.insufficientSpace(remaining, StoreInstallRoot.availableBytes(proposal.folder))) { "Not enough free space for remaining GOG maintenance" } + handle?.diskSize(remaining) + if (!optimized) { + try { GogFallback.transfer(proposal.folder, plan, handle, stop, callback); return } + catch (e: Exception) { + if (stop.get()) return + if (attempt == 4 || Regex("HTTP (401|403|404|500)").find(e.message.orEmpty()) == null) throw e + handle?.log("gog: refreshing file-download links; verified replacements are retained") + continue + } + } + val latch = CountDownLatch(1) + var runWire = 0L + var success = false; var expired = false; var failure = "GOG download failed" + val listener = object : GogNative.Listener { + override fun onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, file: String, fileBytes: Long, verified: Boolean) { + handle?.stage(if (verified) DownloadStage.VERIFY else DownloadStage.INSTALL) + handle?.stageProgress(filesDone.toLong(), filesTotal.toLong(), filesDone, filesTotal) + callback?.onStage(if (verified) "verify" else "install", filesDone.toLong(), filesTotal.toLong(), filesDone, filesTotal) + } + override fun onBytes(bytesDone: Long, bytesTotal: Long) { + runWire = bytesDone + handle?.progress(previousWire + bytesDone, (previousWire + bytesTotal).coerceAtLeast(previousWire + bytesDone)) + callback?.onBytes(previousWire + bytesDone, (previousWire + bytesTotal).coerceAtLeast(previousWire + bytesDone), 0) + } + override fun onLog(line: String) { handle?.log(line); callback?.onLog(line) } + override fun onMaintenance(networkBytes: Long, reusedBytes: Long, appliedBytes: Long) { handle?.maintenance(reusedBytes, appliedBytes) } + override fun onComplete(ok: Boolean, cancelled: Boolean, linkExpiry: Boolean, error: String, bytesWritten: Long, filesDone: Int) { + success = ok; expired = linkExpiry; failure = error; latch.countDown() + } + } + val native = GogNative.start(2, arrayOf(plan.toString()), emptyArray(), proposal.folder.path, emptyArray(), "", 1, 1, false, "gog maintenance", listener) + check(native != 0L) { "GOG native engine could not start" } + try { while (!latch.await(200, TimeUnit.MILLISECONDS)) if (stop.get()) GogNative.cancel(native) } + finally { GogNative.release(native) } + if (stop.get()) return + if (success) { + for (file in proposal.target.files.objects()) if (!file.optBoolean("directory") && file.optString("md5").isBlank() && file.optString("sha256").isBlank()) + file.put("sha256", GogFiles.hash(GogFiles.inside(proposal.folder, file.getString("path")), "SHA-256")) + return + } + if (!expired || attempt == 4) error(failure) + previousWire += runWire + handle?.log("gog: refreshing expired download links; validated files and chunks are retained") + } + } + + @JvmStatic + fun installContent(context: Context, game: GogGame, folder: File, builds: JSONObject, language: String, + callback: GogDownloadManager.Callback, cancelled: AtomicBoolean): GogDownloadManager.Result? { + val build = GogBuild.select(builds) ?: return null + folder.mkdirs() + val old = GogInstallRecord(game.gameId, "depot", language = language) + val target = GogContent.snapshot(old, build, GogContent.token(context)) + GogFiles.validate(target.files, folder) + val bytes = target.files.objects().sumOf { it.optJSONArray("chunks")?.objects()?.sumOf { c -> c.optLong("compressedSize") }?.takeIf { n -> n > 0 } ?: it.optLong("size") } + val disk = target.files.objects().sumOf { it.optLong("size") } + check(!StoreInstallRoot.insufficientSpace(disk, StoreInstallRoot.availableBytes(folder))) { "Not enough free space for GOG installation" } + callback.onSizes(bytes, disk) + transfer(context, Proposal(folder, old, target, GogOperation.UPDATE, bytes, disk), null, cancelled, callback) + if (cancelled.get()) return null + target.write(folder) + val result = GogDownloadManager.Result(GogDownloadManager.pickExe(folder, null, game.title), build.id, disk) + result.language = target.language + result.dependencies = target.manifest.optJSONArray("dependencies")?.strings().orEmpty() + result.dependenciesRead = true + return result + } + + @JvmStatic + fun captureDepot(id: String, folder: File, build: String, buildsJson: String, manifest: JSONObject, depots: JSONArray, language: String) { + val selected = GogBuild.select(JSONObject(buildsJson), pinned = build) + val files = JSONArray() + for (depot in depots.strings()) for (item in JSONObject(depot).getJSONObject("depot").getJSONArray("items").objects()) { + val chunks = item.optJSONArray("chunks") ?: continue + files.put(JSONObject().put("path", item.getString("path").replace('\\', '/')).put("size", chunks.objects().sumOf { it.optLong("size") }) + .put("md5", item.optString("md5")).put("chunks", chunks).put("product", id)) + } + GogFiles.validate(files, folder) + GogInstallRecord(id, "depot", 2, build, selected?.version.orEmpty(), selected?.branch.orEmpty(), language, + manifest = GogContent.sanitize(manifest), files = files).write(folder) + } + + @JvmStatic + fun captureOffline(id: String, folder: File, downloads: JSONObject) { + val selected = GogOfflineDownload.parse(downloads, "installers").firstOrNull() ?: return + GogInstallRecord(id, "offline", version = selected.version, language = selected.language).write(folder) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogNative.kt index f8ab86b14f..80024ab6ac 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogNative.kt @@ -15,6 +15,8 @@ object GogNative { fun onBytes(bytesDone: Long, bytesTotal: Long) + fun onMaintenance(networkBytes: Long, reusedBytes: Long, appliedBytes: Long) + fun onLog(line: String) fun onComplete(success: Boolean, cancelled: Boolean, linkExpiry: Boolean, error: String, bytesWritten: Long, filesDone: Int) @@ -26,7 +28,7 @@ object GogNative { fun isAvailable(): Boolean { available?.let { return it } val ok = try { - StoresNative.ensureLoaded() && nativeProbe() == 1 + StoresNative.ensureLoaded() && nativeProbe() == 2 } catch (t: Throwable) { Log.w(TAG, "native GOG engine unavailable: ${t.javaClass.simpleName}: ${t.message}") false diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogOffline.kt b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogOffline.kt new file mode 100644 index 0000000000..02ff54f26f --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogOffline.kt @@ -0,0 +1,200 @@ +package com.droiddeck.launcher.stores.gog + +import android.content.Context +import com.droiddeck.launcher.agent.AgentGuest +import com.droiddeck.launcher.frontend.AddedGames +import com.droiddeck.launcher.runtime.GuestCommand +import com.droiddeck.launcher.session.SessionService +import com.droiddeck.launcher.session.SessionState +import com.droiddeck.launcher.stores.StoreGameSidecar +import com.droiddeck.launcher.stores.StoreInstallRoot +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadStage +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean + +internal object GogOffline { + private const val HELPER = "/usr/local/bin/droiddeck-gog-maintenance" + + internal data class Layout(val source: File, val prefix: String) + + internal fun layout(extracted: File, installed: File, id: String, executable: String): Layout { + val nested = executable.startsWith("app/", true) && !File(installed, "goggame-$id.info").isFile + return Layout(extracted, if (nested) "app/" else "") + } + + internal fun validateDepotInstaller(extracted: File, target: GogInstallRecord) { + if (target.source != "depot") return + val builds = listOf(extracted, File(extracted, "app")).map { GogInstallRecord.authenticBuild(it, target.id) }.filter { it.isNotBlank() } + require(builds.isNotEmpty() && builds.all { it == target.build } && target.build.isNotBlank()) { + "The offline installer does not identify the selected depot build. Use Update to preserve installed DLC." + } + } + + private fun download(context: Context, proposal: GogMaintenance.Proposal, handle: DownloadQueue.JobHandle, stop: AtomicBoolean): File { + val patch = proposal.patch ?: error("GOG download selection is missing") + val root = File(GogFiles.dir(proposal.folder), "downloads").apply { mkdirs() } + var first: File? = null; var bytes = 0L + for ((index, part) in patch.parts.withIndex()) { + if (stop.get()) return first ?: File(root, "cancelled") + val resolved = GogDownloadManager.resolveRedirect(part.url, GogContent.token(context)) ?: error("The GOG download link is unavailable") + val serverName = GogContent.downloadFilename(resolved).takeIf { it.endsWith(".exe", true) || it.endsWith(".bin", true) } + val name = part.name.ifBlank { serverName ?: error("GOG did not provide the installer part filename") } + require('/' !in name && '\\' !in name && name !in setOf(".", "..")) { "Invalid GOG installer filename" } + val file = GogFiles.inside(root, name) + val receipt = File(root, "part-$index.json") + val saved = GogFiles.read(receipt) + val complete = file.isFile && (part.size <= 0 || file.length() == part.size) && when { + part.md5.isNotBlank() -> GogFiles.hash(file).equals(part.md5, true) + saved?.optString("id") == part.id && saved.optString("filename") == name -> GogFiles.hash(file, "SHA-256") == saved.optString("sha256") + else -> false + } + if (!complete) { + val staging = GogFiles.inside(root, "$name.partial") + GogFallback.download(listOf(resolved), staging, part.size, stop, { n -> bytes += n; handle.progress(bytes, proposal.downloadBytes.coerceAtLeast(bytes)) }) + if (stop.get()) return first ?: file + if (part.md5.isNotBlank()) check(GogFiles.hash(staging).equals(part.md5, true)) { "GOG installer checksum mismatch" } + GogFiles.replace(staging, file) + GogFiles.write(receipt, JSONObject().put("id", part.id).put("filename", name).put("sha256", GogFiles.hash(file, "SHA-256"))) + } + if (index == 0) { check(file.extension.equals("exe", true)) { "GOG patch has no setup executable" }; first = file } + } + return first ?: error("GOG download has no files") + } + + fun refresh(context: Context, proposal: GogMaintenance.Proposal, handle: DownloadQueue.JobHandle, stop: AtomicBoolean) { + val installer = download(context, proposal, handle, stop) + if (stop.get()) return + val extracted = File(GogFiles.dir(proposal.folder), "extracted").apply { mkdirs() } + val source = AddedGames.guestPath(context, installer) ?: error("The GOG installer is not visible to Linux") + val target = AddedGames.guestPath(context, extracted) ?: error("The GOG staging folder is not visible to Linux") + handle.stage(DownloadStage.INSTALL, "Extracting the full offline installer") + val rc = GuestCommand.run(context, listOf("/usr/local/bin/droiddeck-innoextract", "--gog", "--silent", "-d", target, source), + extraBinds = StoreInstallRoot.externalBindSpecs(context), cancelled = stop, logName = "gog-refresh", onLine = handle::log) + if (stop.get()) return + check(rc == 0) { "GOG installer extraction failed (exit $rc)" } + applyRefresh(proposal, extracted, handle, stop) + } + + internal fun applyRefresh(proposal: GogMaintenance.Proposal, extracted: File, handle: DownloadQueue.JobHandle?, stop: AtomicBoolean) { + validateDepotInstaller(extracted, proposal.target) + val sidecar = StoreGameSidecar.read(proposal.folder) ?: error("The original GOG settings are missing") + val layout = layout(extracted, proposal.folder, proposal.before.id, sidecar.exe) + val files = layout.source.walkTopDown().onEnter { directory -> + check(!java.nio.file.Files.isSymbolicLink(directory.toPath())) { "GOG installer contains a symbolic link" } + directory.parentFile != extracted || directory.name !in setOf("tmp", "commonappdata", "userappdata", "localappdata") + }.filter { it != layout.source && it != File(extracted, "app") }.map { file -> + check(!java.nio.file.Files.isSymbolicLink(file.toPath())) + val relative = file.relativeTo(layout.source).invariantSeparatorsPath.removePrefix("app/") + JSONObject().put("path", layout.prefix + relative) + .put("size", if (file.isDirectory) 0 else file.length()).put("directory", file.isDirectory) + .put("md5", if (file.isDirectory) "" else GogFiles.hash(file)).put("product", proposal.before.id) to file + }.toList().groupBy { it.first.getString("path") }.map { (_, entries) -> + check(entries.size == 1 || entries.all { it.first.getBoolean("directory") }) { "GOG installer has conflicting file locations" } + entries.first() + } + check(files.any { !it.first.getBoolean("directory") }) { "GOG installer contained no game files" } + val inventory = JSONArray(files.map { it.first }) + GogFiles.validate(inventory, proposal.folder) + val executable = sidecar.exe.replace('\\', '/') + check(files.any { !it.first.getBoolean("directory") && it.first.getString("path").equals(executable, true) } || + proposal.before.verified && proposal.before.files.length() > 0 && + proposal.before.files.objects().none { it.getString("path").equals(executable, true) } && GogFiles.inside(proposal.folder, executable).isFile) { + "The offline installer did not extract the game's executable. No installed files were changed." + } + val installed = proposal.target.copy(files = inventory) + GogFiles.write(File(GogFiles.dir(proposal.folder), "refresh-target.json"), installed.json()) + for ((index, entry) in files.withIndex()) { + if (stop.get()) return + val (file, sourceFile) = entry + val destination = GogFiles.inside(proposal.folder, file.getString("path")) + if (file.getBoolean("directory")) java.nio.file.Files.createDirectories(destination.toPath()) else { + destination.parentFile!!.mkdirs() + GogFiles.replace(sourceFile, destination) + } + handle?.stageProgress((index + 1).toLong(), files.size.toLong()) + } + if (proposal.target.source != "depot") { + val targetPaths = files.mapTo(HashSet()) { it.first.getString("path").lowercase(java.util.Locale.ROOT) } + for (old in proposal.before.files.objects()) if (old.getString("path").lowercase(java.util.Locale.ROOT) !in targetPaths && GogFiles.matches(proposal.folder, old)) + check(GogFiles.inside(proposal.folder, old.getString("path")).delete()) + installed.write(proposal.folder) + } + } + + fun apply(context: Context, proposal: GogMaintenance.Proposal, handle: DownloadQueue.JobHandle, stop: AtomicBoolean, onApplying: () -> Unit): Boolean { + check(GogFiles.read(GogFiles.journal(proposal.folder))?.optString("phase") in setOf("download", "patch-ready")) { + "The patch result needs recovery. Inspect the installer or use a full offline refresh; the patch will not be run again." + } + val installer = download(context, proposal, handle, stop) + if (stop.get()) return false + val setup = GogContent.setupFiles(proposal.before, GogContent.token(context)) + if (setup != null) { + GogFallback.transfer(proposal.folder, JSONObject().put("files", setup.first).put("links", setup.second) + .put("downloadBytes", setup.first.objects().sumOf { it.getJSONArray("chunks").objects().sumOf { chunk -> chunk.getLong("compressedSize") } }), handle, stop) + if (stop.get()) return false + } + val added = AddedGames.single(context, proposal.folder.path) ?: error("The GOG game shortcut is unavailable") + val id = (added.steamAppId?.toLong() ?: added.appId) and 0xffffffffL + val folder = AddedGames.guestPath(context, proposal.folder) ?: error("The GOG folder is unavailable to Linux") + val executable = AddedGames.guestPath(context, installer) ?: error("The GOG patch is unavailable to Linux") + context.startActivity(android.content.Intent(context, com.droiddeck.launcher.SessionActivity::class.java) + .putExtra(SessionService.EXTRA_MODE, if (SessionState.running) SessionState.mode else SessionService.MODE_DESKTOP) + .addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK or android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP or android.content.Intent.FLAG_ACTIVITY_SINGLE_TOP)) + val deadline = System.currentTimeMillis() + 120_000 + while (AgentGuest.hello(context) == null && System.currentTimeMillis() < deadline) { if (stop.get()) return false; Thread.sleep(250) } + check(AgentGuest.hello(context) != null) { "The Linux session did not become ready; open it and retry" } + val journal = GogFiles.read(GogFiles.journal(proposal.folder)) ?: error("GOG patch journal is missing") + journal.put("phase", "patch-ready") + File(GogFiles.dir(proposal.folder), "patch-status.json").delete() + GogFiles.write(GogFiles.journal(proposal.folder), journal) + val request = JSONObject().put("kind", "exec").put("timeout", 15) + .put("env", JSONObject().put("DROIDDECK_GOG_DESKTOP", if (SessionState.mode == SessionService.MODE_DESKTOP) "1" else "0")) + .put("argv", JSONArray(listOf(HELPER, "start", folder, id.toString(), executable, added.guestExe))) + if (stop.get()) return false + onApplying() + val response = AgentGuest.call(context, request, 20_000, trusted = true) + check(response.optBoolean("ok") && response.optInt("exitCode", -1) == 0) { "The GOG patch installer could not start; inspect the GOG log" } + handle.stage(DownloadStage.INSTALL, "Use the Linux session to finish or cancel the patch installer") + var nextStatus = System.currentTimeMillis() + 5_000 + while (true) { + val status = GogFiles.read(File(GogFiles.dir(proposal.folder), "patch-status.json")) + if (status?.optString("state") in setOf("done", "failed", "uncertain")) { + check(status?.optString("state") == "done") { status?.optString("reason") ?: "GOG patch failed" } + break + } + check(SessionState.running) { "The patch session ended. Its result needs recovery; the patch will not be rerun automatically." } + check(!SessionState.suspended) { "The patch session was suspended. Resume it and verify recovery before retrying." } + if (System.currentTimeMillis() >= nextStatus) { + val query = JSONObject().put("kind", "exec").put("timeout", 5).put("argv", JSONArray(listOf(HELPER, "status", folder))) + val answer = AgentGuest.call(context, query, 10_000, trusted = true) + check(answer.optBoolean("ok") && answer.optInt("exitCode", -1) == 0) { "The patch helper could not be reached. Resume the Linux session and verify recovery." } + nextStatus = System.currentTimeMillis() + 5_000 + } + Thread.sleep(500) + } + val info = GogFiles.read(File(proposal.folder, "goggame-${proposal.before.id}.info")) + val version = info?.optString("versionName").orEmpty() + val build = GogInstallRecord.authenticBuild(proposal.folder, proposal.before.id) + var record = proposal.target.copy(build = build, version = version, files = JSONArray(), manifest = JSONObject(), verified = false) + record.write(proposal.folder) + if (build.isNotBlank()) { + record = runCatching { + val token = GogContent.token(context) + val target = GogBuild.select(GogContent.builds(record.id, token), pinned = build) + val snapshot = target?.let { GogContent.snapshot(record, it, token) } + snapshot?.takeIf { it.files.objects().all { f -> f.optBoolean("directory") || GogFiles.matches(proposal.folder, f) } } ?: record + }.getOrDefault(record) + } + record.write(proposal.folder) + if (!record.verified) { + val before = StoreGameSidecar.read(proposal.folder) ?: error("The GOG installation record is missing") + before.copy(installVersion = "", state = StoreGameSidecar.STATE_UPDATING).write(proposal.folder) + val operation = GogFiles.read(GogFiles.journal(proposal.folder))!! + GogFiles.write(GogFiles.journal(proposal.folder), operation.put("phase", "patch-uncertain")) + } + return true + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/ui/GameDetail.kt b/app/src/main/java/com/droiddeck/launcher/ui/GameDetail.kt index df84306106..03d63cf47f 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/GameDetail.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/GameDetail.kt @@ -351,7 +351,7 @@ internal fun GameActions( val gone = storeKey?.let { StoresState.uninstalling[it] } val xboxBusy = store == Store.XBOX && id != null && XboxContent.busy(id) PrimaryButton( - stringResource(R.string.stores_play), enabled = !s.busy && gone == null && download == null && !xboxBusy, + stringResource(R.string.stores_play), enabled = !s.busy && gone == null && download == null && !xboxBusy && (store != Store.GOG || installed?.sidecar?.isInstalled != false), main = true, icon = Icons.Filled.PlayArrow, modifier = leadOn, ) { if (game != null) a.onSteamGame(game) else if (store != null && id != null) launchStoreGame(ctx, store, id, s, a) @@ -363,6 +363,7 @@ internal fun GameActions( EditArtButton(game) } if (store == Store.ITCH && installed != null) ItchUpdateButton(installed, download == null && StoresState.uninstalling[installed.key] == null) + if (store == Store.GOG && installed != null) GogMaintenanceActions(installed) if (store == Store.XBOX && id != null) XboxInstalledStream(id, game?.name ?: item?.title.orEmpty(), item) if (store == Store.XBOX && installed != null && id != null) { SecondaryButton(stringResource(R.string.stores_xbox_update), enabled = download == null && !XboxRuntime.isActive(id)) { diff --git a/app/src/main/java/com/droiddeck/launcher/ui/GogMaintenanceUi.kt b/app/src/main/java/com/droiddeck/launcher/ui/GogMaintenanceUi.kt new file mode 100644 index 0000000000..cbc7dfcad3 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/ui/GogMaintenanceUi.kt @@ -0,0 +1,188 @@ +package com.droiddeck.launcher.ui + +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.background +import androidx.compose.material3.Text +import androidx.compose.material3.MaterialTheme +import androidx.compose.runtime.* +import androidx.compose.ui.Modifier +import androidx.compose.ui.Alignment +import androidx.compose.ui.draw.clip +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.testTag +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import com.droiddeck.launcher.R +import com.droiddeck.launcher.stores.InstalledStoreGame +import com.droiddeck.launcher.stores.formatBytes +import com.droiddeck.launcher.stores.StoreLanguage +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadState +import com.droiddeck.launcher.stores.gog.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +@Composable +internal fun GogMaintenanceActions(game: InstalledStoreGame) { + val ctx = LocalContext.current + val colors = MaterialTheme.colorScheme + val scope = rememberCoroutineScope() + var menu by remember { mutableStateOf(false) } + var patches by remember { mutableStateOf?>(null) } + var proposal by remember { mutableStateOf(null) } + var error by remember { mutableStateOf(null) } + var busy by remember { mutableStateOf(false) } + var recoverablePatch by remember { mutableStateOf(false) } + val key = "gog:${game.sidecar.id}" + val download = com.droiddeck.launcher.stores.StoresState.download(key) + val active = download?.isActive == true + val check = GogMaintenance.checks[game.sidecar.id] + LaunchedEffect(game.folder.path) { GogMaintenance.check(ctx, game) } + LaunchedEffect(menu, active) { + if (menu) { + recoverablePatch = false + val result = withContext(Dispatchers.IO) { runCatching { + GogFiles.read(GogFiles.journal(game.folder))?.let { + it.optString("operation") == GogOperation.PATCH.name && it.optString("phase") in setOf("patch-starting", "patch-applying", "patch-uncertain") + } == true + } } + result.onSuccess { recoverablePatch = it }.onFailure { error = it.message } + } + } + fun prepare(operation: GogOperation, patch: GogOfflineDownload? = null) { + menu = false; patches = null; busy = true; error = null + scope.launch { + runCatching { withContext(Dispatchers.IO) { GogMaintenance.prepare(ctx, game, operation, patch) } } + .onSuccess { proposal = it }.onFailure { error = it.message } + busy = false + } + } + if (active) { + ActionChip(downloadLabel(download!!), ok = false) + if (download.interruptible) { + SecondaryButton(stringResource(if (download.state == DownloadState.PAUSED) R.string.stores_dl_resume else R.string.stores_dl_pause)) { + if (download.state == DownloadState.PAUSED) DownloadQueue.resume(ctx, key) else DownloadQueue.pause(key) + } + } else Text(stringResource(R.string.gog_patch_in_session), color = MaterialTheme.colorScheme.onSurfaceVariant) + } else if (!game.sidecar.isInstalled || GogFiles.journal(game.folder).exists()) { + SecondaryButton(stringResource(R.string.gog_resume), modifier = Modifier.testTag("gog-resume")) { + busy = true + scope.launch { + runCatching { + withContext(Dispatchers.IO) { + GogMaintenance.resume(ctx, game) + } + }.onSuccess { proposal = it }.onFailure { error = it.message } + busy = false + } + } + } else if (check?.state == GogMaintenance.State.AVAILABLE) { + SecondaryButton(stringResource(R.string.gog_update, check.version.ifBlank { game.sidecar.title }), enabled = !busy, modifier = Modifier.testTag("gog-update")) { prepare(GogOperation.UPDATE) } + } + androidx.compose.foundation.layout.Box { + SecondaryButton(stringResource(if (busy) R.string.common_checking else R.string.gog_maintenance), enabled = !busy && !active, modifier = Modifier.testTag("gog-maintenance")) { menu = !menu } + AnchoredMenu(menu, { menu = false }, title = stringResource(R.string.gog_maintenance), compact = true) { first -> + MenuItem(stringResource(R.string.common_update), checked = false) { prepare(GogOperation.UPDATE) } + MenuItem(stringResource(R.string.gog_check), checked = false, focusRequester = first) { menu = false; GogMaintenance.check(ctx, game, true) } + MenuItem(stringResource(R.string.gog_repair), checked = false, modifier = Modifier.testTag("gog-repair")) { prepare(GogOperation.REPAIR) } + MenuItem(stringResource(R.string.gog_patches), checked = false, modifier = Modifier.testTag("gog-patches")) { + menu = false; busy = true + scope.launch { + runCatching { withContext(Dispatchers.IO) { GogMaintenance.offlinePatches(ctx, game) } } + .onSuccess { patches = it }.onFailure { error = it.message } + busy = false + } + } + MenuItem(stringResource(R.string.gog_refresh), checked = false) { prepare(GogOperation.REFRESH) } + if (recoverablePatch) MenuItem(stringResource(R.string.gog_recover), checked = false, modifier = Modifier.testTag("gog-recover")) { + menu = false; busy = true + scope.launch { + runCatching { withContext(Dispatchers.IO) { GogMaintenance.recoverPatch(ctx, game) } }.onFailure { error = it.message } + busy = false + } + } + } + } + if (check?.state == GogMaintenance.State.FAILED || check?.state == GogMaintenance.State.UNKNOWN) { + if (check.reason.isNotBlank()) Text(check.reason, fontSize = 12.sp, color = colors.onSurfaceVariant) + } + patches?.let { list -> + val shown = rememberShown { patches = null } + val close = { shown.targetState = false } + AppDialog(shown, close, "gogPatches", wide = false, maxWidth = 460.dp) { + DialogTitleRow(stringResource(R.string.gog_patches), close) + Text(game.sidecar.title, fontSize = 12.sp, lineHeight = 16.sp, color = colors.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis) + Text(stringResource(R.string.gog_patch_requirements), fontSize = 12.sp, lineHeight = 18.sp, color = colors.onSurfaceVariant) + if (list.isEmpty()) Text(stringResource(R.string.gog_no_patches), fontSize = 13.sp, color = colors.onSurfaceVariant) + Column(verticalArrangement = Arrangement.spacedBy(6.dp)) { + list.forEach { patch -> + ListRow(onClick = { close(); prepare(GogOperation.PATCH, patch) }) { + Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(3.dp)) { + Text(patch.name, fontSize = 13.sp, lineHeight = 18.sp, fontWeight = FontWeight.Medium, color = colors.onBackground, maxLines = 2, overflow = TextOverflow.Ellipsis) + Text(listOf(StoreLanguage.displayName(patch.language).ifBlank { patch.language }, patch.version, formatBytes(patch.size)).filter { it.isNotBlank() }.joinToString(" · "), + fontSize = 11.sp, lineHeight = 15.sp, color = colors.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + Text(stringResource(R.string.gog_select_patch), fontSize = 12.sp, color = LocalPalette.current.signal) + } + } + } + SecondaryButton(stringResource(R.string.gog_refresh)) { prepare(GogOperation.REFRESH) } + } + } + proposal?.let { selected -> + val shown = rememberShown { proposal = null } + val close = { shown.targetState = false } + val action = when (selected.operation) { + GogOperation.UPDATE -> R.string.common_update + GogOperation.REPAIR -> R.string.gog_repair_action + GogOperation.PATCH -> R.string.gog_open_patch + GogOperation.REFRESH -> R.string.gog_refresh_action + } + AppDialog(shown, close, "gogMaintenance", wide = false, maxWidth = 460.dp) { + DialogTitleRow(stringResource(action), close) + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text(listOf(game.sidecar.title, selected.before.version).filter { it.isNotBlank() }.joinToString(" · "), fontSize = 12.sp, lineHeight = 16.sp, color = colors.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis) + if (selected.operation != GogOperation.REPAIR) Text(selected.patch?.name ?: selected.target.version.ifBlank { selected.target.build }, fontSize = 15.sp, lineHeight = 20.sp, fontWeight = FontWeight.SemiBold, + color = colors.onBackground, maxLines = 2, overflow = TextOverflow.Ellipsis) + } + Row(horizontalArrangement = Arrangement.spacedBy(16.dp), modifier = Modifier.fillMaxWidth().clip(Shape12).background(colors.surface.copy(alpha = 0.65f)).padding(12.dp)) { + GogMetric(stringResource(R.string.gog_download_label), formatBytes(selected.downloadBytes), Modifier.weight(1f)) + GogMetric(stringResource(if (selected.operation == GogOperation.PATCH || selected.operation == GogOperation.REFRESH) R.string.gog_space_estimate_label else R.string.gog_space_label), + formatBytes(selected.diskBytes), Modifier.weight(1f)) + } + Column(verticalArrangement = Arrangement.spacedBy(6.dp)) { + Text(stringResource(R.string.gog_managed_files), fontSize = 12.sp, lineHeight = 18.sp, color = colors.onSurfaceVariant) + if (selected.fullFiles) Text(stringResource(R.string.gog_full_files), fontSize = 12.sp, lineHeight = 18.sp, color = colors.onSurfaceVariant) + if (selected.operation == GogOperation.PATCH) Text(stringResource(R.string.gog_patch_installer_note), fontSize = 12.sp, lineHeight = 18.sp, color = colors.onSurfaceVariant) + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) { + PrimaryButton(stringResource(action), enabled = !busy, modifier = Modifier.testTag("gog-apply")) { + busy = true + close() + scope.launch { + runCatching { withContext(Dispatchers.IO) { GogMaintenance.enqueue(ctx, game, selected) } }.onFailure { error = it.message } + busy = false + } + } + SecondaryButton(stringResource(R.string.common_cancel), onClick = close) + } + } + } + error?.let { Text(it, fontSize = 12.sp, color = colors.error) } +} + +@Composable +private fun GogMetric(label: String, value: String, modifier: Modifier) { + Column(modifier, verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text(label, fontSize = 11.sp, lineHeight = 15.sp, color = MaterialTheme.colorScheme.onSurfaceVariant) + Text(value, fontSize = 15.sp, lineHeight = 20.sp, fontWeight = FontWeight.SemiBold, color = MaterialTheme.colorScheme.onBackground) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/ui/LibraryPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/LibraryPage.kt index 1fae4df5db..243d218532 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/LibraryPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/LibraryPage.kt @@ -92,6 +92,7 @@ import com.droiddeck.launcher.stores.formatBytes import com.droiddeck.launcher.stores.itch.ItchCollection import com.droiddeck.launcher.stores.itch.ItchCollections import com.droiddeck.launcher.stores.itch.ItchUpdates +import com.droiddeck.launcher.stores.gog.GogMaintenance import com.droiddeck.launcher.stores.retainOrder import com.droiddeck.launcher.stores.xbox.XboxLibraryFilter import com.droiddeck.launcher.stores.sortGames @@ -586,13 +587,16 @@ internal fun LibraryMarker(e: LibraryEntry, download: DownloadEntry?) { download != null -> Text("${download.percent}%", fontSize = 10.sp, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) e.item != null && StoresState.isUnfinished(e.item) -> Text(stringResource(R.string.stores_resume), fontSize = 10.sp, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) else -> Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(3.dp)) { - if (e.installed && e.storeKey?.let { it in ItchUpdates.available } == true) Text(stringResource(R.string.common_update), fontSize = 10.sp, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) + if (e.installed && libraryUpdateAvailable(e)) Text(stringResource(R.string.common_update), fontSize = 10.sp, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) if (size != null && size > 0) Text(formatBytes(size), fontSize = 11.sp, fontWeight = FontWeight.Bold, color = Color.White, maxLines = 1) if (!e.installed) Icon(Icons.Outlined.Download, stringResource(R.string.stores_section_not_installed), tint = Color.White, modifier = Modifier.size(12.dp)) } } } +private fun libraryUpdateAvailable(e: LibraryEntry): Boolean = e.storeKey?.let { it in ItchUpdates.available } == true || + e.source == Store.GOG.id && (e.item?.id ?: e.game?.storeId)?.let { GogMaintenance.checks[it]?.state } == GogMaintenance.State.AVAILABLE + @Composable private fun LibraryRow(e: LibraryEntry, first: Boolean, navigation: Modifier, choose: (() -> Unit)?, onOpen: (String) -> Unit) { val colors = MaterialTheme.colorScheme @@ -613,6 +617,7 @@ private fun LibraryRow(e: LibraryEntry, first: Boolean, navigation: Modifier, ch val size = if (e.installed && folder != null) diskSize(inst, folder) else e.item?.let { StoreSizes.size(it) }?.takeIf { it > 0 } val status = when { download != null -> downloadLabel(download) + e.installed && libraryUpdateAvailable(e) -> stringResource(R.string.common_update) !e.installed -> stringResource(R.string.stores_section_not_installed) e.game != null -> playedSpan(e.game.lastPlayed)?.replaceFirstChar { it.uppercase() } ?: stringResource(R.string.games_never_played) else -> stringResource(R.string.stores_installed_chip) diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt index e521856811..bd6898e68f 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt @@ -360,8 +360,10 @@ private fun DownloadCard(d: DownloadEntry, s: FrontEndState, a: FrontEndActions) Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) { when (d.state) { DownloadState.RUNNING, DownloadState.QUEUED -> { - RoundAction(Icons.Filled.Pause, stringResource(R.string.stores_dl_pause)) { DownloadQueue.pause(d.key) } - CancelAction { DownloadQueue.cancel(ctx, d.key) } + if (d.interruptible) { + RoundAction(Icons.Filled.Pause, stringResource(R.string.stores_dl_pause)) { DownloadQueue.pause(d.key) } + CancelAction { DownloadQueue.cancel(ctx, d.key) } + } } DownloadState.PAUSED -> { RoundAction(Icons.Filled.PlayArrow, stringResource(R.string.stores_dl_resume), primary = true) { DownloadQueue.resume(ctx, d.key) } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt index 92b118d54b..19813baf10 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt @@ -1380,6 +1380,9 @@ private fun CardMarker(item: CatalogItem, installed: Boolean, download: com.droi val pal = LocalPalette.current val small = 10.sp when { + item.store == Store.GOG && download != null -> Text("${download.percent}%", fontSize = small, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) + item.store == Store.GOG && installed && com.droiddeck.launcher.stores.gog.GogMaintenance.checks[item.id]?.state == com.droiddeck.launcher.stores.gog.GogMaintenance.State.AVAILABLE -> + Text(stringResource(R.string.common_update), fontSize = small, fontWeight = FontWeight.Bold, color = pal.signal, maxLines = 1) installed -> Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(3.dp)) { val ctx = LocalContext.current val game = StoresState.installedGame(item.store, item.id) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bb9f8061b2..10b2b913c6 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,5 +1,24 @@ + Manage GOG installation + Check for updates + Verify/Repair + Offline patches + Download full offline installer + Update to %1$s + Resume GOG maintenance + Open patch installer + Finish or cancel the patch installer in the Linux session. + Choose a patch for your installed version and language. Supported source versions may not be listed. + GOG has no standalone Windows patches for this game. + Modified game files may be replaced. + The installer opens in your Linux session. + Download + Extra space + Space estimate + Repair files + Refresh installation + Select Luma Ultra cameras — Experimental Joy-Con position tracking Use the glasses cameras to position your paired Joy-Cons. Rotation and button inputs come from the Joy-Cons. Position is estimated when your wrists are hidden. Replaces bare-hand controllers and pauses phone-camera tracking. Point the Joy-Cons forward when connecting or recentering. @@ -3325,4 +3344,6 @@ DroidDeck Home button Pass Home button presses to Steam Guide while a Steam session is focused and DroidDeck is the default Home app. Does not read screen content. Turn on Brightness in Setup → Steam integration + Native optimizations are unavailable. The download includes complete replacement files. + Verify patch result diff --git a/app/src/main/rust/stores/Cargo.lock b/app/src/main/rust/stores/Cargo.lock index 90055ee4c7..2fac37c5aa 100644 --- a/app/src/main/rust/stores/Cargo.lock +++ b/app/src/main/rust/stores/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. version = 4 [[package]] @@ -186,6 +188,7 @@ dependencies = [ name = "droiddeckstores" version = "0.1.0" dependencies = [ + "cc", "flate2", "futures-util", "jni", diff --git a/app/src/main/rust/stores/Cargo.toml b/app/src/main/rust/stores/Cargo.toml index d21d5132be..61d121a22a 100644 --- a/app/src/main/rust/stores/Cargo.toml +++ b/app/src/main/rust/stores/Cargo.toml @@ -21,6 +21,9 @@ serde_json = { version = "1", features = ["preserve_order"] } sha1 = "0.10" sha2 = "0.10" +[build-dependencies] +cc = "1" + [profile.release] codegen-units = 1 lto = "thin" diff --git a/app/src/main/rust/stores/JNI.md b/app/src/main/rust/stores/JNI.md index ee7e5a8278..b87390ac8f 100644 --- a/app/src/main/rust/stores/JNI.md +++ b/app/src/main/rust/stores/JNI.md @@ -77,14 +77,14 @@ after it (marker files, exe resolution, DLC markers, redist orchestration). | `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCancel` | | `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeRelease` | -`nativeProbe()` always returns `1`; it exists so `isAvailable()` can catch `UnsatisfiedLinkError` +`nativeProbe()` returns `2`; it exists so `isAvailable()` can catch `UnsatisfiedLinkError` once at startup instead of mid-download. `nativeStart` parameters, in order: | # | Parameter | Meaning | |---|---|---| -| 1 | `kind: Int` | `0` = `KIND_GEN2_CHUNKS`, `1` = `KIND_GEN1_RANGES` (anything else → gen2) | +| 1 | `kind: Int` | `0` = `KIND_GEN2_CHUNKS`, `1` = `KIND_GEN1_RANGES`, `2` = GOG maintenance (anything else → gen2) | | 2 | `depotManifests: Array` | gen2: the inflated depot-manifest JSON strings, in fetch order, already filtered by product + language; gen1: the inflated build manifest (one element) | | 3 | `cdnBases: Array` | gen2: **every** CDN base of the secure-link answer — one per `urls[]` entry, each resolved the way `parseCdnUrl` resolves `urls[0]` today (query string intact), in GOG's order — or the one unauthenticated dependency store base (an array of one); gen1: `emptyArray()` (file URLs are in the manifest). Blank entries are ignored and a base whose host already appeared is dropped (the first wins). The fetch core spreads the window across the distinct hosts with a per-host cap of `max(6, ceil(maxWorkers / hosts))`, rotating a failed chunk to another CDN; one base behaves exactly like the old single `cdnBase`. The start log line lists them as `hosts=a,b,c`. | | 4 | `installDir: String` | absolute install directory | @@ -113,6 +113,15 @@ On-disk protocol (what a resumed run, by either engine, relies on): a file is as `.bhtmp`, then renamed over `` once its size and MD5 match the manifest; an existing `` whose size+MD5 match is skipped. The engine writes nothing else. +Maintenance (`kind=2`) instead takes a single normalized plan in `depotManifests[0]`: +`files`, `oldFiles`, optional exact-build `patches`, per-product `links` and `patchLinks`, +and estimated `downloadBytes`. Credentials and these temporary links stay in memory. +It stages replacements under `.droiddeck-gog/stage`, revalidates completed chunks on +every run, and verifies each file before atomic replacement. `onBytes` counts actual +network bytes, including failed attempts. `onMaintenance(networkBytes, reusedBytes, +appliedBytes)` (`(JJJ)V`) reports the three separate cumulative counters. Existing +gen1/gen2 first-install progress semantics are unchanged. + ## `com.droiddeck.launcher.stores.epic.EpicNative` Everything Epic runs in the Rust `epic` module: the web-service calls (sign-in, refresh, exchange diff --git a/app/src/main/rust/stores/build.rs b/app/src/main/rust/stores/build.rs new file mode 100644 index 0000000000..f80559c870 --- /dev/null +++ b/app/src/main/rust/stores/build.rs @@ -0,0 +1,21 @@ +fn main() { + cc::Build::new() + .include("vendor/xdelta3") + .file("vendor/xdelta3/xdelta3.c") + .file("src/gog/delta.c") + .define("SIZEOF_SIZE_T", "8") + .define("SIZEOF_UNSIGNED_INT", "4") + .define("SIZEOF_UNSIGNED_LONG", "8") + .define("SIZEOF_UNSIGNED_LONG_LONG", "8") + .define("XD3_USE_LARGEFILE64", "1") + .define("XD3_ENCODER", "0") + .define("SECONDARY_DJW", "0") + .define("SECONDARY_FGK", "0") + .define("SECONDARY_LZMA", "0") + .define("SHELL_TESTS", "0") + .define("XD3_MAIN", "0") + .warnings(false) + .compile("gogdelta"); + println!("cargo:rerun-if-changed=vendor/xdelta3"); + println!("cargo:rerun-if-changed=src/gog/delta.c"); +} diff --git a/app/src/main/rust/stores/src/gog.rs b/app/src/main/rust/stores/src/gog.rs index 3aed660743..e3a049c4cc 100644 --- a/app/src/main/rust/stores/src/gog.rs +++ b/app/src/main/rust/stores/src/gog.rs @@ -2,6 +2,7 @@ pub mod engine; pub mod jni; pub mod plan; +pub mod maintenance; use std::fs::File; use std::io::Read; diff --git a/app/src/main/rust/stores/src/gog/delta.c b/app/src/main/rust/stores/src/gog/delta.c new file mode 100644 index 0000000000..fd64ea924a --- /dev/null +++ b/app/src/main/rust/stores/src/gog/delta.c @@ -0,0 +1,78 @@ +#include "xdelta3.h" + +typedef struct { size_t used; } dd_budget; +typedef struct { size_t size; } dd_allocation; + +static void *dd_alloc(void *opaque, size_t count, usize_t size) { + dd_budget *budget = opaque; + if (size && count > (62u * 1024u * 1024u) / size) return NULL; + size_t n = count * size; + if (n > 62u * 1024u * 1024u - budget->used) return NULL; + dd_allocation *allocation = malloc(sizeof(*allocation) + n); + if (!allocation) return NULL; + allocation->size = n; + budget->used += n; + return allocation + 1; +} + +static void dd_free(void *opaque, void *ptr) { + if (!ptr) return; + dd_allocation *allocation = (dd_allocation *)ptr - 1; + ((dd_budget *)opaque)->used -= allocation->size; + free(allocation); +} + +int dd_gog_delta(const char *source, const char *patch, const char *target, + int (*cancel)(void *), void *opaque) { + FILE *old = NULL, *delta = NULL, *out = NULL; + xd3_stream stream = {0}; + xd3_source src = {0}; + xd3_config config = {0}; + dd_budget budget = {0}; + unsigned char *input = NULL, *block = NULL; + int result = -1, configured = 0; + old = fopen(source, "rb"); delta = fopen(patch, "rb"); out = fopen(target, "wb"); + if (!old || !delta || !out) goto done; + input = malloc(64 * 1024); block = malloc(1024 * 1024); + if (!input || !block) goto done; + config.winsize = 64 * 1024; + config.alloc = dd_alloc; config.freef = dd_free; config.opaque = &budget; + if (xd3_config_stream(&stream, &config)) goto done; + configured = 1; + src.blksize = 1024 * 1024; src.curblk = block; src.curblkno = 0; + src.onblk = fread(block, 1, src.blksize, old); + if (xd3_set_source(&stream, &src)) goto done; + for (;;) { + size_t n = fread(input, 1, 64 * 1024, delta); + if (ferror(delta)) goto done; + if (feof(delta)) xd3_set_flags(&stream, XD3_FLUSH); + xd3_avail_input(&stream, input, n); + int code; + while ((code = xd3_decode_input(&stream)) != XD3_INPUT) { + if (cancel && cancel(opaque)) { result = 130; goto done; } + switch (code) { + case XD3_OUTPUT: + if (fwrite(stream.next_out, 1, stream.avail_out, out) != stream.avail_out) goto done; + xd3_consume_output(&stream); break; + case XD3_GETSRCBLK: + if (fseeko(old, (off_t)(src.getblkno * src.blksize), SEEK_SET)) goto done; + src.curblkno = src.getblkno; + src.onblk = fread(block, 1, src.blksize, old); + if (ferror(old)) goto done; + break; + case XD3_GOTHEADER: case XD3_WINSTART: case XD3_WINFINISH: break; + default: goto done; + } + } + if (feof(delta)) break; + } + if (xd3_close_stream(&stream) || fflush(out)) goto done; + result = 0; +done: + if (configured) xd3_free_stream(&stream); + if (old) fclose(old); + if (delta) fclose(delta); + if (out && fclose(out)) result = -1; + free(input); free(block); + return result; +} diff --git a/app/src/main/rust/stores/src/gog/engine.rs b/app/src/main/rust/stores/src/gog/engine.rs index 9f2f59428e..c1c1a8b111 100644 --- a/app/src/main/rust/stores/src/gog/engine.rs +++ b/app/src/main/rust/stores/src/gog/engine.rs @@ -32,11 +32,14 @@ pub enum PlanKind { #[default] Gen2Chunks, Gen1Ranges, + Maintenance, } impl PlanKind { pub fn from_i32(v: i32) -> Self { - if v == 1 { + if v == 2 { + PlanKind::Maintenance + } else if v == 1 { PlanKind::Gen1Ranges } else { PlanKind::Gen2Chunks @@ -72,6 +75,7 @@ pub trait GogEvents: Sync { ); fn on_log(&self, line: &str); fn on_bytes(&self, _bytes_done: u64, _bytes_total: u64) {} + fn on_maintenance(&self, _network_bytes: u64, _reused_bytes: u64, _applied_bytes: u64) {} } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -672,6 +676,7 @@ pub fn run(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Gog match req.kind { PlanKind::Gen2Chunks => run_gen2(req, cancel, events), PlanKind::Gen1Ranges => run_gen1(req, cancel, events), + PlanKind::Maintenance => super::maintenance::run(req, cancel, events), } } diff --git a/app/src/main/rust/stores/src/gog/fixtures/update.vcdiff b/app/src/main/rust/stores/src/gog/fixtures/update.vcdiff new file mode 100644 index 0000000000000000000000000000000000000000..f053eb5402adfb3bfaad3f480b98637fee0bd592 GIT binary patch literal 66 zcmcb1_y_}wcuI(^b literal 0 HcmV?d00001 diff --git a/app/src/main/rust/stores/src/gog/jni.rs b/app/src/main/rust/stores/src/gog/jni.rs index 9fa1fbf01d..c2e1b0bd59 100644 --- a/app/src/main/rust/stores/src/gog/jni.rs +++ b/app/src/main/rust/stores/src/gog/jni.rs @@ -153,6 +153,15 @@ impl GogEvents for JniEvents { let _ = env.delete_local_ref(text); }); } + + fn on_maintenance(&self, network_bytes: u64, reused_bytes: u64, applied_bytes: u64) { + self.with_env(|env| { + let _ = env.call_method(self.listener.as_obj(), "onMaintenance", "(JJJ)V", &[ + JValue::Long(network_bytes as jlong), JValue::Long(reused_bytes as jlong), JValue::Long(applied_bytes as jlong), + ]); + Self::clear_exception(env); + }); + } } fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { @@ -197,7 +206,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativePr _env: JNIEnv, _class: JClass, ) -> jint { - 1 + 2 } #[no_mangle] diff --git a/app/src/main/rust/stores/src/gog/maintenance.rs b/app/src/main/rust/stores/src/gog/maintenance.rs new file mode 100644 index 0000000000..b2a9338c2e --- /dev/null +++ b/app/src/main/rust/stores/src/gog/maintenance.rs @@ -0,0 +1,523 @@ +use std::collections::{HashMap, HashSet}; +use std::ffi::CString; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; +use serde_json::Value; +use super::engine::{GogEvents, GogRequest, GogRunResult, http_status_in, is_link_expiry_status}; +use super::{md5_hex, md5_hex_file}; + +static DELTA_LOCK: Mutex<()> = Mutex::new(()); +unsafe extern "C" { + fn dd_gog_delta(source: *const libc::c_char, patch: *const libc::c_char, target: *const libc::c_char, + cancel: extern "C" fn(*mut libc::c_void) -> libc::c_int, opaque: *mut libc::c_void) -> libc::c_int; +} +extern "C" fn cancelled(opaque: *mut libc::c_void) -> libc::c_int { + unsafe { (*(opaque as *const AtomicBool)).load(Ordering::Relaxed) as libc::c_int } +} + +fn http_client(read_timeout: std::time::Duration) -> Result { + reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(20)) + .read_timeout(read_timeout) + .build().map_err(|e| e.to_string()) +} + +pub fn inside(root: &Path, relative: &str) -> Result { + if relative.is_empty() || relative.contains(['\\', ':', '\0']) || relative.split('/').any(|p| p.is_empty() || p == "." || p == "..") { return Err("invalid GOG path".into()); } + let mut out = root.to_path_buf(); + for component in Path::new(relative).components() { + let Component::Normal(part) = component else { return Err("GOG path leaves installation".into()) }; + out.push(part); + if fs::symlink_metadata(&out).map(|m| m.file_type().is_symlink()).unwrap_or(false) { + return Err("GOG path is a symbolic link".into()); + } + } + Ok(out) +} + +fn text<'a>(v: &'a Value, key: &str) -> &'a str { v.get(key).and_then(Value::as_str).unwrap_or("") } +fn num(v: &Value, key: &str) -> u64 { v.get(key).and_then(Value::as_u64).unwrap_or(0) } +fn array<'a>(v: &'a Value, key: &str) -> &'a [Value] { v.get(key).and_then(Value::as_array).map(Vec::as_slice).unwrap_or(&[]) } +fn valid_md5(s: &str) -> bool { s.len() == 32 && s.bytes().all(|c| c.is_ascii_hexdigit()) } + +fn matches(path: &Path, file: &Value) -> bool { + if !fs::metadata(path).map(|m| m.is_file() && m.len() == num(file, "size")).unwrap_or(false) { return false; } + let md5 = text(file, "md5"); + if valid_md5(md5) { return md5_hex_file(path).map(|h| h.eq_ignore_ascii_case(md5)).unwrap_or(false); } + let sha = text(file, "sha256"); + if sha.len() == 64 { + use sha2::{Digest, Sha256}; + let Ok(mut input) = File::open(path) else { return false; }; + let mut digest = Sha256::new(); let mut buffer = [0u8; 64 * 1024]; + loop { let Ok(n) = input.read(&mut buffer) else { return false; }; if n == 0 { break; } digest.update(&buffer[..n]); } + return super::to_hex(&digest.finalize()).eq_ignore_ascii_case(sha); + } + let chunks = array(file, "chunks"); + if chunks.is_empty() { return num(file, "size") == 0; } + let Ok(mut input) = File::open(path) else { return false; }; + chunks.iter().all(|chunk| { + let size = num(chunk, "size"); + valid_md5(text(chunk, "md5")) && hash_range(&mut input, size).as_deref() == Some(text(chunk, "md5")) + }) +} + +fn hash_range(input: &mut File, size: u64) -> Option { + let mut digest = crate::md5_small::Md5::new(); + let mut remaining = size; + let mut buffer = [0u8; 64 * 1024]; + while remaining > 0 { + let want = remaining.min(buffer.len() as u64) as usize; + input.read_exact(&mut buffer[..want]).ok()?; + digest.update(&buffer[..want]); remaining -= want as u64; + } + Some(super::to_hex(&digest.finalize())) +} + +#[derive(Clone)] +struct Reuse { path: PathBuf, offset: u64, size: u64 } +type ReuseMap = HashMap<(String, u64), Vec>; + +fn reuse_map(root: &Path, files: &[Value]) -> Result { + let mut result: ReuseMap = HashMap::new(); + for file in files { + if text(file, "path").to_lowercase().starts_with(".droiddeck-") { return Err("GOG content overlaps launcher metadata".into()); } + let path = inside(root, text(file, "path"))?; + let mut offset: u64 = 0; + for chunk in array(file, "chunks") { + let size = num(chunk, "size"); + if valid_md5(text(chunk, "md5")) { + result.entry((text(chunk, "md5").to_ascii_lowercase(), size)).or_default() + .push(Reuse { path: path.clone(), offset, size }); + } + offset = offset.checked_add(size).ok_or("GOG file size overflow")?; + } + } + Ok(result) +} + +fn reusable(chunk: &Value, map: &ReuseMap) -> Option { + let key = (text(chunk, "md5").to_ascii_lowercase(), num(chunk, "size")); + map.get(&key)?.iter().find(|old| { + let Ok(mut file) = File::open(&old.path) else { return false; }; + file.seek(SeekFrom::Start(old.offset)).is_ok() && hash_range(&mut file, old.size).map(|h| h.eq_ignore_ascii_case(&key.0)).unwrap_or(false) + }).cloned() +} + +struct Transfer<'a> { + runtime: tokio::runtime::Runtime, + client: reqwest::Client, + cancel: &'a AtomicBool, + events: &'a dyn GogEvents, + wire: u64, + total: u64, +} + +impl Transfer<'_> { + fn fetch(&mut self, urls: &[String], range: Option<(u64, u64)>, dest: &Path, limit: u64) -> Result<(), String> { + let mut failure = "GOG download has no URLs".to_string(); + for _ in 0..3 { + for url in urls { + if self.cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + let result = self.runtime.block_on(async { + let mut request = self.client.get(url).header("User-Agent", "GOG Galaxy"); + if let Some((start, end)) = range { request = request.header("Range", format!("bytes={start}-{end}")); } + let mut response = request.send().await.map_err(|_| "GOG connection failed".to_string())?; + let status = response.status().as_u16(); + if status != if range.is_some() { 206 } else { 200 } { return Err(format!("HTTP {status}")); } + let mut output = File::create(dest).map_err(|e| e.to_string())?; + let mut size = 0u64; + while let Some(bytes) = response.chunk().await.map_err(|_| "GOG connection interrupted".to_string())? { + if self.cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + size += bytes.len() as u64; + if limit > 0 && size > limit { return Err("GOG response exceeds expected size".into()); } + self.wire += bytes.len() as u64; + self.events.on_bytes(self.wire, self.total.max(self.wire)); + output.write_all(&bytes).map_err(|e| e.to_string())?; + } + if limit > 0 && size != limit { return Err("GOG download is incomplete".into()); } + output.sync_all().map_err(|e| e.to_string()) + }); + match result { Ok(()) => return Ok(()), Err(e) => failure = e } + if http_status_in(&failure).map(is_link_expiry_status).unwrap_or(false) { return Err(failure); } + } + } + Err(failure) + } +} + +fn urls(links: &Value, product: &str, suffix: &str) -> Vec { + links.get(product).and_then(Value::as_array).map(|bases| bases.iter().filter_map(Value::as_str).map(|base| { + if base.contains("{GALAXY_PATH}") { base.replace("{GALAXY_PATH}", suffix) } + else if let Some((path, query)) = base.split_once('?') { format!("{}{suffix}?{query}", path.trim_end_matches('/').to_string() + "/") } + else { format!("{}{suffix}", base.trim_end_matches('/').to_string() + "/") } + }).collect()).unwrap_or_default() +} + +fn chunk_path(chunk: &Value) -> Result { + let hash = text(chunk, "compressedMd5"); + if !valid_md5(hash) { return Err("GOG chunk has no valid compressed hash".into()); } + Ok(super::plan::build_cdn_path(hash)) +} + +fn download_chunk(transfer: &mut Transfer, links: &Value, product: &str, chunk: &Value, scratch: &Path, output: &mut File) -> Result<(), String> { + transfer.fetch(&urls(links, product, &chunk_path(chunk)?), None, scratch, num(chunk, "compressedSize"))?; + if !md5_hex_file(scratch).map(|h| h.eq_ignore_ascii_case(text(chunk, "compressedMd5"))).unwrap_or(false) { return Err("GOG compressed chunk checksum mismatch".into()); } + let mut raw = File::open(scratch).map_err(|e| e.to_string())?; + let mut head = [0u8; 1]; raw.read_exact(&mut head).map_err(|e| e.to_string())?; + raw.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?; + let stored = num(chunk, "size") == num(chunk, "compressedSize") && text(chunk, "md5").eq_ignore_ascii_case(text(chunk, "compressedMd5")); + let input: Box = if !stored && head[0] == 0x78 { Box::new(flate2::read::ZlibDecoder::new(raw)) } else { Box::new(raw) }; + let mut input = input.take(num(chunk, "size") + 1); + let mut digest = crate::md5_small::Md5::new(); + let mut size = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + let n = input.read(&mut buffer).map_err(|_| "GOG chunk decompression failed")?; + if n == 0 { break; } + if transfer.cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + digest.update(&buffer[..n]); size += n as u64; + output.write_all(&buffer[..n]).map_err(|e| e.to_string())?; + } + if size != num(chunk, "size") || !super::to_hex(&digest.finalize()).eq_ignore_ascii_case(text(chunk, "md5")) { return Err("GOG chunk checksum mismatch".into()); } + Ok(()) +} + +fn apply_delta(root: &Path, file: &Value, patch: &Value, plan: &Value, stage: &Path, transfer: &mut Transfer) -> Result<(), String> { + let _guard = DELTA_LOCK.lock().map_err(|_| "GOG delta lock poisoned")?; + let source = inside(root, &text(patch, "path_source").replace('\\', "/"))?; + if !md5_hex_file(&source).map(|h| h.eq_ignore_ascii_case(text(patch, "md5_source"))).unwrap_or(false) { return Err("GOG delta source changed".into()); } + let payload = stage.with_extension("delta"); + let packed = stage.with_extension("packed"); + let mut output = File::create(&payload).map_err(|e| e.to_string())?; + for chunk in array(patch, "chunks") { download_chunk(transfer, &plan["patchLinks"], text(patch, "product"), chunk, &packed, &mut output)?; } + drop(output); + if !md5_hex_file(&payload).map(|h| h.eq_ignore_ascii_case(text(patch, "md5"))).unwrap_or(false) { return Err("GOG delta checksum mismatch".into()); } + let cpath = |p: &Path| CString::new(p.as_os_str().as_encoded_bytes()).map_err(|_| "invalid path"); + let (s, p, t) = (cpath(&source)?, cpath(&payload)?, cpath(stage)?); + let code = unsafe { dd_gog_delta(s.as_ptr(), p.as_ptr(), t.as_ptr(), cancelled, transfer.cancel as *const AtomicBool as *mut libc::c_void) }; + let _ = fs::remove_file(payload); let _ = fs::remove_file(packed); + if code != 0 || !matches(stage, file) || !md5_hex_file(stage).map(|h| h.eq_ignore_ascii_case(text(patch, "md5_target"))).unwrap_or(false) { return Err("GOG delta reconstruction failed".into()); } + Ok(()) +} + +pub fn run(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult { + let mut result = GogRunResult::default(); + let work = || -> Result<(u64, u32), String> { + let plan: Value = serde_json::from_str(req.depot_manifests.first().ok_or("missing GOG maintenance plan")?).map_err(|_| "invalid GOG maintenance plan")?; + let root = fs::canonicalize(&req.install_dir).map_err(|e| e.to_string())?; + let files = array(&plan, "files"); + let mut seen = HashSet::new(); + let mut components = HashMap::new(); + for file in files { + let name = text(file, "path"); inside(&root, name)?; + if name.to_lowercase().starts_with(".droiddeck-") || !seen.insert(name.to_lowercase()) { return Err("ambiguous GOG path".into()); } + let mut parent = root.clone(); + let mut relative = PathBuf::new(); + for part in name.split('/') { + relative.push(part); + let key = relative.to_string_lossy().to_lowercase(); + if let Some(previous) = components.insert(key, relative.clone()) { + if previous != relative { return Err("ambiguous GOG directory".into()); } + } + if let Ok(entries) = fs::read_dir(&parent) { + for entry in entries.flatten() { + let actual = entry.file_name().to_string_lossy().into_owned(); + if actual != part && actual.to_lowercase() == part.to_lowercase() { return Err("existing path differs only by case".into()); } + } + } + parent.push(part); + } + } + let map = reuse_map(&root, array(&plan, "oldFiles"))?; + let stages = inside(&root, ".droiddeck-gog/stage")?; + fs::create_dir_all(&stages).map_err(|e| e.to_string())?; + let mut transfer = Transfer { + runtime: tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(|e| e.to_string())?, + client: http_client(std::time::Duration::from_secs(60))?, + cancel, events, wire: 0, total: num(&plan, "downloadBytes"), + }; + let mut done = 0u32; let mut reused = 0u64; let mut applied = 0u64; + for file in files { + if cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + let path = inside(&root, text(file, "path"))?; + if file.get("directory").and_then(Value::as_bool).unwrap_or(false) { + fs::create_dir_all(&path).map_err(|e| e.to_string())?; done += 1; continue; + } + if matches(&path, file) { done += 1; events.on_file_done(text(file, "path"), num(file, "size"), true, done, files.len() as u32, transfer.wire, transfer.total); continue; } + let stage = stages.join(md5_hex(text(file, "path").as_bytes())); + if !matches(&stage, file) { + let mut start = 0; + let missing_bytes: u64 = array(file, "chunks").iter().map(|chunk| { + let valid = File::open(&stage).ok().map(|mut input| input.seek(SeekFrom::Start(start)).is_ok() && + hash_range(&mut input, num(chunk, "size")).map(|h| h.eq_ignore_ascii_case(text(chunk, "md5"))).unwrap_or(false)).unwrap_or(false); + start += num(chunk, "size"); + if valid || reusable(chunk, &map).is_some() { 0 } else { num(chunk, "compressedSize") } + }).sum(); + let patch = array(&plan, "patches").iter().find(|p| text(p, "path_target").replace('\\', "/") == text(file, "path") && + array(p, "chunks").iter().map(|c| num(c, "compressedSize")).sum::() < missing_bytes); + let delta_ok = if let Some(patch) = patch { + match apply_delta(&root, file, patch, &plan, &stage, &mut transfer) { + Ok(()) => true, + Err(e) => { if cancel.load(Ordering::Relaxed) || http_status_in(&e).map(is_link_expiry_status).unwrap_or(false) { return Err(e); } events.on_log(&format!("gog: delta fallback file={} reason={e}", text(file, "path"))); false } + } + } else { false }; + if !delta_ok { + let chunks = array(file, "chunks"); + if chunks.is_empty() && num(file, "size") > 0 { + let offset = num(file, "offset"); + let remote = text(file, "remotePath"); + let endpoints = if remote.starts_with("https://") { vec![remote.to_string()] } else { urls(&plan["links"], text(file, "product"), remote.trim_start_matches('/')) }; + transfer.fetch(&endpoints, Some((offset, offset + num(file, "size") - 1)), &stage, num(file, "size"))?; + } else { + let mut output = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(&stage).map_err(|e| e.to_string())?; + let mut offset = 0u64; + for chunk in chunks { + if cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + output.seek(SeekFrom::Start(offset)).map_err(|e| e.to_string())?; + let valid_stage = hash_range(&mut output, num(chunk, "size")).map(|h| h.eq_ignore_ascii_case(text(chunk, "md5"))).unwrap_or(false); + output.seek(SeekFrom::Start(offset)).map_err(|e| e.to_string())?; + if valid_stage { reused += num(chunk, "size"); } + else if let Some(old) = reusable(chunk, &map) { + let mut source = File::open(old.path).map_err(|e| e.to_string())?; + source.seek(SeekFrom::Start(old.offset)).map_err(|e| e.to_string())?; + std::io::copy(&mut source.take(old.size), &mut output).map_err(|e| e.to_string())?; + reused += old.size; + } else { + download_chunk(&mut transfer, &plan["links"], text(file, "product"), chunk, &stage.with_extension("packed"), &mut output)?; + } + offset += num(chunk, "size"); + } + output.set_len(num(file, "size")).map_err(|e| e.to_string())?; + output.sync_all().map_err(|e| e.to_string())?; + } + } + } + let unhashed_range = array(file, "chunks").is_empty() && !valid_md5(text(file, "md5")) && text(file, "sha256").is_empty(); + if !matches(&stage, file) && !(unhashed_range && fs::metadata(&stage).map(|m| m.len() == num(file, "size")).unwrap_or(false)) { return Err(format!("GOG target verification failed: {}", text(file, "path"))); } + if cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + inside(&root, text(file, "path"))?; + fs::create_dir_all(path.parent().ok_or("GOG file has no parent")?).map_err(|e| e.to_string())?; + File::open(&stage).and_then(|f| f.sync_all()).map_err(|e| e.to_string())?; + fs::rename(&stage, &path).map_err(|e| e.to_string())?; + applied += num(file, "size"); done += 1; + events.on_file_done(text(file, "path"), num(file, "size"), false, done, files.len() as u32, transfer.wire, transfer.total); + events.on_maintenance(transfer.wire, reused, applied); + events.on_log(&format!("gog: maintenance wire={} reused={reused} applied={applied} files={done}", transfer.wire)); + } + for old in array(&plan, "oldFiles") { + if cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } + if !seen.contains(&text(old, "path").to_lowercase()) { + let path = inside(&root, text(old, "path"))?; + if matches(&path, old) { fs::remove_file(path).map_err(|e| e.to_string())?; } + } + } + events.on_log(&format!("gog: maintenance complete wire={} reused={reused} applied={applied}", transfer.wire)); + Ok((transfer.wire, done)) + }; + match work() { + Ok((bytes, files)) => { result.success = true; result.bytes_written = bytes; result.files_done = files; } + Err(error) => { result.cancelled = cancel.load(Ordering::Relaxed); result.link_expiry = http_status_in(&error).map(is_link_expiry_status).unwrap_or(false); result.error = error; } + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::{write::ZlibEncoder, Compression}; + use serde_json::json; + use std::net::TcpListener; + use std::sync::{Arc, atomic::AtomicU64}; + + struct Events; + impl GogEvents for Events { + fn on_file_done(&self, _: &str, _: u64, _: bool, _: u32, _: u32, _: u64, _: u64) {} + fn on_log(&self, _: &str) {} + } + struct Temp(PathBuf); + impl Temp { + fn new() -> Self { + let path = std::env::temp_dir().join(format!("gog-maintenance-{}-{}", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos())); + fs::create_dir_all(&path).unwrap(); Self(path) + } + } + impl Drop for Temp { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.0); } } + + struct Server { base: String, bytes: Arc, stop: Arc, thread: Option> } + impl Server { + fn new(payloads: HashMap>, range: bool) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); let bytes = Arc::new(AtomicU64::new(0)); + let (stopped, count) = (stop.clone(), bytes.clone()); + let thread = std::thread::spawn(move || { + while !stopped.load(Ordering::Relaxed) { + if let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap(); + let mut request = Vec::new(); let mut one = [0u8]; + while stream.read(&mut one).unwrap_or(0) > 0 { request.push(one[0]); if request.ends_with(b"\r\n\r\n") { break; } } + let request = String::from_utf8_lossy(&request); + let path = request.split_whitespace().nth(1).unwrap_or(""); + if let Some(body) = payloads.get(path) { + let status = if range { "206 Partial Content" } else { "200 OK" }; + let _ = write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + if stream.write_all(body).is_ok() { count.fetch_add(body.len() as u64, Ordering::Relaxed); } + } else { let _ = stream.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); } + } else { std::thread::sleep(std::time::Duration::from_millis(2)); } + } + }); + Self { base, bytes, stop, thread: Some(thread) } + } + } + impl Drop for Server { fn drop(&mut self) { self.stop.store(true, Ordering::Relaxed); if let Some(thread) = self.thread.take() { thread.join().unwrap(); } } } + + #[test] + fn continuing_download_may_exceed_the_read_timeout_in_total() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(3))).unwrap(); + let mut request = Vec::new(); let mut byte = [0u8]; + while stream.read(&mut byte).unwrap() > 0 { request.push(byte[0]); if request.ends_with(b"\r\n\r\n") { break; } } + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n").unwrap(); + for byte in b"abcd" { + stream.write_all(&[*byte]).unwrap(); stream.flush().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(400)); + } + }); + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let client = http_client(std::time::Duration::from_secs(1)).unwrap(); + let body = runtime.block_on(async { client.get(url).send().await.unwrap().bytes().await.unwrap() }); + assert_eq!(&body[..], b"abcd"); + server.join().unwrap(); + } + + fn chunk(data: &[u8]) -> (Value, String, Vec) { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); encoder.write_all(data).unwrap(); + let packed = encoder.finish().unwrap(); let hash = md5_hex(&packed); + (json!({"md5": md5_hex(data), "size": data.len(), "compressedMd5": hash, "compressedSize": packed.len()}), + format!("/{}", super::super::plan::build_cdn_path(&hash)), packed) + } + fn execute(root: &Path, files: Value, old: Value, server: &Server) -> GogRunResult { + let plan = json!({"files": files, "oldFiles": old, "patches": [], "links": {"1": [server.base]}, "downloadBytes": 4096}); + run(&GogRequest { install_dir: root.to_string_lossy().into_owned(), depot_manifests: vec![plan.to_string()], ..Default::default() }, &AtomicBool::new(false), &Events) + } + + #[test] + fn reuses_moved_and_repeated_chunks_and_downloads_only_missing_bytes() { + let temp = Temp::new(); + let (a, _, _) = chunk(b"aaaa"); let (b, path, packed) = chunk(b"bbbb"); + fs::write(temp.0.join("archive"), b"aaaaaaaa").unwrap(); + let server = Server::new(HashMap::from([(path, packed.clone())]), false); + let old = json!([{"path":"archive", "size":8, "md5":md5_hex(b"aaaaaaaa"), "chunks":[a,a]}]); + let files = json!([{"path":"archive", "size":12, "md5":md5_hex(b"bbbbaaaaaaaa"), "chunks":[b,a,a], "product":"1"}]); + let result = execute(&temp.0, files, old, &server); + assert!(result.success, "{}", result.error); + assert_eq!(fs::read(temp.0.join("archive")).unwrap(), b"bbbbaaaaaaaa"); + assert_eq!(server.bytes.load(Ordering::Relaxed), packed.len() as u64); + assert_eq!(result.bytes_written, packed.len() as u64); + } + + #[test] + fn revalidates_partial_staging_and_preserves_original_until_verified() { + let temp = Temp::new(); let (a, _, _) = chunk(b"aaaa"); let (b, path, packed) = chunk(b"bbbb"); + let stages = temp.0.join(".droiddeck-gog/stage"); fs::create_dir_all(&stages).unwrap(); + fs::write(temp.0.join("archive"), b"original").unwrap(); + fs::write(stages.join(md5_hex(b"archive")), b"aaaabadd").unwrap(); + let server = Server::new(HashMap::from([(path, packed.clone())]), false); + let files = json!([{"path":"archive", "size":8, "md5":md5_hex(b"aaaabbbb"), "chunks":[a,b], "product":"1"}]); + let result = execute(&temp.0, files, json!([]), &server); + assert!(result.success, "{}", result.error); + assert_eq!(server.bytes.load(Ordering::Relaxed), packed.len() as u64); + assert_eq!(fs::read(temp.0.join("archive")).unwrap(), b"aaaabbbb"); + } + + #[test] + fn unchanged_files_need_no_network_and_obsolete_modified_files_survive() { + let temp = Temp::new(); let (a, _, _) = chunk(b"aaaa"); + fs::write(temp.0.join("current"), b"aaaa").unwrap(); + fs::write(temp.0.join("obsolete"), b"old").unwrap(); fs::write(temp.0.join("modded"), b"mod").unwrap(); + let server = Server::new(HashMap::new(), false); + let files = json!([{"path":"current", "size":4, "md5":md5_hex(b"aaaa"), "chunks":[a], "product":"1"}, {"path":"empty", "size":0, "chunks":[], "product":"1"}]); + let old = json!([{"path":"obsolete", "size":3, "md5":md5_hex(b"old")}, {"path":"modded", "size":3, "md5":md5_hex(b"old")}]); + let result = execute(&temp.0, files, old, &server); + assert!(result.success, "{}", result.error); + assert_eq!(server.bytes.load(Ordering::Relaxed), 0); + assert!(!temp.0.join("obsolete").exists()); assert!(temp.0.join("modded").exists()); + assert_eq!(fs::metadata(temp.0.join("empty")).unwrap().len(), 0); + } + + #[test] + fn gen1_same_size_corruption_is_replaced_using_hash_and_range() { + let temp = Temp::new(); fs::write(temp.0.join("game"), b"bad!").unwrap(); + let server = Server::new(HashMap::from([("/range".to_string(), b"good".to_vec())]), true); + let files = json!([{"path":"game", "size":4, "md5":md5_hex(b"good"), "chunks":[], "product":"1", "remotePath":"range", "offset":10}]); + let result = execute(&temp.0, files, json!([]), &server); + assert!(result.success, "{}", result.error); assert_eq!(fs::read(temp.0.join("game")).unwrap(), b"good"); + assert_eq!(server.bytes.load(Ordering::Relaxed), 4); + } + + #[test] + fn bad_chunks_never_replace_original_and_expired_links_are_reported() { + let temp = Temp::new(); fs::write(temp.0.join("game"), b"old!").unwrap(); + let (a, path, mut packed) = chunk(b"new!"); packed[0] ^= 1; + let server = Server::new(HashMap::from([(path, packed)]), false); + let files = json!([{"path":"game", "size":4, "md5":md5_hex(b"new!"), "chunks":[a], "product":"1"}]); + let result = execute(&temp.0, files.clone(), json!([]), &server); + assert!(!result.success); assert_eq!(fs::read(temp.0.join("game")).unwrap(), b"old!"); + let missing = Server::new(HashMap::new(), false); + let result = execute(&temp.0, files, json!([]), &missing); + assert!(result.link_expiry, "{}", result.error); + } + + #[test] + fn rejects_traversal_symlinks_and_case_collisions_before_fetching() { + let temp = Temp::new(); let outside = Temp::new(); + std::os::unix::fs::symlink(&outside.0, temp.0.join("linked")).unwrap(); + for path in ["../escape", "/absolute", "a/./b", "a//b", "linked/game"] { assert!(inside(&temp.0, path).is_err(), "{path}"); } + let server = Server::new(HashMap::new(), false); + let result = execute(&temp.0, json!([{"path":"Game"}, {"path":"game"}]), json!([]), &server); + assert!(!result.success); assert_eq!(server.bytes.load(Ordering::Relaxed), 0); + } + + #[test] + fn valid_xdelta_downloads_only_delta_and_corrupt_source_falls_back() { + let old = b"the old archive contents\n".repeat(4096); + let mut new = b"the new archive contents\n".to_vec(); new.extend_from_slice(&old); new.extend_from_slice(b"END\n"); + let delta = include_bytes!("fixtures/update.vcdiff"); + let (patch_chunk, patch_url, packed_delta) = chunk(delta); + let (target_chunk, target_url, packed_target) = chunk(&new); + for mode in 0..3 { + let corrupt = mode == 1; + let temp = Temp::new(); + fs::write(temp.0.join("game"), if corrupt { vec![b'x'; old.len()] } else { old.clone() }).unwrap(); + let (delta_chunk, delta_url, delta_body) = if mode == 2 { chunk(b"unsupported VCDIFF") } else { (patch_chunk.clone(), patch_url.clone(), packed_delta.clone()) }; + let server = Server::new(HashMap::from([(delta_url, delta_body.clone()), (target_url.clone(), packed_target.clone())]), false); + let plan = json!({"files":[{"path":"game", "product":"1", "size":new.len(), "md5":md5_hex(&new), "chunks":[target_chunk]}], + "oldFiles":[{"path":"game", "size":old.len(), "md5":md5_hex(&old), "chunks":[]}], + "patches":[{"path_source":"game", "path_target":"game", "product":"1", "md5_source":md5_hex(&old), "md5_target":md5_hex(&new), "md5":if mode == 2 { md5_hex(b"unsupported VCDIFF") } else { md5_hex(delta) }, "chunks":[delta_chunk]}], + "links":{"1":[server.base]}, "patchLinks":{"1":[server.base]}, "downloadBytes":packed_target.len()}); + let result = run(&GogRequest {install_dir:temp.0.to_string_lossy().into_owned(), depot_manifests:vec![plan.to_string()], ..Default::default()}, &AtomicBool::new(false), &Events); + assert!(result.success, "{}", result.error); + assert_eq!(fs::read(temp.0.join("game")).unwrap(), new); + assert_eq!(server.bytes.load(Ordering::Relaxed), match mode { 1 => packed_target.len(), 2 => delta_body.len() + packed_target.len(), _ => packed_delta.len() } as u64); + } + } + + #[test] + fn stored_chunks_starting_with_x_are_not_mistaken_for_zlib() { + let temp = Temp::new(); let raw = b"xyz stored data"; + let hash = md5_hex(raw); + let path = format!("/{}", super::super::plan::build_cdn_path(&hash)); + let server = Server::new(HashMap::from([(path, raw.to_vec())]), false); + let result = execute(&temp.0, json!([{"path":"game", "product":"1", "size":raw.len(), "md5":hash, + "chunks":[{"md5":hash,"compressedMd5":hash,"size":raw.len(),"compressedSize":raw.len()}]}]), json!([]), &server); + assert!(result.success, "{}", result.error); + assert_eq!(fs::read(temp.0.join("game")).unwrap(), raw); + } +} diff --git a/app/src/main/rust/stores/vendor/xdelta3/LICENSE b/app/src/main/rust/stores/vendor/xdelta3/LICENSE new file mode 100644 index 0000000000..7a774156a6 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/LICENSE @@ -0,0 +1,176 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/app/src/main/rust/stores/vendor/xdelta3/README.droiddeck.md b/app/src/main/rust/stores/vendor/xdelta3/README.droiddeck.md new file mode 100644 index 0000000000..696f3afa76 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/README.droiddeck.md @@ -0,0 +1,7 @@ +Xdelta3 decoder from jmacd/xdelta revision 0525275fe4b553a10f38e455d30c60dc6ed9b45d (release3_0_apl), the +Apache-2.0 revision used by heroic-gogdl. Upstream sources are unmodified. +https://github.com/jmacd/xdelta/tree/0525275/xdelta3 + +DroidDeck's streaming wrapper is src/gog/delta.c. Encoding and secondary +compression are disabled, matching heroic-gogdl's decoder baseline. Unsupported +streams fall back to verified Galaxy chunks. No Python runtime is required. diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3-cfgs.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-cfgs.h new file mode 100644 index 0000000000..84a2221439 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-cfgs.h @@ -0,0 +1,171 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/****************************************************************** + SOFT string matcher + ******************************************************************/ + +#if XD3_BUILD_SOFT + +#define TEMPLATE soft +#define LLOOK stream->smatcher.large_look +#define LSTEP stream->smatcher.large_step +#define SLOOK stream->smatcher.small_look +#define SCHAIN stream->smatcher.small_chain +#define SLCHAIN stream->smatcher.small_lchain +#define MAXLAZY stream->smatcher.max_lazy +#define LONGENOUGH stream->smatcher.long_enough + +#define SOFTCFG 1 +#include "xdelta3.c" +#undef SOFTCFG + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif + +#define SOFTCFG 0 + +/************************************************************ + FASTEST string matcher + **********************************************************/ +#if XD3_BUILD_FASTEST +#define TEMPLATE fastest +#define LLOOK 9 +#define LSTEP 26 +#define SLOOK 4U +#define SCHAIN 1 +#define SLCHAIN 1 +#define MAXLAZY 6 +#define LONGENOUGH 6 + +#include "xdelta3.c" + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif + +/************************************************************ + FASTER string matcher + **********************************************************/ +#if XD3_BUILD_FASTER +#define TEMPLATE faster +#define LLOOK 9 +#define LSTEP 15 +#define SLOOK 4U +#define SCHAIN 1 +#define SLCHAIN 1 +#define MAXLAZY 18 +#define LONGENOUGH 18 + +#include "xdelta3.c" + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif + +/****************************************************** + FAST string matcher + ********************************************************/ +#if XD3_BUILD_FAST +#define TEMPLATE fast +#define LLOOK 9 +#define LSTEP 8 +#define SLOOK 4U +#define SCHAIN 4 +#define SLCHAIN 1 +#define MAXLAZY 18 +#define LONGENOUGH 35 + +#include "xdelta3.c" + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif + +/************************************************** + SLOW string matcher + **************************************************************/ +#if XD3_BUILD_SLOW +#define TEMPLATE slow +#define LLOOK 9 +#define LSTEP 2 +#define SLOOK 4U +#define SCHAIN 44 +#define SLCHAIN 13 +#define MAXLAZY 90 +#define LONGENOUGH 70 + +#include "xdelta3.c" + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif + +/******************************************************** + DEFAULT string matcher + ************************************************************/ +#if XD3_BUILD_DEFAULT +#define TEMPLATE default +#define LLOOK 9 +#define LSTEP 3 +#define SLOOK 4U +#define SCHAIN 8 +#define SLCHAIN 2 +#define MAXLAZY 36 +#define LONGENOUGH 70 + +#include "xdelta3.c" + +#undef TEMPLATE +#undef LLOOK +#undef SLOOK +#undef LSTEP +#undef SCHAIN +#undef SLCHAIN +#undef MAXLAZY +#undef LONGENOUGH +#endif diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3-decode.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-decode.h new file mode 100644 index 0000000000..3933b01579 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-decode.h @@ -0,0 +1,1218 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _XDELTA3_DECODE_H_ +#define _XDELTA3_DECODE_H_ + +#include "xdelta3-internal.h" + +#define SRCORTGT(x) ((((x) & VCD_SRCORTGT) == VCD_SOURCE) ? \ + VCD_SOURCE : ((((x) & VCD_SRCORTGT) == \ + VCD_TARGET) ? VCD_TARGET : 0)) + +static inline int +xd3_decode_byte (xd3_stream *stream, usize_t *val) +{ + if (stream->avail_in == 0) + { + stream->msg = "further input required"; + return XD3_INPUT; + } + + (*val) = stream->next_in[0]; + + DECODE_INPUT (1); + return 0; +} + +static inline int +xd3_decode_bytes (xd3_stream *stream, uint8_t *buf, usize_t *pos, usize_t size) +{ + usize_t want; + usize_t take; + + /* Note: The case where (*pos == size) happens when a zero-length + * appheader or code table is transmitted, but there is nothing in + * the standard against that. */ + while (*pos < size) + { + if (stream->avail_in == 0) + { + stream->msg = "further input required"; + return XD3_INPUT; + } + + want = size - *pos; + take = xd3_min (want, stream->avail_in); + + memcpy (buf + *pos, stream->next_in, (size_t) take); + + DECODE_INPUT (take); + (*pos) += take; + } + + return 0; +} + +/* Initialize the decoder for a new window. The dec_tgtlen value is + * preserved across successive window decodings, and the update to + * dec_winstart is delayed until a new window actually starts. This + * is to avoid throwing an error due to overflow until the last + * possible moment. This makes it possible to encode exactly 4GB + * through a 32-bit encoder. */ +static int +xd3_decode_init_window (xd3_stream *stream) +{ + stream->dec_cpylen = 0; + stream->dec_cpyoff = 0; + stream->dec_cksumbytes = 0; + + xd3_init_cache (& stream->acache); + + return 0; +} + +/* Allocates buffer space for the target window and possibly the + * VCD_TARGET copy-window. Also sets the base of the two copy + * segments. */ +static int +xd3_decode_setup_buffers (xd3_stream *stream) +{ + /* If VCD_TARGET is set then the previous buffer may be reused. */ + if (stream->dec_win_ind & VCD_TARGET) + { + /* Note: this implementation is untested, since Xdelta3 itself + * does not implement an encoder for VCD_TARGET mode. Thus, mark + * unimplemented until needed. */ + if (1) + { + stream->msg = "VCD_TARGET not implemented"; + return XD3_UNIMPLEMENTED; + } + + /* But this implementation only supports copying from the last + * target window. If the offset is outside that range, it can't + * be done. */ + if (stream->dec_cpyoff < stream->dec_laststart) + { + stream->msg = "unsupported VCD_TARGET offset"; + return XD3_INVALID_INPUT; + } + + /* See if the two windows are the same. This indicates the + * first time VCD_TARGET is used. This causes a second buffer + * to be allocated, after that the two are swapped in the + * DEC_FINISH case. */ + if (stream->dec_lastwin == stream->next_out) + { + stream->next_out = NULL; + stream->space_out = 0; + } + + /* TODO: (See note above, this looks incorrect) */ + stream->dec_cpyaddrbase = stream->dec_lastwin + + (usize_t) (stream->dec_cpyoff - stream->dec_laststart); + } + + /* See if the current output window is large enough. */ + if (stream->space_out < stream->dec_tgtlen) + { + xd3_free (stream, stream->dec_buffer); + + stream->space_out = + xd3_round_blksize (stream->dec_tgtlen, XD3_ALLOCSIZE); + + if ((stream->dec_buffer = + (uint8_t*) xd3_alloc (stream, stream->space_out, 1)) == NULL) + { + return ENOMEM; + } + + stream->next_out = stream->dec_buffer; + } + + /* dec_tgtaddrbase refers to an invalid base address, but it is + * always used with a sufficiently large instruction offset (i.e., + * beyond the copy window). This condition is enforced by + * xd3_decode_output_halfinst. */ + stream->dec_tgtaddrbase = stream->next_out - stream->dec_cpylen; + + return 0; +} + +static int +xd3_decode_allocate (xd3_stream *stream, + usize_t size, + uint8_t **buf_ptr, + usize_t *buf_alloc) +{ + IF_DEBUG2 (DP(RINT "[xd3_decode_allocate] size %u alloc %u\n", + size, *buf_alloc)); + + if (*buf_ptr != NULL && *buf_alloc < size) + { + xd3_free (stream, *buf_ptr); + *buf_ptr = NULL; + } + + if (*buf_ptr == NULL) + { + *buf_alloc = xd3_round_blksize (size, XD3_ALLOCSIZE); + + if ((*buf_ptr = (uint8_t*) xd3_alloc (stream, *buf_alloc, 1)) == NULL) + { + return ENOMEM; + } + } + + return 0; +} + +static int +xd3_decode_section (xd3_stream *stream, + xd3_desect *section, + xd3_decode_state nstate, + int copy) +{ + XD3_ASSERT (section->pos <= section->size); + XD3_ASSERT (stream->dec_state != nstate); + + if (section->pos < section->size) + { + usize_t sect_take; + + if (stream->avail_in == 0) + { + return XD3_INPUT; + } + + if ((copy == 0) && (section->pos == 0)) + { + /* No allocation/copy needed */ + section->buf = stream->next_in; + sect_take = section->size; + IF_DEBUG1 (DP(RINT "[xd3_decode_section] zerocopy %u @ %u avail %u\n", + sect_take, section->pos, stream->avail_in)); + } + else + { + usize_t sect_need = section->size - section->pos; + + /* Allocate and copy */ + sect_take = xd3_min (sect_need, stream->avail_in); + + if (section->pos == 0) + { + int ret; + + if ((ret = xd3_decode_allocate (stream, + section->size, + & section->copied1, + & section->alloc1))) + { + return ret; + } + + section->buf = section->copied1; + } + + IF_DEBUG2 (DP(RINT "[xd3_decode_section] take %u @ %u [need %u] avail %u\n", + sect_take, section->pos, sect_need, stream->avail_in)); + XD3_ASSERT (section->pos + sect_take <= section->alloc1); + + memcpy (section->copied1 + section->pos, + stream->next_in, + sect_take); + } + + section->pos += sect_take; + + stream->dec_winbytes += sect_take; + + DECODE_INPUT (sect_take); + } + + if (section->pos < section->size) + { + IF_DEBUG1 (DP(RINT "[xd3_decode_section] further input required %u\n", section->size - section->pos)); + stream->msg = "further input required"; + return XD3_INPUT; + } + + XD3_ASSERT (section->pos == section->size); + + stream->dec_state = nstate; + section->buf_max = section->buf + section->size; + section->pos = 0; + return 0; +} + +/* Decode the size and address for half of an instruction (i.e., a + * single opcode). This updates the stream->dec_position, which are + * bytes already output prior to processing this instruction. Perform + * bounds checking for sizes and copy addresses, which uses the + * dec_position (which is why these checks are done here). */ +static int +xd3_decode_parse_halfinst (xd3_stream *stream, xd3_hinst *inst) +{ + int ret; + + /* If the size from the instruction table is zero then read a size value. */ + if ((inst->size == 0) && + (ret = xd3_read_size (stream, + & stream->inst_sect.buf, + stream->inst_sect.buf_max, + & inst->size))) + { + return XD3_INVALID_INPUT; + } + + /* For copy instructions, read address. */ + if (inst->type >= XD3_CPY) + { + IF_DEBUG2 ({ + static int cnt = 0; + XPR(NT "DECODE:%u: COPY at %"Q"u (winoffset %u) size %u winaddr %u\n", + cnt++, + stream->total_out + (stream->dec_position - + stream->dec_cpylen), + (stream->dec_position - stream->dec_cpylen), + inst->size, + inst->addr); + }); + + if ((ret = xd3_decode_address (stream, + stream->dec_position, + inst->type - XD3_CPY, + & stream->addr_sect.buf, + stream->addr_sect.buf_max, + & inst->addr))) + { + return ret; + } + + /* Cannot copy an address before it is filled-in. */ + if (inst->addr >= stream->dec_position) + { + stream->msg = "address too large"; + return XD3_INVALID_INPUT; + } + + /* Check: a VCD_TARGET or VCD_SOURCE copy cannot exceed the remaining + * buffer space in its own segment. */ + if (inst->addr < stream->dec_cpylen && + inst->addr + inst->size > stream->dec_cpylen) + { + stream->msg = "size too large"; + return XD3_INVALID_INPUT; + } + } + else + { + IF_DEBUG2 ({ + if (inst->type == XD3_ADD) + { + static int cnt; + XPR(NT "DECODE:%d: ADD at %"Q"u (winoffset %u) size %u\n", + cnt++, + (stream->total_out + stream->dec_position - stream->dec_cpylen), + stream->dec_position - stream->dec_cpylen, + inst->size); + } + else + { + static int cnt; + XD3_ASSERT (inst->type == XD3_RUN); + XPR(NT "DECODE:%d: RUN at %"Q"u (winoffset %u) size %u\n", + cnt++, + stream->total_out + stream->dec_position - stream->dec_cpylen, + stream->dec_position - stream->dec_cpylen, + inst->size); + } + }); + } + + /* Check: The instruction will not overflow the output buffer. */ + if (stream->dec_position + inst->size > stream->dec_maxpos) + { + stream->msg = "size too large"; + return XD3_INVALID_INPUT; + } + + stream->dec_position += inst->size; + return 0; +} + +/* Decode a single opcode and then decode the two half-instructions. */ +static int +xd3_decode_instruction (xd3_stream *stream) +{ + int ret; + const xd3_dinst *inst; + + if (stream->inst_sect.buf == stream->inst_sect.buf_max) + { + stream->msg = "instruction underflow"; + return XD3_INVALID_INPUT; + } + + inst = &stream->code_table[*stream->inst_sect.buf++]; + + stream->dec_current1.type = inst->type1; + stream->dec_current2.type = inst->type2; + stream->dec_current1.size = inst->size1; + stream->dec_current2.size = inst->size2; + + /* For each instruction with a real operation, decode the + * corresponding size and addresses if necessary. Assume a + * code-table may have NOOP in either position, although this is + * unlikely. */ + if (inst->type1 != XD3_NOOP && + (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current1))) + { + return ret; + } + if (inst->type2 != XD3_NOOP && + (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current2))) + { + return ret; + } + return 0; +} + +/* Output the result of a single half-instruction. OPT: This the + decoder hotspot. Modifies "hinst", see below. */ +static int +xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) +{ + /* This method is reentrant for copy instructions which may return + * XD3_GETSRCBLK to the caller. Each time through a copy takes the + * minimum of inst->size and the available space on whichever block + * supplies the data */ + usize_t take = inst->size; + + if (USIZE_T_OVERFLOW (stream->avail_out, take) || + stream->avail_out + take > stream->space_out) + { + stream->msg = "overflow while decoding"; + return XD3_INVALID_INPUT; + } + + XD3_ASSERT (inst->type != XD3_NOOP); + + switch (inst->type) + { + case XD3_RUN: + { + /* Only require a single data byte. */ + if (stream->data_sect.buf == stream->data_sect.buf_max) + { + stream->msg = "data underflow"; + return XD3_INVALID_INPUT; + } + + memset (stream->next_out + stream->avail_out, + stream->data_sect.buf[0], + take); + + stream->data_sect.buf += 1; + stream->avail_out += take; + inst->type = XD3_NOOP; + break; + } + case XD3_ADD: + { + /* Require at least TAKE data bytes. */ + if (stream->data_sect.buf + take > stream->data_sect.buf_max) + { + stream->msg = "data underflow"; + return XD3_INVALID_INPUT; + } + + memcpy (stream->next_out + stream->avail_out, + stream->data_sect.buf, + take); + + stream->data_sect.buf += take; + stream->avail_out += take; + inst->type = XD3_NOOP; + break; + } + default: + { + usize_t i; + const uint8_t *src; + uint8_t *dst; + int overlap; + + /* See if it copies from the VCD_TARGET/VCD_SOURCE window or + * the target window. Out-of-bounds checks for the addresses + * and sizes are performed in xd3_decode_parse_halfinst. This + * if/else must set "overlap", "src", and "dst". */ + if (inst->addr < stream->dec_cpylen) + { + /* In both branches we are copying from outside the + * current decoder window, the first (VCD_TARGET) is + * unimplemented. */ + overlap = 0; + + /* This branch sets "src". As a side-effect, we modify + * "inst" so that if we reenter this method after a + * XD3_GETSRCBLK response the state is correct. So if the + * instruction can be fulfilled by a contiguous block of + * memory then we will set: + * + * inst->type = XD3_NOOP; + * inst->size = 0; + */ + if (stream->dec_win_ind & VCD_TARGET) + { + /* TODO: Users have requested long-distance copies of + * similar material within a target (e.g., for dup + * supression in backups). This code path is probably + * dead due to XD3_UNIMPLEMENTED in xd3_decode_setup_buffers */ + inst->size = 0; + inst->type = XD3_NOOP; + stream->msg = "VCD_TARGET not implemented"; + return XD3_UNIMPLEMENTED; + } + else + { + /* In this case we have to read a source block, which + * could return control to the caller. We need to + * know the first block number needed for this + * copy. */ + xd3_source *source = stream->src; + xoff_t block = source->cpyoff_blocks; + usize_t blkoff = source->cpyoff_blkoff; + const usize_t blksize = source->blksize; + int ret; + + xd3_blksize_add (&block, &blkoff, source, inst->addr); + XD3_ASSERT (blkoff < blksize); + + if ((ret = xd3_getblk (stream, block))) + { + /* could be a XD3_GETSRCBLK failure. */ + if (ret == XD3_TOOFARBACK) + { + stream->msg = "non-seekable source in decode"; + ret = XD3_INTERNAL; + } + return ret; + } + + src = source->curblk + blkoff; + + /* This block is either full, or a partial block that + * must contain enough bytes. */ + if ((source->onblk != blksize) && + (blkoff + take > source->onblk)) + { + IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q"u onblk " + "%u blksize %u blkoff %u take %u\n", + block, + source->onblk, + blksize, + blkoff, + take)); + stream->msg = "source file too short"; + return XD3_INVALID_INPUT; + } + + XD3_ASSERT (blkoff != blksize); + + /* Check if we have enough data on this block to + * finish the instruction. */ + if (blkoff + take <= blksize) + { + inst->type = XD3_NOOP; + inst->size = 0; + } + else + { + take = blksize - blkoff; + inst->size -= take; + inst->addr += take; + + /* because (blkoff + take > blksize), above */ + XD3_ASSERT (inst->size != 0); + } + } + } + else + { + /* TODO: the memcpy/overlap optimization, etc. Overlap + * here could be more specific, it's whether (inst->addr - + * srclen) + inst->size > input_pos ? And is the system + * memcpy really any good? */ + overlap = 1; + + /* For a target-window copy, we know the entire range is + * in-memory. The dec_tgtaddrbase is negatively offset by + * dec_cpylen because the addresses start beyond that + * point. */ + src = stream->dec_tgtaddrbase + inst->addr; + inst->type = XD3_NOOP; + inst->size = 0; + } + + dst = stream->next_out + stream->avail_out; + + stream->avail_out += take; + + if (overlap) + { + /* Can't just memcpy here due to possible overlap. */ + for (i = take; i != 0; i -= 1) + { + *dst++ = *src++; + } + } + else + { + memcpy (dst, src, take); + } + } + } + + return 0; +} + +static int +xd3_decode_finish_window (xd3_stream *stream) +{ + stream->dec_winbytes = 0; + stream->dec_state = DEC_FINISH; + + stream->data_sect.pos = 0; + stream->inst_sect.pos = 0; + stream->addr_sect.pos = 0; + + return XD3_OUTPUT; +} + +static int +xd3_decode_secondary_sections (xd3_stream *secondary_stream) +{ +#if SECONDARY_ANY + int ret; +#define DECODE_SECONDARY_SECTION(UPPER,LOWER) \ + ((secondary_stream->dec_del_ind & VCD_ ## UPPER ## COMP) && \ + (ret = xd3_decode_secondary (secondary_stream, \ + & secondary_stream-> LOWER ## _sect, \ + & xd3_sec_ ## LOWER (secondary_stream)))) + + if (DECODE_SECONDARY_SECTION (DATA, data) || + DECODE_SECONDARY_SECTION (INST, inst) || + DECODE_SECONDARY_SECTION (ADDR, addr)) + { + return ret; + } +#undef DECODE_SECONDARY_SECTION +#endif + return 0; +} + +static int +xd3_decode_sections (xd3_stream *stream) +{ + usize_t need, more, take; + int copy, ret; + + if ((stream->flags & XD3_JUST_HDR) != 0) + { + /* Nothing left to do. */ + return xd3_decode_finish_window (stream); + } + + /* To avoid extra copying, allocate three sections at once (but + * check for overflow). */ + need = stream->inst_sect.size; + + if (USIZE_T_OVERFLOW (need, stream->addr_sect.size)) + { + stream->msg = "decoder section size overflow"; + return XD3_INTERNAL; + } + need += stream->addr_sect.size; + + if (USIZE_T_OVERFLOW (need, stream->data_sect.size)) + { + stream->msg = "decoder section size overflow"; + return XD3_INTERNAL; + } + need += stream->data_sect.size; + + /* The window may be entirely processed. */ + XD3_ASSERT (stream->dec_winbytes <= need); + + /* Compute how much more input is needed. */ + more = (need - stream->dec_winbytes); + + /* How much to consume. */ + take = xd3_min (more, stream->avail_in); + + /* See if the input is completely available, to avoid copy. */ + copy = (take != more); + + /* If the window is skipped... */ + if ((stream->flags & XD3_SKIP_WINDOW) != 0) + { + /* Skip the available input. */ + DECODE_INPUT (take); + + stream->dec_winbytes += take; + + if (copy) + { + stream->msg = "further input required"; + return XD3_INPUT; + } + + return xd3_decode_finish_window (stream); + } + + /* Process all but the DATA section. */ + switch (stream->dec_state) + { + default: + stream->msg = "internal error"; + return XD3_INVALID_INPUT; + + case DEC_DATA: + if ((ret = xd3_decode_section (stream, & stream->data_sect, + DEC_INST, copy))) { return ret; } + case DEC_INST: + if ((ret = xd3_decode_section (stream, & stream->inst_sect, + DEC_ADDR, copy))) { return ret; } + case DEC_ADDR: + if ((ret = xd3_decode_section (stream, & stream->addr_sect, + DEC_EMIT, copy))) { return ret; } + } + + XD3_ASSERT (stream->dec_winbytes == need); + + if ((ret = xd3_decode_secondary_sections (stream))) { return ret; } + + if (stream->flags & XD3_SKIP_EMIT) + { + return xd3_decode_finish_window (stream); + } + + /* OPT: A possible optimization is to avoid allocating memory in + * decode_setup_buffers and to avoid a large memcpy when the window + * consists of a single VCD_SOURCE copy instruction. */ + if ((ret = xd3_decode_setup_buffers (stream))) { return ret; } + + return 0; +} + +static int +xd3_decode_emit (xd3_stream *stream) +{ + int ret; + + /* Produce output: originally structured to allow reentrant code + * that fills as much of the output buffer as possible, but VCDIFF + * semantics allows to copy from anywhere from the target window, so + * instead allocate a sufficiently sized buffer after the target + * window length is decoded. + * + * This code still needs to be reentrant to allow XD3_GETSRCBLK to + * return control. This is handled by setting the + * stream->dec_currentN instruction types to XD3_NOOP after they + * have been processed. */ + XD3_ASSERT (! (stream->flags & XD3_SKIP_EMIT)); + XD3_ASSERT (stream->dec_tgtlen <= stream->space_out); + + while (stream->inst_sect.buf != stream->inst_sect.buf_max || + stream->dec_current1.type != XD3_NOOP || + stream->dec_current2.type != XD3_NOOP) + { + /* Decode next instruction pair. */ + if ((stream->dec_current1.type == XD3_NOOP) && + (stream->dec_current2.type == XD3_NOOP) && + (ret = xd3_decode_instruction (stream))) { return ret; } + + /* Output dec_current1 */ + while ((stream->dec_current1.type != XD3_NOOP)) + { + if ((ret = xd3_decode_output_halfinst (stream, & stream->dec_current1))) + { + return ret; + } + } + /* Output dec_current2 */ + while (stream->dec_current2.type != XD3_NOOP) + { + if ((ret = xd3_decode_output_halfinst (stream, & stream->dec_current2))) + { + return ret; + } + } + } + + if (stream->avail_out != stream->dec_tgtlen) + { + IF_DEBUG2 (DP(RINT "AVAIL_OUT(%d) != DEC_TGTLEN(%d)\n", + stream->avail_out, stream->dec_tgtlen)); + stream->msg = "wrong window length"; + return XD3_INVALID_INPUT; + } + + if (stream->data_sect.buf != stream->data_sect.buf_max) + { + stream->msg = "extra data section"; + return XD3_INVALID_INPUT; + } + + if (stream->addr_sect.buf != stream->addr_sect.buf_max) + { + stream->msg = "extra address section"; + return XD3_INVALID_INPUT; + } + + /* OPT: Should cksum computation be combined with the above loop? */ + if ((stream->dec_win_ind & VCD_ADLER32) != 0 && + (stream->flags & XD3_ADLER32_NOVER) == 0) + { + uint32_t a32 = adler32 (1L, stream->next_out, stream->avail_out); + + if (a32 != stream->dec_adler32) + { + stream->msg = "target window checksum mismatch"; + return XD3_INVALID_INPUT; + } + } + + /* Finished with a window. */ + return xd3_decode_finish_window (stream); +} + +int +xd3_decode_input (xd3_stream *stream) +{ + int ret; + + if (stream->enc_state != 0) + { + stream->msg = "encoder/decoder transition"; + return XD3_INVALID_INPUT; + } + +#define BYTE_CASE(expr,x,nstate) \ + do { \ + if ( (expr) && \ + ((ret = xd3_decode_byte (stream, & (x))) != 0) ) { return ret; } \ + stream->dec_state = (nstate); \ + } while (0) + +#define OFFSET_CASE(expr,x,nstate) \ + do { \ + if ( (expr) && \ + ((ret = xd3_decode_offset (stream, & (x))) != 0) ) { return ret; } \ + stream->dec_state = (nstate); \ + } while (0) + +#define SIZE_CASE(expr,x,nstate) \ + do { \ + if ( (expr) && \ + ((ret = xd3_decode_size (stream, & (x))) != 0) ) { return ret; } \ + stream->dec_state = (nstate); \ + } while (0) + + switch (stream->dec_state) + { + case DEC_VCHEAD: + { + if ((ret = xd3_decode_bytes (stream, stream->dec_magic, + & stream->dec_magicbytes, 4))) + { + return ret; + } + + if (stream->dec_magic[0] != VCDIFF_MAGIC1 || + stream->dec_magic[1] != VCDIFF_MAGIC2 || + stream->dec_magic[2] != VCDIFF_MAGIC3) + { + stream->msg = "not a VCDIFF input"; + return XD3_INVALID_INPUT; + } + + if (stream->dec_magic[3] != 0) + { + stream->msg = "VCDIFF input version > 0 is not supported"; + return XD3_INVALID_INPUT; + } + + stream->dec_state = DEC_HDRIND; + } + case DEC_HDRIND: + { + if ((ret = xd3_decode_byte (stream, & stream->dec_hdr_ind))) + { + return ret; + } + + if ((stream->dec_hdr_ind & VCD_INVHDR) != 0) + { + stream->msg = "unrecognized header indicator bits set"; + return XD3_INVALID_INPUT; + } + + stream->dec_state = DEC_SECONDID; + } + + case DEC_SECONDID: + /* Secondary compressor ID: only if VCD_SECONDARY is set */ + if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) + { + BYTE_CASE (1, stream->dec_secondid, DEC_TABLEN); + + switch (stream->dec_secondid) + { + case VCD_FGK_ID: + FGK_CASE (stream); + case VCD_DJW_ID: + DJW_CASE (stream); + case VCD_LZMA_ID: + LZMA_CASE (stream); + default: + stream->msg = "unknown secondary compressor ID"; + return XD3_INVALID_INPUT; + } + } + + case DEC_TABLEN: + /* Length of code table data: only if VCD_CODETABLE is set */ + SIZE_CASE ((stream->dec_hdr_ind & VCD_CODETABLE) != 0, + stream->dec_codetblsz, DEC_NEAR); + + /* The codetblsz counts the two NEAR/SAME bytes */ + if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) { + if (stream->dec_codetblsz <= 2) { + stream->msg = "invalid code table size"; + return ENOMEM; + } + stream->dec_codetblsz -= 2; + } + case DEC_NEAR: + /* Near modes: only if VCD_CODETABLE is set */ + BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, + stream->acache.s_near, DEC_SAME); + case DEC_SAME: + /* Same modes: only if VCD_CODETABLE is set */ + BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, + stream->acache.s_same, DEC_TABDAT); + case DEC_TABDAT: + /* Compressed code table data */ + + if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) + { + stream->msg = "VCD_CODETABLE support was removed"; + return XD3_UNIMPLEMENTED; + } + else + { + /* Use the default table. */ + stream->acache.s_near = __rfc3284_code_table_desc.near_modes; + stream->acache.s_same = __rfc3284_code_table_desc.same_modes; + stream->code_table = xd3_rfc3284_code_table (); + } + + if ((ret = xd3_alloc_cache (stream))) { return ret; } + + stream->dec_state = DEC_APPLEN; + + case DEC_APPLEN: + /* Length of application data */ + SIZE_CASE((stream->dec_hdr_ind & VCD_APPHEADER) != 0, + stream->dec_appheadsz, DEC_APPDAT); + + case DEC_APPDAT: + /* Application data */ + if (stream->dec_hdr_ind & VCD_APPHEADER) + { + /* Note: we add an additional byte for padding, to allow + 0-termination. Check for overflow: */ + if (USIZE_T_OVERFLOW(stream->dec_appheadsz, 1)) + { + stream->msg = "exceptional appheader size"; + return XD3_INVALID_INPUT; + } + + if ((stream->dec_appheader == NULL) && + (stream->dec_appheader = + (uint8_t*) xd3_alloc (stream, + stream->dec_appheadsz+1, 1)) == NULL) + { + return ENOMEM; + } + + stream->dec_appheader[stream->dec_appheadsz] = 0; + + if ((ret = xd3_decode_bytes (stream, stream->dec_appheader, + & stream->dec_appheadbytes, + stream->dec_appheadsz))) + { + return ret; + } + } + + /* xoff_t -> usize_t is safe because this is the first block. */ + stream->dec_hdrsize = (usize_t) stream->total_in; + stream->dec_state = DEC_WININD; + + case DEC_WININD: + { + /* Start of a window: the window indicator */ + if ((ret = xd3_decode_byte (stream, & stream->dec_win_ind))) + { + return ret; + } + + stream->current_window = stream->dec_window_count; + + if (XOFF_T_OVERFLOW (stream->dec_winstart, stream->dec_tgtlen)) + { + stream->msg = "decoder file offset overflow"; + return XD3_INVALID_INPUT; + } + + stream->dec_winstart += stream->dec_tgtlen; + + if ((stream->dec_win_ind & VCD_INVWIN) != 0) + { + stream->msg = "unrecognized window indicator bits set"; + return XD3_INVALID_INPUT; + } + + if ((ret = xd3_decode_init_window (stream))) { return ret; } + + stream->dec_state = DEC_CPYLEN; + + IF_DEBUG2 (DP(RINT "--------- TARGET WINDOW %"Q"u -----------\n", + stream->current_window)); + } + + case DEC_CPYLEN: + /* Copy window length: only if VCD_SOURCE or VCD_TARGET is set */ + SIZE_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpylen, + DEC_CPYOFF); + + /* Set the initial, logical decoder position (HERE address) in + * dec_position. This is set to just after the source/copy + * window, as we are just about to output the first byte of + * target window. */ + stream->dec_position = stream->dec_cpylen; + + case DEC_CPYOFF: + /* Copy window offset: only if VCD_SOURCE or VCD_TARGET is set */ + + OFFSET_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpyoff, + DEC_ENCLEN); + + /* Copy offset and copy length may not overflow. */ + if (XOFF_T_OVERFLOW (stream->dec_cpyoff, stream->dec_cpylen)) + { + stream->msg = "decoder copy window overflows a file offset"; + return XD3_INVALID_INPUT; + } + + /* Check copy window bounds: VCD_TARGET window may not exceed + current position. */ + if ((stream->dec_win_ind & VCD_TARGET) && + (stream->dec_cpyoff + (xoff_t) stream->dec_cpylen > + stream->dec_winstart)) + { + stream->msg = "VCD_TARGET window out of bounds"; + return XD3_INVALID_INPUT; + } + + case DEC_ENCLEN: + /* Length of the delta encoding */ + SIZE_CASE(1, stream->dec_enclen, DEC_TGTLEN); + case DEC_TGTLEN: + /* Length of target window */ + SIZE_CASE(1, stream->dec_tgtlen, DEC_DELIND); + + /* Set the maximum decoder position, beyond which we should not + * decode any data. This is the maximum value for dec_position. + * This may not exceed the size of a usize_t. */ + if (USIZE_T_OVERFLOW (stream->dec_cpylen, stream->dec_tgtlen)) + { + stream->msg = "decoder target window overflows a usize_t"; + return XD3_INVALID_INPUT; + } + + /* Check for malicious files. */ + if (stream->dec_tgtlen > XD3_HARDMAXWINSIZE) + { + stream->msg = "hard window size exceeded"; + return XD3_INVALID_INPUT; + } + + stream->dec_maxpos = stream->dec_cpylen + stream->dec_tgtlen; + + case DEC_DELIND: + /* Delta indicator */ + BYTE_CASE(1, stream->dec_del_ind, DEC_DATALEN); + + if ((stream->dec_del_ind & VCD_INVDEL) != 0) + { + stream->msg = "unrecognized delta indicator bits set"; + return XD3_INVALID_INPUT; + } + + /* Delta indicator is only used with secondary compression. */ + if ((stream->dec_del_ind != 0) && (stream->sec_type == NULL)) + { + stream->msg = "invalid delta indicator bits set"; + return XD3_INVALID_INPUT; + } + + /* Section lengths */ + case DEC_DATALEN: + SIZE_CASE(1, stream->data_sect.size, DEC_INSTLEN); + case DEC_INSTLEN: + SIZE_CASE(1, stream->inst_sect.size, DEC_ADDRLEN); + case DEC_ADDRLEN: + SIZE_CASE(1, stream->addr_sect.size, DEC_CKSUM); + + case DEC_CKSUM: + /* Window checksum. */ + if ((stream->dec_win_ind & VCD_ADLER32) != 0) + { + int i; + + if ((ret = xd3_decode_bytes (stream, stream->dec_cksum, + & stream->dec_cksumbytes, 4))) + { + return ret; + } + + for (i = 0; i < 4; i += 1) + { + stream->dec_adler32 = + (stream->dec_adler32 << 8) | stream->dec_cksum[i]; + } + } + + stream->dec_state = DEC_DATA; + + /* Check dec_enclen for redundency, otherwise it is not really used. */ + { + usize_t enclen_check = + (1 + (xd3_sizeof_size (stream->dec_tgtlen) + + xd3_sizeof_size (stream->data_sect.size) + + xd3_sizeof_size (stream->inst_sect.size) + + xd3_sizeof_size (stream->addr_sect.size)) + + stream->data_sect.size + + stream->inst_sect.size + + stream->addr_sect.size + + ((stream->dec_win_ind & VCD_ADLER32) ? 4 : 0)); + + if (stream->dec_enclen != enclen_check) + { + stream->msg = "incorrect encoding length (redundent)"; + return XD3_INVALID_INPUT; + } + } + + /* Returning here gives the application a chance to inspect the + * header, skip the window, etc. */ + if (stream->current_window == 0) { return XD3_GOTHEADER; } + else { return XD3_WINSTART; } + + case DEC_DATA: + case DEC_INST: + case DEC_ADDR: + /* Next read the three sections. */ + if ((ret = xd3_decode_sections (stream))) { return ret; } + + case DEC_EMIT: + + /* To speed VCD_SOURCE block-address calculations, the source + * cpyoff_blocks and cpyoff_blkoff are pre-computed. */ + if (stream->dec_win_ind & VCD_SOURCE) + { + xd3_source *src = stream->src; + + if (src == NULL) + { + stream->msg = "source input required"; + return XD3_INVALID_INPUT; + } + + xd3_blksize_div(stream->dec_cpyoff, src, + &src->cpyoff_blocks, + &src->cpyoff_blkoff); + + IF_DEBUG2(DP(RINT + "[decode_cpyoff] %"Q"u " + "cpyblkno %"Q"u " + "cpyblkoff %u " + "blksize %u\n", + stream->dec_cpyoff, + src->cpyoff_blocks, + src->cpyoff_blkoff, + src->blksize)); + } + + /* xd3_decode_emit returns XD3_OUTPUT on every success. */ + if ((ret = xd3_decode_emit (stream)) == XD3_OUTPUT) + { + stream->total_out += (xoff_t) stream->avail_out; + } + + return ret; + + case DEC_FINISH: + { + if (stream->dec_win_ind & VCD_TARGET) + { + if (stream->dec_lastwin == NULL) + { + stream->dec_lastwin = stream->next_out; + stream->dec_lastspace = stream->space_out; + } + else + { + xd3_swap_uint8p (& stream->dec_lastwin, + & stream->next_out); + xd3_swap_usize_t (& stream->dec_lastspace, + & stream->space_out); + } + } + + stream->dec_lastlen = stream->dec_tgtlen; + stream->dec_laststart = stream->dec_winstart; + stream->dec_window_count += 1; + + /* Note: the updates to dec_winstart & current_window are + * deferred until after the next DEC_WININD byte is read. */ + stream->dec_state = DEC_WININD; + return XD3_WINFINISH; + } + + default: + stream->msg = "invalid state"; + return XD3_INVALID_INPUT; + } +} + +#endif // _XDELTA3_DECODE_H_ diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3-hash.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-hash.h new file mode 100644 index 0000000000..a20017a315 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-hash.h @@ -0,0 +1,167 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#ifndef _XDELTA3_HASH_H_ +#define _XDELTA3_HASH_H_ + +#if XD3_DEBUG +#define SMALL_HASH_DEBUG1(s,inp) \ + uint32_t debug_state; \ + uint32_t debug_hval = xd3_checksum_hash (& (s)->small_hash, \ + xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look)) +#define SMALL_HASH_DEBUG2(s,inp) \ + XD3_ASSERT (debug_hval == xd3_checksum_hash (& (s)->small_hash, \ + xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look))) +#else +#define SMALL_HASH_DEBUG1(s,inp) +#define SMALL_HASH_DEBUG2(s,inp) +#endif /* XD3_DEBUG */ + +/* This is a good hash multiplier for 32-bit LCGs: see "linear + * congruential generators of different sizes and good lattice + * structure" */ +static const uint32_t hash_multiplier = 1597334677U; + +/*********************************************************************** + Permute stuff + ***********************************************************************/ + +#if HASH_PERMUTE == 0 +#define PERMUTE(x) (x) +#else +#define PERMUTE(x) (__single_hash[(uint32_t)x]) + +extern const uint16_t __single_hash[256]; +#endif + +/* Update the checksum state. */ +#if ADLER_LARGE_CKSUM +inline uint32_t +xd3_large_cksum_update (uint32_t cksum, + const uint8_t *base, + usize_t look) { + uint32_t old_c = PERMUTE(base[0]); + uint32_t new_c = PERMUTE(base[look]); + uint32_t low = ((cksum & 0xffff) - old_c + new_c) & 0xffff; + uint32_t high = ((cksum >> 16) - (old_c * look) + low) & 0xffff; + return (high << 16) | low; +} +#else +/* TODO: revisit this topic */ +#endif + +#if UNALIGNED_OK +#define UNALIGNED_READ32(dest,src) (*(dest)) = (*(uint32_t*)(src)) +#else +#define UNALIGNED_READ32(dest,src) memcpy((dest), (src), 4); +#endif + +/* TODO: small cksum is hard-coded for 4 bytes (i.e., "look" is unused) */ +static inline uint32_t +xd3_scksum (uint32_t *state, + const uint8_t *base, + const usize_t look) +{ + UNALIGNED_READ32(state, base); + return (*state) * hash_multiplier; +} +static inline uint32_t +xd3_small_cksum_update (uint32_t *state, + const uint8_t *base, + usize_t look) +{ + UNALIGNED_READ32(state, base+1); + return (*state) * hash_multiplier; +} + +/*********************************************************************** + Ctable stuff + ***********************************************************************/ + +static inline usize_t +xd3_checksum_hash (const xd3_hash_cfg *cfg, const usize_t cksum) +{ + return (cksum >> cfg->shift) ^ (cksum & cfg->mask); +} + +/*********************************************************************** + Cksum function + ***********************************************************************/ + +#if ADLER_LARGE_CKSUM +static inline uint32_t +xd3_lcksum (const uint8_t *seg, const usize_t ln) +{ + usize_t i = 0; + uint32_t low = 0; + uint32_t high = 0; + + for (; i < ln; i += 1) + { + low += PERMUTE(*seg++); + high += low; + } + + return ((high & 0xffff) << 16) | (low & 0xffff); +} +#else +static inline uint32_t +xd3_lcksum (const uint8_t *seg, const usize_t ln) +{ + usize_t i, j; + uint32_t h = 0; + for (i = 0, j = ln - 1; i < ln; ++i, --j) { + h += PERMUTE(seg[i]) * hash_multiplier_powers[j]; + } + return h; +} +#endif + +#if XD3_ENCODER +static usize_t +xd3_size_log2 (usize_t slots) +{ + int bits = 28; /* This should not be an unreasonable limit. */ + int i; + + for (i = 3; i <= bits; i += 1) + { + if (slots < (1U << i)) + { + /* TODO: this is compaction=1 in checksum_test.cc and maybe should + * not be fixed at -1. */ + bits = i - 1; + break; + } + } + + return bits; +} + +static void +xd3_size_hashtable (xd3_stream *stream, + usize_t slots, + xd3_hash_cfg *cfg) +{ + int bits = xd3_size_log2 (slots); + + /* TODO: there's a 32-bit assumption here */ + cfg->size = (1 << bits); + cfg->mask = (cfg->size - 1); + cfg->shift = 32 - bits; +} +#endif + +#endif diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3-internal.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-internal.h new file mode 100644 index 0000000000..661ce41559 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-internal.h @@ -0,0 +1,370 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#ifndef XDELTA3_INTERNAL_H__ +#define XDELTA3_INTERNAL_H__ + +#include "xdelta3.h" + +typedef struct _main_file main_file; +typedef struct _main_extcomp main_extcomp; + +void main_buffree (void *ptr); +void* main_bufalloc (size_t size); +void main_file_init (main_file *xfile); +int main_file_close (main_file *xfile); +void main_file_cleanup (main_file *xfile); +int main_file_isopen (main_file *xfile); +int main_file_open (main_file *xfile, const char* name, int mode); +int main_file_exists (main_file *xfile); +int xd3_whole_append_window (xd3_stream *stream); +int xd3_main_cmdline (int argc, char **argv); +int main_file_read (main_file *ifile, + uint8_t *buf, + size_t size, + size_t *nread, + const char *msg); +int main_file_write (main_file *ofile, uint8_t *buf, + usize_t size, const char *msg); +int test_compare_files (const char* f0, const char* f1); +usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno); +xoff_t xd3_source_eof(const xd3_source *src); +uint32_t xd3_large_cksum_update (uint32_t cksum, + const uint8_t *base, + usize_t look); +int xd3_emit_byte (xd3_stream *stream, + xd3_output **outputp, + uint8_t code); + +int xd3_emit_bytes (xd3_stream *stream, + xd3_output **outputp, + const uint8_t *base, + usize_t size); +xd3_output* xd3_alloc_output (xd3_stream *stream, + xd3_output *old_output); + +int xd3_encode_init_full (xd3_stream *stream); +size_t xd3_pow2_roundup (size_t x); +int xd3_process_stream (int is_encode, + xd3_stream *stream, + int (*func) (xd3_stream *), + int close_stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max); + +#if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN +int xd3_main_cmdline (int argc, char **argv); +#endif + +/* main_file->mode values */ +typedef enum +{ + XO_READ = 0, + XO_WRITE = 1 +} main_file_modes; + +#ifndef XD3_POSIX +#define XD3_POSIX 0 +#endif +#ifndef XD3_STDIO +#define XD3_STDIO 0 +#endif +#ifndef XD3_WIN32 +#define XD3_WIN32 0 +#endif +#ifndef NOT_MAIN +#define NOT_MAIN 0 +#endif + +/* If none are set, default to posix. */ +#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0 +#undef XD3_POSIX +#define XD3_POSIX 1 +#endif + +struct _main_file +{ +#if XD3_WIN32 + HANDLE file; +#elif XD3_STDIO + FILE *file; +#elif XD3_POSIX + int file; +#endif + + int mode; /* XO_READ and XO_WRITE */ + const char *filename; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + char *filename_copy; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + const char *realname; /* File name or /dev/stdin, + * /dev/stdout, /dev/stderr. */ + const main_extcomp *compressor; /* External compression struct. */ + int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ + xoff_t nread; /* for input position */ + xoff_t nwrite; /* for output position */ + uint8_t *snprintf_buf; /* internal snprintf() use */ + int size_known; /* Set by main_set_souze */ + xoff_t source_position; /* for avoiding seek in getblk_func */ + int seek_failed; /* after seek fails once, try FIFO */ +}; + +#ifdef _WIN32 +#define vsnprintf_func _vsnprintf +#define snprintf_func _snprintf +#else +#define vsnprintf_func vsnprintf +#define snprintf_func snprintf +#endif +#define short_sprintf(sb,fmt,...) \ + snprintf_func((sb).buf,sizeof((sb).buf),fmt,__VA_ARGS__) + +/* Type used for short snprintf calls. */ +typedef struct { + char buf[48]; +} shortbuf; + +/* Prior to SVN 303 this function was only defined in DJGPP and WIN32 + * environments and other platforms would use the builtin snprintf() + * with an arrangement of macros below. In OS X 10.6, Apply made + * snprintf() a macro, which defeated those macros (since snprintf + * would be evaluated before its argument macros were expanded, + * therefore always define xsnprintf_func. */ +#undef PRINTF_ATTRIBUTE +#ifdef __GNUC__ +/* Let's just assume no one uses gcc 2.x! */ +#define PRINTF_ATTRIBUTE(x,y) __attribute__ ((__format__ (__printf__, x, y))) +#else +#define PRINTF_ATTRIBUTE(x,y) +#endif + +/* Underlying xprintf() */ +int xsnprintf_func (char *str, int n, const char *fmt, ...) + PRINTF_ATTRIBUTE(3,4); + +/* XPR(NT "", ...) (used by main) prefixes an "xdelta3: " to the output. */ +void xprintf(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2); +#define XPR xprintf +#define NT "xdelta3: " +#define NTR "" + +#ifndef UINT32_MAX +#define UINT32_MAX 4294967295U +#endif + +#ifndef UINT64_MAX +#define UINT64_MAX 18446744073709551615ULL +#endif + +#define UINT32_OFLOW_MASK 0xfe000000U +#define UINT64_OFLOW_MASK 0xfe00000000000000ULL + +/********************************************************************* + Integer encoder/decoder functions + **********************************************************************/ + +/* Consume N bytes of input, only used by the decoder. */ +#define DECODE_INPUT(n) \ + do { \ + stream->total_in += (xoff_t) (n); \ + stream->avail_in -= (n); \ + stream->next_in += (n); \ + } while (0) + +#define DECODE_INTEGER_TYPE(PART,OFLOW) \ + while (stream->avail_in != 0) \ + { \ + usize_t next = stream->next_in[0]; \ + \ + DECODE_INPUT(1); \ + \ + if (PART & OFLOW) \ + { \ + stream->msg = "overflow in decode_integer"; \ + return XD3_INVALID_INPUT; \ + } \ + \ + PART = (PART << 7) | (next & 127); \ + \ + if ((next & 128) == 0) \ + { \ + (*val) = PART; \ + PART = 0; \ + return 0; \ + } \ + } \ + \ + stream->msg = "further input required"; \ + return XD3_INPUT + +#define READ_INTEGER_TYPE(TYPE, OFLOW) \ + TYPE val = 0; \ + const uint8_t *inp = (*inpp); \ + usize_t next; \ + \ + do \ + { \ + if (inp == maxp) \ + { \ + stream->msg = "end-of-input in read_integer"; \ + return XD3_INVALID_INPUT; \ + } \ + \ + if (val & OFLOW) \ + { \ + stream->msg = "overflow in read_intger"; \ + return XD3_INVALID_INPUT; \ + } \ + \ + next = (*inp++); \ + val = (val << 7) | (next & 127); \ + } \ + while (next & 128); \ + \ + (*valp) = val; \ + (*inpp) = inp; \ + \ + return 0 + +#define EMIT_INTEGER_TYPE() \ + /* max 64-bit value in base-7 encoding is 9.1 bytes */ \ + uint8_t buf[10]; \ + usize_t bufi = 10; \ + \ + /* This loop performs division and turns on all MSBs. */ \ + do \ + { \ + buf[--bufi] = (num & 127) | 128; \ + num >>= 7U; \ + } \ + while (num != 0); \ + \ + /* Turn off MSB of the last byte. */ \ + buf[9] &= 127; \ + \ + return xd3_emit_bytes (stream, output, buf + bufi, 10 - bufi) + +#define IF_SIZEOF32(x) if (num < (1U << (7 * (x)))) return (x); +#define IF_SIZEOF64(x) if (num < (1ULL << (7 * (x)))) return (x); + +#if USE_UINT32 +static inline uint32_t +xd3_sizeof_uint32_t (uint32_t num) +{ + IF_SIZEOF32(1); + IF_SIZEOF32(2); + IF_SIZEOF32(3); + IF_SIZEOF32(4); + return 5; +} + +static inline int +xd3_decode_uint32_t (xd3_stream *stream, uint32_t *val) +{ DECODE_INTEGER_TYPE (stream->dec_32part, UINT32_OFLOW_MASK); } + +static inline int +xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, + const uint8_t *maxp, uint32_t *valp) +{ READ_INTEGER_TYPE (uint32_t, UINT32_OFLOW_MASK); } + +#if XD3_ENCODER +static inline int +xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, uint32_t num) +{ EMIT_INTEGER_TYPE (); } +#endif +#endif + +#if USE_UINT64 +static inline int +xd3_decode_uint64_t (xd3_stream *stream, uint64_t *val) +{ DECODE_INTEGER_TYPE (stream->dec_64part, UINT64_OFLOW_MASK); } + +#if XD3_ENCODER +static inline int +xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num) +{ EMIT_INTEGER_TYPE (); } +#endif + +/* These are tested but not used */ +#if REGRESSION_TEST +static int +xd3_read_uint64_t (xd3_stream *stream, const uint8_t **inpp, + const uint8_t *maxp, uint64_t *valp) +{ READ_INTEGER_TYPE (uint64_t, UINT64_OFLOW_MASK); } + +static uint32_t +xd3_sizeof_uint64_t (uint64_t num) +{ + IF_SIZEOF64(1); + IF_SIZEOF64(2); + IF_SIZEOF64(3); + IF_SIZEOF64(4); + IF_SIZEOF64(5); + IF_SIZEOF64(6); + IF_SIZEOF64(7); + IF_SIZEOF64(8); + IF_SIZEOF64(9); + + return 10; +} +#endif + +#endif + +#if SIZEOF_USIZE_T == 4 +#define USIZE_T_MAX UINT32_MAX +#define USIZE_T_MAXBLKSZ 0x80000000U +#define xd3_decode_size xd3_decode_uint32_t +#define xd3_emit_size xd3_emit_uint32_t +#define xd3_sizeof_size xd3_sizeof_uint32_t +#define xd3_read_size xd3_read_uint32_t +#elif SIZEOF_USIZE_T == 8 +#define USIZE_T_MAX UINT64_MAX +#define USIZE_T_MAXBLKSZ 0x8000000000000000ULL +#define xd3_decode_size xd3_decode_uint64_t +#define xd3_emit_size xd3_emit_uint64_t +#define xd3_sizeof_size xd3_sizeof_uint64_t +#define xd3_read_size xd3_read_uint64_t +#endif + +#if SIZEOF_XOFF_T == 4 +#define XOFF_T_MAX UINT32_MAX +#define xd3_emit_offset xd3_emit_uint32_t +static inline int +xd3_decode_offset (xd3_stream *stream, xoff_t *val) +{ + return xd3_decode_uint32_t (stream, (uint32_t*) val); +} +#elif SIZEOF_XOFF_T == 8 +#define XOFF_T_MAX UINT64_MAX +#define xd3_emit_offset xd3_emit_uint64_t +static inline int +xd3_decode_offset (xd3_stream *stream, xoff_t *val) +{ + return xd3_decode_uint64_t (stream, (uint64_t*) val); +} +#endif + +#define USIZE_T_OVERFLOW(a,b) ((USIZE_T_MAX - (usize_t) (a)) < (usize_t) (b)) +#define XOFF_T_OVERFLOW(a,b) ((XOFF_T_MAX - (xoff_t) (a)) < (xoff_t) (b)) + +#define MAX_LRU_SIZE 32U +#define XD3_MINSRCWINSZ (XD3_ALLOCSIZE * MAX_LRU_SIZE) +#define XD3_MAXSRCWINSZ (1ULL << 31) + +#endif // XDELTA3_INTERNAL_H__ diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3-list.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-list.h new file mode 100644 index 0000000000..23c925eaaf --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3-list.h @@ -0,0 +1,127 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#ifndef __XDELTA3_LIST__ +#define __XDELTA3_LIST__ + +#define XD3_MAKELIST(LTYPE,ETYPE,LNAME) \ + \ +static inline ETYPE* \ +LTYPE ## _entry (LTYPE* l) \ +{ \ + return (ETYPE*) ((char*) l - (ptrdiff_t) &((ETYPE*) 0)->LNAME); \ +} \ + \ +static inline void \ +LTYPE ## _init (LTYPE *l) \ +{ \ + l->next = l; \ + l->prev = l; \ +} \ + \ +static inline void \ +LTYPE ## _add (LTYPE *prev, LTYPE *next, LTYPE *ins) \ +{ \ + next->prev = ins; \ + prev->next = ins; \ + ins->next = next; \ + ins->prev = prev; \ +} \ + \ +static inline void \ +LTYPE ## _push_back (LTYPE *l, ETYPE *i) \ +{ \ + LTYPE ## _add (l->prev, l, & i->LNAME); \ +} \ + \ +static inline void \ +LTYPE ## _del (LTYPE *next, \ + LTYPE *prev) \ +{ \ + next->prev = prev; \ + prev->next = next; \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _remove (ETYPE *f) \ +{ \ + LTYPE *i = f->LNAME.next; \ + LTYPE ## _del (f->LNAME.next, f->LNAME.prev); \ + return LTYPE ## _entry (i); \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _pop_back (LTYPE *l) \ +{ \ + LTYPE *i = l->prev; \ + LTYPE ## _del (i->next, i->prev); \ + return LTYPE ## _entry (i); \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _pop_front (LTYPE *l) \ +{ \ + LTYPE *i = l->next; \ + LTYPE ## _del (i->next, i->prev); \ + return LTYPE ## _entry (i); \ +} \ + \ +static inline int \ +LTYPE ## _empty (LTYPE *l) \ +{ \ + return l == l->next; \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _front (LTYPE *f) \ +{ \ + return LTYPE ## _entry (f->next); \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _back (LTYPE *f) \ +{ \ + return LTYPE ## _entry (f->prev); \ +} \ + \ +static inline int \ +LTYPE ## _end (LTYPE *f, ETYPE *i) \ +{ \ + return f == & i->LNAME; \ +} \ + \ +static inline ETYPE* \ +LTYPE ## _next (ETYPE *f) \ +{ \ + return LTYPE ## _entry (f->LNAME.next); \ +} \ + \ +static inline usize_t \ +LTYPE ## _length (LTYPE *l) \ +{ \ + LTYPE *p; \ + int c = 0; \ + \ + for (p = l->next; p != l; p = p->next) \ + { \ + c += 1; \ + } \ + \ + return c; \ +} \ + \ +typedef int unused_ ## LTYPE + +#endif diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3.c b/app/src/main/rust/stores/vendor/xdelta3/xdelta3.c new file mode 100644 index 0000000000..ce4dadae7a --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3.c @@ -0,0 +1,4835 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ------------------------------------------------------------------- + + Xdelta 3 + + The goal of this library is to to implement both the (stand-alone) + data-compression and delta-compression aspects of VCDIFF encoding, and + to support a programming interface that works like Zlib + (http://www.gzip.org/zlib.html). See RFC3284: The VCDIFF Generic + Differencing and Compression Data Format. + + VCDIFF is a unified encoding that combines data-compression and + delta-encoding ("differencing"). + + VCDIFF has a detailed byte-code instruction set with many features. + The instruction format supports an immediate size operand for small + COPYs and ADDs (e.g., under 18 bytes). There are also instruction + "modes", which are used to compress COPY addresses by using two + address caches. An instruction mode refers to slots in the NEAR + and SAME caches for recent addresses. NEAR remembers the + previous 4 (by default) COPY addresses, and SAME catches + frequent re-uses of the same address using a 3-way (by default) + 256-entry associative cache of [ADDR mod 256], the encoded byte. + A hit in the NEAR/SAME cache requires 0/1 ADDR bytes. + + VCDIFF has a default instruction table, but an alternate + instruction tables may themselves be be delta-compressed and + included in the encoding header. This allows even more freedom. + There are 9 instruction modes in the default code table, 4 near, 3 + same, VCD_SELF (absolute encoding) and VCD_HERE (relative to the + current position). + + ---------------------------------------------------------------------- + + Algorithms + + Aside from the details of encoding and decoding, there are a bunch + of algorithms needed. + + 1. STRING-MATCH. A two-level fingerprinting approach is used. A + single loop computes the two checksums -- small and large -- at + successive offsets in the TARGET file. The large checksum is more + accurate and is used to discover SOURCE matches, which are + potentially very long. The small checksum is used to discover + copies within the TARGET. Small matching, which is more expensive, + usually dominates the large STRING-MATCH costs in this code - the + more exhaustive the search, the better the results. Either of the + two string-matching mechanisms may be disabled. + + 2. INSTRUCTION SELECTION. The IOPT buffer here represents a queue + used to store overlapping copy instructions. There are two possible + optimizations that go beyond a greedy search. Both of these fall + into the category of "non-greedy matching" optimizations. + + The first optimization stems from backward SOURCE-COPY matching. + When a new SOURCE-COPY instruction covers a previous instruction in + the target completely, it is erased from the queue. Randal Burns + originally analyzed these algorithms and did a lot of related work + (\cite the 1.5-pass algorithm). + + The second optimization comes by the encoding of common very-small + COPY and ADD instructions, for which there are special DOUBLE-code + instructions, which code two instructions in a single byte. + + The cost of bad instruction-selection overhead is relatively high + for data-compression, relative to delta-compression, so this second + optimization is fairly important. With "lazy" matching (the name + used in Zlib for a similar optimization), the string-match + algorithm searches after a match for potential overlapping copy + instructions. In Xdelta and by default, VCDIFF, the minimum match + size is 4 bytes, whereas Zlib searches with a 3-byte minimum. This + feature, combined with double instructions, provides a nice + challenge. Search in this file for "black magic", a heuristic. + + 3. STREAM ALIGNMENT. Stream alignment is needed to compress large + inputs in constant space. See xd3_srcwin_move_point(). + + 4. WINDOW SELECTION. When the IOPT buffer flushes, in the first call + to xd3_iopt_finish_encoding containing any kind of copy instruction, + the parameters of the source window must be decided: the offset into + the source and the length of the window. Since the IOPT buffer is + finite, the program may be forced to fix these values before knowing + the best offset/length. + + 5. SECONDARY COMPRESSION. VCDIFF supports a secondary encoding to + be applied to the individual sections of the data format, which are + ADDRess, INSTruction, and DATA. Several secondary compressor + variations are implemented here, although none is standardized yet. + + One is an adaptive huffman algorithm -- the FGK algorithm (Faller, + Gallager, and Knuth, 1985). This compressor is extremely slow. + + The other is a simple static Huffman routine, which is the base + case of a semi-adaptive scheme published by D.J. Wheeler and first + widely used in bzip2 (by Julian Seward). This is a very + interesting algorithm, originally published in nearly cryptic form + by D.J. Wheeler. !!!NOTE!!! Because these are not standardized, + secondary compression remains off by default. + ftp://ftp.cl.cam.ac.uk/users/djw3/bred3.{c,ps} + -------------------------------------------------------------------- + + Other Features + + 1. USER CONVENIENCE + + For user convenience, it is essential to recognize Gzip-compressed + files and automatically Gzip-decompress them prior to + delta-compression (or else no delta-compression will be achieved + unless the user manually decompresses the inputs). The compressed + represention competes with Xdelta, and this must be hidden from the + command-line user interface. The Xdelta-1.x encoding was simple, not + compressed itself, so Xdelta-1.x uses Zlib internally to compress the + representation. + + This implementation supports external compression, which implements + the necessary fork() and pipe() mechanics. There is a tricky step + involved to support automatic detection of a compressed input in a + non-seekable input. First you read a bit of the input to detect + magic headers. When a compressed format is recognized, exec() the + external compression program and create a second child process to + copy the original input stream. [Footnote: There is a difficulty + related to using Gzip externally. It is not possible to decompress + and recompress a Gzip file transparently. If FILE.GZ had a + cryptographic signature, then, after: (1) Gzip-decompression, (2) + Xdelta-encoding, (3) Gzip-compression the signature could be + broken. The only way to solve this problem is to guess at Gzip's + compression level or control it by other means. I recommend that + specific implementations of any compression scheme store + information needed to exactly re-compress the input, that way + external compression is transparent - however, this won't happen + here until it has stabilized.] + + 2. APPLICATION-HEADER + + This feature was introduced in RFC3284. It allows any application + to include a header within the VCDIFF file format. This allows + general inter-application data exchange with support for + application-specific extensions to communicate metadata. + + 3. VCDIFF CHECKSUM + + An optional checksum value is included with each window, which can + be used to validate the final result. This verifies the correct source + file was used for decompression as well as the obvious advantage: + checking the implementation (and underlying) correctness. + + 4. LIGHT WEIGHT + + The code makes efforts to avoid copying data more than necessary. + The code delays many initialization tasks until the first use, it + optimizes for identical (perfectly matching) inputs. It does not + compute any checksums until the first lookup misses. Memory usage + is reduced. String-matching is templatized (by slightly gross use + of CPP) to hard-code alternative compile-time defaults. The code + has few outside dependencies. + ---------------------------------------------------------------------- + + The default rfc3284 instruction table: + (see RFC for the explanation) + + TYPE SIZE MODE TYPE SIZE MODE INDEX + -------------------------------------------------------------------- + 1. Run 0 0 Noop 0 0 0 + 2. Add 0, [1,17] 0 Noop 0 0 [1,18] + 3. Copy 0, [4,18] 0 Noop 0 0 [19,34] + 4. Copy 0, [4,18] 1 Noop 0 0 [35,50] + 5. Copy 0, [4,18] 2 Noop 0 0 [51,66] + 6. Copy 0, [4,18] 3 Noop 0 0 [67,82] + 7. Copy 0, [4,18] 4 Noop 0 0 [83,98] + 8. Copy 0, [4,18] 5 Noop 0 0 [99,114] + 9. Copy 0, [4,18] 6 Noop 0 0 [115,130] + 10. Copy 0, [4,18] 7 Noop 0 0 [131,146] + 11. Copy 0, [4,18] 8 Noop 0 0 [147,162] + 12. Add [1,4] 0 Copy [4,6] 0 [163,174] + 13. Add [1,4] 0 Copy [4,6] 1 [175,186] + 14. Add [1,4] 0 Copy [4,6] 2 [187,198] + 15. Add [1,4] 0 Copy [4,6] 3 [199,210] + 16. Add [1,4] 0 Copy [4,6] 4 [211,222] + 17. Add [1,4] 0 Copy [4,6] 5 [223,234] + 18. Add [1,4] 0 Copy 4 6 [235,238] + 19. Add [1,4] 0 Copy 4 7 [239,242] + 20. Add [1,4] 0 Copy 4 8 [243,246] + 21. Copy 4 [0,8] Add 1 0 [247,255] + -------------------------------------------------------------------- + + Reading the source: Overview + + This file includes itself in several passes to macro-expand certain + sections with variable forms. Just read ahead, there's only a + little confusion. I know this sounds ugly, but hard-coding some of + the string-matching parameters results in a 10-15% increase in + string-match performance. The only time this hurts is when you have + unbalanced #if/endifs. + + A single compilation unit tames the Makefile. In short, this is to + allow the above-described hack without an explodingMakefile. The + single compilation unit includes the core library features, + configurable string-match templates, optional main() command-line + tool, misc optional features, and a regression test. Features are + controled with CPP #defines, see Makefile.am. + + The initial __XDELTA3_C_HEADER_PASS__ starts first, the _INLINE_ and + _TEMPLATE_ sections follow. Easy stuff first, hard stuff last. + + Optional features include: + + xdelta3-main.h The command-line interface, external compression + support, POSIX-specific, info & VCDIFF-debug tools. + xdelta3-second.h The common secondary compression routines. + xdelta3-decoder.h All decoding routines. + xdelta3-djw.h The semi-adaptive huffman secondary encoder. + xdelta3-fgk.h The adaptive huffman secondary encoder. + xdelta3-test.h The unit test covers major algorithms, + encoding and decoding. There are single-bit + error decoding tests. There are 32/64-bit file size + boundary tests. There are command-line tests. + There are compression tests. There are external + compression tests. There are string-matching tests. + There should be more tests... + + Additional headers include: + + xdelta3.h The public header file. + xdelta3-cfgs.h The default settings for default, built-in + encoders. These are hard-coded at + compile-time. There is also a single + soft-coded string matcher for experimenting + with arbitrary values. + xdelta3-list.h A cyclic list template + + Misc little debug utilities: + + badcopy.c Randomly modifies an input file based on two + parameters: (1) the probability that a byte in + the file is replaced with a pseudo-random value, + and (2) the mean change size. Changes are + generated using an expoential distribution + which approximates the expected error_prob + distribution. + -------------------------------------------------------------------- + + This file itself is unusually large. I hope to defend this layout + with lots of comments. Everything in this file is related to + encoding and decoding. I like it all together - the template stuff + is just a hack. */ + +#ifndef __XDELTA3_C_HEADER_PASS__ +#define __XDELTA3_C_HEADER_PASS__ + +#include "xdelta3.h" +#include "xdelta3-internal.h" + +/*********************************************************************** + STATIC CONFIGURATION + ***********************************************************************/ + +#ifndef XD3_MAIN /* the main application */ +#define XD3_MAIN 0 +#endif + +#ifndef VCDIFF_TOOLS +#define VCDIFF_TOOLS XD3_MAIN +#endif + +#ifndef SECONDARY_FGK /* one from the algorithm preservation department: */ +#define SECONDARY_FGK 0 /* adaptive Huffman routines */ +#endif + +#ifndef SECONDARY_DJW /* semi-adaptive/static Huffman for the eventual */ +#define SECONDARY_DJW 0 /* standardization, off by default until such time. */ +#endif + +#ifndef SECONDARY_LZMA +#ifdef HAVE_LZMA_H +#define SECONDARY_LZMA 1 +#else +#define SECONDARY_LZMA 0 +#endif +#endif + +#if XD3_ENCODER +#define IF_ENCODER(x) x +#else +#define IF_ENCODER(x) +#endif + +/***********************************************************************/ + + /* header indicator bits */ +#define VCD_SECONDARY (1U << 0) /* uses secondary compressor */ +#define VCD_CODETABLE (1U << 1) /* supplies code table data */ +#define VCD_APPHEADER (1U << 2) /* supplies application data */ +#define VCD_INVHDR (~0x7U) + + /* window indicator bits */ +#define VCD_SOURCE (1U << 0) /* copy window in source file */ +#define VCD_TARGET (1U << 1) /* copy window in target file */ +#define VCD_ADLER32 (1U << 2) /* has adler32 checksum */ +#define VCD_INVWIN (~0x7U) + +#define VCD_SRCORTGT (VCD_SOURCE | VCD_TARGET) + + /* delta indicator bits */ +#define VCD_DATACOMP (1U << 0) +#define VCD_INSTCOMP (1U << 1) +#define VCD_ADDRCOMP (1U << 2) +#define VCD_INVDEL (~0x7U) + +typedef enum { + VCD_DJW_ID = 1, + VCD_LZMA_ID = 2, + VCD_FGK_ID = 16 /* Note: these are not standard IANA-allocated IDs! */ +} xd3_secondary_ids; + +typedef enum { + SEC_NOFLAGS = 0, + + /* Note: SEC_COUNT_FREQS Not implemented (to eliminate 1st Huffman pass) */ + SEC_COUNT_FREQS = (1 << 0) +} xd3_secondary_flags; + +typedef enum { + DATA_SECTION, /* These indicate which section to the secondary + * compressor. */ + INST_SECTION, /* The header section is not compressed, therefore not + * listed here. */ + ADDR_SECTION +} xd3_section_type; + +typedef unsigned int xd3_rtype; + +/***********************************************************************/ + +#include "xdelta3-list.h" + +XD3_MAKELIST(xd3_rlist, xd3_rinst, link); + +/***********************************************************************/ + +#define SECONDARY_MIN_SAVINGS 2 /* Secondary compression has to save + at least this many bytes. */ +#define SECONDARY_MIN_INPUT 10 /* Secondary compression needs at + least this many bytes. */ + +#define VCDIFF_MAGIC1 0xd6 /* 1st file byte */ +#define VCDIFF_MAGIC2 0xc3 /* 2nd file byte */ +#define VCDIFF_MAGIC3 0xc4 /* 3rd file byte */ +#define VCDIFF_VERSION 0x00 /* 4th file byte */ + +#define VCD_SELF 0 /* 1st address mode */ +#define VCD_HERE 1 /* 2nd address mode */ + +#define CODE_TABLE_STRING_SIZE (6 * 256) /* Should fit a code table string. */ +#define CODE_TABLE_VCDIFF_SIZE (6 * 256) /* Should fit a compressed code + * table string */ + +#define SECONDARY_ANY (SECONDARY_DJW || SECONDARY_FGK || SECONDARY_LZMA) + +#define ALPHABET_SIZE 256 /* Used in test code--size of the secondary + * compressor alphabet. */ + +#define HASH_PERMUTE 1 /* The input is permuted by random nums */ +#define ADLER_LARGE_CKSUM 1 /* Adler checksum vs. RK checksum */ + +#define HASH_CKOFFSET 1U /* Table entries distinguish "no-entry" from + * offset 0 using this offset. */ + +#define MIN_SMALL_LOOK 2U /* Match-optimization stuff. */ +#define MIN_LARGE_LOOK 2U +#define MIN_MATCH_OFFSET 1U +#define MAX_MATCH_SPLIT 18U /* VCDIFF code table: 18 is the default limit + * for direct-coded ADD sizes */ + +#define LEAST_MATCH_INCR 0 /* The least number of bytes an overlapping + * match must beat the preceding match by. This + * is a bias for the lazy match optimization. A + * non-zero value means that an adjacent match + * has to be better by more than the step + * between them. 0. */ + +#define MIN_MATCH 4U /* VCDIFF code table: MIN_MATCH=4 */ +#define MIN_ADD 1U /* 1 */ +#define MIN_RUN 8U /* The shortest run, if it is shorter than this + * an immediate add/copy will be just as good. + * ADD1/COPY6 = 1I+1D+1A bytes, RUN18 = + * 1I+1D+1A. */ + +#define MAX_MODES 9 /* Maximum number of nodes used for + * compression--does not limit decompression. */ + +#define ENC_SECTS 4 /* Number of separate output sections. */ + +#define HDR_TAIL(s) ((s)->enc_tails[0]) +#define DATA_TAIL(s) ((s)->enc_tails[1]) +#define INST_TAIL(s) ((s)->enc_tails[2]) +#define ADDR_TAIL(s) ((s)->enc_tails[3]) + +#define HDR_HEAD(s) ((s)->enc_heads[0]) +#define DATA_HEAD(s) ((s)->enc_heads[1]) +#define INST_HEAD(s) ((s)->enc_heads[2]) +#define ADDR_HEAD(s) ((s)->enc_heads[3]) + +#define TOTAL_MODES(x) (2+(x)->acache.s_same+(x)->acache.s_near) + +/* Template instances. */ +#if XD3_BUILD_SLOW +#define IF_BUILD_SLOW(x) x +#else +#define IF_BUILD_SLOW(x) +#endif +#if XD3_BUILD_FAST +#define IF_BUILD_FAST(x) x +#else +#define IF_BUILD_FAST(x) +#endif +#if XD3_BUILD_FASTER +#define IF_BUILD_FASTER(x) x +#else +#define IF_BUILD_FASTER(x) +#endif +#if XD3_BUILD_FASTEST +#define IF_BUILD_FASTEST(x) x +#else +#define IF_BUILD_FASTEST(x) +#endif +#if XD3_BUILD_SOFT +#define IF_BUILD_SOFT(x) x +#else +#define IF_BUILD_SOFT(x) +#endif +#if XD3_BUILD_DEFAULT +#define IF_BUILD_DEFAULT(x) x +#else +#define IF_BUILD_DEFAULT(x) +#endif + +/* Update the run-length state */ +#define NEXTRUN(c) do { if ((c) == run_c) { run_l += 1; } \ + else { run_c = (c); run_l = 1; } } while (0) + +/* This CPP-conditional stuff can be cleaned up... */ +#if REGRESSION_TEST +#define IF_REGRESSION(x) x +#else +#define IF_REGRESSION(x) +#endif + +/***********************************************************************/ + +#if XD3_ENCODER +static void* xd3_alloc0 (xd3_stream *stream, + usize_t elts, + usize_t size); + + +static int xd3_alloc_iopt (xd3_stream *stream, usize_t elts); + +static void xd3_free_output (xd3_stream *stream, + xd3_output *output); + +static int xd3_emit_double (xd3_stream *stream, xd3_rinst *first, + xd3_rinst *second, usize_t code); +static int xd3_emit_single (xd3_stream *stream, xd3_rinst *single, + usize_t code); + +static usize_t xd3_sizeof_output (xd3_output *output); +static void xd3_encode_reset (xd3_stream *stream); + +static int xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos); +static int xd3_source_extend_match (xd3_stream *stream); +static int xd3_srcwin_setup (xd3_stream *stream); +static usize_t xd3_iopt_last_matched (xd3_stream *stream); +static int xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, + uint32_t num); + +static usize_t xd3_smatch (xd3_stream *stream, + usize_t base, + usize_t scksum, + usize_t *match_offset); +static int xd3_string_match_init (xd3_stream *stream); +static uint32_t xd3_scksum (uint32_t *state, const uint8_t *seg, + const usize_t ln); +static usize_t xd3_comprun (const uint8_t *seg, usize_t slook, uint8_t *run_cp); +static int xd3_srcwin_move_point (xd3_stream *stream, + usize_t *next_move_point); + +static int xd3_emit_run (xd3_stream *stream, usize_t pos, + usize_t size, uint8_t *run_c); +static usize_t xd3_checksum_hash (const xd3_hash_cfg *cfg, + const usize_t cksum); +static xoff_t xd3_source_cksum_offset(xd3_stream *stream, usize_t low); +static void xd3_scksum_insert (xd3_stream *stream, + usize_t inx, + usize_t scksum, + usize_t pos); + + +#if XD3_DEBUG +static void xd3_verify_run_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_run_l, + uint8_t *x_run_c); +static void xd3_verify_large_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum); +static void xd3_verify_small_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum); + +#endif /* XD3_DEBUG */ +#endif /* XD3_ENCODER */ + +static int xd3_decode_allocate (xd3_stream *stream, usize_t size, + uint8_t **copied1, usize_t *alloc1); + +static void* xd3_alloc (xd3_stream *stream, usize_t elts, usize_t size); +static void xd3_free (xd3_stream *stream, void *ptr); + +static int xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, + const uint8_t *max, uint32_t *valp); + +#if REGRESSION_TEST +static int xd3_selftest (void); +#endif + +/***********************************************************************/ + +const char* xd3_strerror (int ret) +{ + switch (ret) + { + case XD3_INPUT: return "XD3_INPUT"; + case XD3_OUTPUT: return "XD3_OUTPUT"; + case XD3_GETSRCBLK: return "XD3_GETSRCBLK"; + case XD3_GOTHEADER: return "XD3_GOTHEADER"; + case XD3_WINSTART: return "XD3_WINSTART"; + case XD3_WINFINISH: return "XD3_WINFINISH"; + case XD3_TOOFARBACK: return "XD3_TOOFARBACK"; + case XD3_INTERNAL: return "XD3_INTERNAL"; + case XD3_INVALID: return "XD3_INVALID"; + case XD3_INVALID_INPUT: return "XD3_INVALID_INPUT"; + case XD3_NOSECOND: return "XD3_NOSECOND"; + case XD3_UNIMPLEMENTED: return "XD3_UNIMPLEMENTED"; + } + return NULL; +} + +/***********************************************************************/ + +#define xd3_sec_data(s) ((s)->sec_stream_d) +#define xd3_sec_inst(s) ((s)->sec_stream_i) +#define xd3_sec_addr(s) ((s)->sec_stream_a) + +struct _xd3_sec_type +{ + int id; + const char *name; + xd3_secondary_flags flags; + + /* xd3_sec_stream is opaque to the generic code */ + xd3_sec_stream* (*alloc) (xd3_stream *stream); + void (*destroy) (xd3_stream *stream, + xd3_sec_stream *sec); + int (*init) (xd3_stream *stream, + xd3_sec_stream *sec_stream, + int is_encode); + int (*decode) (xd3_stream *stream, + xd3_sec_stream *sec_stream, + const uint8_t **input, + const uint8_t *input_end, + uint8_t **output, + const uint8_t *output_end); +#if XD3_ENCODER + int (*encode) (xd3_stream *stream, + xd3_sec_stream *sec_stream, + xd3_output *input, + xd3_output *output, + xd3_sec_cfg *cfg); +#endif +}; + +#define BIT_STATE_ENCODE_INIT { 0, 1 } +#define BIT_STATE_DECODE_INIT { 0, 0x100 } + +typedef struct _bit_state bit_state; +struct _bit_state +{ + usize_t cur_byte; + usize_t cur_mask; +}; + +#if SECONDARY_ANY == 0 +#define IF_SEC(x) +#define IF_NSEC(x) x +#else /* yuck */ +#define IF_SEC(x) x +#define IF_NSEC(x) +static int +xd3_decode_secondary (xd3_stream *stream, + xd3_desect *sect, + xd3_sec_stream **sec_streamp); +#if XD3_ENCODER +static int +xd3_encode_secondary (xd3_stream *stream, + xd3_output **head, + xd3_output **tail, + xd3_sec_stream **sec_streamp, + xd3_sec_cfg *cfg, + int *did_it); +#endif +#endif /* SECONDARY_ANY */ + +#if SECONDARY_FGK +extern const xd3_sec_type fgk_sec_type; +#define IF_FGK(x) x +#define FGK_CASE(s) \ + s->sec_type = & fgk_sec_type; \ + break; +#else +#define IF_FGK(x) +#define FGK_CASE(s) \ + s->msg = "unavailable secondary compressor: FGK Adaptive Huffman"; \ + return XD3_INTERNAL; +#endif + +#if SECONDARY_DJW +extern const xd3_sec_type djw_sec_type; +#define IF_DJW(x) x +#define DJW_CASE(s) \ + s->sec_type = & djw_sec_type; \ + break; +#else +#define IF_DJW(x) +#define DJW_CASE(s) \ + s->msg = "unavailable secondary compressor: DJW Static Huffman"; \ + return XD3_INTERNAL; +#endif + +#if SECONDARY_LZMA +extern const xd3_sec_type lzma_sec_type; +#define IF_LZMA(x) x +#define LZMA_CASE(s) \ + s->sec_type = & lzma_sec_type; \ + break; +#else +#define IF_LZMA(x) +#define LZMA_CASE(s) \ + s->msg = "unavailable secondary compressor: LZMA"; \ + return XD3_INTERNAL; +#endif + +/***********************************************************************/ + +#include "xdelta3-hash.h" + +/* Process template passes - this includes xdelta3.c several times. */ +#define __XDELTA3_C_TEMPLATE_PASS__ +#include "xdelta3-cfgs.h" +#undef __XDELTA3_C_TEMPLATE_PASS__ + +/* Process the inline pass. */ +#define __XDELTA3_C_INLINE_PASS__ +#include "xdelta3.c" +#undef __XDELTA3_C_INLINE_PASS__ + +/* Secondary compression */ +#if SECONDARY_ANY +#include "xdelta3-second.h" +#endif + +#if SECONDARY_FGK +#include "xdelta3-fgk.h" +const xd3_sec_type fgk_sec_type = +{ + VCD_FGK_ID, + "FGK Adaptive Huffman", + SEC_NOFLAGS, + (xd3_sec_stream* (*)(xd3_stream*)) fgk_alloc, + (void (*)(xd3_stream*, xd3_sec_stream*)) fgk_destroy, + (int (*)(xd3_stream*, xd3_sec_stream*, int)) fgk_init, + (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, + uint8_t**, const uint8_t*)) xd3_decode_fgk, + IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, + xd3_output*, xd3_sec_cfg*)) xd3_encode_fgk) +}; +#endif + +#if SECONDARY_DJW +#include "xdelta3-djw.h" +const xd3_sec_type djw_sec_type = +{ + VCD_DJW_ID, + "Static Huffman", + SEC_COUNT_FREQS, + (xd3_sec_stream* (*)(xd3_stream*)) djw_alloc, + (void (*)(xd3_stream*, xd3_sec_stream*)) djw_destroy, + (int (*)(xd3_stream*, xd3_sec_stream*, int)) djw_init, + (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, + uint8_t**, const uint8_t*)) xd3_decode_huff, + IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, + xd3_output*, xd3_sec_cfg*)) xd3_encode_huff) +}; +#endif + +#if SECONDARY_LZMA +#include "xdelta3-lzma.h" +const xd3_sec_type lzma_sec_type = +{ + VCD_LZMA_ID, + "lzma", + SEC_NOFLAGS, + (xd3_sec_stream* (*)(xd3_stream*)) xd3_lzma_alloc, + (void (*)(xd3_stream*, xd3_sec_stream*)) xd3_lzma_destroy, + (int (*)(xd3_stream*, xd3_sec_stream*, int)) xd3_lzma_init, + (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, + uint8_t**, const uint8_t*)) xd3_decode_lzma, + IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, + xd3_output*, xd3_sec_cfg*)) xd3_encode_lzma) +}; +#endif + +#if XD3_MAIN || PYTHON_MODULE || SWIG_MODULE || NOT_MAIN +#include "xdelta3-main.h" +#endif + +#if REGRESSION_TEST +#include "xdelta3-test.h" +#endif + +#endif /* __XDELTA3_C_HEADER_PASS__ */ +#ifdef __XDELTA3_C_INLINE_PASS__ + +const uint16_t __single_hash[256] = +{ + /* Random numbers generated using SLIB's pseudo-random number generator. + * This hashes the input alphabet. */ + 0xbcd1, 0xbb65, 0x42c2, 0xdffe, 0x9666, 0x431b, 0x8504, 0xeb46, + 0x6379, 0xd460, 0xcf14, 0x53cf, 0xdb51, 0xdb08, 0x12c8, 0xf602, + 0xe766, 0x2394, 0x250d, 0xdcbb, 0xa678, 0x02af, 0xa5c6, 0x7ea6, + 0xb645, 0xcb4d, 0xc44b, 0xe5dc, 0x9fe6, 0x5b5c, 0x35f5, 0x701a, + 0x220f, 0x6c38, 0x1a56, 0x4ca3, 0xffc6, 0xb152, 0x8d61, 0x7a58, + 0x9025, 0x8b3d, 0xbf0f, 0x95a3, 0xe5f4, 0xc127, 0x3bed, 0x320b, + 0xb7f3, 0x6054, 0x333c, 0xd383, 0x8154, 0x5242, 0x4e0d, 0x0a94, + 0x7028, 0x8689, 0x3a22, 0x0980, 0x1847, 0xb0f1, 0x9b5c, 0x4176, + 0xb858, 0xd542, 0x1f6c, 0x2497, 0x6a5a, 0x9fa9, 0x8c5a, 0x7743, + 0xa8a9, 0x9a02, 0x4918, 0x438c, 0xc388, 0x9e2b, 0x4cad, 0x01b6, + 0xab19, 0xf777, 0x365f, 0x1eb2, 0x091e, 0x7bf8, 0x7a8e, 0x5227, + 0xeab1, 0x2074, 0x4523, 0xe781, 0x01a3, 0x163d, 0x3b2e, 0x287d, + 0x5e7f, 0xa063, 0xb134, 0x8fae, 0x5e8e, 0xb7b7, 0x4548, 0x1f5a, + 0xfa56, 0x7a24, 0x900f, 0x42dc, 0xcc69, 0x02a0, 0x0b22, 0xdb31, + 0x71fe, 0x0c7d, 0x1732, 0x1159, 0xcb09, 0xe1d2, 0x1351, 0x52e9, + 0xf536, 0x5a4f, 0xc316, 0x6bf9, 0x8994, 0xb774, 0x5f3e, 0xf6d6, + 0x3a61, 0xf82c, 0xcc22, 0x9d06, 0x299c, 0x09e5, 0x1eec, 0x514f, + 0x8d53, 0xa650, 0x5c6e, 0xc577, 0x7958, 0x71ac, 0x8916, 0x9b4f, + 0x2c09, 0x5211, 0xf6d8, 0xcaaa, 0xf7ef, 0x287f, 0x7a94, 0xab49, + 0xfa2c, 0x7222, 0xe457, 0xd71a, 0x00c3, 0x1a76, 0xe98c, 0xc037, + 0x8208, 0x5c2d, 0xdfda, 0xe5f5, 0x0b45, 0x15ce, 0x8a7e, 0xfcad, + 0xaa2d, 0x4b5c, 0xd42e, 0xb251, 0x907e, 0x9a47, 0xc9a6, 0xd93f, + 0x085e, 0x35ce, 0xa153, 0x7e7b, 0x9f0b, 0x25aa, 0x5d9f, 0xc04d, + 0x8a0e, 0x2875, 0x4a1c, 0x295f, 0x1393, 0xf760, 0x9178, 0x0f5b, + 0xfa7d, 0x83b4, 0x2082, 0x721d, 0x6462, 0x0368, 0x67e2, 0x8624, + 0x194d, 0x22f6, 0x78fb, 0x6791, 0xb238, 0xb332, 0x7276, 0xf272, + 0x47ec, 0x4504, 0xa961, 0x9fc8, 0x3fdc, 0xb413, 0x007a, 0x0806, + 0x7458, 0x95c6, 0xccaa, 0x18d6, 0xe2ae, 0x1b06, 0xf3f6, 0x5050, + 0xc8e8, 0xf4ac, 0xc04c, 0xf41c, 0x992f, 0xae44, 0x5f1b, 0x1113, + 0x1738, 0xd9a8, 0x19ea, 0x2d33, 0x9698, 0x2fe9, 0x323f, 0xcde2, + 0x6d71, 0xe37d, 0xb697, 0x2c4f, 0x4373, 0x9102, 0x075d, 0x8e25, + 0x1672, 0xec28, 0x6acb, 0x86cc, 0x186e, 0x9414, 0xd674, 0xd1a5 +}; + +/**************************************************************** + Instruction tables + *****************************************************************/ + +/* The following code implements a parametrized description of the + * code table given above for a few reasons. It is not necessary for + * implementing the standard, to support compression with variable + * tables, so an implementation is only required to know the default + * code table to begin decompression. (If the encoder uses an + * alternate table, the table is included in compressed form inside + * the VCDIFF file.) + * + * Before adding variable-table support there were two functions which + * were hard-coded to the default table above. + * xd3_compute_default_table() would create the default table by + * filling a 256-elt array of xd3_dinst values. The corresponding + * function, xd3_choose_instruction(), would choose an instruction + * based on the hard-coded parameters of the default code table. + * + * Notes: The parametrized code table description here only generates + * tables of a certain regularity similar to the default table by + * allowing to vary the distribution of single- and + * double-instructions and change the number of near and same copy + * modes. More exotic tables are only possible by extending this + * code. + * + * For performance reasons, both the parametrized and non-parametrized + * versions of xd3_choose_instruction remain. The parametrized + * version is only needed for testing multi-table decoding support. + * If ever multi-table encoding is required, this can be optimized by + * compiling static functions for each table. + */ + +/* The XD3_CHOOSE_INSTRUCTION calls xd3_choose_instruction with the + * table description when GENERIC_ENCODE_TABLES are in use. The + * IF_GENCODETBL macro enables generic-code-table specific code + * (removed 10/2014). */ +#define XD3_CHOOSE_INSTRUCTION(stream,prev,inst) \ + xd3_choose_instruction (prev, inst) + +/* This structure maintains information needed by + * xd3_choose_instruction to compute the code for a double instruction + * by first indexing an array of code_table_sizes by copy mode, then + * using (offset + (muliplier * X)) */ +struct _xd3_code_table_sizes { + uint8_t cpy_max; + uint8_t offset; + uint8_t mult; +}; + +/* This contains a complete description of a code table. */ +struct _xd3_code_table_desc +{ + /* Assumes a single RUN instruction */ + /* Assumes that MIN_MATCH is 4 */ + + uint8_t add_sizes; /* Number of immediate-size single adds (default 17) */ + uint8_t near_modes; /* Number of near copy modes (default 4) */ + uint8_t same_modes; /* Number of same copy modes (default 3) */ + uint8_t cpy_sizes; /* Number of immediate-size single copies (default 15) */ + + uint8_t addcopy_add_max; /* Maximum add size for an add-copy double instruction, + all modes (default 4) */ + uint8_t addcopy_near_cpy_max; /* Maximum cpy size for an add-copy double instruction, + up through VCD_NEAR modes (default 6) */ + uint8_t addcopy_same_cpy_max; /* Maximum cpy size for an add-copy double instruction, + VCD_SAME modes (default 4) */ + + uint8_t copyadd_add_max; /* Maximum add size for a copy-add double instruction, + all modes (default 1) */ + uint8_t copyadd_near_cpy_max; /* Maximum cpy size for a copy-add double instruction, + up through VCD_NEAR modes (default 4) */ + uint8_t copyadd_same_cpy_max; /* Maximum cpy size for a copy-add double instruction, + VCD_SAME modes (default 4) */ + + xd3_code_table_sizes addcopy_max_sizes[MAX_MODES]; + xd3_code_table_sizes copyadd_max_sizes[MAX_MODES]; +}; + +/* The rfc3284 code table is represented: */ +static const xd3_code_table_desc __rfc3284_code_table_desc = { + 17, /* add sizes */ + 4, /* near modes */ + 3, /* same modes */ + 15, /* copy sizes */ + + 4, /* add-copy max add */ + 6, /* add-copy max cpy, near */ + 4, /* add-copy max cpy, same */ + + 1, /* copy-add max add */ + 4, /* copy-add max cpy, near */ + 4, /* copy-add max cpy, same */ + + /* addcopy */ + { {6,163,3},{6,175,3},{6,187,3},{6,199,3},{6,211,3},{6,223,3},{4,235,1},{4,239,1},{4,243,1} }, + /* copyadd */ + { {4,247,1},{4,248,1},{4,249,1},{4,250,1},{4,251,1},{4,252,1},{4,253,1},{4,254,1},{4,255,1} }, +}; + +/* Computes code table entries of TBL using the specified description. */ +static void +xd3_build_code_table (const xd3_code_table_desc *desc, xd3_dinst *tbl) +{ + usize_t size1, size2, mode; + usize_t cpy_modes = 2 + desc->near_modes + desc->same_modes; + xd3_dinst *d = tbl; + + (d++)->type1 = XD3_RUN; + (d++)->type1 = XD3_ADD; + + for (size1 = 1; size1 <= desc->add_sizes; size1 += 1, d += 1) + { + d->type1 = XD3_ADD; + d->size1 = size1; + } + + for (mode = 0; mode < cpy_modes; mode += 1) + { + (d++)->type1 = XD3_CPY + mode; + + for (size1 = MIN_MATCH; size1 < MIN_MATCH + desc->cpy_sizes; size1 += 1, d += 1) + { + d->type1 = XD3_CPY + mode; + d->size1 = size1; + } + } + + for (mode = 0; mode < cpy_modes; mode += 1) + { + for (size1 = 1; size1 <= desc->addcopy_add_max; size1 += 1) + { + usize_t max = (mode < 2U + desc->near_modes) ? + desc->addcopy_near_cpy_max : + desc->addcopy_same_cpy_max; + + for (size2 = MIN_MATCH; size2 <= max; size2 += 1, d += 1) + { + d->type1 = XD3_ADD; + d->size1 = size1; + d->type2 = XD3_CPY + mode; + d->size2 = size2; + } + } + } + + for (mode = 0; mode < cpy_modes; mode += 1) + { + usize_t max = (mode < 2U + desc->near_modes) ? + desc->copyadd_near_cpy_max : + desc->copyadd_same_cpy_max; + + for (size1 = MIN_MATCH; size1 <= max; size1 += 1) + { + for (size2 = 1; size2 <= desc->copyadd_add_max; size2 += 1, d += 1) + { + d->type1 = XD3_CPY + mode; + d->size1 = size1; + d->type2 = XD3_ADD; + d->size2 = size2; + } + } + } + + XD3_ASSERT (d - tbl == 256); +} + +/* This function generates the static default code table. */ +static const xd3_dinst* +xd3_rfc3284_code_table (void) +{ + static xd3_dinst __rfc3284_code_table[256]; + + if (__rfc3284_code_table[0].type1 != XD3_RUN) + { + xd3_build_code_table (& __rfc3284_code_table_desc, __rfc3284_code_table); + } + + return __rfc3284_code_table; +} + +#if XD3_ENCODER +/* This version of xd3_choose_instruction is hard-coded for the default + table. */ +static void +xd3_choose_instruction (xd3_rinst *prev, xd3_rinst *inst) +{ + switch (inst->type) + { + case XD3_RUN: + inst->code1 = 0; + break; + + case XD3_ADD: + inst->code1 = 1; + + if (inst->size <= 17) + { + inst->code1 += inst->size; + + if ( (inst->size == 1) && + (prev != NULL) && + (prev->size == 4) && + (prev->type >= XD3_CPY) ) + { + prev->code2 = 247 + (prev->type - XD3_CPY); + } + } + + break; + + default: + { + int mode = inst->type - XD3_CPY; + + XD3_ASSERT (inst->type >= XD3_CPY && inst->type < 12); + + inst->code1 = 19 + 16 * mode; + + if (inst->size <= 18 && inst->size >= 4) + { + inst->code1 += inst->size - 3; + + if ( (prev != NULL) && + (prev->type == XD3_ADD) && + (prev->size <= 4) ) + { + if ( (inst->size <= 6) && + (mode <= 5) ) + { + prev->code2 = 163 + (mode * 12) + (3 * (prev->size - 1)) + (inst->size - 4); + + XD3_ASSERT (prev->code2 <= 234); + } + else if ( (inst->size == 4) && + (mode >= 6) ) + { + prev->code2 = 235 + ((mode - 6) * 4) + (prev->size - 1); + + XD3_ASSERT (prev->code2 <= 246); + } + } + } + + XD3_ASSERT (inst->code1 <= 162); + } + break; + } +} +#endif /* XD3_ENCODER */ + +/***********************************************************************/ + +static inline void +xd3_swap_uint8p (uint8_t** p1, uint8_t** p2) +{ + uint8_t *t = (*p1); + (*p1) = (*p2); + (*p2) = t; +} + +static inline void +xd3_swap_usize_t (usize_t* p1, usize_t* p2) +{ + usize_t t = (*p1); + (*p1) = (*p2); + (*p2) = t; +} + +/* It's not constant time, but it computes the log. */ +static int +xd3_check_pow2 (xoff_t value, usize_t *logof) +{ + xoff_t x = 1; + usize_t nolog; + if (logof == NULL) { + logof = &nolog; + } + + *logof = 0; + + for (; x != 0; x <<= 1, *logof += 1) + { + if (x == value) + { + return 0; + } + } + + return XD3_INTERNAL; +} + +size_t +xd3_pow2_roundup (size_t x) +{ + size_t i = 1; + while (x > i) { + i <<= 1U; + } + return i; +} + +static xoff_t +xd3_xoff_roundup (xoff_t x) +{ + xoff_t i = 1; + while (x > i) { + i <<= 1U; + } + return i; +} + +static usize_t +xd3_round_blksize (usize_t sz, usize_t blksz) +{ + usize_t mod = sz & (blksz-1); + + XD3_ASSERT (xd3_check_pow2 (blksz, NULL) == 0); + + if (mod == 0) + { + return sz; + } + + if (sz > USIZE_T_MAXBLKSZ) + { + return USIZE_T_MAXBLKSZ; + } + + return sz + (blksz - mod); +} + +/*********************************************************************** + Adler32 stream function: code copied from Zlib, defined in RFC1950 + ***********************************************************************/ + +#define A32_BASE 65521L /* Largest prime smaller than 2^16 */ +#define A32_NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ + +#define A32_DO1(buf,i) {s1 += buf[i]; s2 += s1;} +#define A32_DO2(buf,i) A32_DO1(buf,i); A32_DO1(buf,i+1); +#define A32_DO4(buf,i) A32_DO2(buf,i); A32_DO2(buf,i+2); +#define A32_DO8(buf,i) A32_DO4(buf,i); A32_DO4(buf,i+4); +#define A32_DO16(buf) A32_DO8(buf,0); A32_DO8(buf,8); + +static unsigned long adler32 (unsigned long adler, const uint8_t *buf, + usize_t len) +{ + unsigned long s1 = adler & 0xffff; + unsigned long s2 = (adler >> 16) & 0xffff; + int k; + + while (len > 0) + { + k = (len < A32_NMAX) ? len : A32_NMAX; + len -= k; + + while (k >= 16) + { + A32_DO16(buf); + buf += 16; + k -= 16; + } + + if (k != 0) + { + do + { + s1 += *buf++; + s2 += s1; + } + while (--k); + } + + s1 %= A32_BASE; + s2 %= A32_BASE; + } + + return (s2 << 16) | s1; +} + +/*********************************************************************** + Run-length function + ***********************************************************************/ + +#if XD3_ENCODER +static usize_t +xd3_comprun (const uint8_t *seg, usize_t slook, uint8_t *run_cp) +{ + usize_t i; + usize_t run_l = 0; + uint8_t run_c = 0; + + for (i = 0; i < slook; i += 1) + { + NEXTRUN(seg[i]); + } + + (*run_cp) = run_c; + + return run_l; +} +#endif + +/*********************************************************************** + Basic encoder/decoder functions + ***********************************************************************/ + +#if XD3_ENCODER +inline int +xd3_emit_byte (xd3_stream *stream, + xd3_output **outputp, + uint8_t code) +{ + xd3_output *output = (*outputp); + + if (output->next == output->avail) + { + xd3_output *aoutput; + + if ((aoutput = xd3_alloc_output (stream, output)) == NULL) + { + return ENOMEM; + } + + output = (*outputp) = aoutput; + } + + output->base[output->next++] = code; + + return 0; +} + +inline int +xd3_emit_bytes (xd3_stream *stream, + xd3_output **outputp, + const uint8_t *base, + usize_t size) +{ + xd3_output *output = (*outputp); + + do + { + usize_t take; + + if (output->next == output->avail) + { + xd3_output *aoutput; + + if ((aoutput = xd3_alloc_output (stream, output)) == NULL) + { + return ENOMEM; + } + + output = (*outputp) = aoutput; + } + + take = xd3_min (output->avail - output->next, size); + + memcpy (output->base + output->next, base, (size_t) take); + + output->next += take; + size -= take; + base += take; + } + while (size > 0); + + return 0; +} +#endif /* XD3_ENCODER */ + +/*********************************************************************** + Address cache stuff + ***********************************************************************/ + +static int +xd3_alloc_cache (xd3_stream *stream) +{ + if (stream->acache.near_array != NULL) + { + xd3_free (stream, stream->acache.near_array); + } + + if (stream->acache.same_array != NULL) + { + xd3_free (stream, stream->acache.same_array); + } + + if (((stream->acache.s_near > 0) && + (stream->acache.near_array = (usize_t*) + xd3_alloc (stream, stream->acache.s_near, + (usize_t) sizeof (usize_t))) + == NULL) || + ((stream->acache.s_same > 0) && + (stream->acache.same_array = (usize_t*) + xd3_alloc (stream, stream->acache.s_same * 256, + (usize_t) sizeof (usize_t))) + == NULL)) + { + return ENOMEM; + } + + return 0; +} + +void +xd3_init_cache (xd3_addr_cache* acache) +{ + if (acache->s_near > 0) + { + memset (acache->near_array, 0, acache->s_near * sizeof (usize_t)); + acache->next_slot = 0; + } + + if (acache->s_same > 0) + { + memset (acache->same_array, 0, acache->s_same * 256 * sizeof (usize_t)); + } +} + +static void +xd3_update_cache (xd3_addr_cache* acache, usize_t addr) +{ + if (acache->s_near > 0) + { + acache->near_array[acache->next_slot] = addr; + acache->next_slot = (acache->next_slot + 1) % acache->s_near; + } + + if (acache->s_same > 0) + { + acache->same_array[addr % (acache->s_same*256)] = addr; + } +} + +#if XD3_ENCODER +/* OPT: this gets called a lot, can it be optimized? */ +static int +xd3_encode_address (xd3_stream *stream, + usize_t addr, + usize_t here, + uint8_t* mode) +{ + usize_t d, bestd; + usize_t i, bestm, ret; + xd3_addr_cache* acache = & stream->acache; + +#define SMALLEST_INT(x) do { if (((x) & ~127U) == 0) { goto good; } } while (0) + + /* Attempt to find the address mode that yields the smallest integer value + * for "d", the encoded address value, thereby minimizing the encoded size + * of the address. */ + bestd = addr; + bestm = VCD_SELF; + + XD3_ASSERT (addr < here); + + SMALLEST_INT (bestd); + + if ((d = here-addr) < bestd) + { + bestd = d; + bestm = VCD_HERE; + + SMALLEST_INT (bestd); + } + + for (i = 0; i < acache->s_near; i += 1) + { + /* Note: If we used signed computation here, we'd could compte d + * and then check (d >= 0 && d < bestd). */ + if (addr >= acache->near_array[i]) + { + d = addr - acache->near_array[i]; + + if (d < bestd) + { + bestd = d; + bestm = i+2; /* 2 counts the VCD_SELF, VCD_HERE modes */ + + SMALLEST_INT (bestd); + } + } + } + + if (acache->s_same > 0 && + acache->same_array[d = addr%(acache->s_same*256)] == addr) + { + bestd = d%256; + /* 2 + s_near offsets past the VCD_NEAR modes */ + bestm = acache->s_near + 2 + d/256; + + if ((ret = xd3_emit_byte (stream, & ADDR_TAIL (stream), bestd))) + { + return ret; + } + } + else + { + good: + + if ((ret = xd3_emit_size (stream, & ADDR_TAIL (stream), bestd))) + { + return ret; + } + } + + xd3_update_cache (acache, addr); + + (*mode) += bestm; + + return 0; +} +#endif + +static int +xd3_decode_address (xd3_stream *stream, usize_t here, + usize_t mode, const uint8_t **inpp, + const uint8_t *max, uint32_t *valp) +{ + int ret; + usize_t same_start = 2 + stream->acache.s_near; + + if (mode < same_start) + { + if ((ret = xd3_read_size (stream, inpp, max, valp))) { return ret; } + + switch (mode) + { + case VCD_SELF: + break; + case VCD_HERE: + (*valp) = here - (*valp); + break; + default: + (*valp) += stream->acache.near_array[mode - 2]; + break; + } + } + else + { + if (*inpp == max) + { + stream->msg = "address underflow"; + return XD3_INVALID_INPUT; + } + + mode -= same_start; + + (*valp) = stream->acache.same_array[mode*256 + (**inpp)]; + + (*inpp) += 1; + } + + xd3_update_cache (& stream->acache, *valp); + + return 0; +} + +/*********************************************************************** + Alloc/free +***********************************************************************/ + +static void* +__xd3_alloc_func (void* opaque, size_t items, usize_t size) +{ + return malloc (items * (size_t) size); +} + +static void +__xd3_free_func (void* opaque, void* address) +{ + free (address); +} + +static void* +xd3_alloc (xd3_stream *stream, + usize_t elts, + usize_t size) +{ + void *a = stream->alloc (stream->opaque, elts, size); + + if (a != NULL) + { + IF_DEBUG (stream->alloc_cnt += 1); + IF_DEBUG2 (DP(RINT "[stream %p malloc] size %u ptr %p\n", + stream, elts * size, a)); + } + else + { + stream->msg = "out of memory"; + } + + return a; +} + +static void +xd3_free (xd3_stream *stream, + void *ptr) +{ + if (ptr != NULL) + { + IF_DEBUG (stream->free_cnt += 1); + XD3_ASSERT (stream->free_cnt <= stream->alloc_cnt); + IF_DEBUG2 (DP(RINT "[stream %p free] %p\n", + stream, ptr)); + stream->free (stream->opaque, ptr); + } +} + +#if XD3_ENCODER +static void* +xd3_alloc0 (xd3_stream *stream, + usize_t elts, + usize_t size) +{ + void *a = xd3_alloc (stream, elts, size); + + if (a != NULL) + { + memset (a, 0, (size_t) (elts * size)); + } + + return a; +} + +xd3_output* +xd3_alloc_output (xd3_stream *stream, + xd3_output *old_output) +{ + xd3_output *output; + uint8_t *base; + + if (stream->enc_free != NULL) + { + output = stream->enc_free; + stream->enc_free = output->next_page; + } + else + { + if ((output = (xd3_output*) xd3_alloc (stream, 1, + (usize_t) sizeof (xd3_output))) + == NULL) + { + return NULL; + } + + if ((base = (uint8_t*) xd3_alloc (stream, XD3_ALLOCSIZE, + sizeof (uint8_t))) == NULL) + { + xd3_free (stream, output); + return NULL; + } + + output->base = base; + output->avail = XD3_ALLOCSIZE; + } + + output->next = 0; + + if (old_output) + { + old_output->next_page = output; + } + + output->next_page = NULL; + + return output; +} + +static usize_t +xd3_sizeof_output (xd3_output *output) +{ + usize_t s = 0; + + for (; output; output = output->next_page) + { + s += output->next; + } + + return s; +} + +static void +xd3_freelist_output (xd3_stream *stream, + xd3_output *output) +{ + xd3_output *tmp; + + while (output) + { + tmp = output; + output = output->next_page; + + tmp->next = 0; + tmp->next_page = stream->enc_free; + stream->enc_free = tmp; + } +} + +static void +xd3_free_output (xd3_stream *stream, + xd3_output *output) +{ + xd3_output *next; + + again: + if (output == NULL) + { + return; + } + + next = output->next_page; + + xd3_free (stream, output->base); + xd3_free (stream, output); + + output = next; + goto again; +} +#endif /* XD3_ENCODER */ + +void +xd3_free_stream (xd3_stream *stream) +{ + xd3_iopt_buflist *blist = stream->iopt_alloc; + + while (blist != NULL) + { + xd3_iopt_buflist *tmp = blist; + blist = blist->next; + xd3_free (stream, tmp->buffer); + xd3_free (stream, tmp); + } + + xd3_free (stream, stream->large_table); + xd3_free (stream, stream->small_table); + xd3_free (stream, stream->small_prev); + +#if XD3_ENCODER + { + int i; + for (i = 0; i < ENC_SECTS; i += 1) + { + xd3_free_output (stream, stream->enc_heads[i]); + } + xd3_free_output (stream, stream->enc_free); + } +#endif + + xd3_free (stream, stream->acache.near_array); + xd3_free (stream, stream->acache.same_array); + + xd3_free (stream, stream->inst_sect.copied1); + xd3_free (stream, stream->addr_sect.copied1); + xd3_free (stream, stream->data_sect.copied1); + + if (stream->dec_lastwin != stream->dec_buffer) + { + xd3_free (stream, (uint8_t*) stream->dec_lastwin); + } + xd3_free (stream, stream->dec_buffer); + + xd3_free (stream, stream->buf_in); + xd3_free (stream, stream->dec_appheader); + xd3_free (stream, stream->dec_codetbl); + xd3_free (stream, stream->code_table_alloc); + +#if SECONDARY_ANY + xd3_free (stream, stream->inst_sect.copied2); + xd3_free (stream, stream->addr_sect.copied2); + xd3_free (stream, stream->data_sect.copied2); + + if (stream->sec_type != NULL) + { + stream->sec_type->destroy (stream, stream->sec_stream_d); + stream->sec_type->destroy (stream, stream->sec_stream_i); + stream->sec_type->destroy (stream, stream->sec_stream_a); + } +#endif + + xd3_free (stream, stream->whole_target.adds); + xd3_free (stream, stream->whole_target.inst); + xd3_free (stream, stream->whole_target.wininfo); + + XD3_ASSERT (stream->alloc_cnt == stream->free_cnt); + + memset (stream, 0, sizeof (xd3_stream)); +} + +#if (XD3_DEBUG > 1 || VCDIFF_TOOLS) +static const char* +xd3_rtype_to_string (xd3_rtype type, int print_mode) +{ + switch (type) + { + case XD3_NOOP: + return "NOOP "; + case XD3_RUN: + return "RUN "; + case XD3_ADD: + return "ADD "; + default: break; + } + if (! print_mode) + { + return "CPY "; + } + switch (type) + { + case XD3_CPY + 0: return "CPY_0"; + case XD3_CPY + 1: return "CPY_1"; + case XD3_CPY + 2: return "CPY_2"; + case XD3_CPY + 3: return "CPY_3"; + case XD3_CPY + 4: return "CPY_4"; + case XD3_CPY + 5: return "CPY_5"; + case XD3_CPY + 6: return "CPY_6"; + case XD3_CPY + 7: return "CPY_7"; + case XD3_CPY + 8: return "CPY_8"; + case XD3_CPY + 9: return "CPY_9"; + default: return "CPY>9"; + } +} +#endif + +/**************************************************************** + Stream configuration + ******************************************************************/ + +int +xd3_config_stream(xd3_stream *stream, + xd3_config *config) +{ + int ret; + xd3_config defcfg; + xd3_smatcher *smatcher = &stream->smatcher; + + if (config == NULL) + { + config = & defcfg; + memset (config, 0, sizeof (*config)); + } + + /* Initial setup: no error checks yet */ + memset (stream, 0, sizeof (*stream)); + + stream->winsize = config->winsize ? config->winsize : XD3_DEFAULT_WINSIZE; + stream->sprevsz = config->sprevsz ? config->sprevsz : XD3_DEFAULT_SPREVSZ; + + if (config->iopt_size == 0) + { + stream->iopt_size = XD3_ALLOCSIZE / sizeof(xd3_rinst); + stream->iopt_unlimited = 1; + } + else + { + stream->iopt_size = config->iopt_size; + } + + stream->getblk = config->getblk; + stream->alloc = config->alloc ? config->alloc : __xd3_alloc_func; + stream->free = config->freef ? config->freef : __xd3_free_func; + stream->opaque = config->opaque; + stream->flags = config->flags; + + /* Secondary setup. */ + stream->sec_data = config->sec_data; + stream->sec_inst = config->sec_inst; + stream->sec_addr = config->sec_addr; + + stream->sec_data.data_type = DATA_SECTION; + stream->sec_inst.data_type = INST_SECTION; + stream->sec_addr.data_type = ADDR_SECTION; + + /* Check static sizes. */ + if (sizeof (usize_t) != SIZEOF_USIZE_T || + sizeof (xoff_t) != SIZEOF_XOFF_T || + (ret = xd3_check_pow2(XD3_ALLOCSIZE, NULL))) + { + stream->msg = "incorrect compilation: wrong integer sizes"; + return XD3_INTERNAL; + } + + /* Check/set secondary compressor. */ + switch (stream->flags & XD3_SEC_TYPE) + { + case 0: + if (stream->flags & XD3_SEC_NOALL) + { + stream->msg = "XD3_SEC flags require a secondary compressor type"; + return XD3_INTERNAL; + } + break; + case XD3_SEC_FGK: + FGK_CASE (stream); + case XD3_SEC_DJW: + DJW_CASE (stream); + case XD3_SEC_LZMA: + LZMA_CASE (stream); + default: + stream->msg = "too many secondary compressor types set"; + return XD3_INTERNAL; + } + + stream->code_table_desc = & __rfc3284_code_table_desc; + stream->code_table_func = xd3_rfc3284_code_table; + + /* Check sprevsz */ + if (smatcher->small_chain == 1 && + smatcher->small_lchain == 1) + { + stream->sprevsz = 0; + } + else + { + if ((ret = xd3_check_pow2 (stream->sprevsz, NULL))) + { + stream->msg = "sprevsz is required to be a power of two"; + return XD3_INTERNAL; + } + + stream->sprevmask = stream->sprevsz - 1; + } + + /* Default scanner settings. */ +#if XD3_ENCODER + switch (config->smatch_cfg) + { + IF_BUILD_SOFT(case XD3_SMATCH_SOFT: + { + *smatcher = config->smatcher_soft; + smatcher->string_match = __smatcher_soft.string_match; + smatcher->name = __smatcher_soft.name; + if (smatcher->large_look < MIN_MATCH || + smatcher->large_step < 1 || + smatcher->small_look < MIN_MATCH) + { + stream->msg = "invalid soft string-match config"; + return XD3_INVALID; + } + break; + }) + + IF_BUILD_DEFAULT(case XD3_SMATCH_DEFAULT: + *smatcher = __smatcher_default; + break;) + IF_BUILD_SLOW(case XD3_SMATCH_SLOW: + *smatcher = __smatcher_slow; + break;) + IF_BUILD_FASTEST(case XD3_SMATCH_FASTEST: + *smatcher = __smatcher_fastest; + break;) + IF_BUILD_FASTER(case XD3_SMATCH_FASTER: + *smatcher = __smatcher_faster; + break;) + IF_BUILD_FAST(case XD3_SMATCH_FAST: + *smatcher = __smatcher_fast; + break;) + default: + stream->msg = "invalid string match config type"; + return XD3_INTERNAL; + } + + if (config->smatch_cfg == XD3_SMATCH_DEFAULT && + (stream->flags & XD3_COMPLEVEL_MASK) != 0) + { + int level = (stream->flags & XD3_COMPLEVEL_MASK) >> XD3_COMPLEVEL_SHIFT; + + switch (level) + { + case 1: + IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; + break;) + case 2: + IF_BUILD_FASTER(*smatcher = __smatcher_faster; + break;) + case 3: case 4: case 5: + IF_BUILD_FAST(*smatcher = __smatcher_fast; + break;) + case 6: + IF_BUILD_DEFAULT(*smatcher = __smatcher_default; + break;) + default: + IF_BUILD_SLOW(*smatcher = __smatcher_slow; + break;) + IF_BUILD_DEFAULT(*smatcher = __smatcher_default; + break;) + IF_BUILD_FAST(*smatcher = __smatcher_fast; + break;) + IF_BUILD_FASTER(*smatcher = __smatcher_faster; + break;) + IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; + break;) + } + } +#endif + + return 0; +} + +/*********************************************************** + Getblk interface + ***********************************************************/ + +inline +xoff_t xd3_source_eof(const xd3_source *src) +{ + xoff_t r = (src->max_blkno << src->shiftby) + (xoff_t)src->onlastblk; + return r; +} + +inline +usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno) +{ + usize_t r = (blkno == src->max_blkno ? + src->onlastblk : + src->blksize); + return r; +} + +/* This function interfaces with the client getblk function, checks + * its results, updates max_blkno, onlastblk, eof_known. */ +static int +xd3_getblk (xd3_stream *stream, xoff_t blkno) +{ + int ret; + xd3_source *source = stream->src; + + if (source->curblk == NULL || blkno != source->curblkno) + { + source->getblkno = blkno; + + if (stream->getblk == NULL) + { + IF_DEBUG2 (DP(RINT "[getblk] XD3_GETSRCBLK %"Q"u\n", blkno)); + stream->msg = "getblk source input"; + return XD3_GETSRCBLK; + } + + ret = stream->getblk (stream, source, blkno); + if (ret != 0) + { + IF_DEBUG2 (DP(RINT "[getblk] app error blkno %"Q"u: %s\n", + blkno, xd3_strerror (ret))); + return ret; + } + + IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q"u onblk " + "%u blksize %u max_blkno %"Q"u\n", blkno, source->onblk, + source->blksize, source->max_blkno)); + } + + if (blkno > source->max_blkno) + { + source->max_blkno = blkno; + + if (source->onblk == source->blksize) + { + IF_DEBUG1 (DP(RINT "[getblk] full source blkno %"Q"u: " + "source length unknown %"Q"u\n", + blkno, + xd3_source_eof (source))); + } + else if (!source->eof_known) + { + IF_DEBUG1 (DP(RINT "[getblk] eof block has %d bytes; " + "source length known %"Q"u\n", + xd3_bytes_on_srcblk (source, blkno), + xd3_source_eof (source))); + source->eof_known = 1; + } + } + + XD3_ASSERT (source->curblk != NULL); + + if (blkno == source->max_blkno) + { + /* In case the application sets the source as 1 block w/ a + * preset buffer. */ + source->onlastblk = source->onblk; + } + return 0; +} + +/*********************************************************** + Stream open/close + ***************************************************************/ + +int +xd3_set_source (xd3_stream *stream, + xd3_source *src) +{ + usize_t shiftby; + + stream->src = src; + src->srclen = 0; + src->srcbase = 0; + + /* Enforce power-of-two blocksize so that source-block number + * calculations are cheap. */ + if (xd3_check_pow2 (src->blksize, &shiftby) != 0) + { + src->blksize = xd3_pow2_roundup(src->blksize); + xd3_check_pow2 (src->blksize, &shiftby); + IF_DEBUG1 (DP(RINT "raising src_blksz to %u\n", src->blksize)); + } + + src->shiftby = shiftby; + src->maskby = (1 << shiftby) - 1; + + if (xd3_check_pow2 (src->max_winsize, NULL) != 0) + { + src->max_winsize = xd3_xoff_roundup(src->max_winsize); + IF_DEBUG1 (DP(RINT "raising src_maxsize to %u\n", src->blksize)); + } + src->max_winsize = xd3_max (src->max_winsize, XD3_ALLOCSIZE); + return 0; +} + +int +xd3_set_source_and_size (xd3_stream *stream, + xd3_source *user_source, + xoff_t source_size) { + int ret = xd3_set_source (stream, user_source); + if (ret == 0) + { + stream->src->eof_known = 1; + + xd3_blksize_div(source_size, + stream->src, + &stream->src->max_blkno, + &stream->src->onlastblk); + + IF_DEBUG1 (DP(RINT "[set source] size known %"Q"u max_blkno %"Q"u\n", + source_size, stream->src->max_blkno)); + } + return ret; +} + +void +xd3_abort_stream (xd3_stream *stream) +{ + stream->dec_state = DEC_ABORTED; + stream->enc_state = ENC_ABORTED; +} + +int +xd3_close_stream (xd3_stream *stream) +{ + if (stream->enc_state != 0 && stream->enc_state != ENC_ABORTED) + { + if (stream->buf_leftover != NULL) + { + stream->msg = "encoding is incomplete"; + return XD3_INTERNAL; + } + + if (stream->enc_state == ENC_POSTWIN) + { +#if XD3_ENCODER + xd3_encode_reset (stream); +#endif + stream->current_window += 1; + stream->enc_state = ENC_INPUT; + } + + /* If encoding, should be ready for more input but not actually + have any. */ + if (stream->enc_state != ENC_INPUT || stream->avail_in != 0) + { + stream->msg = "encoding is incomplete"; + return XD3_INTERNAL; + } + } + else + { + switch (stream->dec_state) + { + case DEC_VCHEAD: + case DEC_WININD: + /* TODO: Address the zero-byte ambiguity. Does the encoder + * emit a window or not? If so, then catch an error here. + * If not, need another routine to say + * decode_at_least_one_if_empty. */ + case DEC_ABORTED: + break; + default: + /* If decoding, should be ready for the next window. */ + stream->msg = "eof in decode"; + return XD3_INVALID_INPUT; + } + } + + return 0; +} + +/************************************************************** + Application header + ****************************************************************/ + +int +xd3_get_appheader (xd3_stream *stream, + uint8_t **data, + usize_t *size) +{ + if (stream->dec_state < DEC_WININD) + { + stream->msg = "application header not available"; + return XD3_INTERNAL; + } + + (*data) = stream->dec_appheader; + (*size) = stream->dec_appheadsz; + return 0; +} + +/********************************************************** + Decoder stuff + *************************************************/ + +#include "xdelta3-decode.h" + +/**************************************************************** + Encoder stuff + *****************************************************************/ + +#if XD3_ENCODER +void +xd3_set_appheader (xd3_stream *stream, + const uint8_t *data, + usize_t size) +{ + stream->enc_appheader = data; + stream->enc_appheadsz = size; +} + +#if XD3_DEBUG +static int +xd3_iopt_check (xd3_stream *stream) +{ + usize_t ul = xd3_rlist_length (& stream->iopt_used); + usize_t fl = xd3_rlist_length (& stream->iopt_free); + + return (ul + fl + (stream->iout ? 1 : 0)) == stream->iopt_size; +} +#endif + +static xd3_rinst* +xd3_iopt_free (xd3_stream *stream, xd3_rinst *i) +{ + xd3_rinst *n = xd3_rlist_remove (i); + xd3_rlist_push_back (& stream->iopt_free, i); + return n; +} + +static void +xd3_iopt_free_nonadd (xd3_stream *stream, xd3_rinst *i) +{ + if (i->type != XD3_ADD) + { + xd3_rlist_push_back (& stream->iopt_free, i); + } +} + +/* When an instruction is ready to flush from the iopt buffer, this + * function is called to produce an encoding. It writes the + * instruction plus size, address, and data to the various encoding + * sections. */ +static int +xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) +{ + int ret; + + /* Check for input overflow. */ + XD3_ASSERT (inst->pos + inst->size <= stream->avail_in); + + switch (inst->type) + { + case XD3_CPY: + { + /* the address may have an offset if there is a source window. */ + usize_t addr; + xd3_source *src = stream->src; + + if (src != NULL) + { + /* If there is a source copy, the source must have its + * source window decided before we can encode. This can + * be bad -- we have to make this decision even if no + * source matches have been found. */ + if (stream->srcwin_decided == 0) + { + if ((ret = xd3_srcwin_setup (stream))) { return ret; } + } + else + { + stream->srcwin_decided_early = (!stream->src->eof_known || + (stream->srcwin_cksum_pos < + xd3_source_eof (stream->src))); + } + + /* xtra field indicates the copy is from the source */ + if (inst->xtra) + { + XD3_ASSERT (inst->addr >= src->srcbase); + XD3_ASSERT (inst->addr + inst->size <= + src->srcbase + src->srclen); + addr = (usize_t)(inst->addr - src->srcbase); + stream->n_scpy += 1; + stream->l_scpy += (xoff_t) inst->size; + } + else + { + /* with source window: target copy address is offset + * by taroff. */ + addr = stream->taroff + (usize_t) inst->addr; + stream->n_tcpy += 1; + stream->l_tcpy += (xoff_t) inst->size; + } + } + else + { + addr = (usize_t) inst->addr; + stream->n_tcpy += 1; + stream->l_tcpy += inst->size; + } + + /* Note: used to assert inst->size >= MIN_MATCH, but not true + * for merge operations & identical match heuristics. */ + /* the "here" position is always offset by taroff */ + if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, + & inst->type))) + { + return ret; + } + + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt copy:%d] pos %"Q"u-%"Q"u addr %"Q"u-%"Q"u size %u\n", + cnt++, + stream->total_in + inst->pos, + stream->total_in + inst->pos + inst->size, + inst->addr, inst->addr + inst->size, inst->size); + }); + break; + } + case XD3_RUN: + { + if ((ret = xd3_emit_byte (stream, & DATA_TAIL (stream), inst->xtra))) { return ret; } + + stream->n_run += 1; + stream->l_run += inst->size; + + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt run:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); + }); + break; + } + case XD3_ADD: + { + if ((ret = xd3_emit_bytes (stream, & DATA_TAIL (stream), + stream->next_in + inst->pos, inst->size))) { return ret; } + + stream->n_add += 1; + stream->l_add += inst->size; + + IF_DEBUG2 ({ + static int cnt; + DP(RINT "[iopt add:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); + }); + + break; + } + } + + /* This is the only place stream->unencoded_offset is incremented. */ + XD3_ASSERT (stream->unencoded_offset == inst->pos); + stream->unencoded_offset += inst->size; + + inst->code2 = 0; + + XD3_CHOOSE_INSTRUCTION (stream, stream->iout, inst); + + if (stream->iout != NULL) + { + if (stream->iout->code2 != 0) + { + if ((ret = xd3_emit_double (stream, stream->iout, inst, stream->iout->code2))) { return ret; } + + xd3_iopt_free_nonadd (stream, stream->iout); + xd3_iopt_free_nonadd (stream, inst); + stream->iout = NULL; + return 0; + } + else + { + if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } + + xd3_iopt_free_nonadd (stream, stream->iout); + } + } + + stream->iout = inst; + + return 0; +} + +/* This possibly encodes an add instruction, iadd, which must remain + * on the stack until the following call to + * xd3_iopt_finish_encoding. */ +static int +xd3_iopt_add (xd3_stream *stream, usize_t pos, xd3_rinst *iadd) +{ + int ret; + usize_t off = stream->unencoded_offset; + + if (pos > off) + { + iadd->type = XD3_ADD; + iadd->pos = off; + iadd->size = pos - off; + + if ((ret = xd3_iopt_finish_encoding (stream, iadd))) { return ret; } + } + + return 0; +} + +/* This function calls xd3_iopt_finish_encoding to finish encoding an + * instruction, and it may also produce an add instruction for an + * unmatched region. */ +static int +xd3_iopt_add_encoding (xd3_stream *stream, xd3_rinst *inst) +{ + int ret; + xd3_rinst iadd; + + if ((ret = xd3_iopt_add (stream, inst->pos, & iadd))) { return ret; } + + if ((ret = xd3_iopt_finish_encoding (stream, inst))) { return ret; } + + return 0; +} + +/* Generates a final add instruction to encode the remaining input. */ +static int +xd3_iopt_add_finalize (xd3_stream *stream) +{ + int ret; + xd3_rinst iadd; + + if ((ret = xd3_iopt_add (stream, stream->avail_in, & iadd))) { return ret; } + + if (stream->iout) + { + if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } + + xd3_iopt_free_nonadd (stream, stream->iout); + stream->iout = NULL; + } + + return 0; +} + +/* Compact the instruction buffer by choosing the best non-overlapping + * instructions when lazy string-matching. There are no ADDs in the + * iopt buffer because those are synthesized in xd3_iopt_add_encoding + * and during xd3_iopt_add_finalize. */ +static int +xd3_iopt_flush_instructions (xd3_stream *stream, int force) +{ + xd3_rinst *r1 = xd3_rlist_front (& stream->iopt_used); + xd3_rinst *r2; + xd3_rinst *r3; + usize_t r1end; + usize_t r2end; + usize_t r2off; + usize_t r2moff; + usize_t gap; + usize_t flushed; + int ret; + + XD3_ASSERT (xd3_iopt_check (stream)); + + /* Note: once tried to skip this step if it's possible to assert + * there are no overlapping instructions. Doesn't work because + * xd3_opt_erase leaves overlapping instructions. */ + while (! xd3_rlist_end (& stream->iopt_used, r1) && + ! xd3_rlist_end (& stream->iopt_used, r2 = xd3_rlist_next (r1))) + { + r1end = r1->pos + r1->size; + + /* If the instructions do not overlap, continue. */ + if (r1end <= r2->pos) + { + r1 = r2; + continue; + } + + r2end = r2->pos + r2->size; + + /* The min_match adjustments prevent this. */ + XD3_ASSERT (r2end > (r1end + LEAST_MATCH_INCR)); + + /* If r3 is available... */ + if (! xd3_rlist_end (& stream->iopt_used, r3 = xd3_rlist_next (r2))) + { + /* If r3 starts before r1 finishes or just about, r2 is irrelevant */ + if (r3->pos <= r1end + 1) + { + xd3_iopt_free (stream, r2); + continue; + } + } + else if (! force) + { + /* Unless force, end the loop when r3 is not available. */ + break; + } + + r2off = r2->pos - r1->pos; + r2moff = r2end - r1end; + gap = r2end - r1->pos; + + /* If the two matches overlap almost entirely, choose the better match + * and discard the other. The else branch can still create inefficient + * copies, e.g., a 4-byte copy that takes 4 bytes to encode, which + * xd3_smatch() wouldn't allow by its crude efficiency check. However, + * in this case there are adjacent copies which mean the add would cost + * one extra byte. Allow the inefficiency here. */ + if (gap < 2*MIN_MATCH || r2moff <= 2 || r2off <= 2) + { + /* Only one match should be used, choose the longer one. */ + if (r1->size < r2->size) + { + xd3_iopt_free (stream, r1); + r1 = r2; + } + else + { + /* We are guaranteed that r1 does not overlap now, so advance past r2 */ + r1 = xd3_iopt_free (stream, r2); + } + continue; + } + else + { + /* Shorten one of the instructions -- could be optimized + * based on the address cache. */ + usize_t average; + usize_t newsize; + usize_t adjust1; + + XD3_ASSERT (r1end > r2->pos && r2end > r1->pos); + + /* Try to balance the length of both instructions, but avoid + * making both longer than MAX_MATCH_SPLIT . */ + average = gap / 2; + newsize = xd3_min (MAX_MATCH_SPLIT, gap - average); + + /* Should be possible to simplify this code. */ + if (newsize > r1->size) + { + /* shorten r2 */ + adjust1 = r1end - r2->pos; + } + else if (newsize > r2->size) + { + /* shorten r1 */ + adjust1 = r1end - r2->pos; + + XD3_ASSERT (r1->size > adjust1); + + r1->size -= adjust1; + + /* don't shorten r2 */ + adjust1 = 0; + } + else + { + /* shorten r1 */ + adjust1 = r1->size - newsize; + + if (r2->pos > r1end - adjust1) + { + adjust1 -= r2->pos - (r1end - adjust1); + } + + XD3_ASSERT (r1->size > adjust1); + + r1->size -= adjust1; + + /* shorten r2 */ + XD3_ASSERT (r1->pos + r1->size >= r2->pos); + + adjust1 = r1->pos + r1->size - r2->pos; + } + + /* Fallthrough above if-else, shorten r2 */ + XD3_ASSERT (r2->size > adjust1); + + r2->size -= adjust1; + r2->pos += adjust1; + r2->addr += adjust1; + + XD3_ASSERT (r1->size >= MIN_MATCH); + XD3_ASSERT (r2->size >= MIN_MATCH); + + r1 = r2; + } + } + + XD3_ASSERT (xd3_iopt_check (stream)); + + /* If forcing, pick instructions until the list is empty, otherwise + * this empties 50% of the queue. */ + for (flushed = 0; ! xd3_rlist_empty (& stream->iopt_used); ) + { + xd3_rinst *renc = xd3_rlist_pop_front (& stream->iopt_used); + if ((ret = xd3_iopt_add_encoding (stream, renc))) + { + return ret; + } + + if (! force) + { + if (++flushed > stream->iopt_size / 2) + { + break; + } + + /* If there are only two instructions remaining, break, + * because they were not optimized. This means there were + * more than 50% eliminated by the loop above. */ + r1 = xd3_rlist_front (& stream->iopt_used); + if (xd3_rlist_end(& stream->iopt_used, r1) || + xd3_rlist_end(& stream->iopt_used, r2 = xd3_rlist_next (r1)) || + xd3_rlist_end(& stream->iopt_used, r3 = xd3_rlist_next (r2))) + { + break; + } + } + } + + XD3_ASSERT (xd3_iopt_check (stream)); + + XD3_ASSERT (!force || xd3_rlist_length (& stream->iopt_used) == 0); + + return 0; +} + +static int +xd3_iopt_get_slot (xd3_stream *stream, xd3_rinst** iptr) +{ + xd3_rinst *i; + int ret; + + if (xd3_rlist_empty (& stream->iopt_free)) + { + if (stream->iopt_unlimited) + { + usize_t elts = XD3_ALLOCSIZE / sizeof(xd3_rinst); + + if ((ret = xd3_alloc_iopt (stream, elts))) + { + return ret; + } + + stream->iopt_size += elts; + } + else + { + if ((ret = xd3_iopt_flush_instructions (stream, 0))) { return ret; } + + XD3_ASSERT (! xd3_rlist_empty (& stream->iopt_free)); + } + } + + i = xd3_rlist_pop_back (& stream->iopt_free); + + xd3_rlist_push_back (& stream->iopt_used, i); + + (*iptr) = i; + + ++stream->i_slots_used; + + return 0; +} + +/* A copy is about to be emitted that extends backwards to POS, + * therefore it may completely cover some existing instructions in the + * buffer. If an instruction is completely covered by this new match, + * erase it. If the new instruction is covered by the previous one, + * return 1 to skip it. */ +static void +xd3_iopt_erase (xd3_stream *stream, usize_t pos, usize_t size) +{ + while (! xd3_rlist_empty (& stream->iopt_used)) + { + xd3_rinst *r = xd3_rlist_back (& stream->iopt_used); + + /* Verify that greedy is working. The previous instruction + * should end before the new one begins. */ + XD3_ASSERT ((stream->flags & XD3_BEGREEDY) == 0 || (r->pos + r->size <= pos)); + /* Verify that min_match is working. The previous instruction + * should end before the new one ends. */ + XD3_ASSERT ((stream->flags & XD3_BEGREEDY) != 0 || (r->pos + r->size < pos + size)); + + /* See if the last instruction starts before the new + * instruction. If so, there is nothing to erase. */ + if (r->pos < pos) + { + return; + } + + /* Otherwise, the new instruction covers the old one, delete it + and repeat. */ + xd3_rlist_remove (r); + xd3_rlist_push_back (& stream->iopt_free, r); + --stream->i_slots_used; + } +} + +/* This function tells the last matched input position. */ +static usize_t +xd3_iopt_last_matched (xd3_stream *stream) +{ + xd3_rinst *r; + + if (xd3_rlist_empty (& stream->iopt_used)) + { + return 0; + } + + r = xd3_rlist_back (& stream->iopt_used); + + return r->pos + r->size; +} + +/********************************************************* + Emit routines + ***********************************************************/ + +static int +xd3_emit_single (xd3_stream *stream, xd3_rinst *single, usize_t code) +{ + int has_size = stream->code_table[code].size1 == 0; + int ret; + + IF_DEBUG2 (DP(RINT "[emit1] %u %s (%u) code %u\n", + single->pos, + xd3_rtype_to_string ((xd3_rtype) single->type, 0), + single->size, + code)); + + if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) + { + return ret; + } + + if (has_size) + { + if ((ret = xd3_emit_size (stream, & INST_TAIL (stream), single->size))) + { + return ret; + } + } + + return 0; +} + +static int +xd3_emit_double (xd3_stream *stream, xd3_rinst *first, + xd3_rinst *second, usize_t code) +{ + int ret; + + /* All double instructions use fixed sizes, so all we need to do is + * output the instruction code, no sizes. */ + XD3_ASSERT (stream->code_table[code].size1 != 0 && + stream->code_table[code].size2 != 0); + + if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) + { + return ret; + } + + IF_DEBUG2 (DP(RINT "[emit2]: %u %s (%u) %s (%u) code %u\n", + first->pos, + xd3_rtype_to_string ((xd3_rtype) first->type, 0), + first->size, + xd3_rtype_to_string ((xd3_rtype) second->type, 0), + second->size, + code)); + + return 0; +} + +/* This enters a potential run instruction into the iopt buffer. The + * position argument is relative to the target window. */ +static int +xd3_emit_run (xd3_stream *stream, usize_t pos, usize_t size, uint8_t *run_c) +{ + xd3_rinst* ri; + int ret; + + if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } + + ri->type = XD3_RUN; + ri->xtra = *run_c; + ri->pos = pos; + ri->size = size; + + return 0; +} + +/* This enters a potential copy instruction into the iopt buffer. The + * position argument is relative to the target window.. */ +int +xd3_found_match (xd3_stream *stream, usize_t pos, + usize_t size, xoff_t addr, int is_source) +{ + xd3_rinst* ri; + int ret; + + if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } + + ri->type = XD3_CPY; + ri->xtra = is_source; + ri->pos = pos; + ri->size = size; + ri->addr = addr; + + return 0; +} + +static int +xd3_emit_hdr (xd3_stream *stream) +{ + int ret; + int use_secondary = stream->sec_type != NULL; + int use_adler32 = stream->flags & (XD3_ADLER32 | XD3_ADLER32_RECODE); + int vcd_source = xd3_encoder_used_source (stream); + usize_t win_ind = 0; + usize_t del_ind = 0; + usize_t enc_len; + usize_t tgt_len; + usize_t data_len; + usize_t inst_len; + usize_t addr_len; + + if (stream->current_window == 0) + { + usize_t hdr_ind = 0; + int use_appheader = stream->enc_appheader != NULL; + + if (use_secondary) { hdr_ind |= VCD_SECONDARY; } + if (use_appheader) { hdr_ind |= VCD_APPHEADER; } + + if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), + VCDIFF_MAGIC1)) != 0 || + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), + VCDIFF_MAGIC2)) != 0 || + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), + VCDIFF_MAGIC3)) != 0 || + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), + VCDIFF_VERSION)) != 0 || + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), hdr_ind)) != 0) + { + return ret; + } + + /* Secondary compressor ID */ +#if SECONDARY_ANY + if (use_secondary && + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), + stream->sec_type->id))) + { + return ret; + } +#endif + + /* Application header */ + if (use_appheader) + { + if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), + stream->enc_appheadsz)) || + (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), + stream->enc_appheader, + stream->enc_appheadsz))) + { + return ret; + } + } + } + + /* try to compress this window */ +#if SECONDARY_ANY + if (use_secondary) + { + int data_sec = 0; + int inst_sec = 0; + int addr_sec = 0; + +# define ENCODE_SECONDARY_SECTION(UPPER,LOWER) \ + ((stream->flags & XD3_SEC_NO ## UPPER) == 0 && \ + (ret = xd3_encode_secondary (stream, \ + & UPPER ## _HEAD (stream), \ + & UPPER ## _TAIL (stream), \ + & xd3_sec_ ## LOWER (stream), \ + & stream->sec_ ## LOWER, \ + & LOWER ## _sec))) + + if (ENCODE_SECONDARY_SECTION (DATA, data) || + ENCODE_SECONDARY_SECTION (INST, inst) || + ENCODE_SECONDARY_SECTION (ADDR, addr)) + { + return ret; + } + + del_ind |= (data_sec ? VCD_DATACOMP : 0); + del_ind |= (inst_sec ? VCD_INSTCOMP : 0); + del_ind |= (addr_sec ? VCD_ADDRCOMP : 0); + } +#endif + + /* if (vcd_target) { win_ind |= VCD_TARGET; } */ + if (vcd_source) { win_ind |= VCD_SOURCE; } + if (use_adler32) { win_ind |= VCD_ADLER32; } + + /* window indicator */ + if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), win_ind))) + { + return ret; + } + + /* source window */ + if (vcd_source) + { + /* or (vcd_target) { ... } */ + if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), + stream->src->srclen)) || + (ret = xd3_emit_offset (stream, & HDR_TAIL (stream), + stream->src->srcbase))) { return ret; } + } + + tgt_len = stream->avail_in; + data_len = xd3_sizeof_output (DATA_HEAD (stream)); + inst_len = xd3_sizeof_output (INST_HEAD (stream)); + addr_len = xd3_sizeof_output (ADDR_HEAD (stream)); + + /* The enc_len field is a redundency for future extensions.*/ + enc_len = (1 + (xd3_sizeof_size (tgt_len) + + xd3_sizeof_size (data_len) + + xd3_sizeof_size (inst_len) + + xd3_sizeof_size (addr_len)) + + data_len + + inst_len + + addr_len + + (use_adler32 ? 4 : 0)); + + if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), enc_len)) || + (ret = xd3_emit_size (stream, & HDR_TAIL (stream), tgt_len)) || + (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), del_ind)) || + (ret = xd3_emit_size (stream, & HDR_TAIL (stream), data_len)) || + (ret = xd3_emit_size (stream, & HDR_TAIL (stream), inst_len)) || + (ret = xd3_emit_size (stream, & HDR_TAIL (stream), addr_len))) + { + return ret; + } + + if (use_adler32) + { + uint8_t send[4]; + uint32_t a32; + + if (stream->flags & XD3_ADLER32) + { + a32 = adler32 (1L, stream->next_in, stream->avail_in); + } + else + { + a32 = stream->recode_adler32; + } + + /* Four bytes. */ + send[0] = (uint8_t) (a32 >> 24); + send[1] = (uint8_t) (a32 >> 16); + send[2] = (uint8_t) (a32 >> 8); + send[3] = (uint8_t) (a32 & 0x000000FFU); + + if ((ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), send, 4))) + { + return ret; + } + } + + return 0; +} + +/**************************************************************** + Encode routines + ****************************************************************/ + +static int +xd3_encode_buffer_leftover (xd3_stream *stream) +{ + usize_t take; + usize_t room; + + /* Allocate the buffer. */ + if (stream->buf_in == NULL && + (stream->buf_in = (uint8_t*) xd3_alloc (stream, stream->winsize, 1)) == NULL) + { + return ENOMEM; + } + + IF_DEBUG2 (DP(RINT "[leftover] flush?=%s\n", (stream->flags & XD3_FLUSH) ? "yes" : "no")); + + /* Take leftover input first. */ + if (stream->buf_leftover != NULL) + { + XD3_ASSERT (stream->buf_avail == 0); + XD3_ASSERT (stream->buf_leftavail < stream->winsize); + + IF_DEBUG2 (DP(RINT "[leftover] previous %u avail %u\n", stream->buf_leftavail, stream->avail_in)); + + memcpy (stream->buf_in, stream->buf_leftover, stream->buf_leftavail); + + stream->buf_leftover = NULL; + stream->buf_avail = stream->buf_leftavail; + } + + /* Copy into the buffer. */ + room = stream->winsize - stream->buf_avail; + take = xd3_min (room, stream->avail_in); + + memcpy (stream->buf_in + stream->buf_avail, stream->next_in, take); + + stream->buf_avail += take; + + if (take < stream->avail_in) + { + /* Buffer is full */ + stream->buf_leftover = stream->next_in + take; + stream->buf_leftavail = stream->avail_in - take; + } + else if ((stream->buf_avail < stream->winsize) && !(stream->flags & XD3_FLUSH)) + { + /* Buffer has space */ + IF_DEBUG2 (DP(RINT "[leftover] emptied %u\n", take)); + return XD3_INPUT; + } + + /* Use the buffer: */ + IF_DEBUG2 (DP(RINT "[leftover] take %u remaining %u\n", take, stream->buf_leftavail)); + stream->next_in = stream->buf_in; + stream->avail_in = stream->buf_avail; + stream->buf_avail = 0; + + return 0; +} + +/* Allocates one block of xd3_rlist elements */ +static int +xd3_alloc_iopt (xd3_stream *stream, usize_t elts) +{ + usize_t i; + xd3_iopt_buflist* last = + (xd3_iopt_buflist*) xd3_alloc (stream, sizeof (xd3_iopt_buflist), 1); + + if (last == NULL || + (last->buffer = (xd3_rinst*) xd3_alloc (stream, sizeof (xd3_rinst), elts)) == NULL) + { + return ENOMEM; + } + + last->next = stream->iopt_alloc; + stream->iopt_alloc = last; + + for (i = 0; i < elts; i += 1) + { + xd3_rlist_push_back (& stream->iopt_free, & last->buffer[i]); + } + + return 0; +} + +/* This function allocates all memory initially used by the encoder. */ +static int +xd3_encode_init (xd3_stream *stream, int full_init) +{ + int i; + + if (full_init) + { + int large_comp = (stream->src != NULL); + int small_comp = ! (stream->flags & XD3_NOCOMPRESS); + + /* Memory allocations for checksum tables are delayed until + * xd3_string_match_init in the first call to string_match--that way + * identical or short inputs require no table allocation. */ + if (large_comp) + { + usize_t hash_values = stream->src->max_winsize / + stream->smatcher.large_step; + + xd3_size_hashtable (stream, + hash_values, + & stream->large_hash); + } + + if (small_comp) + { + /* TODO: This is under devel: used to have min (sprevsz) here, which sort + * of makes sense, but observed fast performance w/ larger tables, which + * also sort of makes sense. @@@ */ + usize_t hash_values = stream->winsize; + + xd3_size_hashtable (stream, + hash_values, + & stream->small_hash); + } + } + + /* data buffers */ + for (i = 0; i < ENC_SECTS; i += 1) + { + if ((stream->enc_heads[i] = + stream->enc_tails[i] = + xd3_alloc_output (stream, NULL)) == NULL) + { + return ENOMEM; + } + } + + /* iopt buffer */ + xd3_rlist_init (& stream->iopt_used); + xd3_rlist_init (& stream->iopt_free); + + if (xd3_alloc_iopt (stream, stream->iopt_size) != 0) { goto fail; } + + XD3_ASSERT (xd3_rlist_length (& stream->iopt_free) == stream->iopt_size); + XD3_ASSERT (xd3_rlist_length (& stream->iopt_used) == 0); + + /* address cache, code table */ + stream->acache.s_near = stream->code_table_desc->near_modes; + stream->acache.s_same = stream->code_table_desc->same_modes; + stream->code_table = stream->code_table_func (); + + return xd3_alloc_cache (stream); + + fail: + + return ENOMEM; +} + +int +xd3_encode_init_full (xd3_stream *stream) +{ + return xd3_encode_init (stream, 1); +} + +int +xd3_encode_init_partial (xd3_stream *stream) +{ + return xd3_encode_init (stream, 0); +} + +/* Called after the ENC_POSTOUT state, this puts the output buffers + * back into separate lists and re-initializes some variables. (The + * output lists were spliced together during the ENC_FLUSH state.) */ +static void +xd3_encode_reset (xd3_stream *stream) +{ + int i; + xd3_output *olist; + + stream->avail_in = 0; + stream->small_reset = 1; + stream->i_slots_used = 0; + + if (stream->src != NULL) + { + stream->src->srcbase = 0; + stream->src->srclen = 0; + stream->srcwin_decided = 0; + stream->srcwin_decided_early = 0; + stream->match_minaddr = 0; + stream->match_maxaddr = 0; + stream->taroff = 0; + } + + /* Reset output chains. */ + olist = stream->enc_heads[0]; + + for (i = 0; i < ENC_SECTS; i += 1) + { + XD3_ASSERT (olist != NULL); + + stream->enc_heads[i] = olist; + stream->enc_tails[i] = olist; + olist = olist->next_page; + + stream->enc_heads[i]->next = 0; + stream->enc_heads[i]->next_page = NULL; + + stream->enc_tails[i]->next_page = NULL; + stream->enc_tails[i] = stream->enc_heads[i]; + } + + xd3_freelist_output (stream, olist); +} + +/* The main encoding routine. */ +int +xd3_encode_input (xd3_stream *stream) +{ + int ret, i; + + if (stream->dec_state != 0) + { + stream->msg = "encoder/decoder transition"; + return XD3_INTERNAL; + } + + switch (stream->enc_state) + { + case ENC_INIT: + /* Only reached on first time through: memory setup. */ + if ((ret = xd3_encode_init_full (stream))) { return ret; } + + stream->enc_state = ENC_INPUT; + + case ENC_INPUT: + + /* If there is no input yet, just return. This checks for + * next_in == NULL, not avail_in == 0 since zero bytes is a + * valid input. There is an assertion in xd3_avail_input() that + * next_in != NULL for this reason. By returning right away we + * avoid creating an input buffer before the caller has supplied + * its first data. It is possible for xd3_avail_input to be + * called both before and after the first call to + * xd3_encode_input(). */ + if (stream->next_in == NULL) + { + return XD3_INPUT; + } + + enc_flush: + /* See if we should buffer the input: either if there is already + * a leftover buffer, or if the input is short of winsize + * without flush. The label at this point is reached by a goto + * below, when there is leftover input after postout. */ + if ((stream->buf_leftover != NULL) || + (stream->buf_avail != 0) || + (stream->avail_in < stream->winsize && ! (stream->flags & XD3_FLUSH))) + { + if ((ret = xd3_encode_buffer_leftover (stream))) { return ret; } + } + + /* Initalize the address cache before each window. */ + xd3_init_cache (& stream->acache); + + stream->input_position = 0; + stream->min_match = MIN_MATCH; + stream->unencoded_offset = 0; + + stream->enc_state = ENC_SEARCH; + + IF_DEBUG2 (DP(RINT "[WINSTART:%"Q"u] input bytes %u offset %"Q"u\n", + stream->current_window, stream->avail_in, + stream->total_in)); + return XD3_WINSTART; + + case ENC_SEARCH: + IF_DEBUG2 (DP(RINT "[SEARCH] match_state %d avail_in %u %s\n", + stream->match_state, stream->avail_in, + stream->src ? "source" : "no source")); + + /* Reentrant matching. */ + if (stream->src != NULL) + { + switch (stream->match_state) + { + case MATCH_TARGET: + /* Try matching forward at the start of the target. + * This is entered the first time through, to check for + * a perfect match, and whenever there is a source match + * that extends to the end of the previous window. The + * match_srcpos field is initially zero and later set + * during xd3_source_extend_match. */ + + if (stream->avail_in > 0) + { + /* This call can't fail because the source window is + * unrestricted. */ + ret = xd3_source_match_setup (stream, stream->match_srcpos); + XD3_ASSERT (ret == 0); + stream->match_state = MATCH_FORWARD; + } + else + { + stream->match_state = MATCH_SEARCHING; + stream->match_fwd = 0; + } + XD3_ASSERT (stream->match_fwd == 0); + + case MATCH_FORWARD: + case MATCH_BACKWARD: + if (stream->avail_in != 0) + { + if ((ret = xd3_source_extend_match (stream)) != 0) + { + return ret; + } + + /* The search has to make forward progress here + * or else it can get stuck in a match-backward + * (getsrcblk) then match-forward (getsrcblk), + * find insufficient match length, then repeat + + * exactly the same search. + */ + stream->input_position += stream->match_fwd; + } + + case MATCH_SEARCHING: + /* Continue string matching. (It's possible that the + * initial match continued through the entire input, in + * which case we're still in MATCH_FORWARD and should + * remain so for the next input window.) */ + break; + } + } + + /* String matching... */ + if (stream->avail_in != 0 && + (ret = stream->smatcher.string_match (stream))) + { + return ret; + } + + stream->enc_state = ENC_INSTR; + + case ENC_INSTR: + /* Note: Jump here to encode VCDIFF deltas w/o using this + * string-matching code. Merging code enters here. */ + + /* Flush the instrution buffer, then possibly add one more + * instruction, then emit the header. */ + if ((ret = xd3_iopt_flush_instructions (stream, 1)) || + (ret = xd3_iopt_add_finalize (stream))) + { + return ret; + } + + stream->enc_state = ENC_FLUSH; + + case ENC_FLUSH: + /* Note: main_recode_func() bypasses string-matching by setting + * ENC_FLUSH. */ + if ((ret = xd3_emit_hdr (stream))) + { + return ret; + } + + /* Begin output. */ + stream->enc_current = HDR_HEAD (stream); + + /* Chain all the outputs together. After doing this, it looks + * as if there is only one section. The other enc_heads are set + * to NULL to avoid freeing them more than once. */ + for (i = 1; i < ENC_SECTS; i += 1) + { + stream->enc_tails[i-1]->next_page = stream->enc_heads[i]; + stream->enc_heads[i] = NULL; + } + + enc_output: + + stream->enc_state = ENC_POSTOUT; + stream->next_out = stream->enc_current->base; + stream->avail_out = stream->enc_current->next; + stream->total_out += (xoff_t) stream->avail_out; + + /* If there is any output in this buffer, return it, otherwise + * fall through to handle the next buffer or finish the window + * after all buffers have been output. */ + if (stream->avail_out > 0) + { + /* This is the only place xd3_encode returns XD3_OUTPUT */ + return XD3_OUTPUT; + } + + case ENC_POSTOUT: + + if (stream->avail_out != 0) + { + stream->msg = "missed call to consume output"; + return XD3_INTERNAL; + } + + /* Continue outputting one buffer at a time, until the next is NULL. */ + if ((stream->enc_current = stream->enc_current->next_page) != NULL) + { + goto enc_output; + } + + stream->total_in += (xoff_t) stream->avail_in; + stream->enc_state = ENC_POSTWIN; + + IF_DEBUG2 (DP(RINT "[WINFINISH:%"Q"u] in=%"Q"u\n", + stream->current_window, + stream->total_in)); + return XD3_WINFINISH; + + case ENC_POSTWIN: + + xd3_encode_reset (stream); + + stream->current_window += 1; + stream->enc_state = ENC_INPUT; + + /* If there is leftover input to flush, repeat. */ + if (stream->buf_leftover != NULL) + { + goto enc_flush; + } + + /* Ready for more input. */ + return XD3_INPUT; + + default: + stream->msg = "invalid state"; + return XD3_INTERNAL; + } +} +#endif /* XD3_ENCODER */ + +/***************************************************************** + Client convenience functions + ******************************************************************/ + +int +xd3_process_stream (int is_encode, + xd3_stream *stream, + int (*func) (xd3_stream *), + int close_stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max) +{ + usize_t ipos = 0; + usize_t n = xd3_min (stream->winsize, input_size); + + (*output_size) = 0; + + stream->flags |= XD3_FLUSH; + + xd3_avail_input (stream, input + ipos, n); + ipos += n; + + for (;;) + { + int ret; + switch ((ret = func (stream))) + { + case XD3_OUTPUT: { /* memcpy below */ break; } + case XD3_INPUT: { + n = xd3_min(stream->winsize, input_size - ipos); + if (n == 0) + { + goto done; + } + xd3_avail_input (stream, input + ipos, n); + ipos += n; + continue; + } + case XD3_GOTHEADER: { /* ignore */ continue; } + case XD3_WINSTART: { /* ignore */ continue; } + case XD3_WINFINISH: { /* ignore */ continue; } + case XD3_GETSRCBLK: + { + /* When the getblk function is NULL, it is necessary to + * provide the complete source as a single block using + * xd3_set_source_and_size, otherwise this error. The + * library should never ask for another source block. */ + stream->msg = "library requested source block"; + return XD3_INTERNAL; + } + case 0: + { + /* xd3_encode_input/xd3_decode_input never return 0 */ + stream->msg = "invalid return: 0"; + return XD3_INTERNAL; + } + default: + return ret; + } + + if (*output_size + stream->avail_out > output_size_max) + { + stream->msg = "insufficient output space"; + return ENOSPC; + } + + memcpy (output + *output_size, stream->next_out, stream->avail_out); + + *output_size += stream->avail_out; + + xd3_consume_output (stream); + } + done: + return (close_stream == 0) ? 0 : xd3_close_stream (stream); +} + +static int +xd3_process_memory (int is_encode, + int (*func) (xd3_stream *), + const uint8_t *input, + usize_t input_size, + const uint8_t *source, + usize_t source_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max, + int flags) { + xd3_stream stream; + xd3_config config; + xd3_source src; + int ret; + + memset (& stream, 0, sizeof (stream)); + memset (& config, 0, sizeof (config)); + + if (input == NULL || output == NULL) { + stream.msg = "invalid input/output buffer"; + ret = XD3_INTERNAL; + goto exit; + } + + config.flags = flags; + + if (is_encode) + { + config.winsize = xd3_min(input_size, (usize_t) XD3_DEFAULT_WINSIZE); + config.sprevsz = xd3_pow2_roundup (config.winsize); + } + + if ((ret = xd3_config_stream (&stream, &config)) != 0) + { + goto exit; + } + + if (source != NULL) + { + memset (& src, 0, sizeof (src)); + + src.blksize = source_size; + src.onblk = source_size; + src.curblk = source; + src.curblkno = 0; + src.max_winsize = source_size; + + if ((ret = xd3_set_source_and_size (&stream, &src, source_size)) != 0) + { + goto exit; + } + } + + if ((ret = xd3_process_stream (is_encode, + & stream, + func, 1, + input, input_size, + output, + output_size, + output_size_max)) != 0) + { + goto exit; + } + + exit: + if (ret != 0) + { + IF_DEBUG2 (DP(RINT "process_memory: %d: %s\n", ret, stream.msg)); + } + xd3_free_stream(&stream); + return ret; +} + +int +xd3_decode_stream (xd3_stream *stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max) +{ + return xd3_process_stream (0, stream, & xd3_decode_input, 1, + input, input_size, + output, output_size, output_size_max); +} + +int +xd3_decode_memory (const uint8_t *input, + usize_t input_size, + const uint8_t *source, + usize_t source_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max, + int flags) { + return xd3_process_memory (0, & xd3_decode_input, + input, input_size, + source, source_size, + output, output_size, output_size_max, + flags); +} + + +#if XD3_ENCODER +int +xd3_encode_stream (xd3_stream *stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max) +{ + return xd3_process_stream (1, stream, & xd3_encode_input, 1, + input, input_size, + output, output_size, output_size_max); +} + +int +xd3_encode_memory (const uint8_t *input, + usize_t input_size, + const uint8_t *source, + usize_t source_size, + uint8_t *output, + usize_t *output_size, + usize_t output_size_max, + int flags) { + return xd3_process_memory (1, & xd3_encode_input, + input, input_size, + source, source_size, + output, output_size, output_size_max, + flags); +} +#endif + + +/************************************************************* + String matching helpers + *************************************************************/ + +#if XD3_ENCODER +/* Do the initial xd3_string_match() checksum table setup. + * Allocations are delayed until first use to avoid allocation + * sometimes (e.g., perfect matches, zero-length inputs). */ +static int +xd3_string_match_init (xd3_stream *stream) +{ + const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); + const int DO_LARGE = (stream->src != NULL); + + if (DO_LARGE && stream->large_table == NULL) + { + if ((stream->large_table = + (usize_t*) xd3_alloc0 (stream, stream->large_hash.size, sizeof (usize_t))) == NULL) + { + return ENOMEM; + } + } + + if (DO_SMALL) + { + /* Subsequent calls can return immediately after checking reset. */ + if (stream->small_table != NULL) + { + /* The target hash table is reinitialized once per window. */ + /* TODO: This would not have to be reinitialized if absolute + * offsets were being stored. */ + if (stream->small_reset) + { + stream->small_reset = 0; + memset (stream->small_table, 0, + sizeof (usize_t) * stream->small_hash.size); + } + + return 0; + } + + if ((stream->small_table = + (usize_t*) xd3_alloc0 (stream, + stream->small_hash.size, + sizeof (usize_t))) == NULL) + { + return ENOMEM; + } + + /* If there is a previous table needed. */ + if (stream->smatcher.small_lchain > 1 || + stream->smatcher.small_chain > 1) + { + if ((stream->small_prev = + (xd3_slist*) xd3_alloc (stream, + stream->sprevsz, + sizeof (xd3_slist))) == NULL) + { + return ENOMEM; + } + } + } + + return 0; +} + +#if XD3_USE_LARGEFILE64 +/* This function handles the 32/64bit ambiguity -- file positions are 64bit + * but the hash table for source-offsets is 32bit. */ +static xoff_t +xd3_source_cksum_offset(xd3_stream *stream, usize_t low) +{ + xoff_t scp = stream->srcwin_cksum_pos; + xoff_t s0 = scp >> 32; + + usize_t sr = (usize_t) scp; + + if (s0 == 0) { + return low; + } + + /* This should not be >= because srcwin_cksum_pos is the next + * position to index. */ + if (low > sr) { + return (--s0 << 32) | low; + } + + return (s0 << 32) | low; +} +#else +static xoff_t +xd3_source_cksum_offset(xd3_stream *stream, usize_t low) +{ + return (xoff_t) low; +} +#endif + +/* This function sets up the stream->src fields srcbase, srclen. The + * call is delayed until these values are needed to encode a copy + * address. At this point the decision has to be made. */ +static int +xd3_srcwin_setup (xd3_stream *stream) +{ + xd3_source *src = stream->src; + xoff_t length, x; + + /* Check the undecided state. */ + XD3_ASSERT (src->srclen == 0 && src->srcbase == 0); + + /* Avoid repeating this call. */ + stream->srcwin_decided = 1; + + /* If the stream is flushing, then the iopt buffer was able to + * contain the complete encoding. If no copies were issued no + * source window is actually needed. This prevents the VCDIFF + * header from including source base/len. xd3_emit_hdr checks for + * srclen == 0. */ + if (stream->enc_state == ENC_INSTR && stream->match_maxaddr == 0) + { + goto done; + } + + /* Check for overflow, srclen is usize_t - this can't happen unless + * XD3_DEFAULT_SRCBACK and related parameters are extreme - should + * use smaller windows. */ + length = stream->match_maxaddr - stream->match_minaddr; + + x = (xoff_t) USIZE_T_MAX; + if (length > x) + { + stream->msg = "source window length overflow (not 64bit)"; + return XD3_INTERNAL; + } + + /* If ENC_INSTR, then we know the exact source window to use because + * no more copies can be issued. */ + if (stream->enc_state == ENC_INSTR) + { + src->srcbase = stream->match_minaddr; + src->srclen = (usize_t) length; + XD3_ASSERT (src->srclen); + goto done; + } + + /* Otherwise, we have to make a guess. More copies may still be + * issued, but we have to decide the source window base and length + * now. */ + /* TODO: This may not working well in practice, more testing needed. */ + src->srcbase = stream->match_minaddr; + src->srclen = xd3_max ((usize_t) length, + stream->avail_in + (stream->avail_in >> 2)); + + if (src->eof_known) + { + /* Note: if the source size is known, we must reduce srclen or + * code that expects to pass a single block w/ getblk == NULL + * will not function, as the code will return GETSRCBLK asking + * for the second block. */ + src->srclen = xd3_min (src->srclen, xd3_source_eof(src) - src->srcbase); + } + + IF_DEBUG1 (DP(RINT "[srcwin_setup_constrained] base %"Q"u len %u\n", + src->srcbase, src->srclen)); + + XD3_ASSERT (src->srclen); + done: + /* Set the taroff. This convenience variable is used even when + stream->src == NULL. */ + stream->taroff = src->srclen; + return 0; +} + +/* Sets the bounding region for a newly discovered source match, prior + * to calling xd3_source_extend_match(). This sets the match_maxfwd, + * match_maxback variables. Note: srcpos is an absolute position + * (xoff_t) but the match_maxfwd, match_maxback variables are usize_t. + * Returns 0 if the setup succeeds, or 1 if the source position lies + * outside an already-decided srcbase/srclen window. */ +static int +xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) +{ + xd3_source *const src = stream->src; + usize_t greedy_or_not; + + stream->match_maxback = 0; + stream->match_maxfwd = 0; + stream->match_back = 0; + stream->match_fwd = 0; + + /* This avoids a non-blocking endless loop caused by scanning + * backwards across a block boundary, only to find not enough + * matching bytes to beat the current min_match due to a better lazy + * target match: the re-entry to xd3_string_match() repeats the same + * long match because the input position hasn't changed. TODO: if + * ever duplicates are added to the source hash table, this logic + * won't suffice to avoid loops. See testing/regtest.cc's + * TestNonBlockingProgress test! */ + if (srcpos != 0 && srcpos == stream->match_last_srcpos) + { + IF_DEBUG2(DP(RINT "[match_setup] looping failure\n")); + goto bad; + } + + /* Implement src->max_winsize, which prevents the encoder from seeking + * back further than the LRU cache maintaining FIFO discipline, (to + * avoid seeking). */ + if (srcpos < stream->srcwin_cksum_pos && + stream->srcwin_cksum_pos - srcpos > src->max_winsize) + { + IF_DEBUG2(DP(RINT "[match_setup] rejected due to src->max_winsize " + "distance eof=%"Q"u srcpos=%"Q"u max_winsz=%"Q"u\n", + xd3_source_eof (src), + srcpos, src->max_winsize)); + goto bad; + } + + /* There are cases where the above test does not reject a match that + * will experience XD3_TOOFARBACK at the first xd3_getblk call + * because the input may have advanced up to one block beyond the + * actual EOF. */ + IF_DEBUG2(DP(RINT "[match_setup] %"Q"u srcpos %"Q"u, " + "src->max_winsize %"Q"u\n", + stream->total_in + stream->input_position, + srcpos, src->max_winsize)); + + /* Going backwards, the 1.5-pass algorithm allows some + * already-matched input may be covered by a longer source match. + * The greedy algorithm does not allow this. */ + if (stream->flags & XD3_BEGREEDY) + { + /* The greedy algorithm allows backward matching to the last + matched position. */ + greedy_or_not = xd3_iopt_last_matched (stream); + } + else + { + /* The 1.5-pass algorithm allows backward matching to go back as + * far as the unencoded offset, which is updated as instructions + * pass out of the iopt buffer. If this (default) is chosen, it + * means xd3_iopt_erase may be called to eliminate instructions + * when a covering source match is found. */ + greedy_or_not = stream->unencoded_offset; + } + + /* Backward target match limit. */ + XD3_ASSERT (stream->input_position >= greedy_or_not); + stream->match_maxback = stream->input_position - greedy_or_not; + + /* Forward target match limit. */ + XD3_ASSERT (stream->avail_in > stream->input_position); + stream->match_maxfwd = stream->avail_in - stream->input_position; + + /* Now we take the source position into account. It depends whether + * the srclen/srcbase have been decided yet. */ + if (stream->srcwin_decided == 0) + { + /* Unrestricted case: the match can cover the entire source, + * 0--src->size. We compare the usize_t + * match_maxfwd/match_maxback against the xoff_t + * src->size/srcpos values and take the min. */ + if (srcpos < (xoff_t) stream->match_maxback) + { + stream->match_maxback = (usize_t) srcpos; + } + + if (src->eof_known) + { + xoff_t srcavail = xd3_source_eof (src) - srcpos; + + if (srcavail < (xoff_t) stream->match_maxfwd) + { + stream->match_maxfwd = (usize_t) srcavail; + } + } + + IF_DEBUG2(DP(RINT + "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "unrestricted maxback %u maxfwd %u\n", + srcpos, + stream->total_in + stream->input_position, + stream->match_maxback, + stream->match_maxfwd)); + goto good; + } + + /* Decided some source window. */ + XD3_ASSERT (src->srclen > 0); + + /* Restricted case: fail if the srcpos lies outside the source window */ + if ((srcpos < src->srcbase) || + (srcpos > (src->srcbase + (xoff_t) src->srclen))) + { + IF_DEBUG1(DP(RINT "[match_setup] restricted source window failure\n")); + goto bad; + } + else + { + usize_t srcavail; + + srcavail = (usize_t) (srcpos - src->srcbase); + if (srcavail < stream->match_maxback) + { + stream->match_maxback = srcavail; + } + + srcavail = (usize_t) (src->srcbase + (xoff_t) src->srclen - srcpos); + if (srcavail < stream->match_maxfwd) + { + stream->match_maxfwd = srcavail; + } + + IF_DEBUG2(DP(RINT + "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " + "restricted maxback %u maxfwd %u\n", + srcpos, + stream->total_in + stream->input_position, + stream->match_maxback, + stream->match_maxfwd)); + goto good; + } + + good: + stream->match_state = MATCH_BACKWARD; + stream->match_srcpos = srcpos; + stream->match_last_srcpos = srcpos; + return 0; + + bad: + stream->match_state = MATCH_SEARCHING; + stream->match_last_srcpos = srcpos; + return 1; +} + +static inline int +xd3_forward_match(const uint8_t *s1c, const uint8_t *s2c, int n) +{ + int i = 0; +#if UNALIGNED_OK + int nint = n / sizeof(int); + + if (nint >> 3) + { + int j = 0; + const int *s1 = (const int*)s1c; + const int *s2 = (const int*)s2c; + int nint_8 = nint - 8; + + while (i <= nint_8 && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++] && + s1[i++] == s2[j++]) { } + + i = (i - 1) * sizeof(int); + } +#endif + + while (i < n && s1c[i] == s2c[i]) + { + i++; + } + return i; +} + +/* This function expands the source match backward and forward. It is + * reentrant, since xd3_getblk may return XD3_GETSRCBLK, so most + * variables are kept in xd3_stream. There are two callers of this + * function, the string_matching routine when a checksum match is + * discovered, and xd3_encode_input whenever a continuing (or initial) + * match is suspected. The two callers do different things with the + * input_position, thus this function leaves that variable untouched. + * If a match is taken the resulting stream->match_fwd is left + * non-zero. */ +static int +xd3_source_extend_match (xd3_stream *stream) +{ + int ret; + xd3_source *const src = stream->src; + xoff_t matchoff; /* matchoff is the current right/left-boundary of + the source match being tested. */ + usize_t streamoff; /* streamoff is the current right/left-boundary + of the input match being tested. */ + xoff_t tryblk; /* tryblk, tryoff are the block, offset position + of matchoff */ + usize_t tryoff; + usize_t tryrem; /* tryrem is the number of matchable bytes */ + usize_t matched; + + XD3_ASSERT (src != NULL); + + /* Does it make sense to compute backward match AFTER forward match? */ + if (stream->match_state == MATCH_BACKWARD) + { + /* Note: this code is practically duplicated below, substituting + * match_fwd/match_back and direction. */ + matchoff = stream->match_srcpos - stream->match_back; + streamoff = stream->input_position - stream->match_back; + xd3_blksize_div (matchoff, src, &tryblk, &tryoff); + + /* this loops backward over source blocks */ + while (stream->match_back < stream->match_maxback) + { + /* see if we're backing across a source block boundary */ + if (tryoff == 0) + { + tryoff = src->blksize; + tryblk -= 1; + } + + if ((ret = xd3_getblk (stream, tryblk))) + { + if (ret == XD3_TOOFARBACK) + { + IF_DEBUG2(DP(RINT "[maxback] %"Q"u TOOFARBACK: %u INP %"Q"u CKSUM %"Q"u\n", + tryblk, stream->match_back, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos)); + + /* the starting position is too far back. */ + if (stream->match_back == 0) + { + XD3_ASSERT(stream->match_fwd == 0); + goto donefwd; + } + + /* search went too far back, continue forward. */ + goto doneback; + } + + /* could be a XD3_GETSRCBLK failure. */ + return ret; + } + + tryrem = xd3_min (tryoff, stream->match_maxback - stream->match_back); + + IF_DEBUG2(DP(RINT "[maxback] maxback %u trysrc %"Q"u/%u tgt %u tryrem %u\n", + stream->match_maxback, tryblk, tryoff, streamoff, tryrem)); + + /* TODO: This code can be optimized similar to xd3_match_forward() */ + for (; tryrem != 0; tryrem -= 1, stream->match_back += 1) + { + if (src->curblk[tryoff-1] != stream->next_in[streamoff-1]) + { + goto doneback; + } + + tryoff -= 1; + streamoff -= 1; + } + } + + doneback: + stream->match_state = MATCH_FORWARD; + } + + XD3_ASSERT (stream->match_state == MATCH_FORWARD); + + matchoff = stream->match_srcpos + stream->match_fwd; + streamoff = stream->input_position + stream->match_fwd; + xd3_blksize_div (matchoff, src, & tryblk, & tryoff); + + /* Note: practically the same code as backwards case above: same comments */ + while (stream->match_fwd < stream->match_maxfwd) + { + if (tryoff == src->blksize) + { + tryoff = 0; + tryblk += 1; + } + + if ((ret = xd3_getblk (stream, tryblk))) + { + if (ret == XD3_TOOFARBACK) + { + IF_DEBUG2(DP(RINT "[maxfwd] %"Q"u TOOFARBACK: %u INP %"Q"u CKSUM %"Q"u\n", + tryblk, stream->match_fwd, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos)); + goto donefwd; + } + + /* could be a XD3_GETSRCBLK failure. */ + return ret; + } + + tryrem = xd3_min(stream->match_maxfwd - stream->match_fwd, + src->onblk - tryoff); + + if (tryrem == 0) + { + /* Generally, this means we have a power-of-two size source + * and we just found the end-of-file, in this case it's an + * empty block. */ + XD3_ASSERT (src->onblk < src->blksize); + break; + } + + matched = xd3_forward_match(src->curblk + tryoff, + stream->next_in + streamoff, + tryrem); + tryoff += matched; + streamoff += matched; + stream->match_fwd += matched; + + if (tryrem != matched) + { + break; + } + } + + donefwd: + stream->match_state = MATCH_SEARCHING; + + IF_DEBUG2(DP(RINT "[extend match] input %"Q"u srcpos %"Q"u len %u\n", + stream->input_position + stream->total_in, + stream->match_srcpos, + stream->match_fwd)); + + /* If the match ends short of the last instruction end, we probably + * don't want it. There is the possibility that a copy ends short + * of the last copy but also goes further back, in which case we + * might want it. This code does not implement such: if so we would + * need more complicated xd3_iopt_erase logic. */ + if (stream->match_fwd < stream->min_match) + { + stream->match_fwd = 0; + } + else + { + usize_t total = stream->match_fwd + stream->match_back; + + /* Correct the variables to remove match_back from the equation. */ + usize_t target_position = stream->input_position - stream->match_back; + usize_t match_length = stream->match_back + stream->match_fwd; + xoff_t match_position = stream->match_srcpos - stream->match_back; + xoff_t match_end = stream->match_srcpos + stream->match_fwd; + + /* At this point we may have to erase any iopt-buffer + * instructions that are fully covered by a backward-extending + * copy. */ + if (stream->match_back > 0) + { + xd3_iopt_erase (stream, target_position, total); + } + + stream->match_back = 0; + + /* Update ranges. The first source match occurs with both + values set to 0. */ + if (stream->match_maxaddr == 0 || + match_position < stream->match_minaddr) + { + stream->match_minaddr = match_position; + } + + if (match_end > stream->match_maxaddr) + { + /* Note: per-window */ + stream->match_maxaddr = match_end; + } + + if (match_end > stream->maxsrcaddr) + { + /* Note: across windows */ + stream->maxsrcaddr = match_end; + } + + IF_DEBUG2 ({ + static int x = 0; + DP(RINT "[source match:%d] length %u (%s) [ %u bytes ]\n", + x++, + match_length, + stream->total_in + target_position, + stream->total_in + target_position + match_length, + match_position, + match_position + match_length, + (stream->total_in + target_position == match_position) ? "same" : "diff", + match_length); + }); + + if ((ret = xd3_found_match (stream, + /* decoder position */ target_position, + /* length */ match_length, + /* address */ match_position, + /* is_source */ 1))) + { + return ret; + } + + /* If the match ends with the available input: */ + if (target_position + match_length == stream->avail_in) + { + /* Setup continuing match for the next window. */ + stream->match_state = MATCH_TARGET; + stream->match_srcpos = match_end; + } + } + + return 0; +} + +/* Update the small hash. Values in the small_table are offset by + * HASH_CKOFFSET (1) to distinguish empty buckets from real offsets. */ +static void +xd3_scksum_insert (xd3_stream *stream, + usize_t inx, + usize_t scksum, + usize_t pos) +{ + /* If we are maintaining previous duplicates. */ + if (stream->small_prev) + { + usize_t last_pos = stream->small_table[inx]; + xd3_slist *pos_list = & stream->small_prev[pos & stream->sprevmask]; + + /* Note last_pos is offset by HASH_CKOFFSET. */ + pos_list->last_pos = last_pos; + } + + /* Enter the new position into the hash bucket. */ + stream->small_table[inx] = pos + HASH_CKOFFSET; +} + +#if XD3_DEBUG +static int +xd3_check_smatch (const uint8_t *ref0, const uint8_t *inp0, + const uint8_t *inp_max, usize_t cmp_len) +{ + usize_t i; + + for (i = 0; i < cmp_len; i += 1) + { + XD3_ASSERT (ref0[i] == inp0[i]); + } + + if (inp0 + cmp_len < inp_max) + { + XD3_ASSERT (inp0[i] != ref0[i]); + } + + return 1; +} +#endif /* XD3_DEBUG */ + +/* When the hash table indicates a possible small string match, it + * calls this routine to find the best match. The first matching + * position is taken from the small_table, HASH_CKOFFSET is subtracted + * to get the actual position. After checking that match, if previous + * linked lists are in use (because stream->smatcher.small_chain > 1), + * previous matches are tested searching for the longest match. If + * (stream->min_match > MIN_MATCH) then a lazy match is in effect. + */ +static usize_t +xd3_smatch (xd3_stream *stream, + usize_t base, + usize_t scksum, + usize_t *match_offset) +{ + usize_t cmp_len; + usize_t match_length = 0; + usize_t chain = (stream->min_match == MIN_MATCH ? + stream->smatcher.small_chain : + stream->smatcher.small_lchain); + const uint8_t *inp_max = stream->next_in + stream->avail_in; + const uint8_t *inp; + const uint8_t *ref; + + SMALL_HASH_DEBUG1 (stream, stream->next_in + stream->input_position); + + XD3_ASSERT (stream->min_match + stream->input_position <= stream->avail_in); + + base -= HASH_CKOFFSET; + + again: + + IF_DEBUG2 (DP(RINT "smatch at base=%u inp=%u cksum=%u\n", base, + stream->input_position, scksum)); + + /* For small matches, we can always go to the end-of-input because + * the matching position must be less than the input position. */ + XD3_ASSERT (base < stream->input_position); + + ref = stream->next_in + base; + inp = stream->next_in + stream->input_position; + + SMALL_HASH_DEBUG2 (stream, ref); + + /* Expand potential match forward. */ + while (inp < inp_max && *inp == *ref) + { + ++inp; + ++ref; + } + + cmp_len = (usize_t)(inp - (stream->next_in + stream->input_position)); + + /* Verify correctness */ + XD3_ASSERT (xd3_check_smatch (stream->next_in + base, + stream->next_in + stream->input_position, + inp_max, cmp_len)); + + /* Update longest match */ + if (cmp_len > match_length) + { + ( match_length) = cmp_len; + (*match_offset) = base; + + /* Stop if we match the entire input or have a long_enough match. */ + if (inp == inp_max || cmp_len >= stream->smatcher.long_enough) + { + goto done; + } + } + + /* If we have not reached the chain limit, see if there is another + previous position. */ + while (--chain != 0) + { + /* Calculate the previous offset. */ + usize_t prev_pos = stream->small_prev[base & stream->sprevmask].last_pos; + usize_t diff_pos; + + if (prev_pos == 0) + { + break; + } + + prev_pos -= HASH_CKOFFSET; + + if (prev_pos > base) + { + break; + } + + base = prev_pos; + + XD3_ASSERT (stream->input_position > base); + diff_pos = stream->input_position - base; + + /* Stop searching if we go beyond sprevsz, since those entries + * are for unrelated checksum entries. */ + if (diff_pos & ~stream->sprevmask) + { + break; + } + + goto again; + } + + done: + /* Crude efficiency test: if the match is very short and very far back, it's + * unlikely to help, but the exact calculation requires knowing the state of + * the address cache and adjacent instructions, which we can't do here. + * Rather than encode a probably inefficient copy here and check it later + * (which complicates the code a lot), do this: + */ + if (match_length == 4 && stream->input_position - (*match_offset) >= 1<<14) + { + /* It probably takes >2 bytes to encode an address >= 2^14 from here */ + return 0; + } + if (match_length == 5 && stream->input_position - (*match_offset) >= 1<<21) + { + /* It probably takes >3 bytes to encode an address >= 2^21 from here */ + return 0; + } + + /* It's unlikely that a window is large enough for the (match_length == 6 && + * address >= 2^28) check */ + return match_length; +} + +#if XD3_DEBUG +static void +xd3_verify_small_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum) +{ + uint32_t state; + uint32_t cksum = xd3_scksum (&state, inp, stream->smatcher.small_look); + + XD3_ASSERT (cksum == x_cksum); +} + +static void +xd3_verify_large_state (xd3_stream *stream, + const uint8_t *inp, + uint32_t x_cksum) +{ + uint32_t cksum = xd3_lcksum (inp, stream->smatcher.large_look); + XD3_ASSERT (cksum == x_cksum); +} +static void +xd3_verify_run_state (xd3_stream *stream, + const uint8_t *inp, + usize_t x_run_l, + uint8_t *x_run_c) +{ + usize_t slook = stream->smatcher.small_look; + uint8_t run_c; + usize_t run_l = xd3_comprun (inp, slook, &run_c); + + XD3_ASSERT (run_l == 0 || run_c == *x_run_c); + XD3_ASSERT (x_run_l > slook || run_l == x_run_l); +} +#endif /* XD3_DEBUG */ + +/* This function computes more source checksums to advance the window. + * Called at every entrance to the string-match loop and each time + * stream->input_position reaches the value returned as + * *next_move_point. NB: this is one of the most expensive functions + * in this code and also the most critical for good compression. + */ +static int +xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) +{ + /* the source file is indexed until this point */ + xoff_t target_cksum_pos; + /* the absolute target file input position */ + xoff_t absolute_input_pos; + + if (stream->src->eof_known) + { + xoff_t source_size = xd3_source_eof (stream->src); + XD3_ASSERT(stream->srcwin_cksum_pos <= source_size); + + if (stream->srcwin_cksum_pos == source_size) + { + *next_move_point = USIZE_T_MAX; + return 0; + } + } + + absolute_input_pos = stream->total_in + stream->input_position; + + /* Immediately read the entire window. + * + * Note: this reverses a long held policy, at this point in the + * code, of advancing relatively slowly as the input is read, which + * results in better compression for very-similar inputs, but worse + * compression where data is deleted near the beginning of the file. + * + * The new policy is simpler, somewhat slower and can benefit, or + * slightly worsen, compression performance. */ + if (absolute_input_pos < stream->src->max_winsize / 2) + { + target_cksum_pos = stream->src->max_winsize; + } + else + { + /* TODO: The addition of 2 blocks here is arbitrary. Do a + * better job of stream alignment based on observed source copy + * addresses, and when both input sizes are known, the + * difference in size. */ + target_cksum_pos = absolute_input_pos + + stream->src->max_winsize / 2 + + stream->src->blksize * 2; + target_cksum_pos &= ~stream->src->maskby; + } + + /* A long match may have extended past srcwin_cksum_pos. Don't + * start checksumming already-matched source data. */ + if (stream->maxsrcaddr > stream->srcwin_cksum_pos) + { + stream->srcwin_cksum_pos = stream->maxsrcaddr; + } + + if (target_cksum_pos < stream->srcwin_cksum_pos) + { + target_cksum_pos = stream->srcwin_cksum_pos; + } + + while (stream->srcwin_cksum_pos < target_cksum_pos && + (!stream->src->eof_known || + stream->srcwin_cksum_pos < xd3_source_eof (stream->src))) + { + xoff_t blkno; + xoff_t blkbaseoffset; + usize_t blkrem; + ssize_t oldpos; /* Using ssize_t because of a */ + ssize_t blkpos; /* do { blkpos-- } + while (blkpos >= oldpos); */ + int ret; + xd3_blksize_div (stream->srcwin_cksum_pos, + stream->src, &blkno, &blkrem); + oldpos = blkrem; + + if ((ret = xd3_getblk (stream, blkno))) + { + /* TOOFARBACK should never occur here, since we read forward. */ + if (ret == XD3_TOOFARBACK) + { + ret = XD3_INTERNAL; + } + + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] async getblk return for %"Q"u: %s\n", + blkno, xd3_strerror (ret))); + return ret; + } + + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] block %"Q"u T=%"Q"u S=%"Q"u L=%"Q"u EOF=%"Q"u %s\n", + blkno, + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos, + target_cksum_pos, + xd3_source_eof (stream->src), + stream->src->eof_known ? "known" : "unknown")); + + blkpos = xd3_bytes_on_srcblk (stream->src, blkno); + + if (blkpos < (ssize_t) stream->smatcher.large_look) + { + stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; + IF_DEBUG2 (DP(RINT "[srcwin_move_point] continue (end-of-block): %"Q"u\n", blkpos)); + continue; + } + + /* This inserts checksums for the entire block, in reverse, + * starting from the end of the block. This logic does not test + * stream->srcwin_cksum_pos because it always advances it to the + * start of the next block. + * + * oldpos is the srcwin_cksum_pos within this block. blkpos is + * the number of bytes available. Each iteration inspects + * large_look bytes then steps back large_step bytes. The + * if-stmt above ensures at least one large_look of data. */ + blkpos -= stream->smatcher.large_look; + blkbaseoffset = stream->src->blksize * blkno; + + do + { + uint32_t cksum = xd3_lcksum (stream->src->curblk + blkpos, + stream->smatcher.large_look); + usize_t hval = xd3_checksum_hash (& stream->large_hash, cksum); + + stream->large_table[hval] = + (usize_t) (blkbaseoffset + + (xoff_t)(blkpos + HASH_CKOFFSET)); + + IF_DEBUG (stream->large_ckcnt += 1); + + blkpos -= stream->smatcher.large_step; + } + while (blkpos >= oldpos); + + stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; + } + + if (stream->src->eof_known) + { + xoff_t source_size = xd3_source_eof (stream->src); + if (stream->srcwin_cksum_pos >= source_size) + { + /* This invariant is needed for xd3_source_cksum_offset() */ + stream->srcwin_cksum_pos = source_size; + *next_move_point = USIZE_T_MAX; + IF_DEBUG1 (DP(RINT + "[srcwin_move_point] finished with source input\n")); + return 0; + } + } + + /* How long until this function should be called again. */ + XD3_ASSERT(stream->srcwin_cksum_pos >= target_cksum_pos); + + *next_move_point = stream->input_position + + stream->src->blksize - + ((stream->srcwin_cksum_pos - target_cksum_pos) & stream->src->maskby); + + IF_DEBUG2 (DP(RINT + "[srcwin_move_point] finished T=%"Q"u " + "S=%"Q"u L=%"Q"u EOF=%"Q"u %s again in %u\n", + stream->total_in + stream->input_position, + stream->srcwin_cksum_pos, + target_cksum_pos, + xd3_source_eof (stream->src), + stream->src->eof_known ? "known" : "unknown", + *next_move_point - stream->input_position)); + + return 0; +} + +#endif /* XD3_ENCODER */ + +/******************************************************************** + TEMPLATE pass + *********************************************************************/ + +#endif /* __XDELTA3_C_INLINE_PASS__ */ +#ifdef __XDELTA3_C_TEMPLATE_PASS__ + +#if XD3_ENCODER + +/******************************************************************** + Templates + *******************************************************************/ + +/* Template macros */ +#define XD3_TEMPLATE(x) XD3_TEMPLATE2(x,TEMPLATE) +#define XD3_TEMPLATE2(x,n) XD3_TEMPLATE3(x,n) +#define XD3_TEMPLATE3(x,n) x ## n +#define XD3_STRINGIFY(x) XD3_STRINGIFY2(x) +#define XD3_STRINGIFY2(x) #x + +static int XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream); + +static const xd3_smatcher XD3_TEMPLATE(__smatcher_) = +{ + XD3_STRINGIFY(TEMPLATE), + XD3_TEMPLATE(xd3_string_match_), +#if SOFTCFG == 1 + 0, 0, 0, 0, 0, 0, 0 +#else + LLOOK, LSTEP, SLOOK, SCHAIN, SLCHAIN, MAXLAZY, LONGENOUGH +#endif +}; + +static int +XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) +{ + const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); + const int DO_LARGE = (stream->src != NULL); + const int DO_RUN = (1); + + const uint8_t *inp; + uint32_t scksum = 0; + uint32_t scksum_state = 0; + uint32_t lcksum = 0; + usize_t sinx; + usize_t linx; + uint8_t run_c; + usize_t run_l; + int ret; + usize_t match_length; + usize_t match_offset = 0; + usize_t next_move_point; + + IF_DEBUG2(DP(RINT "[string_match] initial entry %u\n", stream->input_position)); + + /* If there will be no compression due to settings or short input, + * skip it entirely. */ + if (! (DO_SMALL || DO_LARGE || DO_RUN) || + stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } + + if ((ret = xd3_string_match_init (stream))) { return ret; } + + /* The restartloop label is reached when the incremental loop state + * needs to be reset. */ + restartloop: + + IF_DEBUG2(DP(RINT "[string_match] restartloop %u\n", stream->input_position)); + + /* If there is not enough input remaining for any kind of match, + skip it. */ + if (stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } + + /* Now reset the incremental loop state: */ + + /* The min_match variable is updated to avoid matching the same lazy + * match over and over again. For example, if you find a (small) + * match of length 9 at one position, you will likely find a match + * of length 8 at the next position. */ + if (xd3_iopt_last_matched (stream) > stream->input_position) + { + stream->min_match = xd3_max (MIN_MATCH, + 1 + xd3_iopt_last_matched(stream) - + stream->input_position); + } + else + { + stream->min_match = MIN_MATCH; + } + + /* The current input byte. */ + inp = stream->next_in + stream->input_position; + + /* Small match state. */ + if (DO_SMALL) + { + scksum = xd3_scksum (&scksum_state, inp, SLOOK); + } + + /* Run state. */ + if (DO_RUN) + { + run_l = xd3_comprun (inp, SLOOK, & run_c); + } + + /* Large match state. We continue the loop even after not enough + * bytes for LLOOK remain, so always check stream->input_position in + * DO_LARGE code. */ + if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) + { + /* Source window: next_move_point is the point that + * stream->input_position must reach before computing more + * source checksum. Note: this is called unconditionally + * the first time after reentry, subsequent calls will be + * avoided if next_move_point is > input_position */ + if ((ret = xd3_srcwin_move_point (stream, & next_move_point))) + { + return ret; + } + + lcksum = xd3_lcksum (inp, LLOOK); + } + + /* TRYLAZYLEN: True if a certain length match should be followed by + * lazy search. This checks that LEN is shorter than MAXLAZY and + * that there is enough leftover data to consider lazy matching. + * "Enough" is set to 2 since the next match will start at the next + * offset, it must match two extra characters. */ +#define TRYLAZYLEN(LEN,POS,MAX) ((MAXLAZY) > 0 && (LEN) < (MAXLAZY) \ + && (POS) + (LEN) <= (MAX) - 2) + + /* HANDLELAZY: This statement is called each time an instruciton is + * emitted (three cases). If the instruction is large enough, the + * loop is restarted, otherwise lazy matching may ensue. */ +#define HANDLELAZY(mlen) \ + if (TRYLAZYLEN ((mlen), (stream->input_position), (stream->avail_in))) \ + { stream->min_match = (mlen) + LEAST_MATCH_INCR; goto updateone; } \ + else \ + { stream->input_position += (mlen); goto restartloop; } + + /* Now loop over one input byte at a time until a match is found... */ + for (;; inp += 1, stream->input_position += 1) + { + /* Now we try three kinds of string match in order of expense: + * run, large match, small match. */ + + /* Expand the start of a RUN. The test for (run_l == SLOOK) + * avoids repeating this check when we pass through a run area + * performing lazy matching. The run is only expanded once when + * the min_match is first reached. If lazy matching is + * performed, the run_l variable will remain inconsistent until + * the first non-running input character is reached, at which + * time the run_l may then again grow to SLOOK. */ + if (DO_RUN && run_l == SLOOK) + { + usize_t max_len = stream->avail_in - stream->input_position; + + IF_DEBUG (xd3_verify_run_state (stream, inp, run_l, &run_c)); + + while (run_l < max_len && inp[run_l] == run_c) { run_l += 1; } + + /* Output a RUN instruction. */ + if (run_l >= stream->min_match && run_l >= MIN_RUN) + { + if ((ret = xd3_emit_run (stream, stream->input_position, + run_l, &run_c))) { return ret; } + + HANDLELAZY (run_l); + } + } + + /* If there is enough input remaining. */ + if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) + { + if ((stream->input_position >= next_move_point) && + (ret = xd3_srcwin_move_point (stream, & next_move_point))) + { + return ret; + } + + linx = xd3_checksum_hash (& stream->large_hash, lcksum); + + IF_DEBUG (xd3_verify_large_state (stream, inp, lcksum)); + + if (stream->large_table[linx] != 0) + { + /* the match_setup will fail if the source window has + * been decided and the match lies outside it. + * OPT: Consider forcing a window at this point to + * permit a new source window. */ + xoff_t adj_offset = + xd3_source_cksum_offset(stream, + stream->large_table[linx] - + HASH_CKOFFSET); + if (xd3_source_match_setup (stream, adj_offset) == 0) + { + if ((ret = xd3_source_extend_match (stream))) + { + return ret; + } + + /* Update stream position. match_fwd is zero if no + * match. */ + if (stream->match_fwd > 0) + { + HANDLELAZY (stream->match_fwd); + } + } + } + } + + /* Small matches. */ + if (DO_SMALL) + { + sinx = xd3_checksum_hash (& stream->small_hash, scksum); + + /* Verify incremental state in debugging mode. */ + IF_DEBUG (xd3_verify_small_state (stream, inp, scksum)); + + /* Search for the longest match */ + if (stream->small_table[sinx] != 0) + { + match_length = xd3_smatch (stream, + stream->small_table[sinx], + scksum, + & match_offset); + } + else + { + match_length = 0; + } + + /* Insert a hash for this string. */ + xd3_scksum_insert (stream, sinx, scksum, stream->input_position); + + /* Maybe output a COPY instruction */ + if (match_length >= stream->min_match) + { + IF_DEBUG2 ({ + static int x = 0; + DP(RINT "[target match:%d] " + "(-%d) [ %u bytes ]\n", + x++, + stream->input_position, + stream->input_position + match_length, + match_offset, + match_offset + match_length, + stream->input_position - match_offset, + match_length); + }); + + if ((ret = xd3_found_match (stream, + /* decoder position */ + stream->input_position, + /* length */ match_length, + /* address */ (xoff_t) match_offset, + /* is_source */ 0))) + { + return ret; + } + + /* Copy instruction. */ + HANDLELAZY (match_length); + } + } + + /* The logic above prevents excess work during lazy matching by + * increasing min_match to avoid smaller matches. Each time we + * advance stream->input_position by one, the minimum match + * shortens as well. */ + if (stream->min_match > MIN_MATCH) + { + stream->min_match -= 1; + } + + updateone: + + /* See if there are no more incremental cksums to compute. */ + if (stream->input_position + SLOOK == stream->avail_in) + { + goto loopnomore; + } + + /* Compute next RUN, CKSUM */ + if (DO_RUN) + { + NEXTRUN (inp[SLOOK]); + } + + if (DO_SMALL) + { + scksum = xd3_small_cksum_update (&scksum_state, inp, SLOOK); + } + + if (DO_LARGE && (stream->input_position + LLOOK < stream->avail_in)) + { + lcksum = xd3_large_cksum_update (lcksum, inp, LLOOK); + } + } + + loopnomore: + return 0; +} + +#endif /* XD3_ENCODER */ +#endif /* __XDELTA3_C_TEMPLATE_PASS__ */ diff --git a/app/src/main/rust/stores/vendor/xdelta3/xdelta3.h b/app/src/main/rust/stores/vendor/xdelta3/xdelta3.h new file mode 100644 index 0000000000..9fbda6fa28 --- /dev/null +++ b/app/src/main/rust/stores/vendor/xdelta3/xdelta3.h @@ -0,0 +1,1396 @@ +/* xdelta3 - delta compression tools and library + Copyright 2016 Joshua MacDonald + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* To know more about Xdelta, start by reading xdelta3.c. If you are + * ready to use the API, continue reading here. There are two + * interfaces -- xd3_encode_input and xd3_decode_input -- plus a dozen + * or so related calls. This interface is styled after Zlib. */ + +#ifndef _XDELTA3_H_ +#define _XDELTA3_H_ + +#define _POSIX_SOURCE +#define _ISOC99_SOURCE +#define _C99_SOURCE + +#if HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +/****************************************************************/ + +/* Default configured value of stream->winsize. If the program + * supplies xd3_encode_input() with data smaller than winsize the + * stream will automatically buffer the input, otherwise the input + * buffer is used directly. + */ +#ifndef XD3_DEFAULT_WINSIZE +#define XD3_DEFAULT_WINSIZE (1U << 23) +#endif + +/* Default total size of the source window used in xdelta3-main.h */ +#ifndef XD3_DEFAULT_SRCWINSZ +#define XD3_DEFAULT_SRCWINSZ (1U << 26) +#endif + +/* When Xdelta requests a memory allocation for certain buffers, it + * rounds up to units of at least this size. The code assumes (and + * asserts) that this is a power-of-two. */ +#ifndef XD3_ALLOCSIZE +#define XD3_ALLOCSIZE (1U<<14) +#endif + +/* The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect + * decoders against malicious files. The decoder will never decode a + * window larger than this. If the file specifies VCD_TARGET the + * decoder may require two buffers of this size. + * + * 8-16MB is reasonable, probably don't need to go larger. */ +#ifndef XD3_HARDMAXWINSIZE +#define XD3_HARDMAXWINSIZE (1U<<24) +#endif +/* The IOPT_SIZE value sets the size of a buffer used to batch + * overlapping copy instructions before they are optimized by picking + * the best non-overlapping ranges. The larger this buffer, the + * longer a forced xd3_srcwin_setup() decision is held off. Setting + * this value to 0 causes an unlimited buffer to be used. */ +#ifndef XD3_DEFAULT_IOPT_SIZE +#define XD3_DEFAULT_IOPT_SIZE (1U<<15) +#endif + +/* The maximum distance backward to search for small matches */ +#ifndef XD3_DEFAULT_SPREVSZ +#define XD3_DEFAULT_SPREVSZ (1U<<18) +#endif + +/* The default compression level + */ +#ifndef XD3_DEFAULT_LEVEL +#define XD3_DEFAULT_LEVEL 3 +#endif + +#ifndef XD3_DEFAULT_SECONDARY_LEVEL +#define XD3_DEFAULT_SECONDARY_LEVEL 6 +#endif + +#ifndef XD3_USE_LARGEFILE64 +#define XD3_USE_LARGEFILE64 1 +#endif + +/* Sizes and addresses within VCDIFF windows are represented as usize_t + * + * For source-file offsets and total file sizes, total input and + * output counts, the xoff_t type is used. The decoder and encoder + * generally check for overflow of the xoff_t size (this is tested at + * the 32bit boundary [xdelta3-test.h]). + */ +#ifndef _WIN32 +#include +#else /* WIN32 case */ +#define WIN32_LEAN_AND_MEAN + +#ifndef WINVER +#if XD3_USE_LARGEFILE64 +/* 64 bit file offsets: uses GetFileSizeEx and SetFilePointerEx. + * requires Win2000 or newer version of WinNT */ +#define WINVER 0x0500 +#define _WIN32_WINNT 0x0500 +#else /* xoff_t is 32bit */ +/* 32 bit (DWORD) file offsets: uses GetFileSize and + * SetFilePointer. compatible with win9x-me and WinNT4 */ +#define WINVER 0x0400 +#define _WIN32_WINNT 0x0400 +#endif /* if XD3_USE_LARGEFILE64 */ +#endif /* ifndef WINVER */ + +#include + +/* _MSV_VER is defined by Microsoft tools, not by mingw32 */ +#ifdef _MSC_VER +/*#define inline*/ +typedef signed int ssize_t; +#if _MSC_VER < 1600 +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned long uint32_t; +typedef ULONGLONG uint64_t; +#else /* _MSC_VER >= 1600 */ +/* For MSVC10 and above */ +#include +#endif /* _MSC_VER < 1600 */ +#else /* _MSC_VER not defined */ +/* mingw32, lcc and watcom provide a proper header */ +#include +#endif /* _MSC_VER defined */ +#endif /* _WIN32 defined */ + +typedef uint32_t usize_t; + +#if XD3_USE_LARGEFILE64 +/* xoff_t is a 64-bit type */ +#define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */ + +#ifndef _LARGEFILE_SOURCE +#define _LARGEFILE_SOURCE +#endif + +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 +#endif + +#if defined(_WIN32) +typedef uint64_t xoff_t; +/* Note: The following generates benign warnings in a mingw + * cross-compiler */ +#define Q "I64" +#elif SIZEOF_UNSIGNED_LONG == 8 +typedef unsigned long xoff_t; +#define Q "l" +#elif SIZEOF_SIZE_T == 8 +typedef size_t xoff_t; +#define Q "z" +#elif SIZEOF_UNSIGNED_LONG_LONG == 8 +typedef unsigned long long xoff_t; +#define Q "ll" +#endif /* #define Q */ + +#define SIZEOF_XOFF_T 8 + +#else /* XD3_USE_LARGEFILE64 == 0 */ + +#if SIZEOF_UNSIGNED_INT == 4 +typedef unsigned int xoff_t; +#elif SIZEOF_UNSIGNED_LONG == 4 +typedef unsigned long xoff_t; +#else +typedef uint32_t xoff_t; +#endif /* xoff_t is 32 bits */ + +#define SIZEOF_XOFF_T 4 +#define Q +#endif /* 64 vs 32 bit xoff_t */ + +/* Note: This gets modified in the 64bithash branch. */ +#define SIZEOF_USIZE_T 4 + +#if SIZEOF_SIZE_T == 4 +#define Z "z" +#elif SIZEOF_SIZE_T == 8 +#ifdef _WIN32 +#define Z "I64" +#else /* !_WIN32 */ +#define Z "z" +#endif /* Windows or not */ +#else +#error Bad configure script +#endif /* size_t printf flags */ + +#define USE_UINT32 (SIZEOF_USIZE_T == 4 || \ + SIZEOF_XOFF_T == 4 || REGRESSION_TEST) +#define USE_UINT64 (SIZEOF_USIZE_T == 8 || \ + SIZEOF_XOFF_T == 8 || REGRESSION_TEST) + +#ifndef UNALIGNED_OK +#ifdef HAVE_ALIGNED_ACCESS_REQUIRED +#define UNALIGNED_OK 0 +#else +/* This generally includes all Windows builds. */ +#define UNALIGNED_OK 1 +#endif +#endif + +/**********************************************************************/ + +/* Whether to build the encoder, otherwise only build the decoder. */ +#ifndef XD3_ENCODER +#define XD3_ENCODER 1 +#endif + +/* The code returned when main() fails, also defined in system + includes. */ +#ifndef EXIT_FAILURE +#define EXIT_FAILURE 1 +#endif + +/* REGRESSION TEST enables the "xdelta3 test" command, which runs a + series of self-tests. */ +#ifndef REGRESSION_TEST +#define REGRESSION_TEST 0 +#endif + +/* XD3_DEBUG=1 enables assertions and various statistics. Levels > 1 + * enable some additional output only useful during development and + * debugging. */ +#ifndef XD3_DEBUG +#define XD3_DEBUG 0 +#endif + +#ifndef PYTHON_MODULE +#define PYTHON_MODULE 0 +#endif + +#ifndef SWIG_MODULE +#define SWIG_MODULE 0 +#endif + +/* There are three string matching functions supplied: one fast, one + * slow (default), and one soft-configurable. To disable any of + * these, use the following definitions. */ +#ifndef XD3_BUILD_SLOW +#define XD3_BUILD_SLOW 1 +#endif +#ifndef XD3_BUILD_FAST +#define XD3_BUILD_FAST 1 +#endif +#ifndef XD3_BUILD_FASTER +#define XD3_BUILD_FASTER 1 +#endif +#ifndef XD3_BUILD_FASTEST +#define XD3_BUILD_FASTEST 1 +#endif +#ifndef XD3_BUILD_SOFT +#define XD3_BUILD_SOFT 1 +#endif +#ifndef XD3_BUILD_DEFAULT +#define XD3_BUILD_DEFAULT 1 +#endif + +#if XD3_DEBUG +#include +#endif + +/* XPRINT. Debug output and VCDIFF_TOOLS functions report to stderr. + * I have used an irregular style to abbreviate [fprintf(stderr, "] as + * [DP(RINT "]. */ +#define DP fprintf +#define RINT stderr, + +typedef struct _xd3_stream xd3_stream; +typedef struct _xd3_source xd3_source; +typedef struct _xd3_hash_cfg xd3_hash_cfg; +typedef struct _xd3_smatcher xd3_smatcher; +typedef struct _xd3_rinst xd3_rinst; +typedef struct _xd3_dinst xd3_dinst; +typedef struct _xd3_hinst xd3_hinst; +typedef struct _xd3_winst xd3_winst; +typedef struct _xd3_rpage xd3_rpage; +typedef struct _xd3_addr_cache xd3_addr_cache; +typedef struct _xd3_output xd3_output; +typedef struct _xd3_desect xd3_desect; +typedef struct _xd3_iopt_buflist xd3_iopt_buflist; +typedef struct _xd3_rlist xd3_rlist; +typedef struct _xd3_sec_type xd3_sec_type; +typedef struct _xd3_sec_cfg xd3_sec_cfg; +typedef struct _xd3_sec_stream xd3_sec_stream; +typedef struct _xd3_config xd3_config; +typedef struct _xd3_code_table_desc xd3_code_table_desc; +typedef struct _xd3_code_table_sizes xd3_code_table_sizes; +typedef struct _xd3_slist xd3_slist; +typedef struct _xd3_whole_state xd3_whole_state; +typedef struct _xd3_wininfo xd3_wininfo; + +/* The stream configuration has three callbacks functions, all of + * which may be supplied with NULL values. If config->getblk is + * provided as NULL, the stream returns XD3_GETSRCBLK. */ + +typedef void* (xd3_alloc_func) (void *opaque, + size_t items, + usize_t size); +typedef void (xd3_free_func) (void *opaque, + void *address); + +typedef int (xd3_getblk_func) (xd3_stream *stream, + xd3_source *source, + xoff_t blkno); + +/* These are internal functions to delay construction of encoding + * tables and support alternate code tables. See the comments & code + * enabled by GENERIC_ENCODE_TABLES. */ + +typedef const xd3_dinst* (xd3_code_table_func) (void); +typedef int (xd3_comp_table_func) (xd3_stream *stream, + const uint8_t **data, + usize_t *size); + + + +#if XD3_DEBUG +#define XD3_ASSERT(x) \ + do { if (! (x)) { DP(RINT "%s:%d: XD3 assertion failed: %s\n", __FILE__, __LINE__, #x); \ + abort (); } } while (0) +#else +#define XD3_ASSERT(x) (void)0 +#endif /* XD3_DEBUG */ + +#define xd3_max(x,y) ((x) < (y) ? (y) : (x)) +#define xd3_min(x,y) ((x) < (y) ? (x) : (y)) + +/**************************************************************** + PUBLIC ENUMS + ******************************************************************/ + +/* These are the five ordinary status codes returned by the + * xd3_encode_input() and xd3_decode_input() state machines. */ +typedef enum { + + /* An application must be prepared to handle these five return + * values from either xd3_encode_input or xd3_decode_input, except + * in the case of no-source compression, in which case XD3_GETSRCBLK + * is never returned. More detailed comments for these are given in + * xd3_encode_input and xd3_decode_input comments, below. */ + XD3_INPUT = -17703, /* need input */ + XD3_OUTPUT = -17704, /* have output */ + XD3_GETSRCBLK = -17705, /* need a block of source input (with no + * xd3_getblk function), a chance to do + * non-blocking read. */ + XD3_GOTHEADER = -17706, /* (decode-only) after the initial VCDIFF & + first window header */ + XD3_WINSTART = -17707, /* notification: returned before a window is + * processed, giving a chance to + * XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that + * window. */ + XD3_WINFINISH = -17708, /* notification: returned after + encode/decode & output for a window */ + XD3_TOOFARBACK = -17709, /* (encoder only) may be returned by + getblk() if the block is too old */ + XD3_INTERNAL = -17710, /* internal error */ + XD3_INVALID = -17711, /* invalid config */ + XD3_INVALID_INPUT = -17712, /* invalid input/decoder error */ + XD3_NOSECOND = -17713, /* when secondary compression finds no + improvement. */ + XD3_UNIMPLEMENTED = -17714 /* currently VCD_TARGET, VCD_CODETABLE */ +} xd3_rvalues; + +/* special values in config->flags */ +typedef enum +{ + XD3_JUST_HDR = (1 << 1), /* used by VCDIFF tools, see + xdelta3-main.h. */ + XD3_SKIP_WINDOW = (1 << 2), /* used by VCDIFF tools, see + xdelta3-main.h. */ + XD3_SKIP_EMIT = (1 << 3), /* used by VCDIFF tools, see + xdelta3-main.h. */ + XD3_FLUSH = (1 << 4), /* flush the stream buffer to + prepare for + xd3_stream_close(). */ + + XD3_SEC_DJW = (1 << 5), /* use DJW static huffman */ + XD3_SEC_FGK = (1 << 6), /* use FGK adaptive huffman */ + XD3_SEC_LZMA = (1 << 24), /* use LZMA secondary */ + + XD3_SEC_TYPE = (XD3_SEC_DJW | XD3_SEC_FGK | XD3_SEC_LZMA), + + XD3_SEC_NODATA = (1 << 7), /* disable secondary compression of + the data section. */ + XD3_SEC_NOINST = (1 << 8), /* disable secondary compression of + the inst section. */ + XD3_SEC_NOADDR = (1 << 9), /* disable secondary compression of + the addr section. */ + + XD3_SEC_NOALL = (XD3_SEC_NODATA | XD3_SEC_NOINST | XD3_SEC_NOADDR), + + XD3_ADLER32 = (1 << 10), /* enable checksum computation in + the encoder. */ + XD3_ADLER32_NOVER = (1 << 11), /* disable checksum verification in + the decoder. */ + + XD3_NOCOMPRESS = (1 << 13), /* disable ordinary data + * compression feature, only search + * the source, not the target. */ + XD3_BEGREEDY = (1 << 14), /* disable the "1.5-pass + * algorithm", instead use greedy + * matching. Greedy is off by + * default. */ + XD3_ADLER32_RECODE = (1 << 15), /* used by "recode". */ + + /* 4 bits to set the compression level the same as the command-line + * setting -1 through -9 (-0 corresponds to the XD3_NOCOMPRESS flag, + * and is independent of compression level). This is for + * convenience, especially with xd3_encode_memory(). */ + + XD3_COMPLEVEL_SHIFT = 20, /* 20 - 23 */ + XD3_COMPLEVEL_MASK = (0xF << XD3_COMPLEVEL_SHIFT), + XD3_COMPLEVEL_1 = (1 << XD3_COMPLEVEL_SHIFT), + XD3_COMPLEVEL_2 = (2 << XD3_COMPLEVEL_SHIFT), + XD3_COMPLEVEL_3 = (3 << XD3_COMPLEVEL_SHIFT), + XD3_COMPLEVEL_6 = (6 << XD3_COMPLEVEL_SHIFT), + XD3_COMPLEVEL_9 = (9 << XD3_COMPLEVEL_SHIFT) + +} xd3_flags; + +/* The values of this enumeration are set in xd3_config using the + * smatch_cfg variable. It can be set to default, slow, fast, etc., + * and soft. */ +typedef enum +{ + XD3_SMATCH_DEFAULT = 0, /* Flags may contain XD3_COMPLEVEL bits, + else default. */ + XD3_SMATCH_SLOW = 1, + XD3_SMATCH_FAST = 2, + XD3_SMATCH_FASTER = 3, + XD3_SMATCH_FASTEST = 4, + XD3_SMATCH_SOFT = 5 +} xd3_smatch_cfg; + +/********************************************************************* + PRIVATE ENUMS +**********************************************************************/ + +/* stream->match_state is part of the xd3_encode_input state machine + * for source matching: + * + * 1. the XD3_GETSRCBLK block-read mechanism means reentrant matching + * 2. this state spans encoder windows: a match and end-of-window + * will continue in the next 3. the initial target byte and source + * byte are a presumed match, to avoid some computation in case the + * inputs are identical. + */ +typedef enum { + + MATCH_TARGET = 0, /* in this state, attempt to match the start of + * the target with the previously set source + * address (initially 0). */ + MATCH_BACKWARD = 1, /* currently expanding a match backward in the + source/target. */ + MATCH_FORWARD = 2, /* currently expanding a match forward in the + source/target. */ + MATCH_SEARCHING = 3 /* currently searching for a match. */ + +} xd3_match_state; + +/* The xd3_encode_input state machine steps through these states in + * the following order. The matcher is reentrant and returns + * XD3_INPUT whenever it requires more data. After receiving + * XD3_INPUT, if the application reads EOF it should call + * xd3_stream_close(). + */ +typedef enum { + + ENC_INIT = 0, /* xd3_encode_input has never been called. */ + ENC_INPUT = 1, /* waiting for xd3_avail_input () to be called. */ + ENC_SEARCH = 2, /* currently searching for matches. */ + ENC_INSTR = 3, /* currently formatting output. */ + ENC_FLUSH = 4, /* currently emitting output. */ + ENC_POSTOUT = 5, /* after an output section. */ + ENC_POSTWIN = 6, /* after all output sections. */ + ENC_ABORTED = 7 /* abort. */ +} xd3_encode_state; + +/* The xd3_decode_input state machine steps through these states in + * the following order. The matcher is reentrant and returns + * XD3_INPUT whenever it requires more data. After receiving + * XD3_INPUT, if the application reads EOF it should call + * xd3_stream_close(). + * + * 0-8: the VCDIFF header + * 9-18: the VCDIFF window header + * 19-21: the three primary sections: data, inst, addr + * 22: producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK, + * 23: return XD3_WINFINISH, set state=9 to decode more input + */ +typedef enum { + + DEC_VCHEAD = 0, /* VCDIFF header */ + DEC_HDRIND = 1, /* header indicator */ + + DEC_SECONDID = 2, /* secondary compressor ID */ + + DEC_TABLEN = 3, /* code table length */ + DEC_NEAR = 4, /* code table near */ + DEC_SAME = 5, /* code table same */ + DEC_TABDAT = 6, /* code table data */ + + DEC_APPLEN = 7, /* application data length */ + DEC_APPDAT = 8, /* application data */ + + DEC_WININD = 9, /* window indicator */ + + DEC_CPYLEN = 10, /* copy window length */ + DEC_CPYOFF = 11, /* copy window offset */ + + DEC_ENCLEN = 12, /* length of delta encoding */ + DEC_TGTLEN = 13, /* length of target window */ + DEC_DELIND = 14, /* delta indicator */ + + DEC_DATALEN = 15, /* length of ADD+RUN data */ + DEC_INSTLEN = 16, /* length of instruction data */ + DEC_ADDRLEN = 17, /* length of address data */ + + DEC_CKSUM = 18, /* window checksum */ + + DEC_DATA = 19, /* data section */ + DEC_INST = 20, /* instruction section */ + DEC_ADDR = 21, /* address section */ + + DEC_EMIT = 22, /* producing data */ + + DEC_FINISH = 23, /* window finished */ + + DEC_ABORTED = 24 /* xd3_abort_stream */ +} xd3_decode_state; + +/************************************************************ + internal types + ************************************************************/ + +/* instruction lists used in the IOPT buffer */ +struct _xd3_rlist +{ + xd3_rlist *next; + xd3_rlist *prev; +}; + +/* the raw encoding of an instruction used in the IOPT buffer */ +struct _xd3_rinst +{ + uint8_t type; + uint8_t xtra; + uint8_t code1; + uint8_t code2; + usize_t pos; + usize_t size; + xoff_t addr; + xd3_rlist link; +}; + +/* the code-table form of an single- or double-instruction */ +struct _xd3_dinst +{ + uint8_t type1; + uint8_t size1; + uint8_t type2; + uint8_t size2; +}; + +/* the decoded form of a single (half) instruction. */ +struct _xd3_hinst +{ + uint8_t type; + uint32_t size; /* TODO: why decode breaks if this is usize_t? */ + uint32_t addr; /* TODO: why decode breaks if this is usize_t? */ +}; + +/* the form of a whole-file instruction */ +struct _xd3_winst +{ + uint8_t type; /* RUN, ADD, COPY */ + uint8_t mode; /* 0, VCD_SOURCE, VCD_TARGET */ + usize_t size; + xoff_t addr; + xoff_t position; /* absolute position of this inst */ +}; + +/* used by the encoder to buffer output in sections. list of blocks. */ +struct _xd3_output +{ + uint8_t *base; + usize_t next; + usize_t avail; + xd3_output *next_page; +}; + +/* used by the decoder to buffer input in sections. */ +struct _xd3_desect +{ + const uint8_t *buf; + const uint8_t *buf_max; + uint32_t size; /* TODO: why decode breaks if this is usize_t? */ + usize_t pos; + + /* used in xdelta3-decode.h */ + uint8_t *copied1; + usize_t alloc1; + + /* used in xdelta3-second.h */ + uint8_t *copied2; + usize_t alloc2; +}; + +/* the VCDIFF address cache, see the RFC */ +struct _xd3_addr_cache +{ + usize_t s_near; + usize_t s_same; + usize_t next_slot; /* the circular index for near */ + usize_t *near_array; /* array of size s_near */ + usize_t *same_array; /* array of size s_same*256 */ +}; + +/* the IOPT buffer list is just a list of buffers, which may be allocated + * during encode when using an unlimited buffer. */ +struct _xd3_iopt_buflist +{ + xd3_rinst *buffer; + xd3_iopt_buflist *next; +}; + +/* This is the record of a pre-compiled configuration, a subset of + xd3_config. */ +struct _xd3_smatcher +{ + const char *name; + int (*string_match) (xd3_stream *stream); + usize_t large_look; + usize_t large_step; + usize_t small_look; + usize_t small_chain; + usize_t small_lchain; + usize_t max_lazy; + usize_t long_enough; +}; + +/* hash table size & power-of-two hash function. */ +struct _xd3_hash_cfg +{ + usize_t size; + usize_t shift; + usize_t mask; +}; + +/* the sprev list */ +struct _xd3_slist +{ + usize_t last_pos; +}; + +/* window info (for whole state) */ +struct _xd3_wininfo { + xoff_t offset; + usize_t length; + uint32_t adler32; +}; + +/* whole state for, e.g., merge */ +struct _xd3_whole_state { + usize_t addslen; + uint8_t *adds; + usize_t adds_alloc; + + usize_t instlen; + xd3_winst *inst; + usize_t inst_alloc; + + usize_t wininfolen; + xd3_wininfo *wininfo; + usize_t wininfo_alloc; + + xoff_t length; +}; + +/******************************************************************** + public types + *******************************************************************/ + +/* Settings for the secondary compressor. */ +struct _xd3_sec_cfg +{ + int data_type; /* Which section. (set automatically) */ + usize_t ngroups; /* Number of DJW Huffman groups. */ + usize_t sector_size; /* Sector size. */ + int inefficient; /* If true, ignore efficiency check [avoid XD3_NOSECOND]. */ +}; + +/* This is the user-visible stream configuration. */ +struct _xd3_config +{ + usize_t winsize; /* The encoder window size. */ + usize_t sprevsz; /* How far back small string + matching goes */ + usize_t iopt_size; /* entries in the + instruction-optimizing + buffer */ + + xd3_getblk_func *getblk; /* The three callbacks. */ + xd3_alloc_func *alloc; + xd3_free_func *freef; + void *opaque; /* Not used. */ + int flags; /* stream->flags are initialized + * from xd3_config & never + * modified by the library. Use + * xd3_set_flags to modify flags + * settings mid-stream. */ + + xd3_sec_cfg sec_data; /* Secondary compressor config: data */ + xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ + xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ + + xd3_smatch_cfg smatch_cfg; /* See enum: use fields below for + soft config */ + xd3_smatcher smatcher_soft; +}; + +/* The primary source file object. You create one of these objects and + * initialize the first four fields. This library maintains the next + * 5 fields. The configured getblk implementation is responsible for + * setting the final 3 fields when called (and/or when XD3_GETSRCBLK + * is returned). + */ +struct _xd3_source +{ + /* you set */ + usize_t blksize; /* block size */ + const char *name; /* its name, for debug/print + purposes */ + void *ioh; /* opaque handle */ + xoff_t max_winsize; /* maximum visible buffer */ + + /* getblk sets */ + xoff_t curblkno; /* current block number: client + sets after getblk request */ + usize_t onblk; /* number of bytes on current + block: client sets, must be >= 0 + and <= blksize */ + const uint8_t *curblk; /* current block array: client + sets after getblk request */ + + /* xd3 sets */ + usize_t srclen; /* length of this source window */ + xoff_t srcbase; /* offset of this source window + in the source itself */ + int shiftby; /* for power-of-two blocksizes */ + int maskby; /* for power-of-two blocksizes */ + xoff_t cpyoff_blocks; /* offset of dec_cpyoff in blocks */ + usize_t cpyoff_blkoff; /* offset of copy window in + blocks, remainder */ + xoff_t getblkno; /* request block number: xd3 sets + current getblk request */ + + /* See xd3_getblk() */ + xoff_t max_blkno; /* Maximum block, if eof is known, + * otherwise, equals frontier_blkno + * (initially 0). */ + usize_t onlastblk; /* Number of bytes on max_blkno */ + int eof_known; /* Set to true when the first + * partial block is read. */ +}; + +/* The primary xd3_stream object, used for encoding and decoding. You + * may access only two fields: avail_out, next_out. Use the methods + * above to operate on xd3_stream. */ +struct _xd3_stream +{ + /* input state */ + const uint8_t *next_in; /* next input byte */ + usize_t avail_in; /* number of bytes available at + next_in */ + xoff_t total_in; /* how many bytes in */ + + /* output state */ + uint8_t *next_out; /* next output byte */ + usize_t avail_out; /* number of bytes available at + next_out */ + usize_t space_out; /* total out space */ + xoff_t current_window; /* number of windows encoded/decoded */ + xoff_t total_out; /* how many bytes out */ + + /* to indicate an error, xd3 sets */ + const char *msg; /* last error message, NULL if + no error */ + + /* source configuration */ + xd3_source *src; /* source array */ + + /* encoder memory configuration */ + usize_t winsize; /* suggested window size */ + usize_t sprevsz; /* small string, previous window + size (power of 2) */ + usize_t sprevmask; /* small string, previous window + size mask */ + usize_t iopt_size; + usize_t iopt_unlimited; + + /* general configuration */ + xd3_getblk_func *getblk; /* set nxtblk, nxtblkno to scanblkno */ + xd3_alloc_func *alloc; /* malloc function */ + xd3_free_func *free; /* free function */ + void* opaque; /* private data object passed to + alloc, free, and getblk */ + int flags; /* various options */ + + /* secondary compressor configuration */ + xd3_sec_cfg sec_data; /* Secondary compressor config: data */ + xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ + xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ + + xd3_smatcher smatcher; + + usize_t *large_table; /* table of large checksums */ + xd3_hash_cfg large_hash; /* large hash config */ + + usize_t *small_table; /* table of small checksums */ + xd3_slist *small_prev; /* table of previous offsets, + circular linked list */ + int small_reset; /* true if small table should + be reset */ + + xd3_hash_cfg small_hash; /* small hash config */ + xd3_addr_cache acache; /* the vcdiff address cache */ + xd3_encode_state enc_state; /* state of the encoder */ + + usize_t taroff; /* base offset of the target input */ + usize_t input_position; /* current input position */ + usize_t min_match; /* current minimum match + length, avoids redundent + matches */ + usize_t unencoded_offset; /* current input, first + * unencoded offset. this value + * is <= the first instruction's + * position in the iopt buffer, + * if there is at least one + * match in the buffer. */ + + /* SRCWIN */ + int srcwin_decided; /* boolean: true if srclen and + srcbase have been + decided. */ + int srcwin_decided_early; /* boolean: true if srclen + and srcbase were + decided early. */ + xoff_t srcwin_cksum_pos; /* Source checksum position */ + + /* MATCH */ + xd3_match_state match_state; /* encoder match state */ + xoff_t match_srcpos; /* current match source + position relative to + srcbase */ + xoff_t match_last_srcpos; /* previously attempted + * srcpos, to avoid loops. */ + xoff_t match_minaddr; /* smallest matching address to + * set window params (reset each + * window xd3_encode_reset) */ + xoff_t match_maxaddr; /* largest matching address to + * set window params (reset each + * window xd3_encode_reset) */ + usize_t match_back; /* match extends back so far */ + usize_t match_maxback; /* match extends back maximum */ + usize_t match_fwd; /* match extends forward so far */ + usize_t match_maxfwd; /* match extends forward maximum */ + + xoff_t maxsrcaddr; /* address of the last source + match (across windows) */ + + uint8_t *buf_in; /* for saving buffered input */ + usize_t buf_avail; /* amount of saved input */ + const uint8_t *buf_leftover; /* leftover content of next_in + (i.e., user's buffer) */ + usize_t buf_leftavail; /* amount of leftover content */ + + xd3_output *enc_current; /* current output buffer */ + xd3_output *enc_free; /* free output buffers */ + xd3_output *enc_heads[4]; /* array of encoded outputs: + head of chain */ + xd3_output *enc_tails[4]; /* array of encoded outputs: + tail of chain */ + uint32_t recode_adler32; /* set the adler32 checksum + * during "recode". */ + + xd3_rlist iopt_used; /* instruction optimizing buffer */ + xd3_rlist iopt_free; + xd3_rinst *iout; /* next single instruction */ + xd3_iopt_buflist *iopt_alloc; + + const uint8_t *enc_appheader; /* application header to encode */ + usize_t enc_appheadsz; /* application header size */ + + /* decoder stuff */ + xd3_decode_state dec_state; /* current DEC_XXX value */ + usize_t dec_hdr_ind; /* VCDIFF header indicator */ + usize_t dec_win_ind; /* VCDIFF window indicator */ + usize_t dec_del_ind; /* VCDIFF delta indicator */ + + uint8_t dec_magic[4]; /* First four bytes */ + usize_t dec_magicbytes; /* Magic position. */ + + usize_t dec_secondid; /* Optional secondary compressor ID. */ + + /* TODO: why decode breaks if this is usize_t? */ + uint32_t dec_codetblsz; /* Optional code table: length. */ + uint8_t *dec_codetbl; /* Optional code table: storage. */ + usize_t dec_codetblbytes; /* Optional code table: position. */ + + /* TODO: why decode breaks if this is usize_t? */ + uint32_t dec_appheadsz; /* Optional application header: + size. */ + uint8_t *dec_appheader; /* Optional application header: + storage */ + usize_t dec_appheadbytes; /* Optional application header: + position. */ + + usize_t dec_cksumbytes; /* Optional checksum: position. */ + uint8_t dec_cksum[4]; /* Optional checksum: storage. */ + uint32_t dec_adler32; /* Optional checksum: value. */ + + /* TODO: why decode breaks if this is usize_t? */ + uint32_t dec_cpylen; /* length of copy window + (VCD_SOURCE or VCD_TARGET) */ + xoff_t dec_cpyoff; /* offset of copy window + (VCD_SOURCE or VCD_TARGET) */ + /* TODO: why decode breaks if this is usize_t? */ + uint32_t dec_enclen; /* length of delta encoding */ + /* TODO: why decode breaks if this is usize_t? */ + uint32_t dec_tgtlen; /* length of target window */ + +#if USE_UINT64 + uint64_t dec_64part; /* part of a decoded uint64_t */ +#endif +#if USE_UINT32 + uint32_t dec_32part; /* part of a decoded uint32_t */ +#endif + + xoff_t dec_winstart; /* offset of the start of + current target window */ + xoff_t dec_window_count; /* == current_window + 1 in + DEC_FINISH */ + usize_t dec_winbytes; /* bytes of the three sections + so far consumed */ + usize_t dec_hdrsize; /* VCDIFF + app header size */ + + const uint8_t *dec_tgtaddrbase; /* Base of decoded target + addresses (addr >= + dec_cpylen). */ + const uint8_t *dec_cpyaddrbase; /* Base of decoded copy + addresses (addr < + dec_cpylen). */ + + usize_t dec_position; /* current decoder position + counting the cpylen + offset */ + usize_t dec_maxpos; /* maximum decoder position + counting the cpylen + offset */ + xd3_hinst dec_current1; /* current instruction */ + xd3_hinst dec_current2; /* current instruction */ + + uint8_t *dec_buffer; /* Decode buffer */ + uint8_t *dec_lastwin; /* In case of VCD_TARGET, the + last target window. */ + usize_t dec_lastlen; /* length of the last target + window */ + xoff_t dec_laststart; /* offset of the start of last + target window */ + usize_t dec_lastspace; /* allocated space of last + target window, for reuse */ + + xd3_desect inst_sect; /* staging area for decoding + window sections */ + xd3_desect addr_sect; + xd3_desect data_sect; + + xd3_code_table_func *code_table_func; + xd3_comp_table_func *comp_table_func; + const xd3_dinst *code_table; + const xd3_code_table_desc *code_table_desc; + xd3_dinst *code_table_alloc; + + /* secondary compression */ + const xd3_sec_type *sec_type; + xd3_sec_stream *sec_stream_d; + xd3_sec_stream *sec_stream_i; + xd3_sec_stream *sec_stream_a; + + /* state for reconstructing whole files (e.g., for merge), this only + * supports loading USIZE_T_MAX instructions, adds, etc. */ + xd3_whole_state whole_target; + + /* statistics */ + xoff_t n_scpy; + xoff_t n_tcpy; + xoff_t n_add; + xoff_t n_run; + + xoff_t l_scpy; + xoff_t l_tcpy; + xoff_t l_add; + xoff_t l_run; + + usize_t i_slots_used; + +#if XD3_DEBUG + usize_t large_ckcnt; + + /* memory usage */ + usize_t alloc_cnt; + usize_t free_cnt; +#endif +}; + +/************************************************************************** + PUBLIC FUNCTIONS + **************************************************************************/ + +/* This function configures an xd3_stream using the provided in-memory + * input buffer, source buffer, output buffer, and flags. The output + * array must be large enough or else ENOSPC will be returned. This + * is the simplest in-memory encoding interface. */ +int xd3_encode_memory (const uint8_t *input, + usize_t input_size, + const uint8_t *source, + usize_t source_size, + uint8_t *output_buffer, + usize_t *output_size, + usize_t avail_output, + int flags); + +/* The reverse of xd3_encode_memory. */ +int xd3_decode_memory (const uint8_t *input, + usize_t input_size, + const uint8_t *source, + usize_t source_size, + uint8_t *output_buf, + usize_t *output_size, + usize_t avail_output, + int flags); + +/* This function encodes an in-memory input using a pre-configured + * xd3_stream. This allows the caller to set a variety of options + * which are not available in the xd3_encode/decode_memory() + * functions. + * + * The output array must be large enough to hold the output or else + * ENOSPC is returned. The source (if any) should be set using + * xd3_set_source_and_size() with a single-block xd3_source. This + * calls the underlying non-blocking interfaces, + * xd3_encode/decode_input(), handling the necessary input/output + * states. This method may be considered a reference for any + * application using xd3_encode_input() directly. + * + * xd3_stream stream; + * xd3_config config; + * xd3_source src; + * + * memset (& src, 0, sizeof (src)); + * memset (& stream, 0, sizeof (stream)); + * memset (& config, 0, sizeof (config)); + * + * if (source != NULL) + * { + * src.size = source_size; + * src.blksize = source_size; + * src.curblkno = 0; + * src.onblk = source_size; + * src.curblk = source; + * src.max_winsize = source_size; + * xd3_set_source(&stream, &src); + * } + * + * config.flags = flags; + * config.winsize = input_size; + * + * ... set smatcher, appheader, encoding-table, compression-level, etc. + * + * xd3_config_stream(&stream, &config); + * xd3_encode_stream(&stream, ...); + * xd3_free_stream(&stream); + */ +int xd3_encode_stream (xd3_stream *stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t avail_output); + +/* The reverse of xd3_encode_stream. */ +int xd3_decode_stream (xd3_stream *stream, + const uint8_t *input, + usize_t input_size, + uint8_t *output, + usize_t *output_size, + usize_t avail_size); + +/* This is the non-blocking interface. + * + * Handling input and output states is the same for encoding or + * decoding using the xd3_avail_input() and xd3_consume_output() + * routines, inlined below. + * + * Return values: + * + * XD3_INPUT: the process requires more input: call + * xd3_avail_input() then repeat + * + * XD3_OUTPUT: the process has more output: read stream->next_out, + * stream->avail_out, then call xd3_consume_output(), + * then repeat + * + * XD3_GOTHEADER: (decoder-only) notification returned following the + * VCDIFF header and first window header. the decoder + * may use the header to configure itself. + * + * XD3_WINSTART: a general notification returned once for each + * window except the 0-th window, which is implied by + * XD3_GOTHEADER. It is recommended to use a + * switch-stmt such as: + * + * ... + * again: + * switch ((ret = xd3_decode_input (stream))) { + * case XD3_GOTHEADER: { + * assert(stream->current_window == 0); + * stuff; + * } + * // fallthrough + * case XD3_WINSTART: { + * something(stream->current_window); + * goto again; + * } + * ... + * + * XD3_WINFINISH: a general notification, following the complete + * input & output of a window. at this point, + * stream->total_in and stream->total_out are consistent + * for either encoding or decoding. + * + * XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value + * is returned to initiate a non-blocking source read. + */ +int xd3_decode_input (xd3_stream *stream); +int xd3_encode_input (xd3_stream *stream); + +/* The xd3_config structure is used to initialize a stream - all data + * is copied into stream so config may be a temporary variable. See + * the [documentation] or comments on the xd3_config structure. */ +int xd3_config_stream (xd3_stream *stream, + xd3_config *config); + +/* Since Xdelta3 doesn't open any files, xd3_close_stream is just an + * error check that the stream is in a proper state to be closed: this + * means the encoder is flushed and the decoder is at a window + * boundary. The application is responsible for freeing any of the + * resources it supplied. */ +int xd3_close_stream (xd3_stream *stream); + +/* This arranges for closes the stream to succeed. Does not free the + * stream.*/ +void xd3_abort_stream (xd3_stream *stream); + +/* xd3_free_stream frees all memory allocated for the stream. The + * application is responsible for freeing any of the resources it + * supplied. */ +void xd3_free_stream (xd3_stream *stream); + +/* This function informs the encoder or decoder that source matching + * (i.e., delta-compression) is possible. For encoding, this should + * be called before the first xd3_encode_input. A NULL source is + * ignored. For decoding, this should be called before the first + * window is decoded, but the appheader may be read first + * (XD3_GOTHEADER). After decoding the header, call xd3_set_source() + * if you have a source file. Note: if (stream->dec_win_ind & VCD_SOURCE) + * is true, it means the first window expects there to be a source file. + */ +int xd3_set_source (xd3_stream *stream, + xd3_source *source); + +/* If the source size is known, call this instead of xd3_set_source(). + * to avoid having stream->getblk called (and/or to avoid XD3_GETSRCBLK). + * + * Follow these steps: + xd3_source source; + memset(&source, 0, sizeof(source)); + source.blksize = size; + source.onblk = size; + source.curblk = buf; + source.curblkno = 0; + int ret = xd3_set_source_and_size(&stream, &source, size); + ... + */ +int xd3_set_source_and_size (xd3_stream *stream, + xd3_source *source, + xoff_t source_size); + +/* This should be called before the first call to xd3_encode_input() + * to include application-specific data in the VCDIFF header. */ +void xd3_set_appheader (xd3_stream *stream, + const uint8_t *data, + usize_t size); + +/* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER. + * For convenience, the decoder always adds a single byte padding to + * the end of the application header, which is set to zero in case the + * application header is a string. */ +int xd3_get_appheader (xd3_stream *stream, + uint8_t **data, + usize_t *size); + +/* To generate a VCDIFF encoded delta with xd3_encode_init() from + * another format, use: + * + * xd3_encode_init_partial() -- initialze encoder state (w/o hash tables) + * xd3_init_cache() -- reset VCDIFF address cache + * xd3_found_match() -- to report a copy instruction + * + * set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual. + */ +int xd3_encode_init_partial (xd3_stream *stream); +void xd3_init_cache (xd3_addr_cache* acache); +int xd3_found_match (xd3_stream *stream, + usize_t pos, usize_t size, + xoff_t addr, int is_source); + +/* Gives an error string for xdelta3-speficic errors, returns NULL for + system errors */ +const char* xd3_strerror (int ret); + +/* For convenience, zero & initialize the xd3_config structure with + specified flags. */ +static inline +void xd3_init_config (xd3_config *config, + int flags) +{ + memset (config, 0, sizeof (*config)); + config->flags = flags; +} + +/* This supplies some input to the stream. + * + * For encoding, if the input is larger than the configured window + * size (xd3_config.winsize), the entire input will be consumed and + * encoded anyway. If you wish to strictly limit the window size, + * limit the buffer passed to xd3_avail_input to the window size. + * + * For encoding, if the input is smaller than the configured window + * size (xd3_config.winsize), the library will create a window-sized + * buffer and accumulate input until a full-sized window can be + * encoded. XD3_INPUT will be returned. The input must remain valid + * until the next time xd3_encode_input() returns XD3_INPUT. + * + * For decoding, the input will be consumed entirely before XD3_INPUT + * is returned again. + */ +static inline +void xd3_avail_input (xd3_stream *stream, + const uint8_t *idata, + usize_t isize) +{ + /* Even if isize is zero, the code expects a non-NULL idata. Why? + * It uses this value to determine whether xd3_avail_input has ever + * been called. If xd3_encode_input is called before + * xd3_avail_input it will return XD3_INPUT right away without + * allocating a stream->winsize buffer. This is to avoid an + * unwanted allocation. */ + XD3_ASSERT (idata != NULL || isize == 0); + + stream->next_in = idata; + stream->avail_in = isize; +} + +/* This acknowledges receipt of output data, must be called after any + * XD3_OUTPUT return. */ +static inline +void xd3_consume_output (xd3_stream *stream) +{ + stream->avail_out = 0; +} + +/* These are set for each XD3_WINFINISH return. */ +static inline +int xd3_encoder_used_source (xd3_stream *stream) { + return stream->src != NULL && stream->src->srclen > 0; +} +static inline +xoff_t xd3_encoder_srcbase (xd3_stream *stream) { + return stream->src->srcbase; +} +static inline +usize_t xd3_encoder_srclen (xd3_stream *stream) { + return stream->src->srclen; +} + +/* Checks for legal flag changes. */ +static inline +void xd3_set_flags (xd3_stream *stream, int flags) +{ + /* The bitwise difference should contain only XD3_FLUSH or + XD3_SKIP_WINDOW */ + XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0); + stream->flags = flags; +} + +/* Gives some extra information about the latest library error, if any + * is known. */ +static inline +const char* xd3_errstring (xd3_stream *stream) +{ + return stream->msg ? stream->msg : ""; +} + + +/* 64-bit divisions are expensive, which is why we require a + * power-of-two source->blksize. To relax this restriction is + * relatively easy, see the history for xd3_blksize_div(). */ +static inline +void xd3_blksize_div (const xoff_t offset, + const xd3_source *source, + xoff_t *blkno, + usize_t *blkoff) { + *blkno = (xoff_t) (offset >> source->shiftby); + *blkoff = (usize_t) (offset & source->maskby); + XD3_ASSERT (*blkoff < source->blksize); +} + +static inline +void xd3_blksize_add (xoff_t *blkno, + usize_t *blkoff, + const xd3_source *source, + const usize_t add) +{ + usize_t blkdiff; + + /* Does not check for overflow, checked in xdelta3-decode.h. */ + *blkoff += add; + blkdiff = *blkoff >> source->shiftby; + + if (blkdiff) + { + *blkno += blkdiff; + *blkoff &= source->maskby; + } + + XD3_ASSERT (*blkoff < source->blksize); +} + +#define XD3_NOOP 0U +#define XD3_ADD 1U +#define XD3_RUN 2U +#define XD3_CPY 3U /* XD3_CPY rtypes are represented as (XD3_CPY + + * copy-mode value) */ + +#if XD3_DEBUG +#define IF_DEBUG(x) x +#else +#define IF_DEBUG(x) +#endif +#if XD3_DEBUG > 1 +#define IF_DEBUG1(x) x +#else +#define IF_DEBUG1(x) +#endif +#if XD3_DEBUG > 2 +#define IF_DEBUG2(x) x +#else +#define IF_DEBUG2(x) +#endif + +#define SIZEOF_ARRAY(x) (sizeof(x) / sizeof(x[0])) + +#endif /* _XDELTA3_H_ */ diff --git a/app/src/test/java/com/droiddeck/launcher/stores/gog/GogMaintenanceTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/gog/GogMaintenanceTest.kt new file mode 100644 index 0000000000..1d755a6db0 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/gog/GogMaintenanceTest.kt @@ -0,0 +1,402 @@ +package com.droiddeck.launcher.stores.gog + +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreGameSidecar +import com.droiddeck.launcher.stores.StoreLanguage +import com.droiddeck.launcher.stores.CatalogItem +import com.droiddeck.launcher.stores.StoreInstallRoot +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadState +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.* +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.io.File +import java.nio.file.Files +import java.util.concurrent.atomic.AtomicBoolean + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class GogMaintenanceTest { + @get:Rule val temporary = TemporaryFolder() + + @Test fun selectsPublicBranchAndTreatsBuildIdsAsOpaque() { + val builds = JSONObject("""{"items":[ + {"os":"windows","branch":"beta","build_id":"99","generation":2,"link":"https://example.invalid/beta"}, + {"os":"windows","branch":null,"build_id":"0002","generation":2,"version_name":"RC-final","link":"https://example.invalid/current"}, + {"os":"windows","branch":null,"build_id":"10","generation":2,"link":"https://example.invalid/old"}]}""") + assertEquals("0002", GogBuild.select(builds)!!.id) + assertEquals("99", GogBuild.select(builds, branch = "beta")!!.id) + assertEquals("10", GogBuild.select(builds, pinned = "10")!!.id) + assertNull(GogBuild.select(builds, pinned = "missing")) + assertNull(GogBuild.select(builds, branch = "private")) + } + + @Test fun migrationRetainsKnownBuildLanguageAndInstalledProducts() { + val folder = temporary.newFolder() + File(folder, "_gog_manifest.json").writeText("""{"gameId":"1","buildId":"000123"}""") + File(folder, "goggame-2.info").writeText("{}") + val sidecar = StoreGameSidecar(Store.GOG, "1", "Game", "game.exe", installVersion = "000123", + args = listOf("--custom"), env = mapOf("DD_TEST" to "1"), cloud = false, achievements = false, + extra = mapOf(StoreLanguage.EXTRA to "pl-PL")) + val record = GogInstallRecord.migrate(folder, sidecar) + assertEquals("000123", record.build) + assertEquals("pl-PL", record.language) + assertEquals(listOf("1", "2"), record.products) + assertEquals(record.json().toString(), GogInstallRecord.read(folder)!!.json().toString()) + assertEquals(listOf("--custom"), sidecar.args) + assertFalse(sidecar.cloud) + } + + @Test fun missingManifestNeverInventsLatestBuild() { + val folder = temporary.newFolder() + val record = GogInstallRecord.migrate(folder, StoreGameSidecar(Store.GOG, "1", "Game", "game.exe")) + assertEquals("offline", record.source) + assertEquals("", record.build) + assertEquals(0, record.files.length()) + } + + @Test fun offlineRefreshWithDlcRequiresAnAuthenticMatchingDepotBuild() { + val extracted = temporary.newFolder() + val app = File(extracted, "app").apply { mkdirs() } + val target = GogInstallRecord("1", "depot", build = "0000123", products = listOf("1", "2")) + assertThrows(IllegalArgumentException::class.java) { GogOffline.validateDepotInstaller(extracted, target) } + File(app, "goggame-1.info").writeText("""{"buildId":"0000999"}""") + assertThrows(IllegalArgumentException::class.java) { GogOffline.validateDepotInstaller(extracted, target) } + File(app, "goggame-1.info").writeText("""{"buildId":"0000123"}""") + GogOffline.validateDepotInstaller(extracted, target) + File(app, "goggame-1.info").delete() + GogOffline.validateDepotInstaller(extracted, target.copy(source = "offline", build = "")) + } + + @Test fun offlineRefreshRetainsTheExistingExecutableLayout() { + val installed = temporary.newFolder() + val extracted = temporary.newFolder() + val app = File(extracted, "app").apply { mkdirs() } + File(app, "game.exe").writeText("replacement") + File(installed, "goggame-1.info").writeText("{}") + assertEquals(extracted, GogOffline.layout(extracted, installed, "1", "game.exe").source) + assertEquals("", GogOffline.layout(extracted, installed, "1", "game.exe").prefix) + File(installed, "goggame-1.info").delete() + assertEquals(extracted, GogOffline.layout(extracted, installed, "1", "app/game.exe").source) + val flat = temporary.newFolder() + val legacy = GogOffline.layout(flat, installed, "1", "app/game.exe") + assertEquals(flat, legacy.source) + assertEquals("app/", legacy.prefix) + } + + @Test fun offlineInstallerMatchesRegionalLanguageWithoutChoosingAnotherLanguage() { + val downloads = JSONObject("""{"installers":[ + {"id":"de","os":"windows","language":"de","files":[{"downlink":"https://example.invalid/de"}]}, + {"id":"en","os":"windows","language":"en","files":[{"downlink":"https://example.invalid/en"}]}]}""") + assertEquals("en", GogMaintenance.offlineInstaller(downloads, "en-US").id) + assertThrows(IllegalStateException::class.java) { GogMaintenance.offlineInstaller(downloads, "pl-PL") } + downloads.getJSONArray("installers").put(0, JSONObject("""{"id":"neutral","os":"windows","language":"Neutral","files":[{"downlink":"https://example.invalid/neutral"}]}""")) + assertEquals("en", GogMaintenance.offlineInstaller(downloads, "en-US").id) + assertEquals("neutral", GogMaintenance.offlineInstaller(downloads, "pl-PL").id) + } + + @Test fun incompleteOfflineExtractionCannotReplaceOrRemoveInstalledFiles() { + val folder = temporary.newFolder() + StoreGameSidecar(Store.GOG, "1", "Game", "game.exe").write(folder) + val game = File(folder, "game.exe").apply { writeText("installed game") } + val old = GogInstallRecord("1", "depot", files = JSONArray().put(JSONObject().put("path", game.name) + .put("size", game.length()).put("md5", GogFiles.hash(game)))) + old.write(folder) + val extracted = temporary.newFolder() + File(extracted, "goggame-1.ico").writeText("ancillary file") + assertThrows(IllegalStateException::class.java) { + GogOffline.applyRefresh(GogMaintenance.Proposal(folder, old, old.copy(source = "offline"), GogOperation.REFRESH, 0, 0), extracted, null, AtomicBoolean(false)) + } + assertEquals("installed game", game.readText()) + assertFalse(File(folder, "goggame-1.ico").exists()) + assertEquals("depot", GogInstallRecord.read(folder)!!.source) + val migrated = old.copy(files = JSONArray()) + migrated.write(folder) + assertThrows(IllegalStateException::class.java) { + GogOffline.applyRefresh(GogMaintenance.Proposal(folder, migrated, migrated.copy(source = "offline"), GogOperation.REFRESH, 0, 0), extracted, null, AtomicBoolean(false)) + } + assertEquals("installed game", game.readText()) + assertFalse(File(folder, "goggame-1.ico").exists()) + assertEquals("depot", GogInstallRecord.read(folder)!!.source) + assertEquals(0, GogInstallRecord.read(folder)!!.files.length()) + } + + @Test fun offlineRefreshCombinesRootPayloadAndAppPlaceholdersInBothLayouts() { + for (nested in listOf(false, true)) { + val folder = temporary.newFolder() + val executable = if (nested) "app/game.exe" else "game.exe" + StoreGameSidecar(Store.GOG, "1", "Game", executable).write(folder) + val game = File(folder, executable).apply { parentFile!!.mkdirs(); writeText("old game") } + val old = GogInstallRecord("1", "offline", files = JSONArray().put(JSONObject().put("path", executable) + .put("size", game.length()).put("md5", GogFiles.hash(game)))) + old.write(folder) + val extracted = temporary.newFolder() + File(extracted, "game.exe").writeText("complete new game") + File(extracted, "Data").mkdirs() + File(extracted, "app/Data/SaveGame").mkdirs() + File(extracted, "app/Data/SaveGame/dummy.txt").writeBytes(byteArrayOf()) + File(extracted, "tmp").mkdirs() + File(extracted, "tmp/installer.exe").writeText("installer support") + GogOffline.applyRefresh(GogMaintenance.Proposal(folder, old, old, GogOperation.REFRESH, 0, 0), extracted, null, AtomicBoolean(false)) + assertEquals("complete new game", game.readText()) + assertTrue(File(folder, (if (nested) "app/" else "") + "Data/SaveGame/dummy.txt").isFile) + assertFalse(File(folder, "tmp").exists()) + assertTrue(GogInstallRecord.read(folder)!!.files.objects().any { it.getString("path") == executable }) + } + } + + @Test fun offlineRefreshDefersDlcCleanupAndRecordPublicationUntilDepotCompletion() { + val folder = temporary.newFolder() + StoreGameSidecar(Store.GOG, "1", "Game", "base.exe").write(folder) + val base = File(folder, "base.exe").apply { writeText("old base") } + val dlc = File(folder, "dlc.bin").apply { writeText("old DLC") } + val obsolete = File(folder, "old.bin").apply { writeText("obsolete") } + val save = File(folder, "save.dat").apply { writeText("player progress") } + fun entry(file: File, product: String) = JSONObject().put("path", file.name).put("size", file.length()) + .put("md5", GogFiles.hash(file)).put("product", product).put("offset", 0).put("remotePath", file.name) + val old = GogInstallRecord("1", "depot", build = "old", products = listOf("1", "2"), + files = JSONArray().put(entry(base, "1")).put(entry(dlc, "2")).put(entry(obsolete, "1"))) + old.write(folder) + val recordFile = File(GogFiles.dir(folder), "installed.json") + val record = recordFile.readText() + val extracted = temporary.newFolder() + val app = File(extracted, "app").apply { mkdirs() } + File(app, "goggame-1.info").writeText("""{"buildId":"new"}""") + val replacement = File(app, "base.exe").apply { writeText("new base") } + val expectedDlc = File(temporary.newFolder(), "dlc.bin").apply { writeText("new DLC") } + val target = old.copy(build = "new", files = JSONArray().put(entry(replacement, "1")).put(entry(expectedDlc, "2"))) + GogOffline.applyRefresh(GogMaintenance.Proposal(folder, old, target, GogOperation.REFRESH, 0, 0), extracted, null, AtomicBoolean(false)) + assertEquals("new base", base.readText()) + assertEquals("old DLC", dlc.readText()) + assertTrue(obsolete.isFile) + assertEquals(record, recordFile.readText()) + val server = java.net.ServerSocket(0, 1, java.net.InetAddress.getByName("127.0.0.1")) + val worker = Thread { + server.accept().use { client -> + val reader = client.getInputStream().bufferedReader() + while (!reader.readLine().isNullOrEmpty()) Unit + val content = expectedDlc.readBytes() + client.getOutputStream().apply { + write("HTTP/1.1 206 Partial Content\r\nContent-Length: ${content.size}\r\nConnection: close\r\n\r\n".toByteArray()) + write(content); flush() + } + } + }.apply { isDaemon = true; start() } + try { + GogFallback.transfer(folder, JSONObject().put("files", target.files).put("oldFiles", old.files) + .put("links", JSONObject().put("2", JSONArray().put("http://127.0.0.1:${server.localPort}"))), null, AtomicBoolean(false)) + assertEquals("new DLC", dlc.readText()) + assertFalse(obsolete.exists()) + assertEquals("player progress", save.readText()) + assertEquals(record, recordFile.readText()) + } finally { server.close(); worker.join(2000) } + } + + @Test fun authenticIdentityOverridesAStaleLegacyMarkerWithoutTreatingSchemaAsVersion() { + val folder = temporary.newFolder() + File(folder, "_gog_manifest.json").writeText("""{"buildId":"old"}""") + File(folder, "goggame-1.id").writeText("""{"buildId":"patched"}""") + File(folder, "goggame-1.info").writeText("""{"version":1}""") + val record = GogInstallRecord.migrate(folder, StoreGameSidecar(Store.GOG, "1", "Game", "game.exe", installVersion = "old")) + assertEquals("patched", record.build) + assertEquals("", record.version) + } + + @Test fun authenticGenerationOneResolvesProductDepotsAndRetainsHashesAndBuild() { + fun fixture(name: String) = JSONObject(javaClass.classLoader!!.getResource("gog/$name")!!.readText()) + val build = GogBuild("3301", "", 1, "", "https://example.invalid/repository.json") + val requested = ArrayList() + val snapshot = GogContent.snapshot(GogInstallRecord("1207658940", "depot"), build, "unused") { url, _ -> + requested.add(url) + when { + url == build.link -> fixture("rayman-gen1-repository.json") + url.endsWith("a01f6fb8-55a8-4f33-bc5b-5f5d2f591775.json") -> fixture("rayman-gen1-neutral.json") + url.endsWith("ec88140c-2589-41a0-bbf3-8fc1f215fe5a.json") -> fixture("rayman-gen1-english-excerpt.json") + else -> error("Unexpected metadata: $url") + } + } + assertEquals("3301", snapshot.build) + assertEquals(1, snapshot.generation) + assertEquals(4, snapshot.files.length()) + val support = snapshot.files.getJSONObject(0) + assertEquals("support/1207658940/galaxy_rayman2_2.1.0.39.exe", support.getString("path")) + assertEquals("ee2c247373733f29782395f1f92622f6", support.getString("md5")) + val game = snapshot.files.getJSONObject(1) + assertEquals("Data/World/Levels/Raycap/Raycap.snd", game.getString("path")) + assertEquals(384206096L, game.getLong("offset")) + assertEquals("1207658940/main.bin", game.getString("remotePath")) + assertTrue(requested.drop(1).all { "/1207658940/windows/31847977/" in it }) + GogFiles.validate(snapshot.files, temporary.newFolder()) + } + + @Test fun multipartPatchNamesVersionsLanguageAndChecksumsSurviveParsing() { + val downloads = JSONObject("""{"patches":[ + {"id":"patch-1","name":"Patch A to B","version":"B","language":"pl","os":"windows","files":[ + {"id":"part0","filename":"patch_original.exe","downlink":"https://example.invalid/0","size":3,"md5":"abc"}, + {"id":"part1","filename":"patch_original-1.bin","downlink":"https://example.invalid/1","size":4}]}, + {"id":"linux","os":"linux","files":[{"downlink":"https://example.invalid/linux"}]}]}""") + val patch = GogOfflineDownload.parse(downloads, "patches").single() + assertEquals("Patch A to B", patch.name) + assertEquals("B", patch.version) + assertEquals("pl", patch.language) + assertEquals(7, patch.size) + assertEquals(listOf("patch_original.exe", "patch_original-1.bin"), patch.parts.map { it.name }) + assertEquals("abc", patch.parts.first().md5) + assertEquals("patch_(52404)+special.exe", GogContent.downloadFilename("https://example.invalid/patch_%2852404%29+special.exe?token=temporary")) + } + + @Test fun pathsRejectTraversalSymlinksCaseCollisionsAndMetadataOverlap() { + val folder = temporary.newFolder() + for (path in listOf("../escape", "/absolute", "C:/file", "a/./file", "a//file")) { + assertThrows(IllegalArgumentException::class.java) { GogFiles.inside(folder, path) } + } + val files = JSONArray("""[{"path":"Game.exe"},{"path":"game.exe"}]""") + assertThrows(IllegalArgumentException::class.java) { GogFiles.validate(files, folder) } + assertThrows(IllegalArgumentException::class.java) { GogFiles.validate(JSONArray("""[{"path":"Data/a"},{"path":"data/b"}]"""), folder) } + File(folder, "Game.bin").writeText("original") + assertThrows(IllegalArgumentException::class.java) { GogFiles.validate(JSONArray("""[{"path":"game.bin"}]"""), folder) } + assertThrows(IllegalArgumentException::class.java) { GogFiles.validate(JSONArray("""[{"path":".droiddeck-store.json"}]"""), folder) } + val outside = temporary.newFolder() + try { Files.createSymbolicLink(File(folder, "link").toPath(), outside.toPath()) } + catch (_: java.nio.file.FileSystemException) { return } + assertThrows(IllegalArgumentException::class.java) { GogFiles.inside(folder, "link/game.exe") } + } + + @Test fun sameSizeCorruptionFailsVerificationAndLanguageVariantsRemainDistinct() { + val folder = temporary.newFolder() + val file = File(folder, "game.bin").apply { writeText("original") } + val expected = JSONObject().put("path", file.name).put("size", file.length()).put("md5", GogFiles.hash(file)) + file.writeText("modified") + assertFalse(GogFiles.matches(folder, expected)) + assertTrue(GogContent.languageSelected(listOf("*"), "zh-Hant")) + assertFalse(GogContent.languageSelected(listOf("zh-Hans"), "zh-Hant")) + assertTrue(GogContent.languageSelected(listOf("Polish"), "pl")) + } + + @Test fun persistedManifestsContainNoCredentialsOrTemporaryLinks() { + val input = JSONObject("""{"buildId":"1","clientSecret":"secret","urls":[{"url":"https://example.invalid/?token=secret"}], + "products":[{"productId":"2","clientSecret":"nested"}],"dependencies":["ISI"]}""") + val text = GogContent.sanitize(input).toString() + assertFalse(text.contains("secret")) + assertFalse(text.contains("https://")) + assertTrue(text.contains("ISI")) + } + + @Test fun interruptedExternalPatchRestoresRecoveryWithoutExecutingAgain() { + val app = RuntimeEnvironment.getApplication() + val folder = File(StoreInstallRoot.storeDir(StoreInstallRoot.internalRoot(app), Store.GOG), "Patch recovery").apply { mkdirs() } + try { + val sidecar = StoreGameSidecar(Store.GOG, "recovery-test", "Patch recovery", "game.exe", + args = listOf("--custom"), env = mapOf("SETTING" to "value"), cloud = false, + achievements = false, state = StoreGameSidecar.STATE_UPDATING) + sidecar.write(folder) + val previous = sidecar.toJson().toString() + GogFiles.write(GogFiles.journal(folder), JSONObject().put("operation", "PATCH").put("phase", "patch-applying")) + val item = CatalogItem(Store.GOG, sidecar.id, sidecar.title, imageUrl = null, extra = mapOf( + GogMaintenance.OPERATION to "PATCH", GogMaintenance.FOLDER to folder.path)) + GogMaintenance.restore(app, item) + assertEquals(DownloadState.FAILED, DownloadQueue.entry(item.key)?.state) + assertTrue(DownloadQueue.entry(item.key)!!.error!!.contains("recovery")) + assertEquals(previous, StoreGameSidecar.read(folder)!!.toJson().toString()) + assertEquals("patch-applying", GogFiles.read(GogFiles.journal(folder))!!.getString("phase")) + DownloadQueue.clear(app, item.key) + } finally { folder.deleteRecursively() } + } + + @Test fun signedOutCheckRemainsAFailureInsteadOfCurrent() { + val app = RuntimeEnvironment.getApplication() + com.droiddeck.launcher.stores.StoreAccounts.clear(app, Store.GOG) + GogMaintenance.reset() + val folder = temporary.newFolder() + val game = com.droiddeck.launcher.stores.InstalledStoreGame( + StoreGameSidecar(Store.GOG, "signed-out-check", "Check", "game.exe", installVersion = "opaque-build"), folder) + GogMaintenance.check(app, game, force = true) + val deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5) + while (System.nanoTime() < deadline) { + org.robolectric.shadows.ShadowLooper.idleMainLooper() + if (GogMaintenance.checks[game.sidecar.id]?.state == GogMaintenance.State.FAILED) break + Thread.sleep(10) + } + assertEquals(GogMaintenance.State.FAILED, GogMaintenance.checks[game.sidecar.id]?.state) + assertTrue(GogMaintenance.checks[game.sidecar.id]!!.reason.contains("Sign in")) + } + + @Test fun pauseBeforeInstallerClaimCannotStartExternalApplication() { + val app = RuntimeEnvironment.getApplication() + val ready = java.util.concurrent.CountDownLatch(1) + val proceed = java.util.concurrent.CountDownLatch(1) + val finished = java.util.concurrent.CountDownLatch(1) + val applied = AtomicBoolean(false) + val item = com.droiddeck.launcher.stores.download.DownloadEntry(Store.GOG, "installer-claim", "Installer claim") + DownloadQueue.enqueue(app, item) { + object : DownloadQueue.DownloadJob { + override fun run(handle: DownloadQueue.JobHandle): String? { + ready.countDown() + try { + check(proceed.await(5, java.util.concurrent.TimeUnit.SECONDS)) + handle.interruptible(false) + applied.set(true) + return null + } finally { finished.countDown() } + } + override fun cancel(deleteFiles: Boolean) {} + } + } + try { + assertTrue(ready.await(5, java.util.concurrent.TimeUnit.SECONDS)) + DownloadQueue.pause(item.key) + proceed.countDown() + assertTrue(finished.await(5, java.util.concurrent.TimeUnit.SECONDS)) + assertFalse(applied.get()) + } finally { + proceed.countDown() + val deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5) + while (DownloadQueue.entry(item.key)?.state == DownloadState.RUNNING && System.nanoTime() < deadline) Thread.sleep(10) + DownloadQueue.clear(app, item.key) + } + } + + @Test fun fileFallbackVerifiesRemoteBytesAndPreservesOriginalOnCorruption() { + val folder = temporary.newFolder() + val content = "verified remote content".toByteArray() + val original = File(folder, "game.bin").apply { writeText("local data") } + val server = java.net.ServerSocket(0, 10, java.net.InetAddress.getByName("127.0.0.1")) + var reply = content + var transferred = 0 + val worker = Thread { + while (!server.isClosed) { + val client = try { server.accept() } catch (_: java.io.IOException) { break } + client.use { + val reader = it.getInputStream().bufferedReader() + while (!reader.readLine().isNullOrEmpty()) Unit + it.getOutputStream().apply { + write("HTTP/1.1 206 Partial Content\r\nContent-Length: ${reply.size}\r\nConnection: close\r\n\r\n".toByteArray()) + write(reply); flush(); transferred += reply.size + } + } + } + }.apply { isDaemon = true; start() } + try { + val expected = File(folder, "expected").apply { writeBytes(content) } + val file = JSONObject().put("path", "game.bin").put("size", content.size).put("md5", GogFiles.hash(expected)) + .put("product", "1").put("offset", 0).put("remotePath", "data") + val plan = JSONObject().put("files", JSONArray().put(file)).put("links", JSONObject().put("1", JSONArray().put("http://127.0.0.1:${server.localPort}"))) + reply = ByteArray(content.size) { 65 } + assertThrows(IllegalStateException::class.java) { GogFallback.transfer(folder, plan, null, AtomicBoolean(false)) } + assertEquals("local data", original.readText()) + reply = content + GogFallback.transfer(folder, plan, null, AtomicBoolean(false)) + assertArrayEquals(content, original.readBytes()) + val before = transferred + GogFallback.transfer(folder, plan, null, AtomicBoolean(false)) + assertEquals(before, transferred) + } finally { server.close(); worker.join(2000) } + } +} diff --git a/app/src/test/resources/gog/README.md b/app/src/test/resources/gog/README.md new file mode 100644 index 0000000000..88e237a116 --- /dev/null +++ b/app/src/test/resources/gog/README.md @@ -0,0 +1,8 @@ +Generation 1 fixtures from GOG's public Rayman 2 repository, timestamp 31847977, +retrieved 2026-10-28. The repository and Neutral depot are complete. The English +depot is an excerpt containing its first three original file entries. + +https://gog-cdn-fastly.gog.com/content-system/v1/manifests/1207658940/windows/31847977/repository.json + +The fixtures retain provider paths, support flags, range offsets, and original +hexadecimal hashes. They contain no game payload, credentials, or signed links. diff --git a/app/src/test/resources/gog/rayman-gen1-english-excerpt.json b/app/src/test/resources/gog/rayman-gen1-english-excerpt.json new file mode 100644 index 0000000000..dfca15da87 --- /dev/null +++ b/app/src/test/resources/gog/rayman-gen1-english-excerpt.json @@ -0,0 +1,29 @@ +{ + "version": 1, + "depot": { + "name": "Rayman 2 - The Great Escape", + "files": [ + { + "offset": 384206096, + "hash": "a08070ecde7f746779259f7ce5775747", + "url": "1207658940/main.bin", + "path": "/Data/World/Levels/Raycap/Raycap.snd", + "size": 32 + }, + { + "offset": 114255832, + "hash": "67165fe729ffda351a04e8bd92e05531", + "url": "1207658940/main.bin", + "path": "/Data/World/Sound/Bnk_29.bnm", + "size": 76 + }, + { + "offset": 97474601, + "hash": "2d2e0b308cae5ff44a620fd32bf6cb4d", + "url": "1207658940/main.bin", + "path": "/Data/World/Sound/Bnk_43.bnm", + "size": 76 + } + ] + } +} diff --git a/app/src/test/resources/gog/rayman-gen1-neutral.json b/app/src/test/resources/gog/rayman-gen1-neutral.json new file mode 100644 index 0000000000..fde2b69291 --- /dev/null +++ b/app/src/test/resources/gog/rayman-gen1-neutral.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "depot": { + "name": "Rayman 2 - The Great Escape", + "files": [ + { + "offset": 0, + "path": "1207658940/galaxy_rayman2_2.1.0.39.exe", + "size": 1258944, + "support": true, + "hash": "ee2c247373733f29782395f1f92622f6", + "url": "1207658940/main.bin" + } + ] + } +} diff --git a/app/src/test/resources/gog/rayman-gen1-repository.json b/app/src/test/resources/gog/rayman-gen1-repository.json new file mode 100644 index 0000000000..33922d842b --- /dev/null +++ b/app/src/test/resources/gog/rayman-gen1-repository.json @@ -0,0 +1,65 @@ +{ + "product": { + "timestamp": 31847977, + "depots": [ + { + "languages": [ + "Neutral" + ], + "size": "1258944", + "gameIDs": [ + "1207658940" + ], + "systems": [ + "Windows" + ], + "manifest": "a01f6fb8-55a8-4f33-bc5b-5f5d2f591775.json" + }, + { + "languages": [ + "English" + ], + "size": "505011865", + "gameIDs": [ + "1207658940" + ], + "systems": [ + "Windows" + ], + "manifest": "ec88140c-2589-41a0-bbf3-8fc1f215fe5a.json" + }, + { + "redist": "nGlide", + "size": "0", + "targetDir": "/" + } + ], + "support_commands": [ + { + "languages": [ + "Neutral" + ], + "executable": "/galaxy_rayman2_2.1.0.39.exe", + "gameID": "1207658940", + "argument": "", + "systems": [ + "Windows" + ] + } + ], + "installDirectory": "Rayman 2", + "rootGameID": "1207658940", + "gameIDs": [ + { + "gameID": "1207658940", + "name": { + "en": "Rayman 2 - The Great Escape" + }, + "dependencies": [], + "standalone": true + } + ], + "projectName": "Rayman 2 - The Great Escape" + }, + "version": 1 +} diff --git a/docs/development/gog-maintenance.md b/docs/development/gog-maintenance.md new file mode 100644 index 0000000000..9676db5906 --- /dev/null +++ b/docs/development/gog-maintenance.md @@ -0,0 +1,114 @@ +# GOG maintenance + +Installed GOG games are checked after library refresh and when details open. +Successful automatic checks are throttled to 30 minutes; two metadata workers +run at most. Manual checks bypass the throttle. Downloads and application always +require a user action. Unknown metadata, authentication failures, and network +errors remain distinct from a current installation. + +## Records and recovery + +`.droiddeck-gog/installed.json` is a schema-versioned installation inventory. +It retains source, generation, opaque build identifier, branch, display version, +language, installed products, sanitized manifests, and file hashes/chunks. +Migration uses the existing marker, sidecar, and authentic GOG metadata; missing +historical content is never replaced with the latest manifest as an assumed +description of installed files. +Authentic `goggame-*.id` build metadata takes precedence over legacy markers; +the numeric `version` field in a GOG info file is a schema, not a game version. + +`operation.json` pins the operation, exact folder, previous sidecar/inventory, +target, and phase. Staged chunks are durable work and are rehashed on resume. +The existing download queue persists GOG operation/folder details; older queue +entries keep their install behavior. Uncertain external patch operations cannot +be retried as ordinary downloads. Existing sidecar choices and prefix locations +are retained. Newly entitled DLC is not added automatically. + +Depot updates and repair verify files, reuse matching chunks, and stage changed +files on the installation filesystem. Replacements are atomic per file. There +is no whole-game rollback: interrupted application keeps the journal and blocks +launch until recovery. Obsolete managed files are deleted only when they still +match their previous content. Untracked and modified obsolete files survive. + +The native maintenance contract separates network, reused, and applied bytes. +Generation 1 range downloads retain authentic hashes and never trust equal +size alone. A verified Java file-download path remains available when the native +library is unavailable. Free-space checks include staged replacements and +transfer payloads; a full offline refresh uses a conservative size estimate. + +## Deltas and installers + +Galaxy deltas are requested for the exact source/target pair, and used only +when smaller than missing chunks. Source, payload, and reconstructed target are +verified independently. Decoder errors, unsupported features, or changed source +content fall back to verified chunks. Reconstruction is serialized and bounded +below 64 MiB. Upstream decoder sources and notices are pinned under +`app/src/main/rust/stores/vendor/xdelta3`; Gradle tracks those build inputs. + +Offline patches preserve provider grouping and filenames. Download metadata is +resolved before confirmation; no patch chain is inferred from a filename. +Installer application is interactive through the selected game's Proton tool +and actual prefix. A detached Linux helper owns the installer, tracks prefix +children, writes durable status, and excludes cloud/achievement game-launch +hooks. Registry setup uses the existing GOG script path and authentic setup +payloads. Installer exit alone does not establish the installed version: the +app reconciles the resulting build against a manifest and checks its files. +Unverifiable results require recovery/full refresh instead of silently restoring +the previous manifest. External installer cancellation belongs to the installer. +Setup runs the script interpreter only when an actual GOG script exists, and +registers the authentic installation path, version, and build in the selected +prefix. Patch filenames are decoded from provider URLs without changing literal +plus signs or multipart names. Offline refresh retains the existing executable +layout and includes empty files and directories. Installers can put payload files +at the extraction root while placing placeholders under `app/`; refresh combines +both locations and verifies the selected executable before changing installed +files. Installer language selection accepts the installed locale's base language +and never silently chooses an unrelated language. Refresh with selected DLC also +requires authentic matching depot metadata and completes the selected depots +before publishing the installed record or deleting obsolete content. + +Short helper status requests detect a dead installer owner or abandoned start. +Session loss or suspension requires recovery; it never triggers automatic replay. +The maintenance menu offers result verification for interrupted patch operations. + +The same POSIX record lease excludes maintenance and game launches from DD or +Steam. Session staging publishes the helper to existing Linux installations. +Maintenance processes use full x87 precision. Some installers copy 64-bit data +through x87 integer loads/stores; FEX's reduced-precision mode rounds those values +and corrupts path strings. This override is local to maintenance and leaves the +saved game performance profile unchanged. See [FEX issue #5827](https://github.com/FEX-Emu/FEX/issues/5827). + +## References and validation + +The manifest/chunk/delta approach follows +[heroic-gogdl](https://github.com/Heroic-Games-Launcher/heroic-gogdl/blob/main/gogdl/dl/objects/v2.py). +Generation 1 layout is checked against actual public Rayman 2 manifests in +`app/src/test/resources/gog`. Setup requirements are informed by +[Heroic's GOG setup](https://github.com/Heroic-Games-Launcher/HeroicGamesLauncher/blob/main/src/backend/storeManagers/gog/setup.ts). +The separation of depot and offline maintenance is also reflected in +[Lutris](https://github.com/lutris/lutris/blob/master/lutris/services/gog.py). +[innoextract](https://constexpr.org/innoextract/) extracts data; it does not run +installer scripts and checks. + +Automated coverage includes hash verification, moved/repeated chunk reuse, +interrupted staging, corrupt deltas and fallback, generation 1 same-size +corruption, path protection, multipart metadata, external patch recovery, +launch leases, prefix process detection, and the Java fallback. Run Rust and +Linux helper suites in WSL. JVM results, release/native/runtime-input checks, +CI results, and actual-device evidence must be reported separately in the PR. + +Thor testing has verified a real Rayman 2 depot update, repair of deliberately +corrupted Rayman content, process-death download recovery, shared Java/Linux +launch exclusion, and a Neo Cab repair on external SD storage with its sidecar +unchanged. The owned ELDERBORN 1.08-to-1.09 standalone patch initially failed with +`Path not found`: a Wine file trace showed corrupted path characters under reduced +x87 precision. With full precision, the actual patch completed through DD's normal +installer UI, returned zero, and its files matched all 381 entries in target build +`55745895026856358`. The saved performance configuration was byte-for-byte unchanged. +The patched game then reached its main menu through the existing Steam shortcut; +its process used the ordinary reduced-precision setting rather than the installer's +full-precision override. The original sidecar remained identical and the operation +journal cleared after verification. +Rayman 2 also launches through its existing Steam shortcut after maintenance and +reaches its interactive language menu. Its existing cloud download remains on +the ordinary game-launch path; patch execution bypasses that path. diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-gog-maintenance b/tools/linuxfs/overlay/usr/local/bin/droiddeck-gog-maintenance new file mode 100644 index 0000000000..4a56fd1226 --- /dev/null +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-gog-maintenance @@ -0,0 +1,344 @@ +#!/usr/bin/python3 +"""GOG patch installers and the lease shared by maintenance and game launches.""" +import fcntl +import errno +import json +import os +from pathlib import Path +import runpy +import subprocess +import sys +import tempfile +import time + +BIN = Path(__file__).resolve().parent +IDLE = {"wineserver", "services.exe", "winedevice.exe", "explorer.exe", "rpcss.exe", "plugplay.exe", + "conhost.exe", "lsass.exe", "svchost.exe", "steam.exe", "wineboot.exe", "winemenubuilder.exe"} + + +def control(folder): + folder = Path(folder).resolve(strict=True) + root = folder / ".droiddeck-gog" + if root.is_symlink(): + raise ValueError("GOG control directory is a symbolic link") + root.mkdir(exist_ok=True) + return root + + +def read(path): + if path.is_symlink(): + raise ValueError("GOG metadata is a symbolic link") + return json.loads(path.read_text(encoding="utf-8")) + + +def write(path, value): + if path.is_symlink(): + raise ValueError("GOG metadata is a symbolic link") + descriptor, name = tempfile.mkstemp(prefix=path.name + ".", suffix=".new", dir=path.parent) + temporary = Path(name) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + try: + os.fsync(directory) + except OSError as error: + if error.errno not in (errno.EINVAL, errno.ENOTSUP): + raise + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + + +def lock_file(root): + return os.fdopen(os.open(root / "lease", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), "r+") + + +def generation(pid): + try: + return Path("/proc/%s/stat" % pid).read_text().rsplit(")", 1)[1].split()[19] + except (OSError, IndexError): + return None + + +def lease(folder, owner): + root = control(folder) + with lock_file(root) as stream: + fcntl.lockf(stream, fcntl.LOCK_SH | fcntl.LOCK_NB) + sidecar = read(Path(folder) / ".droiddeck-store.json") + if sidecar.get("state", "installed") != "installed": + raise ValueError("Resume or recover the GOG update before playing") + journal = root / "operation.json" + if journal.exists() and read(journal).get("phase") != "download": + raise ValueError("GOG maintenance is incomplete") + start = generation(owner) + if start is None: + raise ValueError("Game launcher has exited") + print("ready", flush=True) + sys.stdout.close() + while generation(owner) == start: + time.sleep(0.25) + return 0 + + +def desktop_environment(base, proc=Path("/proc")): + env = dict(base) + if env.get("DROIDDECK_GOG_DESKTOP") != "1": + return env + deadline = time.monotonic() + 45 + while True: + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + command = (entry / "cmdline").read_bytes().split(b"\0")[0] + if command != b"/usr/bin/plasmashell": + continue + values = dict(os.fsdecode(item).split("=", 1) for item in (entry / "environ").read_bytes().split(b"\0") if b"=" in item) + if values.get("BL_DESKTOP") != "1" or values.get("XDG_RUNTIME_DIR") != env.get("XDG_RUNTIME_DIR"): + continue + if not values.get("DISPLAY") or values.get("WAYLAND_DISPLAY") == env.get("WAYLAND_DISPLAY"): + continue + for key in ("DISPLAY", "WAYLAND_DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "BL_DESKTOP"): + if key in values: + env[key] = values[key] + return env + except (OSError, ValueError): + continue + if time.monotonic() >= deadline: + raise ValueError("The desktop display is not ready; resume the Linux session and verify recovery before retrying") + time.sleep(0.25) + + +def target_for(folder, appid, game): + run = runpy.run_path(str(BIN / "droiddeck-proton-run"))["main"].__globals__ + root = os.environ.get("STEAM_COMPAT_CLIENT_INSTALL_PATH") or run["STEAM_ROOT"] + target = run["shortcut_target"](str(int(appid) & 0xffffffff), str(game), str(folder), root) + target.install = str(folder) + return run, root, target + + +def runner(folder, appid, game, graphical=False): + run, root, target = target_for(folder, appid, game) + available = run["tools"](root) + name, _ = run["choose_tool"](target.appid, run["mapping"](root), available) + if name is None: + raise ValueError("No Proton tool is installed for this game") + tool, command = available[name] + base = desktop_environment(os.environ) if graphical else dict(os.environ) + env = run["environment"](base, target, tool, root, window=base.get("BL_DESKTOP") == "1") + env["DROIDDECK_GOG_MAINTENANCE"] = "1" + # Installer copy routines use x87 integer loads/stores; reduced precision corrupts paths (FEX #5827). + env["FEX_X87REDUCEDPRECISION"] = "0" + env.pop("BL_STORE_EXIT", None) + return str(Path(tool) / command), env, target.compatdata + + +def active(prefix, proc=Path("/proc")): + result = [] + prefix = os.path.realpath(os.path.join(prefix, "pfx")) + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + env = dict(part.split(b"=", 1) for part in (entry / "environ").read_bytes().split(b"\0") if b"=" in part) + if os.path.realpath(os.fsdecode(env.get(b"WINEPREFIX", b""))) != prefix: + continue + args = (entry / "cmdline").read_bytes().split(b"\0") + name = os.fsdecode(args[0]).replace("\\", "/").rsplit("/", 1)[-1].lower() + if name not in IDLE: + result.append(int(entry.name)) + except (OSError, ValueError): + continue + return result + + +def idle(folder, appid, game): + _, _, target = target_for(folder, appid, game) + if active(target.compatdata): + raise ValueError("Close the GOG game before maintenance") + return 0 + + +def validate_installer(folder, installer): + root = control(folder) + path = Path(installer) + if path.is_symlink() or not path.resolve(strict=True).is_relative_to(root / "downloads") or path.suffix.lower() != ".exe": + raise ValueError("The patch executable is outside GOG download storage") + with path.open("rb") as stream: + if stream.read(2) != b"MZ": + raise ValueError("The GOG patch is not a Windows executable") + return path + + +def installer_arguments(path): + with path.open("rb") as stream: + data = stream.read(4 * 1024 * 1024) + return ["/NOGUI"] if b"Inno Setup Setup Data" in data else [] + + +def start(folder, appid, installer, game): + root = control(folder) + validate_installer(folder, installer) + with lock_file(root) as stream: + fcntl.lockf(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + journal = read(root / "operation.json") + if journal.get("phase") != "patch-ready": + raise ValueError("The patch must be selected again before it can run") + idle(folder, appid, game) + journal["phase"] = "patch-starting" + write(root / "operation.json", journal) + write(root / "patch-status.json", {"state": "starting", "startedAt": time.time()}) + with (root / "patch.log").open("ab") as log: + subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "run", folder, appid, installer, game], + stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True, close_fds=True) + return 0 + + +def status(folder): + root = control(folder) + path = root / "patch-status.json" + result = read(path) + ended = result.get("state") == "running" and generation(result.get("pid")) != result.get("generation") + abandoned = result.get("state") == "starting" and time.time() - result.get("startedAt", 0) > 60 + if ended or abandoned: + result = {"state": "uncertain", "reason": "The patch helper ended without a confirmed result; verify recovery before retrying"} + write(path, result) + journal = read(root / "operation.json") + journal["phase"] = "patch-uncertain" + write(root / "operation.json", journal) + print(json.dumps(result), flush=True) + return 0 + + +def run_patch(folder, appid, installer, game): + root = control(folder) + try: + with lock_file(root) as stream: + fcntl.lockf(stream, fcntl.LOCK_EX) + path = validate_installer(folder, installer) + command, env, prefix = runner(folder, appid, game, True) + if active(prefix): + raise ValueError("Close the GOG game before applying the patch") + journal = read(root / "operation.json") + if journal.get("phase") != "patch-starting": + raise ValueError("The patch is already running or needs recovery") + journal["phase"] = "patch-applying" + write(root / "operation.json", journal) + sidecar_path = Path(folder) / ".droiddeck-store.json" + sidecar = read(sidecar_path) + sidecar["state"] = "updating" + write(sidecar_path, sidecar) + installed_path = root / "installed.json" + if installed_path.is_file(): + installed = read(installed_path) + installed.update(verified=False, manifest={}, files=[]) + write(installed_path, installed) + write(root / "patch-status.json", {"state": "running", "pid": os.getpid(), "generation": generation(os.getpid())}) + os.makedirs(prefix, exist_ok=True) + setup = runpy.run_path(str(BIN / "droiddeck-store-launch"))["main"].__globals__ + previous = dict(os.environ) + try: + os.environ.update(env) + pfx = Path(prefix) / "pfx" + if not (pfx / "system.reg").is_file() or not (pfx / "drive_c/windows/system32/reg.exe").is_file(): + subprocess.run([command, "run", "wineboot.exe", "-u"], env=env, timeout=90, check=True) + if not (pfx / "drive_c/windows/system32/reg.exe").is_file(): + raise ValueError("Proton did not initialize the game's registry tool") + result = setup["apply_gog_registry"](Path(folder), prefix, command, installation=journal.get("before")) + if not result.startswith(("ready", "written=")): + raise ValueError("GOG registry preparation failed; the patch was not applied") + prepare_setup(folder, command, env) + finally: + os.environ.clear(); os.environ.update(previous) + proc = subprocess.Popen([command, "run", str(path), *installer_arguments(path)], cwd=str(path.parent), env=env) + code = proc.wait() + quiet_at = time.monotonic() + while True: + if active(prefix): + quiet_at = time.monotonic() + elif time.monotonic() - quiet_at >= 2: + break + time.sleep(0.25) + journal["phase"] = "patch-uncertain" + write(root / "operation.json", journal) + if code: + raise ValueError("GOG patch installer exited with code %d; inspect patch.log" % code) + write(root / "patch-status.json", {"state": "done", "exitCode": code}) + return 0 + except Exception as error: + write(root / "patch-status.json", {"state": "failed", "reason": str(error)}) + return 1 + + +def prepare_setup(folder, command, env): + root = control(folder) + record_path = root / "installed.json" + if not record_path.is_file(): + return + journal = read(root / "operation.json") + record = journal.get("before") or read(record_path) + manifest = record.get("manifest", {}) + if not record.get("build"): + return + info_path = Path(folder) / ("goggame-%s.info" % record["id"]) + language = read(info_path).get("language", "") if info_path.is_file() else "" + legacy = manifest.get("product", {}) + products = manifest.get("products", []) if not legacy else [ + {"productId": str(item["gameID"]), "temp_executable": item["executable"].lstrip("/\\")} + for item in legacy.get("support_commands", []) + if "Neutral" in item.get("languages", []) or record.get("language") in item.get("languages", []) or language in item.get("languages", [])] + for product in products: + if str(product.get("productId")) not in record.get("products", []): + continue + program = None + if manifest.get("scriptInterpreter"): + if not (Path(folder) / ("goggame-%s.script" % product["productId"])).is_file(): + continue + program = Path(folder) / "support/__redist/ISI/scriptinterpreter.exe" + elif product.get("temp_executable"): + relative = product["temp_executable"].replace("\\", "/") + for candidate in (Path(folder) / "support" / str(product["productId"]) / relative, Path(folder) / relative): + if candidate.is_file(): + program = candidate + break + if program is None: + continue + if not program.resolve(strict=True).is_relative_to(Path(folder).resolve()): + raise ValueError("GOG setup executable leaves the installation") + windows = "Z:" + str(Path(folder).resolve()).replace("/", "\\") + args = ["/VERYSILENT", "/DIR=" + windows, "/ProductId=" + str(product["productId"]), + "/galaxyclient", "/buildId=" + str(legacy.get("timestamp", record["build"])), "/versionName=" + record.get("version", ""), + "/lang-code=" + record.get("language", "en-US"), "/supportDir=" + windows + "\\support", "/nodesktopshortcut"] + if language: + args += ["/Language=" + language, "/LANG=" + language.lower()] + result = subprocess.run([command, "run", str(program), *args], env=env, cwd=folder, timeout=120) + if result.returncode: + raise ValueError("GOG installation setup failed; the patch was not applied") + + +def main(argv): + if len(argv) == 3 and argv[0] == "lease": + return lease(argv[1], int(argv[2])) + if len(argv) == 4 and argv[0] == "idle": + return idle(*argv[1:]) + if len(argv) == 2 and argv[0] == "status": + return status(argv[1]) + if len(argv) == 5 and argv[0] == "start": + return start(*argv[1:]) + if len(argv) == 5 and argv[0] == "run": + return run_patch(*argv[1:]) + raise ValueError("Invalid GOG maintenance command") + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except Exception as error: + print("GOG maintenance: " + str(error), file=sys.stderr) + sys.exit(1) diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-store-launch b/tools/linuxfs/overlay/usr/local/bin/droiddeck-store-launch index 718498dff8..682cfc0edf 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-store-launch +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-store-launch @@ -40,6 +40,7 @@ import time import subprocess import tempfile import hashlib +import threading from pathlib import Path LAUNCHER = ".droiddeck-launch.bat" @@ -494,8 +495,27 @@ def gog_registry(folder): return entries, skipped -def apply_gog_registry(folder, compat_data, proton, timeout=90): +def apply_gog_registry(folder, compat_data, proton, timeout=90, installation=None): entries, skipped = gog_registry(folder) + if installation: + product = str(installation["id"]) + if not product.isdigit(): + raise ValueError("Invalid GOG product identifier") + values = {"PATH": windows_path(str(folder))} + identity = Path(folder) / ("goggame-%s.id" % product) + if identity.is_file(): + actual = json.loads(identity.read_text(encoding="utf-8-sig")).get("buildId") + if actual and installation.get("build") and str(actual) != str(installation["build"]): + raise ValueError("GOG build metadata changed before patch setup") + if installation.get("build"): + values["BUILDID"] = installation["build"] + if installation.get("version"): + values["VER"] = installation["version"] + for key, value in values.items(): + entry = gog_entry({"root": "HKLM", "subkey": "Software\\GOG.com\\Games\\" + product, + "valueName": key, "valueData": value, "valueType": "string"}, {}) + if entry: + entries.append(entry) note = " skipped=%d" % skipped if skipped else "" if not entries: return "none" + note @@ -512,7 +532,8 @@ def apply_gog_registry(folder, compat_data, proton, timeout=90): with tempfile.TemporaryDirectory(prefix="droiddeck-gog-") as tmp: source = Path(tmp) / "registry.reg" source.write_bytes(b"\xff\xfe" + text.encode("utf-16-le")) - run = subprocess.run([proton, "run", "reg.exe", "import", windows_path(str(source)), "/reg:32"], + executable = pfx / "drive_c/windows/system32/reg.exe" + run = subprocess.run([proton, "run", str(executable) if executable.is_file() else "reg.exe", "import", windows_path(str(source)), "/reg:32"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout) if run.returncode or not (pfx / "system.reg").is_file(): return "import-failed" + note @@ -525,6 +546,8 @@ def apply_gog_registry(folder, compat_data, proton, timeout=90): def main(argv): + if os.environ.get("DROIDDECK_GOG_MAINTENANCE") == "1": + return 0 if argv and argv[0] == "--exited": rest, code = argv[1:], 0 if rest[:1] == ["--code"] and len(rest) > 1: @@ -543,6 +566,18 @@ def main(argv): return 0 folder, sidecar = found store, game = str(sidecar.get("store")), str(sidecar.get("id")) + if store == "gog": + helper = Path(__file__).resolve().with_name("droiddeck-gog-maintenance") + if helper.is_file(): + child = subprocess.Popen([sys.executable, str(helper), "lease", str(folder), str(os.getppid())], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, + start_new_session=True) + ready = child.stdout.readline().decode().strip() + child.stdout.close() + threading.Thread(target=child.wait, daemon=True).start() + if ready != "ready": + log("gog: close or recover the maintenance operation before playing") + return RAN_FAILED compat_data = os.environ.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/") prefix_id = os.path.basename(compat_data) if compat_data else "" launch_dir = os.environ.get("BL_LAUNCH_DIR", "") diff --git a/tools/release/runtime_inputs.py b/tools/release/runtime_inputs.py index ccce1f0f0d..6db216383a 100644 --- a/tools/release/runtime_inputs.py +++ b/tools/release/runtime_inputs.py @@ -76,6 +76,7 @@ def required(): for name in ['turnip26.3.0-a6xx', 'turnip26.3.0-quest2']] paths += ['assets/linuxfs/' + name for name in ['libfakeinput.so', 'libblsession.so', 'libblfastpath.so', 'libssbs.so', 'usr/local/bin/droiddeck-clipboard', + 'usr/local/bin/droiddeck-gog-maintenance', 'usr/local/bin/gamescope', 'usr/local/lib/droiddeck/uruntime', 'usr/local/lib/droiddeck-pipewire.tzst']] for arch in ['x86_64', 'i386']: diff --git a/tools/tests/test_game_environment.py b/tools/tests/test_game_environment.py index 0585fc57f1..0fa6b07d93 100644 --- a/tools/tests/test_game_environment.py +++ b/tools/tests/test_game_environment.py @@ -30,6 +30,24 @@ def config_text(entries): class GameDebugLoggingTest(unittest.TestCase): + def test_maintenance_precision_survives_run_with_performance_game_settings(self): + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) + config = home / ".config/droiddeck/game-environment.json" + config.parent.mkdir(parents=True) + config.write_text(json.dumps({"version": 1, "shared": {"FEX_X87REDUCEDPRECISION": "1"}, + "games": {"42": {"FEX_X87REDUCEDPRECISION": "1"}}})) + probe = home / "proton" + probe.write_text("#!" + sys.executable + "\nimport os\nprint(os.environ['FEX_X87REDUCEDPRECISION'])\n") + probe.chmod(0o755) + base = {**os.environ, "HOME": tmp, "STEAM_COMPAT_DATA_PATH": str(home / "compatdata/42")} + maintenance = subprocess.run([sys.executable, str(BIN / "droiddeck-game-env"), str(probe), "run"], + env={**base, "DROIDDECK_GOG_MAINTENANCE": "1", "FEX_X87REDUCEDPRECISION": "0"}, text=True, capture_output=True, check=True) + game = subprocess.run([sys.executable, str(BIN / "droiddeck-game-env"), str(probe), "waitforexitandrun"], + env={**base, "FEX_X87REDUCEDPRECISION": "1"}, text=True, capture_output=True, check=True) + self.assertEqual("0", maintenance.stdout.strip()) + self.assertEqual("1", game.stdout.strip()) + def test_real_game_launch_without_config_file_keeps_the_base_environment(self): with tempfile.TemporaryDirectory() as tmp: home = Path(tmp) diff --git a/tools/tests/test_gog_maintenance.py b/tools/tests/test_gog_maintenance.py new file mode 100644 index 0000000000..0465e91b42 --- /dev/null +++ b/tools/tests/test_gog_maintenance.py @@ -0,0 +1,216 @@ +import contextlib +import concurrent.futures +import errno +import json +import os +from pathlib import Path +import runpy +import subprocess +import sys +import stat +import tempfile +import unittest +from unittest import mock + +BIN = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin" +G = runpy.run_path(str(BIN / "droiddeck-gog-maintenance"))["main"].__globals__ + + +class GogMaintenance(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + self.control = G["control"](self.folder) + (self.folder / ".droiddeck-store.json").write_text(json.dumps({"store": "gog", "id": "1"})) + + def test_only_downloaded_windows_executables_can_be_patches(self): + download = self.control / "downloads" + download.mkdir() + patch = download / "patch.exe" + patch.write_bytes(b"MZtest") + self.assertEqual(patch, G["validate_installer"](self.folder, patch)) + outside = self.folder / "outside.exe" + outside.write_bytes(b"MZtest") + with self.assertRaises(ValueError): + G["validate_installer"](self.folder, outside) + patch.write_bytes(b"not PE") + with self.assertRaises(ValueError): + G["validate_installer"](self.folder, patch) + + def test_overlapping_metadata_writes_publish_complete_documents(self): + target = self.control / "operation.json" + def publish(value): + G["write"](target, {"value": value, "payload": str(value) * 10000}) + result = G["read"](target) + self.assertEqual(str(result["value"]) * 10000, result["payload"]) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(publish, range(32))) + self.assertEqual([], list(self.control.glob("*.new"))) + + def test_directory_sync_unsupported_by_external_storage_keeps_the_record(self): + original = os.fsync + def fsync(fd): + if stat.S_ISDIR(os.fstat(fd).st_mode): + raise OSError(errno.EINVAL, "directory sync unsupported") + return original(fd) + target = self.control / "operation.json" + with mock.patch.object(G["os"], "fsync", side_effect=fsync): + G["write"](target, {"phase": "download"}) + self.assertEqual("download", G["read"](target)["phase"]) + + def test_unsuccessful_registry_preparation_never_runs_the_installer(self): + download = self.control / "downloads"; download.mkdir() + patch = download / "patch.exe"; patch.write_bytes(b"MZtest") + prefix = self.folder / "prefix/pfx" + (prefix / "drive_c/windows/system32").mkdir(parents=True) + (prefix / "drive_c/windows/system32/reg.exe").write_bytes(b"MZreg") + (prefix / "system.reg").write_text("registry") + for result in ("timeout", "import-failed", "no-proton", "none", "unexpected"): + with self.subTest(result=result): + G["write"](self.control / "operation.json", {"phase": "patch-starting", "before": {"id": "1"}}) + def setup_main(): + pass + with mock.patch.dict(setup_main.__globals__, {"apply_gog_registry": lambda *a, **k: result}), \ + mock.patch.dict(G, {"runner": lambda *a: ("/proton", {}, str(self.folder / "prefix")), "active": lambda *a: []}), \ + mock.patch.object(G["runpy"], "run_path", return_value={"main": setup_main}), \ + mock.patch.object(G["subprocess"], "Popen") as spawn: + self.assertEqual(1, G["run_patch"](str(self.folder), "1", str(patch), "game.exe")) + spawn.assert_not_called() + self.assertIn("registry preparation failed", G["read"](self.control / "patch-status.json")["reason"]) + + def test_only_recognized_inno_installers_get_the_gog_ui_compatibility_argument(self): + path = self.folder / "patch.exe" + path.write_bytes(b"MZunknown installer") + self.assertEqual([], G["installer_arguments"](path)) + path.write_bytes(b"MZdata Inno Setup Setup Data (6.2.1)") + self.assertEqual(["/NOGUI"], G["installer_arguments"](path)) + + def test_game_launch_lease_excludes_updater_and_releases_on_owner_exit(self): + import fcntl + owner = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + self.addCleanup(lambda: owner.poll() is None and owner.kill()) + lease = subprocess.Popen([sys.executable, str(BIN / "droiddeck-gog-maintenance"), "lease", str(self.folder), str(owner.pid)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(lease.stdout.close) + self.addCleanup(lease.stderr.close) + self.addCleanup(lambda: lease.poll() is None and lease.kill()) + self.assertEqual(b"ready\n", lease.stdout.readline()) + with G["lock_file"](self.control) as stream: + with self.assertRaises(BlockingIOError): + fcntl.lockf(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + owner.terminate(); owner.wait() + lease.wait(timeout=5) + with G["lock_file"](self.control) as stream: + fcntl.lockf(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + + def test_incomplete_update_cannot_launch(self): + (self.folder / ".droiddeck-store.json").write_text(json.dumps({"store": "gog", "id": "1", "state": "updating"})) + with self.assertRaises(ValueError): + G["lease"](self.folder, os.getpid()) + + def test_running_game_and_wrong_prefix_are_distinguished(self): + proc = self.folder / "proc" + for pid, prefix, name in [(10, "/prefix/pfx", "game.exe"), (11, "/other/pfx", "other.exe"), (12, "/prefix/pfx", "services.exe")]: + row = proc / str(pid); row.mkdir(parents=True) + (row / "environ").write_bytes(b"WINEPREFIX=" + prefix.encode() + b"\0") + (row / "cmdline").write_bytes(name.encode() + b"\0") + self.assertEqual([10], G["active"]("/prefix", proc)) + + def test_desktop_installer_inherits_the_nested_display_not_the_bridge_display(self): + proc = self.folder / "proc" + row = proc / "10"; row.mkdir(parents=True) + (row / "cmdline").write_bytes(b"/usr/bin/plasmashell\0") + (row / "environ").write_bytes(b"BL_DESKTOP=1\0XDG_RUNTIME_DIR=/run/session\0WAYLAND_DISPLAY=wayland-1\0DISPLAY=:0\0XAUTHORITY=/run/session/auth\0UNRELATED=value\0") + env = G["desktop_environment"]({"DROIDDECK_GOG_DESKTOP": "1", "XDG_RUNTIME_DIR": "/run/session", "WAYLAND_DISPLAY": "wayland-0"}, proc) + self.assertEqual("wayland-1", env["WAYLAND_DISPLAY"]) + self.assertEqual(":0", env["DISPLAY"]) + self.assertEqual("/run/session/auth", env["XAUTHORITY"]) + self.assertEqual("1", env["BL_DESKTOP"]) + self.assertNotIn("UNRELATED", env) + + def test_maintenance_uses_full_x87_precision_without_changing_game_preferences(self): + target = mock.Mock(compatdata="/selected/prefix", appid="2526420341") + run = {"tools": lambda root: {"selected": ("/selected/proton", "proton")}, + "mapping": lambda root: {}, "choose_tool": lambda *args: ("selected", None), + "environment": lambda env, *args, **kwargs: dict(env)} + with mock.patch.dict(G, {"target_for": lambda *args: (run, "/steam", target)}), \ + mock.patch.dict(os.environ, {"FEX_X87REDUCEDPRECISION": "1"}): + command, env, prefix = G["runner"]("/game", "2526420341", "/game/game.exe") + self.assertEqual("/selected/proton/proton", command) + self.assertEqual("/selected/prefix", prefix) + self.assertEqual("0", env["FEX_X87REDUCEDPRECISION"]) + self.assertEqual("1", os.environ["FEX_X87REDUCEDPRECISION"]) + + def test_legacy_setup_uses_authentic_timestamp_and_support_layout(self): + program = self.folder / "support/1/setup.exe" + program.parent.mkdir(parents=True); program.write_bytes(b"MZtest") + record = {"id": "1", "build": "3301", "products": ["1"], "language": "en-US", "manifest": {"product": { + "timestamp": 31847977, "support_commands": [{"gameID": "1", "executable": "/setup.exe", "languages": ["English"]}]}}} + G["write"](self.control / "installed.json", {"verified": False}) + G["write"](self.control / "operation.json", {"before": record}) + G["write"](self.folder / "goggame-1.info", {"language": "English"}) + with mock.patch.object(G["subprocess"], "run", return_value=mock.Mock(returncode=0)) as run: + G["prepare_setup"](str(self.folder), "/proton", {}) + args = run.call_args.args[0] + self.assertEqual(str(program), args[2]) + self.assertIn("/buildId=31847977", args) + self.assertNotIn("/buildId=3301", args) + self.assertIn("/Language=English", args) + + def test_interrupted_patch_is_not_started_again(self): + (self.control / "operation.json").write_text(json.dumps({"phase": "patch-applying"})) + with mock.patch.dict(G, {"validate_installer": lambda *a: self.folder / "patch.exe"}): + with self.assertRaises(ValueError): + G["start"](str(self.folder), "2147483666", "patch.exe", "game.exe") + + def test_dead_installer_helper_requires_recovery_without_replay(self): + G["write"](self.control / "operation.json", {"phase": "patch-applying"}) + G["write"](self.control / "patch-status.json", {"state": "running", "pid": 99999999, "generation": "123"}) + with mock.patch.dict(G, {"generation": lambda pid: None}), mock.patch.object(G["subprocess"], "Popen") as spawn: + self.assertEqual(0, G["status"](str(self.folder))) + self.assertFalse(spawn.called) + self.assertEqual("uncertain", G["read"](self.control / "patch-status.json")["state"]) + self.assertEqual("patch-uncertain", G["read"](self.control / "operation.json")["phase"]) + + def test_abandoned_start_and_confirmed_completion_are_distinct(self): + G["write"](self.control / "operation.json", {"phase": "patch-starting"}) + G["write"](self.control / "patch-status.json", {"state": "starting", "startedAt": 10}) + with mock.patch.object(G["time"], "time", return_value=100): + G["status"](str(self.folder)) + self.assertEqual("uncertain", G["read"](self.control / "patch-status.json")["state"]) + G["write"](self.control / "patch-status.json", {"state": "done", "exitCode": 0}) + G["status"](str(self.folder)) + self.assertEqual({"state": "done", "exitCode": 0}, G["read"](self.control / "patch-status.json")) + + def test_registry_and_installer_use_same_prefix_without_cloud_launch(self): + download = self.control / "downloads"; download.mkdir() + patch = download / "patch.exe"; patch.write_bytes(b"MZtest") + prefix = self.folder / "prefix/pfx" + (prefix / "drive_c/windows/system32").mkdir(parents=True) + (prefix / "drive_c/windows/system32/reg.exe").write_bytes(b"MZreg") + (prefix / "system.reg").write_text("registry") + G["write"](self.control / "operation.json", {"phase": "patch-starting"}) + calls = [] + setup = {"main": lambda: None} + def setup_main(): + pass + setup_main.__globals__["apply_gog_registry"] = lambda *a, **k: calls.append(("registry", a[1])) or "written=1" + fake = mock.Mock(); fake.wait.return_value = 0 + def spawn(argv, **kwargs): + self.assertEqual("run", argv[1]) + self.assertEqual("1", kwargs["env"]["DROIDDECK_GOG_MAINTENANCE"]) + calls.append(("patch", kwargs["env"]["STEAM_COMPAT_DATA_PATH"])) + return fake + with mock.patch.dict(G, {"runner": lambda *a: ("/proton", {"STEAM_COMPAT_DATA_PATH": str(self.folder / "prefix"), "DROIDDECK_GOG_MAINTENANCE": "1"}, str(self.folder / "prefix")), + "active": lambda *a: []}), \ + mock.patch.object(G["runpy"], "run_path", return_value={"main": setup_main}), \ + mock.patch.object(G["subprocess"], "Popen", side_effect=spawn): + self.assertEqual(0, G["run_patch"](str(self.folder), "2147483666", str(patch), "game.exe")) + self.assertEqual([("registry", str(self.folder / "prefix")), ("patch", str(self.folder / "prefix"))], calls) + self.assertEqual("done", G["read"](self.control / "patch-status.json")["state"]) + self.assertEqual("updating", G["read"](self.folder / ".droiddeck-store.json")["state"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_store_launch.py b/tools/tests/test_store_launch.py index c42448424a..5dadcfb2fb 100644 --- a/tools/tests/test_store_launch.py +++ b/tools/tests/test_store_launch.py @@ -391,6 +391,30 @@ def set_registry(**arguments): class GogRegistry(unittest.TestCase): + def test_patch_registration_uses_authentic_build_and_the_prefix_registry_executable(self): + with tempfile.TemporaryDirectory() as root: + folder = game(root, store_id='gog', ident='7') + (folder / 'goggame-7.id').write_text(json.dumps({'buildId': 'opaque-0001'})) + proton = Path(root) / 'proton'; proton.write_text('selected Proton') + data = Path(root) / 'compat' + executable = data / 'pfx/drive_c/windows/system32/reg.exe' + executable.parent.mkdir(parents=True); executable.write_bytes(b'MZreg') + (data / 'pfx/system.reg').write_text('registry') + imported = [] + def run(argv, **kwargs): + self.assertEqual([str(proton), 'run', str(executable), 'import'], argv[:4]) + text = Path(argv[4][2:].replace('\\', '/')).read_bytes()[2:].decode('utf-16-le') + imported.append(text) + return __import__('subprocess').CompletedProcess(argv, 0) + record = {'id': '7', 'build': 'opaque-0001', 'version': 'RC-final'} + with patch.object(store.subprocess, 'run', side_effect=run): + self.assertTrue(store.apply_gog_registry(folder, str(data), str(proton), installation=record).startswith('written=')) + self.assertIn('[HKEY_LOCAL_MACHINE\\Software\\GOG.com\\Games\\7]', imported[0]) + self.assertIn('"BUILDID"="opaque-0001"', imported[0]) + self.assertIn('"VER"="RC-final"', imported[0]) + with self.assertRaises(ValueError): + store.apply_gog_registry(folder, str(data), str(proton), installation=dict(record, build='invented')) + def test_the_scripts_registry_values_are_read_with_the_games_paths(self): with tempfile.TemporaryDirectory() as root: folder = game(root, store_id='gog', ident='1454587428')