feat: apply itch updates as butler patches (#1829)

This commit is contained in:
FX1922 authored and GitHub committed 2026-10-30 07:38:06 -04:00
1 parent 4770d11e39
commit 67d391958f
12 files changed
+388 -12

No files matched your search

@@ -146,5 +146,7 @@ internal class AddonPack(
listOf("version", "xodus-cli", "xodus-service"))
val COMET = AddonPack("comet", BuildConfig.PACK_COMET_SHA256, BuildConfig.PACK_COMET_SIZE,
listOf("version", "comet"))
val BUTLER = AddonPack("butler", BuildConfig.PACK_BUTLER_SHA256, BuildConfig.PACK_BUTLER_SIZE,
listOf("version", "butler", "7z.so", "libc7zip.so"))
}
}
@@ -0,0 +1,114 @@
package com.droiddeck.launcher.stores.itch
import com.droiddeck.launcher.stores.StoreNet
import org.json.JSONArray
import org.json.JSONObject
import java.io.File
import java.io.FileOutputStream
import java.net.HttpURLConnection
import java.net.URL
import java.util.concurrent.atomic.AtomicBoolean
internal object ItchBuilds {
private const val BASE = "https://api.itch.io"
private const val MAX_STEPS = 4
class Step(val buildId: Long, val patchFile: Long, val patchSize: Long, val signatureFile: Long)
fun buildId(upload: JSONObject): String {
val value = upload.opt("build_id") ?: upload.opt("buildId")
return when (value) {
is Number -> value.toLong().takeIf { it > 0 }?.toString().orEmpty()
is String -> value.takeIf { it.all(Char::isDigit) }?.toLongOrNull()?.takeIf { it > 0 }?.toString().orEmpty()
else -> ""
}
}
fun path(key: String, from: String, to: String, downloadKeyId: String): List<Step>? {
val fromId = from.toLongOrNull() ?: return null
val toId = to.toLongOrNull() ?: return null
val reply = ItchApi.get("/builds/$fromId/upgrade-paths/$toId" + keyQuery(downloadKeyId), key) ?: return null
return steps(reply, fromId, toId) { filesOf(key, it, downloadKeyId) }
}
fun steps(reply: JSONObject, fromId: Long, toId: Long, filesFor: (JSONObject) -> JSONArray?): List<Step>? {
val builds = (reply.optJSONObject("upgrade_path") ?: reply.optJSONObject("upgradePath"))?.optJSONArray("builds") ?: return null
val ids = (0 until builds.length()).mapNotNull { builds.optJSONObject(it) }
val after = ids.filter { it.optLong("id") != fromId }
if (after.isEmpty() || after.size > MAX_STEPS || after.last().optLong("id") != toId) return null
val steps = after.map { build ->
val files = filesFor(build) ?: return null
val patch = pick(files, "patch") ?: return null
val signature = pick(files, "signature")
Step(build.optLong("id"), patch.optLong("id"), patch.optLong("size"), signature?.optLong("id") ?: 0L)
}
return steps.takeIf { it.last().signatureFile > 0 }
}
fun fileUrl(buildId: Long, fileId: Long, downloadKeyId: String): String =
"$BASE/wharf/builds/$buildId/files/$fileId/download" + keyQuery(downloadKeyId)
fun download(url: String, key: String, target: File, cancelled: AtomicBoolean, progress: (Long) -> Unit): Boolean {
val first = open(url, key)
try {
val code = first.responseCode
val location = first.getHeaderField("Location")
if (code in 300..399 && location != null) {
first.disconnect()
val next = URL(URL(url), location).toString()
if (!next.startsWith("https://")) return false
return save(open(next, null), target, cancelled, progress)
}
return save(first, target, cancelled, progress)
} finally {
runCatching { first.disconnect() }
}
}
private fun save(conn: HttpURLConnection, target: File, cancelled: AtomicBoolean, progress: (Long) -> Unit): Boolean {
try {
if (conn.responseCode !in 200..299) return false
var done = 0L
val buffer = ByteArray(128 * 1024)
conn.inputStream.use { input ->
FileOutputStream(target).use { out ->
while (true) {
if (cancelled.get()) return false
val read = input.read(buffer)
if (read < 0) break
out.write(buffer, 0, read)
done += read
progress(done)
}
}
}
return true
} finally {
runCatching { conn.disconnect() }
}
}
private fun open(url: String, key: String?): HttpURLConnection =
(URL(url).openConnection() as HttpURLConnection).apply {
connectTimeout = 30_000
readTimeout = 60_000
instanceFollowRedirects = false
setRequestProperty("User-Agent", StoreNet.BROWSER_UA)
setRequestProperty("Accept", "*/*")
if (key != null) setRequestProperty("Authorization", "Bearer $key")
}
private fun filesOf(key: String, build: JSONObject, downloadKeyId: String): JSONArray? =
build.optJSONArray("files")
?: ItchApi.get("/builds/${build.optLong("id")}" + keyQuery(downloadKeyId), key)?.optJSONObject("build")?.optJSONArray("files")
private fun pick(files: JSONArray, type: String): JSONObject? {
val all = (0 until files.length()).mapNotNull { files.optJSONObject(it) }
.filter { it.optString("type") == type && it.optString("state", "uploaded") == "uploaded" }
return all.firstOrNull { subType(it) == "optimized" } ?: all.firstOrNull { subType(it) == "default" }
}
private fun subType(file: JSONObject): String = file.optString("sub_type", file.optString("subType", "default"))
private fun keyQuery(downloadKeyId: String): String = if (downloadKeyId.isBlank()) "" else "?download_key_id=$downloadKeyId"
}
@@ -62,12 +62,31 @@ internal class ItchInstaller(
val scratch = StoreInstallRoot.scratchDir(app, Store.ITCH, item.id)
scratch.mkdirs()
val archive = File(scratch, filename.ifBlank { "download.bin" })
val buildId = ItchBuilds.buildId(upload)
val exe = try {
handle.stage(DownloadStage.DOWNLOAD)
download(key, downloadKeyId, uploadId, upload.optString("updated_at", upload.optString("updatedAt")), archive, size, handle)
if (cancelled.get()) return null
handle.stage(DownloadStage.INSTALL)
install(archive, filename, handle, resuming)
var patched = updating && !resuming && previous?.isInstalled == true &&
patch(key, downloadKeyId, uploadId, buildId, size, scratch, handle)
if (cancelled.get()) {
if (patched) dropCaches(scratch)
return null
}
var picked: String? = null
if (patched) {
picked = install(archive, filename, handle, resuming, true)
if (picked == null) {
if (cancelled.get()) return null
handle.log("itch.io: the patched copy of ${item.title} has no exe, doing a full update")
patched = false
}
}
if (!patched) {
handle.stage(DownloadStage.DOWNLOAD)
download(key, downloadKeyId, uploadId, upload.optString("updated_at", upload.optString("updatedAt")), archive, size, handle)
if (cancelled.get()) return null
handle.stage(DownloadStage.INSTALL)
picked = install(archive, filename, handle, resuming, false)
}
picked
} catch (t: Throwable) {
if (!cancelled.get()) dropCaches(scratch)
throw t
@@ -90,7 +109,7 @@ internal class ItchInstaller(
installedAt = System.currentTimeMillis(),
cover = item.imageUrl,
hero = item.imageUrl,
extra = mapOf("gameId" to gameId, "downloadKeyId" to downloadKeyId, "uploadId" to uploadId),
extra = mapOf("gameId" to gameId, "downloadKeyId" to downloadKeyId, "uploadId" to uploadId, "buildId" to buildId),
cloud = false,
)
StoreInstalls.complete(app, folder, sidecar) { done, steps -> handle.stageProgress(done.toLong(), steps.toLong()) }
@@ -180,13 +199,37 @@ internal class ItchInstaller(
}
}
private fun install(archive: File, filename: String, handle: DownloadQueue.JobHandle, resuming: Boolean): String? {
private fun patch(key: String, downloadKeyId: String, uploadId: String, buildId: String, fullSize: Long, scratch: File, handle: DownloadQueue.JobHandle): Boolean {
val installed = previous?.extra.orEmpty()
val from = installed["buildId"].orEmpty()
if (installed["uploadId"] != uploadId || from.isBlank() || buildId.isBlank() || from == buildId) return false
val out = File(folder, ItchStaging.DIR)
ItchStaging.delete(out)
scratch.mkdirs()
val outcome = try {
ItchPatch.run(app, key, downloadKeyId, from, buildId, fullSize, folder, out, scratch, handle, cancelled)
} catch (t: Exception) {
ItchPatch.Outcome.Failed("${t.javaClass.simpleName} ${t.message}")
}
when (outcome) {
ItchPatch.Outcome.Applied -> {
handle.log("itch.io: patched ${item.title} from build $from to $buildId")
return true
}
ItchPatch.Outcome.Cancelled -> Unit
is ItchPatch.Outcome.Failed -> handle.log("itch.io: no patch for ${item.title}, doing a full update (${outcome.reason})")
}
ItchStaging.delete(out)
return false
}
private fun install(archive: File, filename: String, handle: DownloadQueue.JobHandle, resuming: Boolean, patched: Boolean = false): String? {
work = if (updating) File(folder, ItchStaging.DIR) else folder
if (updating) ItchStaging.delete(work)
if (updating && !patched) ItchStaging.delete(work)
work.mkdirs()
if (resuming) clearPartialInstall()
val exe = try {
unpack(archive, filename, handle)
if (!patched) unpack(archive, filename, handle)
val hint = filename.substringBeforeLast('.')
var picked = StoreExe.pick(work, hint.ifBlank { null }, item.title)
var passes = 0
@@ -0,0 +1,124 @@
package com.droiddeck.launcher.stores.itch
import android.content.Context
import com.droiddeck.launcher.core.AddonPack
import com.droiddeck.launcher.runtime.GuestCommand
import com.droiddeck.launcher.runtime.LinuxRuntime
import com.droiddeck.launcher.stores.StoreInstalls
import com.droiddeck.launcher.stores.download.DownloadQueue
import com.droiddeck.launcher.stores.download.DownloadStage
import java.io.File
import java.util.concurrent.atomic.AtomicBoolean
internal object ItchPatch {
private const val DIR = "/usr/local/lib/droiddeck-butler"
sealed interface Outcome {
data object Applied : Outcome
data object Cancelled : Outcome
data class Failed(val reason: String) : Outcome
}
fun ready(context: Context): Boolean {
if (!AddonPack.BUTLER.available || !LinuxRuntime.isInstalled(context)) return false
if (!AddonPack.BUTLER.installed(context)) {
AddonPack.BUTLER.ensure(context)
return false
}
return stage(context)
}
private val stageLock = Any()
private fun stage(context: Context): Boolean = synchronized(stageLock) { stageLocked(context) }
private fun stageLocked(context: Context): Boolean {
val pack = AddonPack.BUTLER.dir(context)
val root = File(LinuxRuntime.rootDir(context), DIR.removePrefix("/"))
val packaged = File(pack, "version").readText()
if (runCatching { File(root, "version").readText() }.getOrNull() == packaged && File(root, "butler").isFile) return true
root.mkdirs()
pack.listFiles().orEmpty().filter { it.isFile && it.name != "version" && it.name != AddonPack.COMPLETE }.forEach { source ->
val staged = File(root, source.name + ".staged")
source.inputStream().use { input -> staged.outputStream().use { input.copyTo(it) } }
staged.setExecutable(true, false)
check(staged.renameTo(File(root, source.name)))
}
File(root, "version").writeText(packaged)
return true
}
fun run(
app: Context, key: String, downloadKeyId: String, from: String, to: String, fullSize: Long, installed: File, out: File, scratch: File,
handle: DownloadQueue.JobHandle, cancelled: AtomicBoolean,
): Outcome {
if (!ready(app)) return Outcome.Failed("the patch tool is not ready")
val steps = ItchBuilds.path(key, from, to, downloadKeyId) ?: return Outcome.Failed("no patch path from build $from to $to")
if (fullSize > 0 && steps.sumOf { it.patchSize } > fullSize) return Outcome.Failed("the patches are larger than a full download")
val dir = File(scratch, "patches").apply { deleteRecursively(); mkdirs() }
try {
handle.stage(DownloadStage.DOWNLOAD)
val total = steps.sumOf { it.patchSize }
var finished = 0L
val patches = steps.mapIndexed { index, step ->
val file = File(dir, "$index.pwr")
val base = finished
val ok = ItchBuilds.download(ItchBuilds.fileUrl(step.buildId, step.patchFile, downloadKeyId), key, file, cancelled) {
handle.progress(base + it, total)
}
if (cancelled.get()) return Outcome.Cancelled
if (!ok) return Outcome.Failed("build ${step.buildId}: the patch did not download")
finished += step.patchSize
file
}
val last = steps.last()
val signature = File(dir, "final.sig")
val got = ItchBuilds.download(ItchBuilds.fileUrl(last.buildId, last.signatureFile, downloadKeyId), key, signature, cancelled) {}
if (cancelled.get()) return Outcome.Cancelled
if (!got) return Outcome.Failed("build ${last.buildId}: the signature did not download")
handle.stage(DownloadStage.INSTALL)
return apply(app, patches, signature, installed, out, scratch, cancelled)
} finally {
StoreInstalls.deleteTree(dir)
}
}
private fun apply(app: Context, patches: List<File>, signature: File, installed: File, out: File, scratch: File, cancelled: AtomicBoolean): Outcome {
var source = installed
var intermediate: File? = null
try {
patches.forEachIndexed { index, patch ->
val finalStep = index == patches.lastIndex
val target = if (finalStep) out else File(scratch, "patched-$index")
ItchStaging.delete(target)
target.mkdirs()
val staging = File(scratch, "butler-staging-$index").apply { deleteRecursively(); mkdirs() }
val args = mutableListOf("$DIR/butler", "apply", "--assume-yes", "--staging-dir", staging.absolutePath, "-d", target.absolutePath)
if (finalStep) args += listOf("--signature", signature.absolutePath)
args += listOf(patch.absolutePath, source.absolutePath)
val binds = listOf(installed, out, scratch).map { it.absolutePath }
.filter { path -> listOf(app.filesDir, app.cacheDir).none { path.startsWith(it.absolutePath + "/") } }
.distinct().takeIf { it.isNotEmpty() }
val tail = ArrayDeque<String>()
val code = try {
GuestCommand.run(app, args, logName = "itch-butler", extraBinds = binds, fastPath = true, cancelled = cancelled) { line ->
if (!line.startsWith("proot ")) {
if (tail.size == 4) tail.removeFirst()
tail.addLast(line)
}
}
} finally {
StoreInstalls.deleteTree(staging)
}
if (cancelled.get()) return Outcome.Cancelled
if (code != 0) return Outcome.Failed("butler exited $code: ${tail.joinToString(" | ")}")
intermediate?.let { StoreInstalls.deleteTree(it) }
intermediate = if (finalStep) null else target
source = target
}
return Outcome.Applied
} finally {
intermediate?.let { StoreInstalls.deleteTree(it) }
}
}
}