diff --git a/app/src/main/rust/stores/Cargo.lock b/app/src/main/rust/stores/Cargo.lock new file mode 100644 index 0000000000..232f425812 --- /dev/null +++ b/app/src/main/rust/stores/Cargo.lock @@ -0,0 +1,2104 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bl-steam-client" +version = "0.1.0" +dependencies = [ + "aes", + "cbc", + "crc32fast", + "flate2", + "futures-util", + "hmac", + "jni", + "libc", + "lzma-rs", + "rand 0.8.6", + "reqwest", + "rsa", + "ruzstd", + "serde_json", + "sha1", + "sha2", + "tokio", + "tungstenite", + "zeroize", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.7", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ruzstd" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad02996bfc73da3e301efe90b1837be9ed8f4a462b6ed410aa35d00381de89f" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.18", + "utf-8", + "webpki-roots 0.26.11", +] + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/app/src/main/rust/stores/Cargo.toml b/app/src/main/rust/stores/Cargo.toml new file mode 100644 index 0000000000..0c3f7d9910 --- /dev/null +++ b/app/src/main/rust/stores/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "droiddeckstores" +version = "0.1.0" +edition = "2021" +# The engines are a port of Bannerlator's bl-steam-client store modules (GPL-3.0-or-later, derived +# from WinNative's wn-steam-client; the Epic selection/verify rules trace back to GameNative and +# Legendary, both GPL-3.0). DroidDeck is GPL-3.0, so the crate keeps the licence it came with. +license = "GPL-3.0-or-later" +publish = false + +[lib] +name = "droiddeckstores" +crate-type = ["cdylib"] + +[dependencies] +jni = "0.21" +# zlib inflate for GOG gen2 chunks and Epic chunk bodies. The pure-Rust backend keeps the .so free +# of a C zlib and of the NDK sysroot's libz, which the apk does not ship. +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +# The fetch core drives every request through ONE tokio current-thread runtime and feeds a sync +# process pool; rustls (via ring) is the TLS stack so there is no OpenSSL to cross-compile. `gzip` +# transparently inflates a CDN answer that comes back content-encoded. No `blocking` client: the +# three engines are async-fetch only. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip"] } +tokio = { version = "1", default-features = false, features = ["rt", "net", "time", "sync"] } +futures-util = { version = "0.3", default-features = false, features = ["std"] } +# preserve_order keeps JSON object keys in source order, as in Bannerlator, so the GOG depot plan +# walks a manifest's files in the order the manifest lists them (largest-first sorting happens +# after, explicitly). +serde_json = { version = "1", features = ["preserve_order"] } +# Epic chunks are SHA-1 verified, Amazon files SHA-256; GOG's MD5 is the in-tree md5_small. +sha1 = "0.10" +sha2 = "0.10" + +[profile.release] +codegen-units = 1 +lto = "thin" +strip = "symbols" diff --git a/app/src/main/rust/stores/JNI.md b/app/src/main/rust/stores/JNI.md new file mode 100644 index 0000000000..ab969483e7 --- /dev/null +++ b/app/src/main/rust/stores/JNI.md @@ -0,0 +1,211 @@ +# `libdroiddeckstores.so` — the JNI contract + +The Kotlin side of the store download engines mirrors this file exactly. Every symbol the library +exports is listed here with the class it binds to, the JNI signature, when it is called and on +which thread. The engines came from Bannerlator's `bl-steam-client` crate; the method names and +signatures are identical to Bannerlator's `BlGogDownload` / `BlEpicDownload` / `BlAmazonDownload`, +only the package and class names changed. + +Load it once with `System.loadLibrary("droiddeckstores")`. The library is built by Gradle +(`buildRustStores`, see `app/build.gradle`) and is absent from a `-PskipRust=true` build, so every +call site must be ready for `UnsatisfiedLinkError` on `loadLibrary` or on the first `external` +call and degrade to "store unavailable". + +## Conventions that hold for all three engines + +- The `external` functions are declared `@JvmStatic` in a Kotlin `object` (a `JClass` arrives as + the second JNI argument). Declaring them in a plain `object` without `@JvmStatic` also binds — + the symbol name is the same — but keep `@JvmStatic` so the contract matches Bannerlator's. +- No overloads: the exported names are the short JNI form (`Java___`), so + adding a second `nativeStart` with a different parameter list would NOT bind. +- Listener objects are only ever reached through `CallVoidMethod` by name + signature on the + object that was passed to `nativeStart`. The library never does `FindClass`, so the listener's + class may be any class (interface, nested interface, anonymous object) that declares the listed + methods with exactly these signatures. The names suggested below are what the Rust doc comments + use. +- Every listener method runs on a native thread (the run's worker thread, a process-pool thread + or the throughput reporter) that is attached to the JVM as a daemon. Never touch Views directly; + post to the main thread. A Java exception thrown by a listener method is cleared and ignored by + the native side — it does not stop the run. +- `nativeStart` returns an opaque handle (`long`). `0` means the run did not start. The handle is + a heap pointer: call `nativeRelease(handle)` exactly once per non-zero handle (after + `onComplete`), never twice, never with a made-up value. +- `nativeCancel(handle)` is idempotent and safe at any time between `nativeStart` and + `nativeRelease`. The run stops at its next scheduling point; `onComplete` still fires. +- `nativeRelease(handle)` is safe even while a cancelled run is still winding down: the worker + thread owns its own references (cancel flag, listener `GlobalRef`). +- `caBundlePath`: a PEM bundle the HTTPS client adds as extra root certificates; `""` = the roots + rustls ships with (webpki). Bannerlator extracts the system CA store to a file for this; on + DroidDeck `""` is the sensible default unless a CDN fails TLS verification. +- `maxWorkers` is the ceiling of the adaptive in-flight request window; `processWorkers` is the + number of sync inflate+hash+write threads. Bannerlator passes its "download threads" preference + for both (8 by default). + +## `com.droiddeck.launcher.stores.StoresNative` + +| Method | JNI signature | Symbol | +|---|---|---| +| `nativeVersion(): String` | `()Ljava/lang/String;` | `Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion` | + +Returns the crate version from `Cargo.toml` (`"0.1.0"`). Call it from Setup, after +`System.loadLibrary("droiddeckstores")`, to prove the library loads and binds; show the string +next to the other component versions. May return `null` only if the JVM cannot allocate a string. + +```kotlin +package com.droiddeck.launcher.stores + +object StoresNative { + @JvmStatic external fun nativeVersion(): String +} +``` + +## `com.droiddeck.launcher.stores.gog.GogNative` + +GOG gen2 chunk engine (depot manifests → chunk fetch + zlib inflate + MD5) and gen1 (build manifest +→ per-file HTTP Range GET streamed to disk). One `nativeStart` = one download loop (base install, +DLC install, or a dependency-redist assembly). The Kotlin manager owns everything before (token, +builds, manifest head, product + language depot selection, secure-link resolution, disk guard) and +after it (marker files, exe resolution, DLC markers, redist orchestration). + +| Method | JNI signature | Symbol | +|---|---|---| +| `nativeProbe(): Int` | `()I` | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeProbe` | +| `nativeStart(...)`: Long | `(I[Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;Ljava/lang/String;IIZLjava/lang/String;Ljava/lang/Object;)J` (last parameter is the listener; its static type is the app's choice) | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart` | +| `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCancel` | +| `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_gog_GogNative_nativeRelease` | + +`nativeProbe()` always returns `1`; it exists so `isAvailable()` can catch `UnsatisfiedLinkError` +once at startup instead of mid-download. + +`nativeStart` parameters, in order: + +| # | Parameter | Meaning | +|---|---|---| +| 1 | `kind: Int` | `0` = `KIND_GEN2_CHUNKS`, `1` = `KIND_GEN1_RANGES` (anything else → gen2) | +| 2 | `depotManifests: Array` | gen2: the inflated depot-manifest JSON strings, in fetch order, already filtered by product + language; gen1: the inflated build manifest (one element) | +| 3 | `cdnBase: String` | gen2: the resolved secure-link base (query string intact) or the unauthenticated dependency store base; gen1: `""` (file URLs are in the manifest) | +| 4 | `installDir: String` | absolute install directory | +| 5 | `skipPaths: Array` | relative paths already completed by an earlier run of this same download (secure-link refresh re-run): counted done without re-hashing, no progress event | +| 6 | `caBundlePath: String` | see conventions | +| 7 | `maxWorkers: Int` | window ceiling for this loop (Bannerlator: its download-threads preference, 8, or 1 for redists) | +| 8 | `processWorkers: Int` | inflate/hash/write threads | +| 9 | `sortLargestFirst: Boolean` | base install = `true` (largest files first); DLC / dependency = `false` | +| 10 | `label: String` | log prefix, e.g. `"gog base"`; `""` → `"gog"` | +| 11 | `listener` | see below | + +Returns `0` without any callback when: the listener is null, the JavaVM cannot be obtained, the +install dir or manifest list is empty, `cdnBase` is empty for gen2, or the worker thread cannot be +spawned. Otherwise the run is on a thread named `gog-dl` and `onComplete` fires exactly once. + +Suggested listener: `com.droiddeck.launcher.stores.gog.GogNative.Listener` (an interface). + +| Listener method | JNI signature | When | +|---|---|---| +| `onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, file: String, fileBytes: Long, verified: Boolean)` | `(JJIILjava/lang/String;JZ)V` | one file reached its final state. `verified = true` = resume-skip (existing file passed size+MD5; no bytes credited, `fileBytes` = its size); `false` = freshly assembled, size+MD5-verified and renamed (`fileBytes` = its decompressed size). `filesDone` counts both, `bytesDone` counts assembled bytes only. Process-pool / worker thread. | +| `onLog(line: String)` | `(Ljava/lang/String;)V` | engine diagnostics; the same line already went to logcat under the tag `GogNative`. Any native thread. | +| `onComplete(success: Boolean, cancelled: Boolean, linkExpiry: Boolean, error: String, bytesWritten: Long, filesDone: Int)` | `(ZZZLjava/lang/String;JI)V` | exactly once, from the worker thread, last. `linkExpiry = true` = the run died on an HTTP 401/403/404/500, the codes the GOG secure link returns once it expired: refresh the link and call `nativeStart` again with the finished files in `skipPaths`. `error` is `""` on success. | + +On-disk protocol (what a resumed run, by either engine, relies on): a file is assembled into +`.bhtmp`, then renamed over `` once its size and MD5 match the manifest; an existing +`` whose size+MD5 match is skipped. The engine writes nothing else. + +## `com.droiddeck.launcher.stores.epic.EpicNative` + +The chunk-fetch inner loop of an Epic install: fills `/.chunks/` with verified, +decompressed chunks for the files the manager says are pending. The Kotlin manager still fetches +and parses the manifest API JSON, downloads the manifest, selects files (install tags), runs the +delta/verify pass, assembles files from the chunk cache and does every post-install step. The +engine re-parses the same manifest bytes (ChunksV4 binary or legacy JSON), rebuilds the chunk plan +for the pending files, skips chunks already in the cache, and writes verified chunks with a +`.part` + rename protocol, so the on-disk state is the same whichever side fetched. + +| Method | JNI signature | Symbol | +|---|---|---| +| `nativeStart(...)`: Long | `([BLjava/lang/String;[Ljava/lang/String;[IIJLjava/lang/String;IILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart` | +| `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCancel` | +| `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeRelease` | + +There is no `nativeProbe`; Bannerlator probes by catching the `Throwable` from `loadLibrary` / +the first `nativeStart`. + +`nativeStart` parameters, in order: + +| # | Parameter | Meaning | +|---|---|---| +| 1 | `manifest: ByteArray` | the raw manifest bytes the manager downloaded (binary ChunksV4 or JSON) | +| 2 | `installDir: String` | absolute install directory; the chunk cache is `/.chunks` | +| 3 | `cdnPrefixes: Array` | the CDN base URLs from the manifest API answer (`.../ChunksV4//` style); one or many, distinct hosts widen the window | +| 4 | `pendingFileIdx: IntArray` | indices into the manifest's file list of the files still to fetch (the manager's delta/verify result) | +| 5 | `expectedChunks: Int` | the manager's own needed-chunk count for a cross-check; `-1` = no cross-check | +| 6 | `expectedBytes: Long` | the manager's `Σ max(fileSize, 1)` for a cross-check; `-1` = no cross-check | +| 7 | `caBundlePath: String` | see conventions | +| 8 | `maxWorkers: Int` | window ceiling (Bannerlator: 8) | +| 9 | `processWorkers: Int` | inflate/verify/write threads | +| 10 | `listener` | see below | + +Planning (manifest parse + plan + cross-check) runs synchronously on the calling thread, so a +plan failure returns `0` BEFORE any fetch; the manager then runs its own fallback loop. On `0`, +`onComplete(false, reason, 0)` has already been called synchronously (except for a null listener). +On success, `onPlan` has been called synchronously before `nativeStart` returns, then the fetch +runs on a new thread. + +Suggested listener: `com.droiddeck.launcher.stores.epic.EpicNative.Listener` (nested interface, +as in Bannerlator). + +| Listener method | JNI signature | When | +|---|---|---| +| `onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String)` | `(IJLjava/lang/String;)V` | once, synchronously inside `nativeStart`, before any fetch: the plan the engine derived | +| `onProgress(bytesDone: Long, bytesTotal: Long, chunksDone: Int, chunksTotal: Int)` | `(JJII)V` | per accounted chunk (cached-skip or fetched); process-pool thread | +| `onLog(line: String)` | `(Ljava/lang/String;)V` | engine log line, already in logcat under `EpicNative`; any native thread (also synchronously from `nativeStart` for the start line) | +| `onComplete(success: Boolean, error: String, bytesCredited: Long)` | `(ZLjava/lang/String;J)V` | terminal, exactly once (worker thread; or synchronously when `nativeStart` returns `0`) | + +Bannerlator's `BlEpicDownload.run(...)` wraps this in a blocking call that polls an +`AtomicBoolean` cancel flag every 250 ms, calls `nativeCancel` on cancel, waits up to 5 s for +`onComplete`, then `nativeRelease` in `finally`. That shape is worth keeping. + +## `com.droiddeck.launcher.stores.amazon.AmazonNative` + +The file-fetch pool of an Amazon Games install: one streamed HTTP GET per manifest file, SHA-256 +checked, written as `.tmp` and renamed. The manager keeps manifest/auth/markers/post-install +and hands the engine a JSON plan. + +| Method | JNI signature | Symbol | +|---|---|---| +| `nativeStart(...)`: Long | `(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;IILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart` | +| `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel` | +| `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease` | + +`nativeStart` parameters, in order: + +| # | Parameter | Meaning | +|---|---|---| +| 1 | `planJson: String` | a JSON array, one object per manifest file: `{"relPath": "...", "url": "https://...", "size": , "sha256hex": "..."}` (`sha256hex` may be `""` = no hash check). Every manifest file goes in: the engine applies the size-based resume-skip itself. | +| 2 | `installDir: String` | absolute install directory | +| 3 | `caBundlePath: String` | see conventions | +| 4 | `maxWorkers: Int` | window size; `<= 0` = the engine default (8, Amazon's own `MAX_PARALLEL`) | +| 5 | `processWorkers: Int` | sync verify+write threads (`<= 0` → 1) | +| 6 | `listener` | see below | + +Returns `0` when the listener is null (no callback), or when `planJson` / `installDir` is empty +(`onComplete(false, "empty plan or install dir", 0)` fires synchronously first), or when the +listener `GlobalRef` cannot be taken (no callback). The engine keeps the `JavaVM` it takes from +the first `nativeStart`. + +Suggested listener: `com.droiddeck.launcher.stores.amazon.AmazonNative.Listener` (an interface). + +| Listener method | JNI signature | When | +|---|---|---| +| `onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Long, filesTotal: Long)` | `(JJJJ)V` | once up front with the resume-skipped credit, then after each committed file (and as pieces stream in); `bytesDone` includes skipped files. Native threads. | +| `onLog(line: String)` | `(Ljava/lang/String;)V` | one diagnostic line (`engine=rust …`, `fetch-window …`, `summary …`). The Amazon engine does NOT write to logcat itself: the Kotlin side is the one place that logs these, so each line appears once. | +| `onComplete(success: Boolean, error: String, bytesWritten: Long)` | `(ZLjava/lang/String;J)V` | exactly once, when the run succeeds, fails or is cancelled (`error = "cancelled"`). | + +Bannerlator's `BlAmazonDownload.runBlocking(...)` polls a cancel probe every 100 ms, calls +`nativeCancel` once, and `nativeRelease` in `finally` after `onComplete`; on thread interrupt it +cancels and waits up to 30 s for the run to wind down before releasing. + +## Log tags + +Lines go to logcat under `GogNative` and `EpicNative` (the Amazon engine logs only through +`onLog`). Bannerlator used `BL_GOG_DL` / `BL_EPIC_DL` / `BL_AMAZON_DL`; the line grammar +(`fetch-start`, `fetch-window`, `throughput`, `fetch-end`, `summary`) is unchanged, so its tuning +notes still read against a DroidDeck log. diff --git a/app/src/main/rust/stores/src/amazon.rs b/app/src/main/rust/stores/src/amazon.rs new file mode 100644 index 0000000000..7ca66798b9 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon.rs @@ -0,0 +1,989 @@ +//! Amazon Games download adapter over the shared fetch core. +//! +//! Replaces ONLY the byte-fetching pool of `AmazonDownloadManager.install()` (Step 4). +//! Java still resolves the download spec, fetches and parses `manifest.proto`, and hands +//! this module the resolved file list (one JSON entry per manifest file with the exact +//! URL the Java loop would have opened). The adapter mirrors the Java rules 1:1 where the +//! core allows it — see Bannerlator's `docs/RUST_AMAZON_PARITY.md` §4 / §9 for the table and the +//! documented deviations: +//! +//! * one `FetchItem` per WHOLE file, streamed (`FetchOptions.stream = true`): pieces are +//! appended to `.tmp` as they arrive, 8 files wide like Java's pool, memory bounded +//! to the pieces not yet written; +//! * resume-skip = `st_size == manifest size`, no hash check on skip (Java `:250`); +//! * SHA-256 is accumulated while streaming and checked in `on_finish` when the manifest +//! carries a hash; a mismatch deletes the tmp and is a retryable failure (Java `:269`); +//! * `.tmp` → delete existing dest → rename (Java `:280`); rename failure is fatal +//! without retry (Java `:281`); +//! * a piece credits its wire length to progress as it is written (Java `:347`), and pieces +//! of an attempt that later fails stay counted (Java never rolls `totalDownloaded` back). + +pub mod jni; + +use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// Java `AmazonDownloadManager.MAX_PARALLEL`. +pub const MAX_PARALLEL: usize = 8; +/// Java `AmazonDownloadManager.DOWNLOAD_USER_AGENT`. +pub const DOWNLOAD_USER_AGENT: &str = "nile/0.1 Amazon"; +/// Java `conn.setReadTimeout(120000)`; in stream mode the core applies this to the response +/// headers and then as an idle deadline per received piece — the same shape as a read timeout. +pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +/// Java `new File(installDir, file.unixPath() + ".tmp")`. +pub const TMP_SUFFIX: &str = ".tmp"; +/// Log label used in the core's `fetch-window` lines. +pub const LABEL: &str = "amazon"; + +/// One resolved manifest file as serialised by Java (`AmazonDownloadManager.buildRustPlan`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlanEntry { + /// `ManifestFile.unixPath()` — relative to the install dir, forward slashes. + pub rel_path: String, + /// `AmazonApiClient.appendPath(downloadUrl, "files/" + hashHex)` — signed, whole file. + pub url: String, + /// `ManifestFile.size`. + pub size: u64, + /// SHA-256 to verify against; EMPTY = Java would not verify (`hashAlgorithm != 0` or + /// no hash bytes). + pub sha256: Vec, +} + +/// Parse the JSON array `[{relPath, url, size, sha256hex}]`. `relPath` is normalised the same +/// way as `ManifestFile.unixPath()` so a caller that forgot is still byte-identical. +pub fn parse_plan(json: &str) -> Result, String> { + let value: serde_json::Value = + serde_json::from_str(json).map_err(|err| format!("plan json: {err}"))?; + let Some(array) = value.as_array() else { + return Err("plan json: expected an array".to_string()); + }; + let mut entries = Vec::with_capacity(array.len()); + for (index, item) in array.iter().enumerate() { + let rel_path = item + .get("relPath") + .and_then(|v| v.as_str()) + .map(unix_path) + .unwrap_or_default(); + if rel_path.is_empty() { + return Err(format!("plan json: entry {index} has no relPath")); + } + let url = item + .get("url") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + if url.is_empty() { + return Err(format!("plan json: entry {index} ({rel_path}) has no url")); + } + let size = item + .get("size") + .and_then(|v| v.as_u64().or_else(|| v.as_i64().map(|s| s.max(0) as u64))) + .unwrap_or(0); + let sha_hex = item + .get("sha256hex") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let sha256 = if sha_hex.is_empty() { + Vec::new() + } else { + hex_decode(sha_hex) + .ok_or_else(|| format!("plan json: entry {index} ({rel_path}) bad sha256hex"))? + }; + entries.push(PlanEntry { + rel_path, + url, + size, + sha256, + }); + } + Ok(entries) +} + +/// `ManifestFile.unixPath()`. +pub fn unix_path(path: &str) -> String { + path.replace('\\', "/") +} + +/// Lowercase/uppercase hex → bytes; `None` on odd length or a non-hex digit. +pub fn hex_decode(hex: &str) -> Option> { + let bytes = hex.as_bytes(); + if bytes.len() % 2 != 0 { + return None; + } + let nibble = |b: u8| -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } + }; + let mut out = Vec::with_capacity(bytes.len() / 2); + for pair in bytes.chunks(2) { + out.push((nibble(pair[0])? << 4) | nibble(pair[1])?); + } + Some(out) +} + +/// `new File(installDir, rel)` — Java resolves the child textually under the parent, so +/// build the path by concatenation (never `Path::join`, which would let a leading `/` escape). +pub fn dest_path(install_dir: &str, rel_path: &str) -> PathBuf { + let base = install_dir.trim_end_matches('/'); + PathBuf::from(format!("{base}/{rel_path}")) +} + +/// `new File(installDir, rel + ".tmp")`. +pub fn tmp_path(install_dir: &str, rel_path: &str) -> PathBuf { + dest_path(install_dir, &format!("{rel_path}{TMP_SUFFIX}")) +} + +/// Java `:250`: `destFile.exists() && destFile.length() == file.size`. `File.length()` is +/// `st_size`, which is what `Metadata::len()` returns — including for a directory, so the +/// (degenerate) behaviours match too. No hash is checked on skip, on purpose. +pub fn is_present_and_complete(install_dir: &str, entry: &PlanEntry) -> bool { + fs::metadata(dest_path(install_dir, &entry.rel_path)) + .map(|meta| meta.len() == entry.size) + .unwrap_or(false) +} + +/// `scheme://host[:port]` of a URL — the per-host cap key. Amazon serves every file from the +/// one signed CDN base, so this yields a single host. +pub fn host_key(url: &str) -> String { + let (scheme, rest) = match url.find("://") { + Some(idx) => (&url[..idx], &url[idx + 3..]), + None => ("https", url), + }; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + format!("{scheme}://{}", &rest[..end]) +} + +/// Improvements round 1: `max(6, ceil(max_workers / distinct_hosts))` — a single-host store +/// (Amazon's one signed a2z base) gets the whole ceiling on its one host; the core clamps the +/// window to `hosts × per_host_cap`, so anything smaller would silently cut the ceiling. +pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize { + let hosts = distinct_hosts.max(1); + let per_host = max_workers.max(1).div_ceil(hosts); + per_host.max(crate::fetch_tuning::PER_HOST_CAP) +} + +/// The resolved work for one run. +#[derive(Debug, Default)] +pub struct Plan { + /// Items to fetch (index into `entries` via `FetchItem.id`). + pub items: Vec, + /// Every manifest file, in manifest order (fetched and skipped alike). + pub entries: Vec, + /// Σ size of all entries = `ParsedManifest.totalInstallSize`. + pub total_bytes: u64, + /// Bytes credited up front for files that passed the resume check. + pub skipped_bytes: u64, + /// Number of files that passed the resume check. + pub skipped_files: u64, + /// Distinct host keys in fetch order. + pub hosts: Vec, +} + +/// Apply the Java resume-skip rule to every entry and turn the rest into whole-file items. +pub fn build_plan(entries: Vec, install_dir: &str) -> Plan { + let mut plan = Plan { + total_bytes: entries.iter().map(|e| e.size).sum(), + ..Plan::default() + }; + for (index, entry) in entries.iter().enumerate() { + if is_present_and_complete(install_dir, entry) { + plan.skipped_bytes = plan.skipped_bytes.saturating_add(entry.size); + plan.skipped_files += 1; + continue; + } + let host = host_key(&entry.url); + if !plan.hosts.contains(&host) { + plan.hosts.push(host); + } + plan.items.push(FetchItem { + id: index as u64, + urls: vec![entry.url.clone()], + reserve: entry.size, + range: None, + }); + } + if plan.hosts.is_empty() { + plan.hosts.push("https://amazon".to_string()); + } + plan.entries = entries; + plan +} + +/// One `.tmp` being streamed: the open handle, the running hash and the bytes written +/// by the current attempt. +struct OpenTmp { + file: File, + hasher: Sha256, + written: u64, +} + +/// Sink: stream pieces into `.tmp` (SHA-256 accumulated on the fly), verify and rename +/// in `on_finish`. `process` (whole-body mode) does the same in one step. +pub struct AmazonSink { + install_dir: String, + entries: Vec, + cancel: Arc, + files_done: AtomicU64, + /// Per-item open tmp files. The map lock is held only to look the entry up; the write + /// itself happens under the per-item lock so 8 streams never serialise on one mutex. + open: Mutex>>>, +} + +impl AmazonSink { + pub fn new(install_dir: &str, entries: Vec, cancel: Arc) -> Self { + Self { + install_dir: install_dir.to_string(), + entries, + cancel, + files_done: AtomicU64::new(0), + open: Mutex::new(HashMap::new()), + } + } + + pub fn files_done(&self) -> u64 { + self.files_done.load(Ordering::Relaxed) + } + + fn entry(&self, item: &FetchItem) -> Result<&PlanEntry, SinkError> { + self.entries + .get(item.id as usize) + .ok_or_else(|| SinkError::Fatal(format!("plan index {} out of range", item.id))) + } + + /// Java `:260` — `destFile.getParentFile().mkdirs()` before each attempt; a failure + /// surfaces on the create/write below, like Java's. + fn ensure_parent(&self, entry: &PlanEntry) { + if let Some(parent) = dest_path(&self.install_dir, &entry.rel_path).parent() { + let _ = fs::create_dir_all(parent); + } + } + + /// Java `:280-284`: delete an existing dest, rename tmp → dest; rename failure fails the + /// file with NO retry (→ whole install fails). + fn rename_into_place(&self, entry: &PlanEntry) -> Result<(), SinkError> { + let dest = dest_path(&self.install_dir, &entry.rel_path); + let tmp = tmp_path(&self.install_dir, &entry.rel_path); + if dest.exists() { + let _ = fs::remove_file(&dest); + } + if let Err(err) = fs::rename(&tmp, &dest) { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Fatal(format!( + "Failed to rename tmp → {} ({err})", + dest.display() + ))); + } + self.files_done.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + /// Java `:267-277`: SHA-256 check when the manifest carries a hash; a mismatch deletes the + /// tmp and is retried. + fn verify(&self, entry: &PlanEntry, digest: &[u8]) -> Result<(), SinkError> { + if !entry.sha256.is_empty() && digest != entry.sha256.as_slice() { + let _ = fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path)); + return Err(SinkError::Retry(format!( + "SHA-256 mismatch for: {}", + entry.rel_path + ))); + } + Ok(()) + } + + /// Whole-body path (`FetchOptions.stream = false`): verify → write tmp → rename. + pub fn commit(&self, entry: &PlanEntry, body: &[u8]) -> Result { + if !entry.sha256.is_empty() { + let digest = Sha256::digest(body); + if digest.as_slice() != entry.sha256.as_slice() { + return Err(SinkError::Retry(format!( + "SHA-256 mismatch for: {}", + entry.rel_path + ))); + } + } + self.ensure_parent(entry); + let tmp = tmp_path(&self.install_dir, &entry.rel_path); + if let Err(err) = fs::write(&tmp, body) { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!("write {}: {err}", tmp.display()))); + } + self.rename_into_place(entry)?; + Ok(body.len() as u64) + } + + /// After the run: close and delete every tmp that never reached `on_finish` (a stream that + /// failed after max attempts, or was cut by cancel / a fatal error). Java deletes the tmp + /// of every failed or cancelled attempt (`:271`, `:288`, `:293`, `:297`), so no `.tmp` + /// may survive a run on either engine. Returns how many were removed. + pub fn cleanup_partials(&self) -> usize { + let leftovers: Vec<(u64, Arc>)> = match self.open.lock() { + Ok(mut open) => open.drain().collect(), + Err(_) => return 0, + }; + let mut removed = 0; + for (id, open) in leftovers { + drop(open); + if let Some(entry) = self.entries.get(id as usize) { + if fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path)).is_ok() { + removed += 1; + } + } + } + removed + } +} + +impl FetchSink for AmazonSink { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + if self.cancel.load(Ordering::Relaxed) { + return Err(SinkError::Fatal("cancelled".to_string())); + } + let entry = self.entry(item)?; + self.commit(entry, &body) + } + + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + if self.cancel.load(Ordering::Relaxed) { + // Java `:342-344`: cancel inside the read loop → tmp deleted (cleanup_partials). + return Err(SinkError::Fatal("cancelled".to_string())); + } + let entry = self.entry(item)?; + let tmp = tmp_path(&self.install_dir, &entry.rel_path); + let open = if offset == 0 { + // (Re)start of an attempt: Java opens a fresh FileOutputStream (truncating). + self.ensure_parent(entry); + let file = File::create(&tmp).map_err(|err| { + let _ = fs::remove_file(&tmp); + SinkError::Retry(format!("create {}: {err}", tmp.display())) + })?; + let open = Arc::new(Mutex::new(OpenTmp { + file, + hasher: Sha256::new(), + written: 0, + })); + match self.open.lock() { + Ok(mut map) => { + map.insert(item.id, Arc::clone(&open)); + } + Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), + } + open + } else { + let found = match self.open.lock() { + Ok(map) => map.get(&item.id).cloned(), + Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), + }; + match found { + Some(open) => open, + None => { + return Err(SinkError::Retry(format!( + "piece at offset {offset} without a stream start for: {}", + entry.rel_path + ))) + } + } + }; + let Ok(mut state) = open.lock() else { + return Err(SinkError::Fatal("sink item poisoned".to_string())); + }; + if state.written != offset { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!( + "piece out of order ({} != {offset}) for: {}", + state.written, entry.rel_path + ))); + } + if let Err(err) = state.file.write_all(data) { + // Java: IOException inside downloadFile → tmp deleted → next attempt. + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!("write {}: {err}", tmp.display()))); + } + state.hasher.update(data); + state.written = state.written.saturating_add(data.len() as u64); + Ok(()) + } + + fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result { + let entry = self.entry(item)?; + let tmp = tmp_path(&self.install_dir, &entry.rel_path); + let taken = match self.open.lock() { + Ok(mut map) => map.remove(&item.id), + Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), + }; + let (written, digest) = match taken { + Some(open) => { + let Ok(mut state) = open.lock() else { + return Err(SinkError::Fatal("sink item poisoned".to_string())); + }; + if let Err(err) = state.file.flush() { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!("flush {}: {err}", tmp.display()))); + } + let written = state.written; + let digest = std::mem::replace(&mut state.hasher, Sha256::new()).finalize(); + (written, digest) + } + None => { + // Zero-byte body: no piece ever arrived. Java would have created an empty tmp. + if total_len != 0 { + return Err(SinkError::Retry(format!( + "finish without a stream for: {}", + entry.rel_path + ))); + } + self.ensure_parent(entry); + if let Err(err) = File::create(&tmp) { + return Err(SinkError::Retry(format!("create {}: {err}", tmp.display()))); + } + (0, Sha256::new().finalize()) + } + }; + if written != total_len { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!( + "stream length {written} != {total_len} for: {}", + entry.rel_path + ))); + } + self.verify(entry, digest.as_slice())?; + self.rename_into_place(entry)?; + Ok(0) + } +} + +/// Outcome of one `run_download`. +#[derive(Clone, Debug, Default)] +pub struct RunResult { + pub success: bool, + pub cancelled: bool, + pub error: String, + /// Bytes fetched by this run (skipped files excluded; failed attempts' pieces included, + /// like Java's `totalDownloaded`). + pub bytes_written: u64, + /// Files committed by this run plus files skipped by the resume check. + pub files_done: u64, + pub files_total: u64, +} + +/// Throughput bookkeeping for the final summary line (avg over the run, peak over ≥500 ms +/// progress intervals — the same sampling window the Java speed label uses). +struct SpeedMeter { + started: Instant, + last_at: Instant, + last_bytes: u64, + peak_bps: f64, +} + +impl SpeedMeter { + fn new() -> Self { + let now = Instant::now(); + Self { + started: now, + last_at: now, + last_bytes: 0, + peak_bps: 0.0, + } + } + + fn observe(&mut self, bytes: u64) { + let now = Instant::now(); + let dt = now.duration_since(self.last_at).as_secs_f64(); + if dt >= 0.5 { + let bps = bytes.saturating_sub(self.last_bytes) as f64 / dt; + if bps > self.peak_bps { + self.peak_bps = bps; + } + self.last_at = now; + self.last_bytes = bytes; + } + } + + fn summary(&self, bytes: u64) -> (f64, f64, f64) { + let elapsed = self.started.elapsed().as_secs_f64().max(0.001); + let avg = bytes as f64 / elapsed; + (elapsed, avg * 8.0 / 1_000_000.0, self.peak_bps * 8.0 / 1_000_000.0) + } +} + +/// Blocking: plan → stream-fetch → verify/rename, calling `progress(bytes_done, bytes_total, +/// files_done, files_total)` after each written piece and each finished file (skipped files +/// are pre-credited) and `log` for every diagnostic line. `bytes_done` includes skipped bytes, +/// matching Java's `totalDownloaded` (which credits skipped files at `:251`). +pub fn run_download( + plan_json: &str, + install_dir: &str, + ca_bundle_path: &str, + max_workers: usize, + process_workers: usize, + cancel: Arc, + progress: &(dyn Fn(u64, u64, u64, u64) + Sync), + log: &(dyn Fn(&str) + Sync), +) -> RunResult { + let max_workers = if max_workers == 0 { MAX_PARALLEL } else { max_workers }; + let process_workers = process_workers.max(1); + let entries = match parse_plan(plan_json) { + Ok(entries) => entries, + Err(err) => { + log(&format!("engine=rust plan error: {err}")); + return RunResult { + error: err, + ..RunResult::default() + }; + } + }; + let plan = build_plan(entries, install_dir); + let files_total = plan.entries.len() as u64; + let fetch_bytes: u64 = plan.items.iter().map(|item| item.reserve).sum(); + let per_host_cap = per_host_cap_for(max_workers, plan.hosts.len()); + log(&format!( + "engine=rust mode=stream plan={} files skip={} ({} bytes) fetch={} ({} bytes) hosts={} workers={} per_host_cap={} process={} dir={}", + files_total, + plan.skipped_files, + plan.skipped_bytes, + plan.items.len(), + fetch_bytes, + plan.hosts.len(), + max_workers, + per_host_cap, + process_workers, + install_dir + )); + progress(plan.skipped_bytes, plan.total_bytes, plan.skipped_files, files_total); + if plan.items.is_empty() { + log("summary bytes=0 elapsed=0.000 avg_mbps=0.0 peak_mbps=0.0 files=all-present"); + return RunResult { + success: true, + files_done: plan.skipped_files, + files_total, + ..RunResult::default() + }; + } + if cancel.load(Ordering::Relaxed) { + return RunResult { + cancelled: true, + error: "cancelled".to_string(), + files_done: plan.skipped_files, + files_total, + ..RunResult::default() + }; + } + + let opts = FetchOptions { + max_workers, + // Every Amazon file comes from the one signed CDN base: give that host the whole + // ceiling (`per_host_cap_for`), or the core would clamp the window to hosts × cap. + per_host_cap, + timeout: REQUEST_TIMEOUT, + headers: vec![("User-Agent".to_string(), DOWNLOAD_USER_AGENT.to_string())], + ca_bundle_path: ca_bundle_path.to_string(), + process_workers, + label: LABEL.to_string(), + // Whole files, streamed into the tmp as they arrive: 8 wide like Java's pool, memory + // bounded to the pieces not yet written. + stream: true, + ..FetchOptions::default() + }; + let sink = AmazonSink::new(install_dir, plan.entries.clone(), Arc::clone(&cancel)); + let meter = Mutex::new(SpeedMeter::new()); + let skipped_bytes = plan.skipped_bytes; + let skipped_files = plan.skipped_files; + let total_bytes = plan.total_bytes; + let progress_cb = |credited: u64, items_ok: u64| { + if let Ok(mut meter) = meter.lock() { + meter.observe(credited); + } + progress( + skipped_bytes.saturating_add(credited), + total_bytes, + skipped_files.saturating_add(items_ok), + files_total, + ); + }; + let outcome = run_fetch( + plan.items, + &plan.hosts, + &opts, + &sink, + cancel.as_ref(), + &progress_cb, + log, + ); + let leftovers = sink.cleanup_partials(); + let (elapsed, avg_mbps, peak_mbps) = meter + .lock() + .map(|meter| meter.summary(outcome.bytes_credited)) + .unwrap_or((0.0, 0.0, 0.0)); + let files_done = skipped_files.saturating_add(sink.files_done()); + log(&format!( + "summary bytes={} elapsed={:.3} avg_mbps={:.1} peak_mbps={:.1} files={}/{} items_ok={} tmp_removed={} cancelled={} error={}", + outcome.bytes_credited, + elapsed, + avg_mbps, + peak_mbps, + files_done, + files_total, + outcome.items_ok, + leftovers, + outcome.cancelled, + outcome.error.as_deref().unwrap_or("") + )); + let cancelled = outcome.cancelled || cancel.load(Ordering::Relaxed); + let error = if cancelled { + outcome.error.unwrap_or_else(|| "cancelled".to_string()) + } else { + outcome.error.unwrap_or_default() + }; + RunResult { + success: !cancelled && error.is_empty(), + cancelled, + error, + bytes_written: outcome.bytes_credited, + files_done, + files_total, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + fn scratch_dir() -> String { + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!( + "amazon-test-{}-{}", + std::process::id(), + n + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir.to_string_lossy().into_owned() + } + + fn sha_hex(data: &[u8]) -> String { + crate::gog::to_hex(Sha256::digest(data).as_slice()) + } + + fn item_for(entry: &PlanEntry, id: u64) -> FetchItem { + FetchItem { + id, + urls: vec![entry.url.clone()], + reserve: entry.size, + range: None, + } + } + + const PLAN: &str = r#"[ + {"relPath":"Binaries\\Win64\\Game.exe","url":"https://d1.cdn.example/base/files/aa?Sig=1","size":4,"sha256hex":"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"}, + {"relPath":"data/level1.pak","url":"https://d1.cdn.example/base/files/bb?Sig=1","size":2,"sha256hex":""}, + {"relPath":"readme.txt","url":"https://d2.cdn.example/base/files/cc?Sig=1","size":0,"sha256hex":""} + ]"#; + + #[test] + fn parses_plan_and_normalises_paths() { + let entries = parse_plan(PLAN).unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].rel_path, "Binaries/Win64/Game.exe"); + assert_eq!(entries[0].size, 4); + assert_eq!(entries[0].sha256.len(), 32); + assert!(entries[1].sha256.is_empty()); + assert_eq!(entries[2].size, 0); + } + + #[test] + fn rejects_bad_plan() { + assert!(parse_plan("{}").is_err()); + assert!(parse_plan(r#"[{"url":"x","size":1}]"#).is_err()); + assert!(parse_plan(r#"[{"relPath":"a","size":1}]"#).is_err()); + assert!(parse_plan(r#"[{"relPath":"a","url":"u","size":1,"sha256hex":"zz"}]"#).is_err()); + } + + #[test] + fn hex_roundtrip() { + assert_eq!(hex_decode("00ff10"), Some(vec![0x00, 0xff, 0x10])); + assert_eq!(hex_decode("00FF"), Some(vec![0x00, 0xff])); + assert_eq!(hex_decode("0"), None); + assert_eq!(hex_decode("0g"), None); + } + + #[test] + fn per_host_cap_gives_single_host_the_whole_ceiling() { + assert_eq!(per_host_cap_for(32, 1), 32); + assert_eq!(per_host_cap_for(32, 3), 11); + assert_eq!(per_host_cap_for(8, 1), 8); + assert_eq!(per_host_cap_for(4, 1), 6); + assert_eq!(per_host_cap_for(0, 0), 6); + } + + #[test] + fn host_key_strips_path_and_query() { + assert_eq!( + host_key("https://d1.cdn.example/base/files/aa?Sig=1"), + "https://d1.cdn.example" + ); + assert_eq!(host_key("http://h:8080/x"), "http://h:8080"); + assert_eq!(host_key("nohost"), "https://nohost"); + } + + #[test] + fn dest_path_never_escapes_install_dir() { + assert_eq!( + dest_path("/inst/", "a/b.txt"), + PathBuf::from("/inst/a/b.txt") + ); + assert_eq!(dest_path("/inst", "/abs.txt"), PathBuf::from("/inst//abs.txt")); + assert_eq!(tmp_path("/inst", "a.bin"), PathBuf::from("/inst/a.bin.tmp")); + } + + #[test] + fn build_plan_skips_only_size_matched_files() { + let dir = scratch_dir(); + let entries = parse_plan(PLAN).unwrap(); + // Entry 1 present with the right size → skipped (no hash check even if wrong bytes). + fs::create_dir_all(format!("{dir}/data")).unwrap(); + fs::write(format!("{dir}/data/level1.pak"), b"zz").unwrap(); + // Entry 0 present with the wrong size → fetched. + fs::create_dir_all(format!("{dir}/Binaries/Win64")).unwrap(); + fs::write(format!("{dir}/Binaries/Win64/Game.exe"), b"tes").unwrap(); + let plan = build_plan(entries, &dir); + assert_eq!(plan.total_bytes, 6); + assert_eq!(plan.skipped_files, 1); + assert_eq!(plan.skipped_bytes, 2); + assert_eq!(plan.items.len(), 2); + assert_eq!(plan.items[0].id, 0); + assert_eq!(plan.items[0].reserve, 4); + assert!(plan.items[0].range.is_none()); + assert_eq!(plan.items[1].id, 2); + assert_eq!( + plan.hosts, + vec!["https://d1.cdn.example".to_string(), "https://d2.cdn.example".to_string()] + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn zero_size_missing_file_is_fetched_and_present_zero_is_skipped() { + let dir = scratch_dir(); + let entry = PlanEntry { + rel_path: "empty.txt".into(), + url: "https://h/x".into(), + size: 0, + sha256: Vec::new(), + }; + assert!(!is_present_and_complete(&dir, &entry)); + fs::write(format!("{dir}/empty.txt"), b"").unwrap(); + assert!(is_present_and_complete(&dir, &entry)); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn whole_body_sink_verifies_writes_and_renames() { + let dir = scratch_dir(); + let body = b"test".to_vec(); + let entry = PlanEntry { + rel_path: "sub/Game.exe".into(), + url: "https://h/x".into(), + size: 4, + sha256: hex_decode(&sha_hex(&body)).unwrap(), + }; + let cancel = Arc::new(AtomicBool::new(false)); + let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::clone(&cancel)); + let item = item_for(&entry, 0); + // Mismatch → Retry, nothing on disk. + match sink.process(&item, b"nope".to_vec()) { + Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")), + Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"), + Ok(_) => panic!("expected Retry, got Ok"), + } + assert!(!dest_path(&dir, "sub/Game.exe").exists()); + assert!(!tmp_path(&dir, "sub/Game.exe").exists()); + // Match → written, renamed, tmp gone, credited body length. + assert_eq!(sink.process(&item, body.clone()).unwrap(), 4); + assert_eq!(fs::read(dest_path(&dir, "sub/Game.exe")).unwrap(), body); + assert!(!tmp_path(&dir, "sub/Game.exe").exists()); + assert_eq!(sink.files_done(), 1); + // Existing dest is replaced (Java `:280`). + assert_eq!(sink.process(&item, body.clone()).unwrap(), 4); + // Cancel → Fatal, no write. + cancel.store(true, Ordering::Relaxed); + assert!(matches!( + sink.process(&item, body), + Err(SinkError::Fatal(_)) + )); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_sink_appends_verifies_and_renames() { + let dir = scratch_dir(); + let body = b"hello world".to_vec(); + let entry = PlanEntry { + rel_path: "sub/big.bin".into(), + url: "https://h/x".into(), + size: body.len() as u64, + sha256: hex_decode(&sha_hex(&body)).unwrap(), + }; + let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); + let item = item_for(&entry, 0); + sink.on_chunk(&item, 0, b"hello").unwrap(); + assert!(tmp_path(&dir, "sub/big.bin").exists()); + sink.on_chunk(&item, 5, b" wor").unwrap(); + sink.on_chunk(&item, 9, b"ld").unwrap(); + assert_eq!(sink.on_finish(&item, 11).unwrap(), 0); + assert_eq!(fs::read(dest_path(&dir, "sub/big.bin")).unwrap(), body); + assert!(!tmp_path(&dir, "sub/big.bin").exists()); + assert_eq!(sink.files_done(), 1); + assert_eq!(sink.cleanup_partials(), 0); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_sink_restart_truncates_and_mismatch_retries() { + let dir = scratch_dir(); + let body = b"abcdef".to_vec(); + let entry = PlanEntry { + rel_path: "f.bin".into(), + url: "https://h/x".into(), + size: 6, + sha256: hex_decode(&sha_hex(&body)).unwrap(), + }; + let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); + let item = item_for(&entry, 0); + // Attempt 1 delivers garbage, then the core restarts at offset 0. + sink.on_chunk(&item, 0, b"zzzz").unwrap(); + sink.on_chunk(&item, 0, b"abc").unwrap(); + sink.on_chunk(&item, 3, b"xyz").unwrap(); + match sink.on_finish(&item, 6) { + Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")), + Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"), + Ok(_) => panic!("expected Retry, got Ok"), + } + assert!(!tmp_path(&dir, "f.bin").exists()); + assert!(!dest_path(&dir, "f.bin").exists()); + // Out-of-order piece → Retry. + sink.on_chunk(&item, 0, b"abc").unwrap(); + assert!(matches!( + sink.on_chunk(&item, 5, b"f"), + Err(SinkError::Retry(_)) + )); + // Length mismatch at finish → Retry. + sink.on_chunk(&item, 0, b"abc").unwrap(); + assert!(matches!(sink.on_finish(&item, 6), Err(SinkError::Retry(_)))); + // Piece without a start → Retry. + assert!(matches!( + sink.on_chunk(&item, 3, b"def"), + Err(SinkError::Retry(_)) + )); + // Good attempt. + sink.on_chunk(&item, 0, b"abc").unwrap(); + sink.on_chunk(&item, 3, b"def").unwrap(); + assert_eq!(sink.on_finish(&item, 6).unwrap(), 0); + assert_eq!(fs::read(dest_path(&dir, "f.bin")).unwrap(), body); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_sink_cleanup_removes_unfinished_tmp_and_zero_len_finishes() { + let dir = scratch_dir(); + let big = PlanEntry { + rel_path: "cut.bin".into(), + url: "https://h/x".into(), + size: 10, + sha256: Vec::new(), + }; + let empty = PlanEntry { + rel_path: "empty.bin".into(), + url: "https://h/y".into(), + size: 0, + sha256: Vec::new(), + }; + let cancel = Arc::new(AtomicBool::new(false)); + let sink = AmazonSink::new(&dir, vec![big.clone(), empty.clone()], Arc::clone(&cancel)); + sink.on_chunk(&item_for(&big, 0), 0, b"12345").unwrap(); + assert!(tmp_path(&dir, "cut.bin").exists()); + // Zero-byte body: on_finish with no pieces creates and commits an empty file. + assert_eq!(sink.on_finish(&item_for(&empty, 1), 0).unwrap(), 0); + assert_eq!(fs::read(dest_path(&dir, "empty.bin")).unwrap(), b""); + // Cancel mid-stream → Fatal; cleanup deletes the partial tmp like Java does. + cancel.store(true, Ordering::Relaxed); + assert!(matches!( + sink.on_chunk(&item_for(&big, 0), 5, b"6"), + Err(SinkError::Fatal(_)) + )); + assert_eq!(sink.cleanup_partials(), 1); + assert!(!tmp_path(&dir, "cut.bin").exists()); + assert!(!dest_path(&dir, "cut.bin").exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_sink_skips_hash_when_manifest_has_none() { + let dir = scratch_dir(); + let entry = PlanEntry { + rel_path: "a.bin".into(), + url: "https://h/x".into(), + size: 3, + sha256: Vec::new(), + }; + let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); + let item = item_for(&entry, 0); + sink.on_chunk(&item, 0, b"abc").unwrap(); + assert_eq!(sink.on_finish(&item, 3).unwrap(), 0); + assert_eq!(fs::read(dest_path(&dir, "a.bin")).unwrap(), b"abc"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn run_download_all_present_short_circuits() { + let dir = scratch_dir(); + fs::write(format!("{dir}/a.bin"), b"abc").unwrap(); + let plan = r#"[{"relPath":"a.bin","url":"https://h/x","size":3,"sha256hex":""}]"#; + let seen = Mutex::new(Vec::new()); + let logs = Mutex::new(Vec::new()); + let result = run_download( + plan, + &dir, + "", + 0, + 1, + Arc::new(AtomicBool::new(false)), + &|done, total, fd, ft| seen.lock().unwrap().push((done, total, fd, ft)), + &|line| logs.lock().unwrap().push(line.to_string()), + ); + assert!(result.success); + assert_eq!(result.files_done, 1); + assert_eq!(result.files_total, 1); + assert_eq!(seen.lock().unwrap().as_slice(), &[(3, 3, 1, 1)]); + assert!(logs.lock().unwrap()[0].starts_with("engine=rust mode=stream")); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn run_download_reports_plan_error() { + let result = run_download( + "not json", + "/nonexistent", + "", + 8, + 2, + Arc::new(AtomicBool::new(false)), + &|_, _, _, _| {}, + &|_| {}, + ); + assert!(!result.success); + assert!(result.error.contains("plan json")); + } +} diff --git a/app/src/main/rust/stores/src/amazon/jni.rs b/app/src/main/rust/stores/src/amazon/jni.rs new file mode 100644 index 0000000000..7ce2155585 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/jni.rs @@ -0,0 +1,228 @@ +//! JNI facade for the Amazon download adapter. +//! +//! Kotlin side: `com.droiddeck.launcher.stores.amazon.AmazonNative` (`nativeStart` / +//! `nativeCancel` / `nativeRelease`) with an `AmazonNative.Listener` +//! (`onProgress(JJJJ)V`, `onLog(String)V`, `onComplete(ZString;J)V`). Mirrors the shape of +//! the GOG and Epic facades: the run happens on a spawned native +//! thread, callbacks attach the JavaVM as daemon threads and hold a `GlobalRef` to the +//! listener; cancel flips an `AtomicBool`. This module keeps its own `JavaVM` handle (taken +//! from the calling `JNIEnv` on first use) so it depends on nothing outside this module. + +use crate::amazon::{run_download, MAX_PARALLEL}; +use jni::objects::{GlobalRef, JClass, JObject, JString, JValue}; +use jni::sys::{jint, jlong, JNI_FALSE, JNI_TRUE}; +use jni::{JNIEnv, JavaVM}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::thread; + +/// Logcat tag (`adb logcat -s AmazonNative`). Every engine line reaches logcat exactly ONCE: +/// this module only forwards lines to the listener (`onLog`), and the Java manager is the one +/// place that writes them to `android.util.Log` + its debug file. (Round 1 fix: the native side +/// used to also call `__android_log_write`, so each line appeared twice.) +pub const LOG_TAG: &str = "AmazonNative"; + +static JVM: OnceLock = OnceLock::new(); + +fn clear_pending_exception(env: &mut JNIEnv) { + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } +} + +fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { + env.get_string(value) + .ok() + .map(|s| s.to_string_lossy().into_owned()) +} + +/// One live run; boxed and handed to Kotlin as an opaque `jlong`. +struct AmazonRun { + cancel: Arc, +} + +fn to_handle(run: Box) -> jlong { + Box::into_raw(run) as jlong +} + +unsafe fn from_handle<'a>(handle: jlong) -> Option<&'a AmazonRun> { + if handle == 0 { + None + } else { + Some(&*(handle as *const AmazonRun)) + } +} + +fn call_log(env: &mut JNIEnv, listener: &JObject, line: &str) { + let Ok(text) = env.new_string(line) else { + clear_pending_exception(env); + return; + }; + let text_obj = JObject::from(text); + let _ = env.call_method( + listener, + "onLog", + "(Ljava/lang/String;)V", + &[JValue::Object(&text_obj)], + ); + clear_pending_exception(env); +} + +fn call_progress( + env: &mut JNIEnv, + listener: &JObject, + bytes_done: u64, + bytes_total: u64, + files_done: u64, + files_total: u64, +) { + let _ = env.call_method( + listener, + "onProgress", + "(JJJJ)V", + &[ + JValue::Long(bytes_done as jlong), + JValue::Long(bytes_total as jlong), + JValue::Long(files_done as jlong), + JValue::Long(files_total as jlong), + ], + ); + clear_pending_exception(env); +} + +fn call_complete( + env: &mut JNIEnv, + listener: &JObject, + success: bool, + error: &str, + bytes_written: u64, +) { + let Ok(error) = env.new_string(error) else { + clear_pending_exception(env); + return; + }; + let error_obj = JObject::from(error); + let _ = env.call_method( + listener, + "onComplete", + "(ZLjava/lang/String;J)V", + &[ + JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), + JValue::Object(&error_obj), + JValue::Long(bytes_written as jlong), + ], + ); + clear_pending_exception(env); +} + +/// Attach the current (native) thread and run `f` with the listener object. +fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)) { + let Some(vm) = JVM.get() else { + return; + }; + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + f(&mut env, listener.as_obj()); +} + +/// `AmazonNative.nativeStart(planJson, installDir, caBundlePath, maxWorkers, +/// processWorkers, listener) -> handle`. Returns 0 (and fires `onComplete(false, …)`) when the +/// inputs are unusable. The run itself happens on a spawned thread. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart( + mut env: JNIEnv, + _class: JClass, + plan_json: JString, + install_dir: JString, + ca_bundle_path: JString, + max_workers: jint, + process_workers: jint, + listener: JObject, +) -> jlong { + if listener.is_null() { + return 0; + } + if JVM.get().is_none() { + if let Ok(vm) = env.get_java_vm() { + let _ = JVM.set(vm); + } + } + let plan_json = jstring_to_string(&mut env, &plan_json).unwrap_or_default(); + let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); + let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); + if plan_json.is_empty() || install_dir.is_empty() { + call_complete(&mut env, &listener, false, "empty plan or install dir", 0); + return 0; + } + let Ok(listener) = env.new_global_ref(&listener) else { + clear_pending_exception(&mut env); + return 0; + }; + let max_workers = if max_workers <= 0 { + MAX_PARALLEL + } else { + max_workers as usize + }; + let process_workers = process_workers.max(1) as usize; + let cancel = Arc::new(AtomicBool::new(false)); + let run = Box::new(AmazonRun { + cancel: Arc::clone(&cancel), + }); + + thread::spawn(move || { + let log_listener = listener.clone(); + let log = move |line: &str| { + with_attached_env(&log_listener, |env, obj| call_log(env, obj, line)); + }; + let progress_listener = listener.clone(); + let progress = move |bytes_done: u64, bytes_total: u64, files_done: u64, files_total: u64| { + with_attached_env(&progress_listener, |env, obj| { + call_progress(env, obj, bytes_done, bytes_total, files_done, files_total) + }); + }; + let result = run_download( + &plan_json, + &install_dir, + &ca_bundle_path, + max_workers, + process_workers, + cancel, + &progress, + &log, + ); + with_attached_env(&listener, |env, obj| { + call_complete(env, obj, result.success, &result.error, result.bytes_written) + }); + }); + + to_handle(run) +} + +/// `AmazonNative.nativeCancel(handle)` — flips the run's cancel flag; the core stops +/// issuing requests, in-flight bodies are dropped, `onComplete(false, "cancelled", …)` follows. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if let Some(run) = unsafe { from_handle(handle) } { + run.cancel.store(true, Ordering::Relaxed); + } +} + +/// `AmazonNative.nativeRelease(handle)` — frees the handle. Safe at any time: the worker +/// thread owns its own `Arc` of the cancel flag and the listener `GlobalRef`. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if handle != 0 { + unsafe { + drop(Box::from_raw(handle as *mut AmazonRun)); + } + } +} diff --git a/app/src/main/rust/stores/src/epic.rs b/app/src/main/rust/stores/src/epic.rs new file mode 100644 index 0000000000..ccd8717045 --- /dev/null +++ b/app/src/main/rust/stores/src/epic.rs @@ -0,0 +1,42 @@ +//! Epic Games Store download adapter for the shared fetch core (`crate::fetch_core`). +//! +//! Scope = the chunk-fetch inner loop of `EpicDownloadManager.java` (the fixed 8-thread pool that +//! fills `/.chunks/`), nothing more. The Java manager still parses the manifest +//! API JSON, downloads the manifest, selects files (install tags), runs the delta/verify pass, +//! assembles files from the chunk cache and does every post-install step. The adapter re-parses +//! the same manifest bytes, rebuilds the same chunk plan for the pending files Java hands it, +//! skips chunks already in the cache and writes verified chunks with the same `.part` + rename +//! protocol, so the on-disk state is identical whichever engine ran (see +//! Bannerlator's `docs/RUST_EPIC_PARITY.md` for the rule-by-rule table). +//! +//! Submodules: +//! - [`manifest`] — ChunksV4 binary manifest + legacy JSON manifest parsers (1:1 with the Java +//! `parseManifest` / `parseJsonManifest`). +//! - [`plan`] — install-tag selection, unique-chunk plan, chunk paths / CDN URLs. +//! - [`chunk`] — chunk header parse, zlib inflate, SHA-1 verify, cache write. +//! - [`driver`] — plan → `FetchItem`s → `fetch_core::run_fetch` with the chunk-cache sink. +//! - [`jni`] — `Java_com_droiddeck_launcher_stores_epic_EpicNative_native*` exports. +//! +//! The selective-install-tag, delta/resume and chunk verification rules mirrored here were ported +//! into the Java manager from GameNative (`service/epic/EpicDownloadManager.kt`, +//! `manifest/ManifestUtils.kt`), itself derived from Legendary; both are GPL-3.0 and this crate +//! carries the same licence (see `Cargo.toml`). + +pub mod chunk; +pub mod driver; +pub mod jni; +pub mod manifest; +pub mod plan; + +/// User-Agent the Java manager sends on every chunk request (`EpicDownloadManager.UA`). +pub const USER_AGENT: &str = + "UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; + +/// Name of the chunk cache directory under the install dir (`new File(installDir, ".chunks")`). +pub const CHUNK_CACHE_DIR: &str = ".chunks"; + +/// Fixed pool width of the Java chunk downloader (`Executors.newFixedThreadPool(8)`). +pub const JAVA_POOL_THREADS: usize = 8; + +/// logcat tag for every engine line (the Kotlin manager mirrors it into its own debug log). +pub const LOG_TAG: &str = "EpicNative"; diff --git a/app/src/main/rust/stores/src/epic/chunk.rs b/app/src/main/rust/stores/src/epic/chunk.rs new file mode 100644 index 0000000000..22ebb864db --- /dev/null +++ b/app/src/main/rust/stores/src/epic/chunk.rs @@ -0,0 +1,375 @@ +//! One chunk: header parse → (zlib inflate | copy) → SHA-1 → `.part` → rename into the cache. +//! +//! Port of `EpicDownloadManager.downloadChunkStreaming` minus the HTTP part (the fetch core +//! hands us the whole response body). Java's protocol, kept exactly: +//! - 41-byte header: magic(4) headerVersion(4) headerSize(4) compressedSize(4) GUID(16) hash(8) +//! storedAs(1); payload starts at `max(headerSize, 41)` (Java never seeks back); +//! - payload = the next `compressedSize` bytes, or whatever the body still has (a short body +//! is NOT an error — the SHA-1 check decides); +//! - `storedAs & 1` → zlib; corrupt zlib = failure; a stream that ends early = partial output; +//! - the decompressed bytes go to `.part`; the SHA-1 (when the manifest has one) must +//! match or the `.part` is deleted; then an atomic rename publishes ``. +//! A cache file therefore exists under its final name ONLY if it fully downloaded and verified, +//! on either engine. + +use std::fs::{self, File}; +use std::io::{self, BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use sha1::{Digest, Sha1}; + +/// Chunk file magic (`0xB1FE3AA2`). +pub const CHUNK_MAGIC: u32 = 0xB1FE_3AA2; + +/// Bytes Java reads before looking at the header (`byte[] hdrBuf = new byte[41]`). +pub const CHUNK_HEADER_PREFIX: usize = 41; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChunkHeader { + pub header_version: i32, + pub header_size: i32, + pub compressed_size: i32, + pub stored_as: u8, +} + +impl ChunkHeader { + pub fn is_compressed(&self) -> bool { + (self.stored_as & 1) != 0 + } + + /// Where the payload begins in the body (`if (headerSize > 41) skipFully(headerSize - 41)`). + pub fn payload_start(&self) -> usize { + if self.header_size > CHUNK_HEADER_PREFIX as i32 { + self.header_size as usize + } else { + CHUNK_HEADER_PREFIX + } + } +} + +/// `readFully(hdrBuf)` + header decode. `Err` = the Java `continue`/exception paths +/// (short stream, bad magic). +pub fn parse_chunk_header(body: &[u8]) -> Result { + if body.len() < CHUNK_HEADER_PREFIX { + return Err(format!( + "Stream ended after {}/{} bytes", + body.len(), + CHUNK_HEADER_PREFIX + )); + } + let le = |o: usize| i32::from_le_bytes([body[o], body[o + 1], body[o + 2], body[o + 3]]); + let magic = le(0) as u32; + if magic != CHUNK_MAGIC { + return Err(format!("Bad chunk magic (streaming): 0x{magic:x}")); + } + Ok(ChunkHeader { + header_version: le(4), + header_size: le(8), + compressed_size: le(12), + stored_as: body[40], + }) +} + +/// The payload slice Java's read loop would consume: `compressedSize` bytes after the header, +/// truncated to what the body holds (`if (n <= 0) break;`). `Err` when the header skip runs +/// past the body (`skipFully` throws). +pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8], String> { + let start = hdr.payload_start(); + if start > body.len() { + return Err("Stream ended during skip".to_string()); + } + let want = if hdr.compressed_size <= 0 { + 0 + } else { + hdr.compressed_size as usize + }; + let avail = body.len() - start; + let take = want.min(avail); + Ok(&body[start..start + take]) +} + +/// `Writer` that mirrors `fos.write(...)` + `sha.update(...)` on every block. +struct HashingWriter { + inner: W, + sha: Option, + written: u64, +} + +impl Write for HashingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let n = self.inner.write(buf)?; + if let Some(sha) = self.sha.as_mut() { + sha.update(&buf[..n]); + } + self.written += n as u64; + Ok(n) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// `new File(outFile.getPath() + ".part")`. +pub fn part_path(final_path: &Path) -> PathBuf { + let mut s = final_path.as_os_str().to_os_string(); + s.push(".part"); + PathBuf::from(s) +} + +/// Process one downloaded body into `final_path`. Returns the number of DECOMPRESSED bytes +/// written. `Err(reason)` = Java's "try the next CDN" outcome; the `.part` never survives an +/// error and `final_path` is only ever created by the rename after verification. +pub fn write_verified_chunk( + body: &[u8], + expected_sha1: Option<&[u8; 20]>, + final_path: &Path, +) -> Result { + let tmp = part_path(final_path); + // `tmp.delete()` — clear any stale partial from a prior interrupted attempt. + let _ = fs::remove_file(&tmp); + + let result = write_part(body, expected_sha1, &tmp, final_path); + if result.is_err() { + let _ = fs::remove_file(&tmp); + } + result +} + +fn write_part( + body: &[u8], + expected_sha1: Option<&[u8; 20]>, + tmp: &Path, + final_path: &Path, +) -> Result { + let hdr = parse_chunk_header(body)?; + let payload = chunk_payload(body, &hdr)?; + + let file = File::create(tmp).map_err(|e| format!("open {}: {e}", tmp.display()))?; + let mut out = HashingWriter { + inner: BufWriter::with_capacity(128 * 1024, file), + sha: expected_sha1.map(|_| Sha1::new()), + written: 0, + }; + + if hdr.is_compressed() { + // Java: inflate until the stream finishes or the input runs out; a corrupt stream throws + // (DataFormatException) → this CDN attempt fails. flate2 behaves the same way: EOF on a + // truncated stream is a clean end (partial output), corruption is an error. + let mut dec = flate2::read::ZlibDecoder::new(payload); + io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?; + } else { + out.write_all(payload).map_err(|e| format!("write: {e}"))?; + } + out.flush().map_err(|e| format!("flush: {e}"))?; + let written = out.written; + let digest = out.sha.take().map(|s| s.finalize()); + drop(out); // close the file before the rename + + if let (Some(expected), Some(actual)) = (expected_sha1, digest) { + if actual.as_slice() != &expected[..] { + return Err("Chunk SHA-1 mismatch (streaming)".to_string()); + } + } + + // Verified → publish atomically (same directory → same filesystem). + fs::rename(tmp, final_path).map_err(|e| format!("Chunk rename failed: {e}"))?; + Ok(written) +} + +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + + /// Build a chunk body the way the CDN serves it. + pub fn build_chunk_body( + decompressed: &[u8], + compress: bool, + extra_header: usize, + declared_compressed_size: Option, + ) -> Vec { + let payload = if compress { + let mut enc = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(decompressed).unwrap(); + enc.finish().unwrap() + } else { + decompressed.to_vec() + }; + let header_size = (CHUNK_HEADER_PREFIX + extra_header) as i32; + let mut out = Vec::new(); + out.extend_from_slice(&CHUNK_MAGIC.to_le_bytes()); + out.extend_from_slice(&3i32.to_le_bytes()); // headerVersion + out.extend_from_slice(&header_size.to_le_bytes()); + out.extend_from_slice( + &declared_compressed_size + .unwrap_or(payload.len() as i32) + .to_le_bytes(), + ); + out.extend_from_slice(&[0xAB; 16]); // GUID (unused by the reader) + out.extend_from_slice(&[0xCD; 8]); // hash (unused by the reader) + out.push(if compress { 1 } else { 0 }); + out.extend(std::iter::repeat(0x5A).take(extra_header)); + assert_eq!(out.len(), header_size as usize); + out.extend_from_slice(&payload); + out + } + + pub fn sha1_of(data: &[u8]) -> [u8; 20] { + let mut h = Sha1::new(); + h.update(data); + let d = h.finalize(); + let mut out = [0u8; 20]; + out.copy_from_slice(&d); + out + } + + pub fn temp_dir(tag: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let dir = std::env::temp_dir().join(format!( + "epic-{tag}-{}-{nanos}", + std::process::id() + )); + fs::create_dir_all(&dir).unwrap(); + dir + } +} + +#[cfg(test)] +mod tests { + use super::test_support::*; + use super::*; + + fn sample_data() -> Vec { + (0..200_000u32).map(|i| (i % 251) as u8).collect() + } + + #[test] + fn header_parse_matches_java_fields() { + let body = build_chunk_body(b"hello", true, 5, None); + let h = parse_chunk_header(&body).unwrap(); + assert_eq!(h.header_version, 3); + assert_eq!(h.header_size, 46); + assert!(h.is_compressed()); + assert_eq!(h.payload_start(), 46); + let short = ChunkHeader { + header_version: 0, + header_size: 20, + compressed_size: 1, + stored_as: 0, + }; + assert_eq!(short.payload_start(), 41, "headerSize < 41 → payload still starts at 41"); + } + + #[test] + fn short_or_wrong_magic_bodies_are_rejected() { + assert!(parse_chunk_header(&[0u8; 40]).is_err()); + let mut body = build_chunk_body(b"x", false, 0, None); + body[0] ^= 0xFF; + assert!(parse_chunk_header(&body).unwrap_err().contains("Bad chunk magic")); + } + + #[test] + fn payload_is_truncated_to_the_body_not_an_error() { + let body = build_chunk_body(b"abcdef", false, 0, Some(100)); + let h = parse_chunk_header(&body).unwrap(); + assert_eq!(chunk_payload(&body, &h).unwrap(), b"abcdef"); + let neg = build_chunk_body(b"abcdef", false, 0, Some(-1)); + let h = parse_chunk_header(&neg).unwrap(); + assert_eq!(chunk_payload(&neg, &h).unwrap(), b""); + // headerSize past the body → skipFully throws. + let mut far = build_chunk_body(b"abcdef", false, 0, None); + far[8..12].copy_from_slice(&5000i32.to_le_bytes()); + let h = parse_chunk_header(&far).unwrap(); + assert!(chunk_payload(&far, &h).is_err()); + } + + #[test] + fn compressed_chunk_is_inflated_verified_and_renamed() { + let dir = temp_dir("ok"); + let data = sample_data(); + let body = build_chunk_body(&data, true, 3, None); + let final_path = dir.join("00000001000000020000000300000004"); + let n = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap(); + assert_eq!(n, data.len() as u64); + assert_eq!(fs::read(&final_path).unwrap(), data); + assert!(!part_path(&final_path).exists(), ".part renamed away"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stored_chunk_is_copied_as_is() { + let dir = temp_dir("stored"); + let data = b"stored-as-is payload".to_vec(); + let body = build_chunk_body(&data, false, 0, None); + let final_path = dir.join("chunk"); + let n = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap(); + assert_eq!(n, data.len() as u64); + assert_eq!(fs::read(&final_path).unwrap(), data); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn sha1_mismatch_leaves_nothing_behind() { + let dir = temp_dir("sha"); + let data = sample_data(); + let body = build_chunk_body(&data, true, 0, None); + let final_path = dir.join("chunk"); + let err = write_verified_chunk(&body, Some(&[0x42; 20]), &final_path).unwrap_err(); + assert!(err.contains("SHA-1 mismatch"), "{err}"); + assert!(!final_path.exists()); + assert!(!part_path(&final_path).exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn unverifiable_chunk_is_accepted_without_a_hash() { + let dir = temp_dir("nosha"); + let data = sample_data(); + let body = build_chunk_body(&data, true, 0, None); + let final_path = dir.join("chunk"); + assert_eq!( + write_verified_chunk(&body, None, &final_path).unwrap(), + data.len() as u64 + ); + assert_eq!(fs::read(&final_path).unwrap(), data); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn corrupt_zlib_fails_and_truncated_zlib_fails_the_hash() { + let dir = temp_dir("zlib"); + let data = sample_data(); + let mut body = build_chunk_body(&data, true, 0, None); + let final_path = dir.join("chunk"); + // Truncate the body: inflate ends early → partial output → SHA-1 mismatch. + let cut = body.len() - 1000; + let err = write_verified_chunk(&body[..cut], Some(&sha1_of(&data)), &final_path).unwrap_err(); + assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}"); + assert!(!final_path.exists()); + // Corrupt the deflate stream itself. + for b in body.iter_mut().skip(60).take(64) { + *b ^= 0x55; + } + let err = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap_err(); + assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}"); + assert!(!final_path.exists()); + assert!(!part_path(&final_path).exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stale_part_file_is_cleared_before_the_attempt() { + let dir = temp_dir("stale"); + let final_path = dir.join("chunk"); + fs::write(part_path(&final_path), b"stale").unwrap(); + let body = [0u8; 10]; // too short → header failure + assert!(write_verified_chunk(&body, None, &final_path).is_err()); + assert!(!part_path(&final_path).exists()); + assert!(!final_path.exists()); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/app/src/main/rust/stores/src/epic/driver.rs b/app/src/main/rust/stores/src/epic/driver.rs new file mode 100644 index 0000000000..6740185fce --- /dev/null +++ b/app/src/main/rust/stores/src/epic/driver.rs @@ -0,0 +1,455 @@ +//! Plan → `FetchItem`s → `crate::fetch_core::run_fetch` with a sink that fills the chunk cache. +//! +//! This is the replacement for the body of the Java pool block in `EpicDownloadManager.install` +//! ("Download unique chunks — 8 parallel threads"). Inputs are exactly what that block sees: +//! the parsed manifest (re-parsed here from the same bytes), the pending file set (indices Java +//! computed after its delta/verify pass), the CDN prefixes (`baseUrl + cloudDir`, cloudflare +//! already skipped) and the chunk cache dir. Output = `/.chunks/` for every +//! needed chunk; Java assembles the files afterwards exactly as before. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; + +use super::manifest::{parse_manifest, Manifest}; +use super::plan::{ + cached_chunk_path, chunk_cache_dir, chunk_url, distinct_prefixes, per_host_cap, + total_credit_bytes, unique_chunks_for_files, +}; + +/// Java `conn.setReadTimeout(60000)` — the longer of the two Java timeouts (connect was 30 s). +pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); + +/// Everything the JNI layer hands over for one run. +pub struct EpicRequest { + pub manifest_bytes: Vec, + pub install_dir: String, + /// `cdn.baseUrl + cdn.cloudDir` per manifest-API CDN entry, in Java's order. + pub cdn_prefixes: Vec, + /// Indices into `manifest.files` of Java's `pendingFiles` (post delta/verify), in order. + pub pending_file_indices: Vec, + /// Java's `neededChunks.size()` / `totalBytes` for the plan cross-check (`None` = skip). + pub expected_chunks: Option, + pub expected_bytes: Option, + pub ca_bundle_path: String, + /// Window ceiling: the Steam speed tier's network window (Fast = 32) since improvements + /// round 1; the Java fallback pool keeps its fixed 8 (`super::JAVA_POOL_THREADS`). + pub max_workers: usize, + pub process_workers: usize, + pub label: String, +} + +/// The chunk plan for one run. +#[derive(Debug)] +pub struct EpicPlan { + pub manifest: Manifest, + pub cache_dir: PathBuf, + /// Chunk indices (into `manifest.unique_chunks`) in Java's submission order. + pub needed: Vec, + /// `Σ max(fileSize, 1)` over `needed`. + pub total_bytes: u64, + /// Distinct CDN prefixes = fetch-core host keys. + pub hosts: Vec, +} + +/// Terminal result of a run, in the shape Java's pool block needs to reproduce its own exit +/// paths (`CANCELLED during chunk download (n/total chunks)`, `N chunks failed`, `chunksOK=`). +#[derive(Clone, Debug, Default)] +pub struct EpicOutcome { + pub success: bool, + pub cancelled: bool, + pub error: String, + /// Credited (manifest `max(fileSize,1)`) bytes: skipped-cached + fetched. + pub bytes_credited: u64, + /// Chunks accounted for (skipped-cached + fetched), Java's `completedCount`. + pub chunks_done: u64, + pub chunks_total: u64, + pub bytes_total: u64, + /// Decompressed bytes actually written to the cache this run. + pub decompressed_written: u64, +} + +/// Parse + plan (no I/O beyond reading the cache dir). `Err` = "engine could not start"; Java +/// then runs its own pool, since nothing has been fetched yet. +pub fn build_plan(req: &EpicRequest) -> Result { + let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("plan: {e}"))?; + for &i in &req.pending_file_indices { + if i >= manifest.files.len() { + return Err(format!( + "plan: pending file index {i} out of range ({} files)", + manifest.files.len() + )); + } + } + let needed = unique_chunks_for_files(&manifest, &req.pending_file_indices); + let total_bytes = total_credit_bytes(&manifest, &needed); + if let Some(expected) = req.expected_chunks { + if expected != needed.len() as u64 { + return Err(format!( + "plan: chunk count mismatch java={expected} rust={}", + needed.len() + )); + } + } + if let Some(expected) = req.expected_bytes { + if expected != total_bytes { + return Err(format!( + "plan: byte total mismatch java={expected} rust={total_bytes}" + )); + } + } + let hosts = distinct_prefixes(&req.cdn_prefixes); + if hosts.is_empty() { + return Err("plan: no CDN prefixes".to_string()); + } + let cache_dir = chunk_cache_dir(&req.install_dir); + std::fs::create_dir_all(&cache_dir) + .map_err(|e| format!("plan: mkdirs {}: {e}", cache_dir.display()))?; + Ok(EpicPlan { + manifest, + cache_dir, + needed, + total_bytes, + hosts, + }) +} + +/// Sink: one downloaded body → verified cache file. `id` indexes `fetch_chunks`. +struct ChunkCacheSink<'a> { + plan: &'a EpicPlan, + /// Chunk indices (into `manifest.unique_chunks`) of the items actually being fetched. + fetch_chunks: &'a [usize], + decompressed: AtomicU64, +} + +impl<'a> FetchSink for ChunkCacheSink<'a> { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + let Some(&ci) = self.fetch_chunks.get(item.id as usize) else { + return Err(SinkError::Fatal(format!("item id {} out of range", item.id))); + }; + let chunk = &self.plan.manifest.unique_chunks[ci]; + let final_path = cached_chunk_path(&self.plan.cache_dir, chunk); + match super::chunk::write_verified_chunk(&body, chunk.verifiable_sha1(), &final_path) { + Ok(written) => { + self.decompressed.fetch_add(written, Ordering::Relaxed); + Ok(chunk.credit_bytes()) + } + // Every per-attempt failure in Java is "try the next CDN"; the core's Retry rotates + // hosts and backs off the same way (bounded at its attempt cap). + Err(reason) => Err(SinkError::Retry(format!("{} {reason}", chunk.guid_str()))), + } + } +} + +/// Rolling speed sampler for the end-of-run summary: samples at ≥500 ms like the Java pool's +/// `lastSpeedMs` logic, keeps the peak. +struct SpeedMeter { + start: Instant, + last_at: Instant, + last_bytes: u64, + peak_bps: f64, +} + +impl SpeedMeter { + fn new() -> Self { + let now = Instant::now(); + Self { + start: now, + last_at: now, + last_bytes: 0, + peak_bps: 0.0, + } + } + + fn sample(&mut self, bytes_now: u64) { + let now = Instant::now(); + let dt = now.duration_since(self.last_at); + if dt >= Duration::from_millis(500) { + let delta = bytes_now.saturating_sub(self.last_bytes) as f64; + let bps = delta / dt.as_secs_f64(); + if bps > self.peak_bps { + self.peak_bps = bps; + } + self.last_at = now; + self.last_bytes = bytes_now; + } + } + + fn summary(&self, bytes: u64, decompressed: u64, skipped: u64) -> String { + let secs = self.start.elapsed().as_secs_f64().max(0.001); + let avg_bps = bytes as f64 / secs; + format!( + "summary bytes={bytes} decompressed={decompressed} skipped_chunks={skipped} elapsed={secs:.1}s avg_mbps={:.2} peak_mbps={:.2} avg_MBps={:.2}", + avg_bps * 8.0 / 1_000_000.0, + self.peak_bps * 8.0 / 1_000_000.0, + avg_bps / (1024.0 * 1024.0) + ) + } +} + +/// Run the plan. `progress(bytes_done, chunks_done)` fires once per accounted chunk (cached-skip +/// or fetched), exactly the cadence of the Java pool's per-task `progress(...)` call. `log` +/// receives engine lines (`fetch-window`, `summary`, failures). +pub fn run_plan( + plan: &EpicPlan, + req: &EpicRequest, + cancel: &AtomicBool, + progress: &(dyn Fn(u64, u64) + Sync), + log: &(dyn Fn(&str) + Sync), +) -> EpicOutcome { + let chunks_total = plan.needed.len() as u64; + let mut outcome = EpicOutcome { + chunks_total, + bytes_total: plan.total_bytes, + ..EpicOutcome::default() + }; + let host_cap = per_host_cap(req.max_workers, plan.hosts.len()); + log(&format!( + "plan chunk_dir={} files_pending={} chunks={} bytes={} hosts={} workers={} per_host_cap={host_cap} process_workers={}", + plan.manifest.chunk_dir, + req.pending_file_indices.len(), + chunks_total, + plan.total_bytes, + plan.hosts.len(), + req.max_workers, + req.process_workers + )); + + // Java: each pool task first checks the cancel flag, then `cachedFile.exists()` → credit + // without fetching. Account for the cached chunks up front (same credit, same callback). + let mut fetch_chunks: Vec = Vec::with_capacity(plan.needed.len()); + let mut pre_bytes: u64 = 0; + let mut pre_chunks: u64 = 0; + for &ci in &plan.needed { + if cancel.load(Ordering::Relaxed) { + outcome.cancelled = true; + outcome.error = "cancelled".to_string(); + outcome.bytes_credited = pre_bytes; + outcome.chunks_done = pre_chunks; + log(&format!( + "cancelled before fetch ({pre_chunks}/{chunks_total} chunks)" + )); + return outcome; + } + let chunk = &plan.manifest.unique_chunks[ci]; + if cached_chunk_path(&plan.cache_dir, chunk).exists() { + pre_bytes += chunk.credit_bytes(); + pre_chunks += 1; + progress(pre_bytes, pre_chunks); + } else { + fetch_chunks.push(ci); + } + } + for (i, h) in plan.hosts.iter().enumerate() { + log(&format!("host[{i}]={h}")); + } + log(&format!( + "skip cached={pre_chunks} bytes={pre_bytes} to_fetch={}", + fetch_chunks.len() + )); + + if fetch_chunks.is_empty() { + outcome.success = true; + outcome.bytes_credited = pre_bytes; + outcome.chunks_done = pre_chunks; + log("summary bytes=0 decompressed=0 elapsed=0.0s avg_mbps=0.00 peak_mbps=0.00 (nothing to fetch)"); + return outcome; + } + + let items: Vec = fetch_chunks + .iter() + .enumerate() + .map(|(id, &ci)| { + let chunk = &plan.manifest.unique_chunks[ci]; + FetchItem { + id: id as u64, + urls: plan + .hosts + .iter() + .map(|h| chunk_url(h, &plan.manifest.chunk_dir, chunk)) + .collect(), + reserve: if chunk.file_size > 0 { + chunk.file_size as u64 + } else { + 0 + }, + range: None, + } + }) + .collect(); + + let opts = FetchOptions { + max_workers: req.max_workers.max(1), + // Improvements round 1: the tier ceiling split across the distinct CDNs (floor 6), so + // the whole window is reachable on 1 host or on all 3. + per_host_cap: host_cap, + timeout: REQUEST_TIMEOUT, + headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())], + ca_bundle_path: req.ca_bundle_path.clone(), + process_workers: req.process_workers.max(1), + label: req.label.clone(), + // Whole-body mode (Epic chunks are ≤ ~1 MiB compressed; the core's byte budget scales + // with the window); `stream` and any future field keep the core's defaults. + ..FetchOptions::default() + }; + + let sink = ChunkCacheSink { + plan, + fetch_chunks: &fetch_chunks, + decompressed: AtomicU64::new(0), + }; + + let meter = Mutex::new(SpeedMeter::new()); + let progress_adapter = |bytes: u64, items_ok: u64| { + if let Ok(mut m) = meter.lock() { + m.sample(bytes); + } + progress(pre_bytes + bytes, pre_chunks + items_ok); + }; + + let fetched = run_fetch( + items, + &plan.hosts, + &opts, + &sink, + cancel, + &progress_adapter, + log, + ); + + let decompressed = sink.decompressed.load(Ordering::Relaxed); + outcome.bytes_credited = pre_bytes + fetched.bytes_credited; + outcome.chunks_done = pre_chunks + fetched.items_ok; + outcome.decompressed_written = decompressed; + if let Ok(m) = meter.lock() { + log(&m.summary(fetched.bytes_credited, decompressed, pre_chunks)); + } + + if fetched.cancelled { + outcome.cancelled = true; + outcome.error = "cancelled".to_string(); + log(&format!( + "cancelled during chunk download ({}/{chunks_total} chunks)", + outcome.chunks_done + )); + return outcome; + } + if let Some(err) = fetched.error { + outcome.error = err; + log(&format!( + "FAIL {} ({}/{chunks_total} chunks ok)", + outcome.error, outcome.chunks_done + )); + return outcome; + } + if fetched.items_ok != fetch_chunks.len() as u64 { + outcome.error = format!( + "fetch ended with {}/{} chunks", + fetched.items_ok, + fetch_chunks.len() + ); + log(&format!("FAIL {}", outcome.error)); + return outcome; + } + outcome.success = true; + log(&format!("chunksOK={}", outcome.chunks_done)); + outcome +} + +#[cfg(test)] +mod tests { + use super::super::manifest::test_support::*; + use super::*; + + fn request(dir: &str, pending: Vec) -> EpicRequest { + EpicRequest { + manifest_bytes: build_manifest(&sample_chunks(), &sample_files(), 21, true), + install_dir: dir.to_string(), + cdn_prefixes: vec![ + "https://fastly-download.epicgames.com/Builds/o/x/default".to_string(), + "https://download.epicgames.com/Builds/o/x/default".to_string(), + "https://fastly-download.epicgames.com/Builds/o/x/default".to_string(), + ], + pending_file_indices: pending, + expected_chunks: None, + expected_bytes: None, + ca_bundle_path: String::new(), + max_workers: super::super::JAVA_POOL_THREADS, + process_workers: 2, + label: "epic test".to_string(), + } + } + + #[test] + fn plan_matches_java_for_the_pending_set() { + let dir = super::super::chunk::test_support::temp_dir("plan"); + let req = request(dir.to_str().unwrap(), vec![0, 1]); + let plan = build_plan(&req).unwrap(); + assert_eq!(plan.needed, vec![0, 1]); + assert_eq!(plan.total_bytes, 700_000 + 1); + assert_eq!(plan.hosts.len(), 2, "duplicate CDN prefix collapsed"); + assert!(plan.cache_dir.ends_with(".chunks")); + assert!(plan.cache_dir.is_dir()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn plan_cross_check_rejects_mismatches() { + let dir = super::super::chunk::test_support::temp_dir("xcheck"); + let mut req = request(dir.to_str().unwrap(), vec![0, 1]); + req.expected_chunks = Some(3); + assert!(build_plan(&req).unwrap_err().contains("chunk count mismatch")); + req.expected_chunks = Some(2); + req.expected_bytes = Some(5); + assert!(build_plan(&req).unwrap_err().contains("byte total mismatch")); + req.expected_bytes = Some(700_001); + assert!(build_plan(&req).is_ok()); + req.pending_file_indices = vec![7]; + assert!(build_plan(&req).unwrap_err().contains("out of range")); + req.pending_file_indices = vec![0]; + req.cdn_prefixes.clear(); + assert!(build_plan(&req).unwrap_err().contains("no CDN")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn fully_cached_plan_completes_without_fetching() { + let dir = super::super::chunk::test_support::temp_dir("cached"); + let req = request(dir.to_str().unwrap(), vec![0]); + let plan = build_plan(&req).unwrap(); + for &ci in &plan.needed { + std::fs::write( + cached_chunk_path(&plan.cache_dir, &plan.manifest.unique_chunks[ci]), + b"cached", + ) + .unwrap(); + } + let cancel = AtomicBool::new(false); + let calls = Mutex::new(Vec::new()); + let progress = |b: u64, c: u64| calls.lock().unwrap().push((b, c)); + let log = |_: &str| {}; + let out = run_plan(&plan, &req, &cancel, &progress, &log); + assert!(out.success); + assert_eq!(out.chunks_done, 2); + assert_eq!(out.bytes_credited, 700_001); + assert_eq!(*calls.lock().unwrap(), vec![(700_000, 1), (700_001, 2)]); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn cancel_before_fetch_reports_cancelled() { + let dir = super::super::chunk::test_support::temp_dir("cancel"); + let req = request(dir.to_str().unwrap(), vec![0]); + let plan = build_plan(&req).unwrap(); + let cancel = AtomicBool::new(true); + let progress = |_: u64, _: u64| {}; + let log = |_: &str| {}; + let out = run_plan(&plan, &req, &cancel, &progress, &log); + assert!(out.cancelled); + assert!(!out.success); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/app/src/main/rust/stores/src/epic/jni.rs b/app/src/main/rust/stores/src/epic/jni.rs new file mode 100644 index 0000000000..9f7cf2f52a --- /dev/null +++ b/app/src/main/rust/stores/src/epic/jni.rs @@ -0,0 +1,388 @@ +//! JNI exports for `com.droiddeck.launcher.stores.epic.EpicNative`. +//! +//! ```text +//! nativeStart(manifest: byte[], installDir: String, cdnPrefixes: String[], pendingFileIdx: int[], +//! expectedChunks: int, expectedBytes: long, caBundlePath: String, maxWorkers: int, +//! processWorkers: int, listener: EpicNative.Listener) -> long handle (0 = not started) +//! nativeCancel(handle) — flips the run's AtomicBool; the fetch core stops promptly +//! nativeRelease(handle) — frees the handle (after onComplete) +//! ``` +//! Listener (all methods run on native threads): +//! `onPlan(int chunksTotal, long bytesTotal, String chunkDir)` once before any fetch, +//! `onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal)` per chunk, +//! `onLog(String line)`, `onComplete(boolean success, String error, long bytesCredited)`. +//! +//! Planning (manifest parse + cross-check) runs synchronously on the calling thread so a plan +//! failure returns 0 BEFORE any fetch and Java can fall back to its own pool; the fetch runs on +//! a new thread. JavaVM attach / GlobalRef handling is the same shape as the GOG facade. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; + +use jni::objects::{GlobalRef, JByteArray, JClass, JIntArray, JObject, JObjectArray, JString, JValue}; +use jni::sys::{jint, jlong, JNI_FALSE, JNI_TRUE}; +use jni::{JNIEnv, JavaVM}; + +use super::driver::{build_plan, run_plan, EpicRequest}; + +#[cfg(target_os = "android")] +#[link(name = "log")] +unsafe extern "C" { + fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32; +} + +fn android_log(message: &str) { + #[cfg(target_os = "android")] + { + use std::ffi::CString; + let Ok(tag) = CString::new(super::LOG_TAG) else { + return; + }; + let sanitized = message.replace('\0', " "); + let Ok(text) = CString::new(sanitized) else { + return; + }; + unsafe { + let _ = __android_log_write(4, tag.as_ptr().cast(), text.as_ptr().cast()); + } + } + #[cfg(not(target_os = "android"))] + { + let _ = message; + } +} + +/// Per-run native state behind the `jlong` handle. +pub struct EpicDownloadHandle { + cancel: Arc, +} + +fn clear_pending_exception(env: &mut JNIEnv) { + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } +} + +fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { + env.get_string(value) + .ok() + .map(|s| s.to_string_lossy().into_owned()) +} + +fn string_array_to_vec(env: &mut JNIEnv, array: &JObjectArray) -> Vec { + let len = env.get_array_length(array).unwrap_or(0); + let mut out = Vec::with_capacity(len.max(0) as usize); + let mut i = 0; + while i < len { + match env.get_object_array_element(array, i) { + Ok(obj) => { + if obj.is_null() { + out.push(String::new()); + } else { + let js = JString::from(obj); + out.push(jstring_to_string(env, &js).unwrap_or_default()); + } + } + Err(_) => { + clear_pending_exception(env); + out.push(String::new()); + } + } + i += 1; + } + out +} + +fn int_array_to_vec(env: &JNIEnv, array: &JIntArray) -> Vec { + let len = env.get_array_length(array).unwrap_or(0); + if len <= 0 { + return Vec::new(); + } + let mut values = vec![0i32; len as usize]; + if env.get_int_array_region(array, 0, &mut values).is_err() { + return Vec::new(); + } + values +} + +fn call_on_log(env: &mut JNIEnv, listener: &JObject, line: &str) { + if listener.is_null() { + return; + } + let Ok(s) = env.new_string(line) else { + clear_pending_exception(env); + return; + }; + let obj = JObject::from(s); + let _ = env.call_method( + listener, + "onLog", + "(Ljava/lang/String;)V", + &[JValue::Object(&obj)], + ); + clear_pending_exception(env); +} + +fn call_on_plan(env: &mut JNIEnv, listener: &JObject, chunks_total: u64, bytes_total: u64, chunk_dir: &str) { + if listener.is_null() { + return; + } + let Ok(s) = env.new_string(chunk_dir) else { + clear_pending_exception(env); + return; + }; + let obj = JObject::from(s); + let _ = env.call_method( + listener, + "onPlan", + "(IJLjava/lang/String;)V", + &[ + JValue::Int(chunks_total.min(i32::MAX as u64) as jint), + JValue::Long(bytes_total as jlong), + JValue::Object(&obj), + ], + ); + clear_pending_exception(env); +} + +fn call_on_progress( + env: &mut JNIEnv, + listener: &JObject, + bytes_done: u64, + bytes_total: u64, + chunks_done: u64, + chunks_total: u64, +) { + if listener.is_null() { + return; + } + let _ = env.call_method( + listener, + "onProgress", + "(JJII)V", + &[ + JValue::Long(bytes_done as jlong), + JValue::Long(bytes_total as jlong), + JValue::Int(chunks_done.min(i32::MAX as u64) as jint), + JValue::Int(chunks_total.min(i32::MAX as u64) as jint), + ], + ); + clear_pending_exception(env); +} + +fn call_on_complete(env: &mut JNIEnv, listener: &JObject, success: bool, error: &str, bytes: u64) { + if listener.is_null() { + return; + } + let Ok(s) = env.new_string(error) else { + clear_pending_exception(env); + return; + }; + let obj = JObject::from(s); + let _ = env.call_method( + listener, + "onComplete", + "(ZLjava/lang/String;J)V", + &[ + JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), + JValue::Object(&obj), + JValue::Long(bytes as jlong), + ], + ); + clear_pending_exception(env); +} + +/// Log to logcat AND to the listener (the Kotlin manager mirrors it into its own debug log). +fn log_both(env: &mut JNIEnv, listener: &JObject, line: &str) { + android_log(line); + call_on_log(env, listener, line); +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart( + mut env: JNIEnv, + _class: JClass, + manifest: JByteArray, + install_dir: JString, + cdn_prefixes: JObjectArray, + pending_file_idx: JIntArray, + expected_chunks: jint, + expected_bytes: jlong, + ca_bundle_path: JString, + max_workers: jint, + process_workers: jint, + listener: JObject, +) -> jlong { + if listener.is_null() { + android_log("nativeStart: null listener"); + return 0; + } + let Ok(manifest_bytes) = env.convert_byte_array(manifest) else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "manifest bytes unreadable", 0); + return 0; + }; + let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); + let cdn_prefixes = string_array_to_vec(&mut env, &cdn_prefixes); + let pending_file_indices: Vec = int_array_to_vec(&env, &pending_file_idx) + .into_iter() + .map(|i| i.max(0) as usize) + .collect(); + let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); + let max_workers = max_workers.max(1) as usize; + let process_workers = process_workers.max(1) as usize; + let label = { + let base = std::path::Path::new(&install_dir) + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + format!("epic app={base}") + }; + + let host_count = super::plan::distinct_prefixes(&cdn_prefixes).len(); + let host_cap = super::plan::per_host_cap(max_workers, host_count); + log_both( + &mut env, + &listener, + &format!( + "engine=rust label=\"{label}\" install_dir={install_dir} cdns={} hosts={host_count} pending_files={} workers={max_workers} per_host_cap={host_cap} process_workers={process_workers} manifest_bytes={}", + cdn_prefixes.len(), + pending_file_indices.len(), + manifest_bytes.len() + ), + ); + + let req = EpicRequest { + manifest_bytes, + install_dir, + cdn_prefixes, + pending_file_indices, + expected_chunks: if expected_chunks >= 0 { + Some(expected_chunks as u64) + } else { + None + }, + expected_bytes: if expected_bytes >= 0 { + Some(expected_bytes as u64) + } else { + None + }, + ca_bundle_path, + max_workers, + process_workers, + label, + }; + + let plan = match build_plan(&req) { + Ok(plan) => plan, + Err(err) => { + log_both(&mut env, &listener, &format!("not started: {err}")); + call_on_complete(&mut env, &listener, false, &err, 0); + return 0; + } + }; + call_on_plan( + &mut env, + &listener, + plan.needed.len() as u64, + plan.total_bytes, + &plan.manifest.chunk_dir, + ); + + let Ok(vm) = env.get_java_vm() else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); + return 0; + }; + let Ok(listener) = env.new_global_ref(&listener) else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "listener ref failed", 0); + return 0; + }; + + let cancel = Arc::new(AtomicBool::new(false)); + let thread_cancel = Arc::clone(&cancel); + let handle = Box::new(EpicDownloadHandle { cancel }); + + thread::spawn(move || { + run_on_thread(vm, listener, plan, req, thread_cancel); + }); + + Box::into_raw(handle) as jlong +} + +fn run_on_thread( + vm: JavaVM, + listener: GlobalRef, + plan: super::driver::EpicPlan, + req: EpicRequest, + cancel: Arc, +) { + let bytes_total = plan.total_bytes; + let chunks_total = plan.needed.len() as u64; + + // Progress fires from the core's process-pool threads; attach each as a daemon like the + // Steam engine's `dispatch_download_progress` does. + let progress = |bytes_done: u64, chunks_done: u64| { + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_progress( + &mut env, + listener.as_obj(), + bytes_done, + bytes_total, + chunks_done, + chunks_total, + ); + }; + let log = |line: &str| { + android_log(line); + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_log(&mut env, listener.as_obj(), line); + }; + + let outcome = run_plan(&plan, &req, cancel.as_ref(), &progress, &log); + + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_complete( + &mut env, + listener.as_obj(), + outcome.success, + &outcome.error, + outcome.bytes_credited, + ); +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCancel( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if handle == 0 { + return; + } + let handle = unsafe { &*(handle as *const EpicDownloadHandle) }; + handle.cancel.store(true, Ordering::Relaxed); + android_log("cancel requested"); +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeRelease( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if handle == 0 { + return; + } + // The worker thread owns its own Arc clone of the cancel flag and its own GlobalRef, so + // freeing the handle is safe even if a (cancelled) run is still winding down. + drop(unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) }); +} diff --git a/app/src/main/rust/stores/src/epic/manifest.rs b/app/src/main/rust/stores/src/epic/manifest.rs new file mode 100644 index 0000000000..579a56c320 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/manifest.rs @@ -0,0 +1,987 @@ +//! Epic build manifest parsers — a 1:1 port of `EpicDownloadManager.parseManifest` (binary, +//! magic `0x44BEC00C`) and `parseJsonManifest` (legacy JSON manifests). +//! +//! Every quirk of the Java reader is kept on purpose (the Java manager stays the reference and +//! the two engines must derive the same plan from the same bytes): section skips by declared +//! size, `readFString` UTF-16LE/ASCII decoding, duplicate-GUID handling (last value, first +//! position), the JSON `Long.parseLong(.., 16)` / `parseInt` fallbacks, and "any exception → +//! parse failed". + +use std::collections::HashMap; +use std::io::Read; + +/// Binary manifest magic (`EpicDownloadManager.parseManifest`). +pub const MANIFEST_MAGIC: u32 = 0x44BE_C00C; + +/// One entry of the ChunkDataList section (`EpicDownloadManager.ChunkInfo`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChunkInfo { + /// Four uint32 in binary read order. + pub guid: [u32; 4], + /// uint64 "hash" that forms the first half of the chunk file name. + pub hash: u64, + /// SHA-1 of the DECOMPRESSED chunk; `None` for JSON manifests (Java: `sha1 == null`). + pub sha1: Option<[u8; 20]>, + /// Subfolder number (DECIMAL `%02d`). `i32` so the JSON parser's `Integer.parseInt` range is + /// representable exactly (binary manifests always yield 0..=255). + pub group_num: i32, + /// Uncompressed size (0 for JSON manifests). + pub window_size: i32, + /// Compressed (wire) size. Signed like the Java `long`; JSON manifests may yield anything. + pub file_size: i64, +} + +impl ChunkInfo { + /// `String.format("%08X%08X%08X%08X", guid[0..4])`. + pub fn guid_str(&self) -> String { + guid_to_string(&self.guid) + } + + /// `ChunkInfo.getPath`: `"/<%02d groupNum>/<%016X hash>_.chunk"`. + pub fn path(&self, chunk_dir: &str) -> String { + format!( + "{}/{:02}/{:016X}_{}.chunk", + chunk_dir, + self.group_num, + self.hash, + self.guid_str() + ) + } + + /// Java `chunk.sha1 != null && length == 20 && !allZero` — the condition under which the + /// streaming downloader verifies the decompressed chunk. + pub fn verifiable_sha1(&self) -> Option<&[u8; 20]> { + match &self.sha1 { + Some(h) if h.iter().any(|b| *b != 0) => Some(h), + _ => None, + } + } + + /// `Math.max(chunk.fileSize, 1)` — the byte credit the Java pool uses for totals/progress. + pub fn credit_bytes(&self) -> u64 { + if self.file_size < 1 { + 1 + } else { + self.file_size as u64 + } + } +} + +/// One file part referencing a chunk (`EpicDownloadManager.ChunkPart`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChunkPart { + pub guid: [u32; 4], + /// Offset into the DECOMPRESSED chunk (Java `int`). + pub offset: i32, + /// Part length (Java `int`; summed as `size & 0xFFFFFFFFL`). + pub size: i32, +} + +impl ChunkPart { + pub fn guid_str(&self) -> String { + guid_to_string(&self.guid) + } +} + +/// One entry of the FileManifestList section (`EpicDownloadManager.FileInfo`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FileInfo { + pub filename: String, + pub parts: Vec, + /// Install-tag names; empty = required/base file. + pub install_tags: Vec, + /// Full-file SHA-1; `None` for JSON manifests. + pub sha1: Option<[u8; 20]>, +} + +impl FileInfo { + /// `FileInfo.fileSize()` = Σ `(part.size & 0xFFFFFFFFL)`. + pub fn file_size(&self) -> u64 { + self.parts.iter().map(|p| p.size as u32 as u64).sum() + } +} + +/// Parsed manifest (`EpicDownloadManager.EpicManifest.ParsedManifest`, minus the CDN list which +/// the Java side still owns). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Manifest { + pub chunk_dir: String, + pub unique_chunks: Vec, + pub files: Vec, +} + +impl Manifest { + /// `byGuid` map as `uniqueChunksForFiles` builds it: `LinkedHashMap.put` per chunk, so a + /// duplicate GUID resolves to the LAST `ChunkInfo`. + pub fn chunk_index_by_guid(&self) -> HashMap { + let mut map = HashMap::with_capacity(self.unique_chunks.len()); + for (i, c) in self.unique_chunks.iter().enumerate() { + map.insert(c.guid_str(), i); + } + map + } +} + +pub fn guid_to_string(guid: &[u32; 4]) -> String { + format!("{:08X}{:08X}{:08X}{:08X}", guid[0], guid[1], guid[2], guid[3]) +} + +/// `chunkDir` from the binary header version (`parseManifest`). +pub fn chunk_dir_for_binary_version(version: i32) -> &'static str { + if version >= 15 { + "ChunksV4" + } else if version >= 6 { + "ChunksV3" + } else if version >= 3 { + "ChunksV2" + } else { + "Chunks" + } +} + +/// `chunkDir` from `ManifestFileVersion` (`parseJsonManifest`; note the different `< 3` default). +pub fn chunk_dir_for_json_version(version: i32) -> &'static str { + if version >= 15 { + "ChunksV4" + } else if version >= 6 { + "ChunksV3" + } else if version >= 3 { + "ChunksV2" + } else { + "ChunksV4" + } +} + +// ── little-endian cursor with ByteBuffer-like failure semantics ───────────────────────────── + +/// Mirrors the `ByteBuffer` calls the Java parser makes: every under-run or out-of-range +/// `position()` is an exception there, which the Java code turns into "parse failed" (null). +struct Cursor<'a> { + buf: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(buf: &'a [u8]) -> Self { + Self { buf, pos: 0 } + } + + fn position(&self) -> usize { + self.pos + } + + /// `ByteBuffer.position(p)` — `p` must be within `0..=limit`. + fn set_position(&mut self, p: i64) -> Result<(), String> { + if p < 0 || p as u64 > self.buf.len() as u64 { + return Err(format!("position {p} out of range (limit {})", self.buf.len())); + } + self.pos = p as usize; + Ok(()) + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], String> { + let end = self + .pos + .checked_add(n) + .ok_or_else(|| "buffer overflow".to_string())?; + if end > self.buf.len() { + return Err(format!( + "buffer underflow: need {n} at {} (limit {})", + self.pos, + self.buf.len() + )); + } + let out = &self.buf[self.pos..end]; + self.pos = end; + Ok(out) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn i32(&mut self) -> Result { + let b = self.take(4)?; + Ok(i32::from_le_bytes([b[0], b[1], b[2], b[3]])) + } + + fn u32(&mut self) -> Result { + Ok(self.i32()? as u32) + } + + fn u64(&mut self) -> Result { + let b = self.take(8)?; + Ok(u64::from_le_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ])) + } + + fn sha1(&mut self) -> Result<[u8; 20], String> { + let b = self.take(20)?; + let mut out = [0u8; 20]; + out.copy_from_slice(b); + Ok(out) + } + + /// `ByteBuffer.position(position + n)` with the same range check. + fn skip(&mut self, n: i64) -> Result<(), String> { + let target = self.pos as i64 + n; + self.set_position(target) + } + + fn remaining(&self) -> &'a [u8] { + &self.buf[self.pos..] + } + + /// `EpicDownloadManager.readFString`: int32 length; `0` → ""; negative → `(-len)-1` UTF-16LE + /// code units + 2-byte terminator; positive → `len-1` US-ASCII bytes + 1-byte terminator. + fn fstring(&mut self) -> Result { + let length = self.i32()?; + if length == 0 { + return Ok(String::new()); + } + if length < 0 { + // Java: `int chars = (-length) - 1;` — `-Integer.MIN_VALUE` wraps to itself, then -1 + // gives Integer.MAX_VALUE; `new byte[chars * 2]` overflows negative → exception. + let chars = length.wrapping_neg().wrapping_sub(1); + let byte_len = chars.checked_mul(2).ok_or("fstring length overflow")?; + if byte_len < 0 { + return Err("negative fstring length".to_string()); + } + let bytes = self.take(byte_len as usize)?; + self.take(2)?; // null terminator + let units: Vec = bytes + .chunks_exact(2) + .map(|p| u16::from_le_bytes([p[0], p[1]])) + .collect(); + Ok(String::from_utf16_lossy(&units)) + } else { + let bytes = self.take((length - 1) as usize)?; + self.take(1)?; // null terminator + // `new String(bytes, US_ASCII)`: every byte > 0x7F decodes to U+FFFD. + Ok(bytes + .iter() + .map(|&b| if b < 0x80 { b as char } else { '\u{FFFD}' }) + .collect()) + } + } +} + +// ── binary manifest ────────────────────────────────────────────────────────────────────── + +/// `EpicDownloadManager.parseManifest(bytes)`: binary when the magic matches, otherwise the JSON +/// parser. `Err` wherever Java returns `null`. +pub fn parse_manifest(bytes: &[u8]) -> Result { + let mut cur = Cursor::new(bytes); + let magic = cur.u32()?; + if magic != MANIFEST_MAGIC { + return parse_json_manifest(bytes); + } + parse_binary_after_magic(&mut cur) +} + +fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result { + let header_size = cur.i32()?; + let size_uncompressed = cur.i32()?; + let _size_compressed = cur.i32()?; + cur.skip(20)?; // SHA-1 of the body, not checked by the Java parser + let stored_as = cur.u8()?; + let version = cur.i32()?; + let chunk_dir = chunk_dir_for_binary_version(version); + + cur.set_position(header_size as i64)?; + let body_slice = cur.remaining(); + + let inflated; + let body: &[u8] = if (stored_as & 1) != 0 { + if size_uncompressed < 0 { + return Err("negative sizeUncompressed".to_string()); + } + // Java: `inflater.inflate(new byte[sizeUncompressed])` — a single call that fills at most + // `sizeUncompressed` bytes; anything else (short, corrupt) is a size mismatch → null. + let mut out = Vec::with_capacity(size_uncompressed as usize); + let mut dec = flate2::read::ZlibDecoder::new(body_slice).take(size_uncompressed as u64); + dec.read_to_end(&mut out) + .map_err(|e| format!("manifest inflate failed: {e}"))?; + if out.len() != size_uncompressed as usize { + return Err(format!( + "decomp size mismatch: expected {size_uncompressed} got {}", + out.len() + )); + } + inflated = out; + &inflated + } else { + body_slice + }; + + let mut b = Cursor::new(body); + + // ManifestMeta: skipped by its declared size (the size field counts itself). + let meta_size = b.i32()?; + let meta_end = b.position() as i64 - 4 + meta_size as i64; + b.set_position(meta_end)?; + + // ChunkDataList + let cdl_start = b.position(); + let cdl_size = b.i32()?; + let _cdl_version = b.u8()?; + let chunk_count = b.i32()?; + if chunk_count < 0 { + return Err("negative chunkCount".to_string()); // `new ArrayList<>(negative)` throws + } + let chunk_count = chunk_count as usize; + let mut chunks: Vec = (0..chunk_count) + .map(|_| ChunkInfo { + guid: [0; 4], + hash: 0, + sha1: None, + group_num: 0, + window_size: 0, + file_size: 0, + }) + .collect(); + for c in chunks.iter_mut() { + c.guid = [b.u32()?, b.u32()?, b.u32()?, b.u32()?]; + } + for c in chunks.iter_mut() { + c.hash = b.u64()?; + } + for c in chunks.iter_mut() { + c.sha1 = Some(b.sha1()?); + } + for c in chunks.iter_mut() { + c.group_num = b.u8()? as i32; + } + for c in chunks.iter_mut() { + c.window_size = b.i32()?; + } + for c in chunks.iter_mut() { + c.file_size = b.u64()? as i64; + } + b.set_position(cdl_start as i64 + cdl_size as i64)?; + + // FileManifestList + let fml_start = b.position(); + let fml_size = b.i32()?; + let _fml_version = b.u8()?; + let file_count = b.i32()?; + if file_count < 0 { + return Err("negative fileCount".to_string()); + } + let file_count = file_count as usize; + let mut files: Vec = (0..file_count).map(|_| FileInfo::default()).collect(); + for f in files.iter_mut() { + f.filename = b.fstring()?; + } + for _ in 0..file_count { + b.fstring()?; // symlink targets + } + for f in files.iter_mut() { + f.sha1 = Some(b.sha1()?); + } + b.skip(file_count as i64)?; // per-file flags byte + for f in files.iter_mut() { + let tag_count = b.i32()?; + let mut j = 0; + while j < tag_count { + let tag = b.fstring()?; + if !tag.is_empty() { + f.install_tags.push(tag); + } + j += 1; + } + } + for f in files.iter_mut() { + let part_count = b.i32()?; + let mut j = 0; + while j < part_count { + let part_start = b.position(); + let part_struct_size = b.i32()?; + let guid = [b.u32()?, b.u32()?, b.u32()?, b.u32()?]; + let offset = b.i32()?; + let size = b.i32()?; + f.parts.push(ChunkPart { guid, offset, size }); + b.set_position(part_start as i64 + part_struct_size as i64)?; + j += 1; + } + } + b.set_position(fml_start as i64 + fml_size as i64)?; + + Ok(Manifest { + chunk_dir: chunk_dir.to_string(), + unique_chunks: dedup_last_wins_first_position(chunks), + files, + }) +} + +/// `LinkedHashMap` keyed by GUID string: a repeated key keeps its FIRST +/// position but takes the LAST value. +fn dedup_last_wins_first_position(chunks: Vec) -> Vec { + let mut order: Vec = Vec::with_capacity(chunks.len()); + let mut index: HashMap = HashMap::with_capacity(chunks.len()); + for c in chunks { + let key = c.guid_str(); + match index.get(&key).copied() { + Some(i) => order[i] = c, + None => { + index.insert(key, order.len()); + order.push(c); + } + } + } + order +} + +// ── JSON manifest (older games) ────────────────────────────────────────────────────────── + +/// `org.json` value → string coercion as `optString` / `getString` do on Android (numbers and +/// booleans print as-is, JSON null prints "null", containers print their JSON). +fn json_to_string(v: &serde_json::Value) -> String { + match v { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Null => "null".to_string(), + other => other.to_string(), + } +} + +/// `obj.optString(key, fallback)`. +fn opt_string(obj: &serde_json::Map, key: &str, fallback: &str) -> String { + match obj.get(key) { + Some(v) => json_to_string(v), + None => fallback.to_string(), + } +} + +/// `(int) Long.parseLong(hex8, 16)` — the Java GUID quarter parse; invalid hex throws out of the +/// whole parse. +fn parse_guid_quarter(s: &str) -> Result { + let v = i64::from_str_radix(s, 16).map_err(|e| format!("bad guid hex '{s}': {e}"))?; + Ok(v as i32 as u32) +} + +fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> { + // Java substrings by UTF-16 unit and `parseLong` throws on anything non-hex; a non-ASCII + // prefix can never parse, so refuse it before slicing (keeps the slice on char boundaries). + if s.len() < 32 || !s.as_bytes()[..32].is_ascii() { + return Err(format!("bad guid hex '{s}'")); + } + Ok([ + parse_guid_quarter(&s[0..8])?, + parse_guid_quarter(&s[8..16])?, + parse_guid_quarter(&s[16..24])?, + parse_guid_quarter(&s[24..32])?, + ]) +} + +/// `EpicDownloadManager.parseJsonManifest`. +pub fn parse_json_manifest(bytes: &[u8]) -> Result { + let text = String::from_utf8_lossy(bytes); + let root: serde_json::Value = + serde_json::from_str(&text).map_err(|e| format!("JSON manifest: {e}"))?; + let root = root + .as_object() + .ok_or_else(|| "JSON manifest: root is not an object".to_string())?; + + let manifest_version: i32 = opt_string(root, "ManifestFileVersion", "0") + .parse::() + .unwrap_or(0); + let chunk_dir = chunk_dir_for_json_version(manifest_version); + + let chunk_hash_list = root + .get("ChunkHashList") + .and_then(|v| v.as_object()) + .ok_or_else(|| "JSON manifest: no ChunkHashList".to_string())?; + let data_group_list = root.get("DataGroupList").and_then(|v| v.as_object()); + let chunk_filesize_list = root.get("ChunkFilesizeList").and_then(|v| v.as_object()); + + // `LinkedHashMap` keyed by the RAW key string (not the normalized GUID). + let mut chunk_order: Vec = Vec::new(); + let mut chunk_index: HashMap = HashMap::new(); + for (guid_hex, hash_value) in chunk_hash_list.iter() { + if guid_hex.len() < 32 || !guid_hex.is_char_boundary(32) { + continue; + } + let hash_hex = json_to_string(hash_value); + let guid = parse_guid_from_hex(guid_hex)?; + let hash = if hash_hex.len() >= 16 && hash_hex.is_char_boundary(16) { + u64::from_str_radix(&hash_hex[0..16], 16).unwrap_or(0) + } else { + 0 + }; + let group_num = match data_group_list { + Some(dgl) => opt_string(dgl, guid_hex, "0").parse::().unwrap_or(0), + None => 0, + }; + let file_size = match chunk_filesize_list { + Some(cfl) => i64::from_str_radix(&opt_string(cfl, guid_hex, "0"), 16).unwrap_or(0), + None => 0, + }; + let c = ChunkInfo { + guid, + hash, + sha1: None, + group_num, + window_size: 0, + file_size, + }; + match chunk_index.get(guid_hex).copied() { + Some(i) => chunk_order[i] = c, + None => { + chunk_index.insert(guid_hex.clone(), chunk_order.len()); + chunk_order.push(c); + } + } + } + + let file_list = root + .get("FileManifestList") + .and_then(|v| v.as_array()) + .ok_or_else(|| "JSON manifest: no FileManifestList".to_string())?; + + let mut files = Vec::with_capacity(file_list.len()); + for entry in file_list { + let file_obj = entry + .as_object() + .ok_or_else(|| "JSON manifest: FileManifestList entry is not an object".to_string())?; + let mut fi = FileInfo { + filename: opt_string(file_obj, "Filename", ""), + ..FileInfo::default() + }; + if let Some(tags) = file_obj.get("InstallTags").and_then(|v| v.as_array()) { + for t in tags { + let tag = json_to_string(t); + if !tag.is_empty() { + fi.install_tags.push(tag); + } + } + } + if let Some(parts) = file_obj.get("FileChunkParts").and_then(|v| v.as_array()) { + for p in parts { + let part_obj = p.as_object().ok_or_else(|| { + "JSON manifest: FileChunkParts entry is not an object".to_string() + })?; + let mut part = ChunkPart { + guid: [0; 4], + offset: 0, + size: 0, + }; + let part_guid = opt_string(part_obj, "Guid", ""); + if part_guid.len() >= 32 && part_guid.is_char_boundary(32) { + part.guid = parse_guid_from_hex(&part_guid)?; + } + part.offset = opt_string(part_obj, "Offset", "0").parse::().unwrap_or(0); + part.size = opt_string(part_obj, "Size", "0").parse::().unwrap_or(0); + fi.parts.push(part); + } + } + files.push(fi); + } + + Ok(Manifest { + chunk_dir: chunk_dir.to_string(), + unique_chunks: chunk_order, + files, + }) +} + +// ── tests ──────────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +pub(crate) mod test_support { + //! Synthetic binary-manifest builder used by this module's tests and by `plan`/`driver` + //! tests. Emits exactly the layout `parseManifest` walks (meta / CDL / FML with sizes). + + use super::*; + use std::io::Write; + + pub struct TestChunk { + pub guid: [u32; 4], + pub hash: u64, + pub sha1: [u8; 20], + pub group: u8, + pub window: i32, + pub file_size: u64, + } + + pub struct TestFile { + pub name: String, + pub sha1: [u8; 20], + pub tags: Vec, + /// (guid, offset, size) + pub parts: Vec<([u32; 4], i32, i32)>, + } + + pub fn fstring_ascii(s: &str) -> Vec { + let mut v = Vec::new(); + if s.is_empty() { + v.extend_from_slice(&0i32.to_le_bytes()); + return v; + } + v.extend_from_slice(&((s.len() as i32) + 1).to_le_bytes()); + v.extend_from_slice(s.as_bytes()); + v.push(0); + v + } + + pub fn fstring_utf16(s: &str) -> Vec { + let units: Vec = s.encode_utf16().collect(); + let mut v = Vec::new(); + v.extend_from_slice(&(-((units.len() as i32) + 1)).to_le_bytes()); + for u in units { + v.extend_from_slice(&u.to_le_bytes()); + } + v.extend_from_slice(&[0, 0]); + v + } + + /// Body (meta + CDL + FML) without the outer header. + pub fn build_body(chunks: &[TestChunk], files: &[TestFile], meta_padding: usize) -> Vec { + let mut body = Vec::new(); + // ManifestMeta: size field counts itself. + let meta_size = 4 + meta_padding; + body.extend_from_slice(&(meta_size as i32).to_le_bytes()); + body.extend(std::iter::repeat(0xEEu8).take(meta_padding)); + + // ChunkDataList + let mut cdl = Vec::new(); + cdl.push(0u8); // version + cdl.extend_from_slice(&(chunks.len() as i32).to_le_bytes()); + for c in chunks { + for g in c.guid { + cdl.extend_from_slice(&g.to_le_bytes()); + } + } + for c in chunks { + cdl.extend_from_slice(&c.hash.to_le_bytes()); + } + for c in chunks { + cdl.extend_from_slice(&c.sha1); + } + for c in chunks { + cdl.push(c.group); + } + for c in chunks { + cdl.extend_from_slice(&c.window.to_le_bytes()); + } + for c in chunks { + cdl.extend_from_slice(&c.file_size.to_le_bytes()); + } + // Trailing padding the parser must skip via cdlSize. + cdl.extend_from_slice(&[0xAA, 0xBB, 0xCC]); + body.extend_from_slice(&((cdl.len() + 4) as i32).to_le_bytes()); + body.extend_from_slice(&cdl); + + // FileManifestList + let mut fml = Vec::new(); + fml.push(0u8); // version + fml.extend_from_slice(&(files.len() as i32).to_le_bytes()); + for f in files { + if f.name.is_ascii() { + fml.extend_from_slice(&fstring_ascii(&f.name)); + } else { + fml.extend_from_slice(&fstring_utf16(&f.name)); + } + } + for _ in files { + fml.extend_from_slice(&fstring_ascii("")); // symlink target + } + for f in files { + fml.extend_from_slice(&f.sha1); + } + for _ in files { + fml.push(0); // flags + } + for f in files { + fml.extend_from_slice(&(f.tags.len() as i32).to_le_bytes()); + for t in &f.tags { + fml.extend_from_slice(&fstring_ascii(t)); + } + } + for f in files { + fml.extend_from_slice(&(f.parts.len() as i32).to_le_bytes()); + for (guid, offset, size) in &f.parts { + // struct size 4 + 16 + 4 + 4 = 28, plus 2 bytes of padding the parser skips. + fml.extend_from_slice(&30i32.to_le_bytes()); + for g in guid { + fml.extend_from_slice(&g.to_le_bytes()); + } + fml.extend_from_slice(&offset.to_le_bytes()); + fml.extend_from_slice(&size.to_le_bytes()); + fml.extend_from_slice(&[0x11, 0x22]); + } + } + body.extend_from_slice(&((fml.len() + 4) as i32).to_le_bytes()); + body.extend_from_slice(&fml); + body + } + + /// Full manifest bytes: header + (optionally zlib-compressed) body. + pub fn build_manifest( + chunks: &[TestChunk], + files: &[TestFile], + version: i32, + compress: bool, + ) -> Vec { + let body = build_body(chunks, files, 7); + let payload = if compress { + let mut enc = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&body).unwrap(); + enc.finish().unwrap() + } else { + body.clone() + }; + let header_size: i32 = 41 + 3; // 3 bytes of extra header the parser must jump over + let mut out = Vec::new(); + out.extend_from_slice(&MANIFEST_MAGIC.to_le_bytes()); + out.extend_from_slice(&header_size.to_le_bytes()); + out.extend_from_slice(&(body.len() as i32).to_le_bytes()); + out.extend_from_slice(&(payload.len() as i32).to_le_bytes()); + out.extend_from_slice(&[0u8; 20]); + out.push(if compress { 1 } else { 0 }); + out.extend_from_slice(&version.to_le_bytes()); + out.extend_from_slice(&[0xF0, 0xF1, 0xF2]); + assert_eq!(out.len(), header_size as usize); + out.extend_from_slice(&payload); + out + } + + pub fn sample_chunks() -> Vec { + vec![ + TestChunk { + guid: [0x0000_0001, 0x0000_0002, 0x0000_0003, 0x0000_0004], + hash: 0x0123_4567_89AB_CDEF, + sha1: [0x11; 20], + group: 5, + window: 1_048_576, + file_size: 700_000, + }, + TestChunk { + guid: [0xDEAD_BEEF, 0xFFFF_FFFF, 0x0000_0000, 0x8000_0000], + hash: 0xFFFF_FFFF_FFFF_FFFE, + sha1: [0x22; 20], + group: 123, + window: 4096, + file_size: 0, + }, + TestChunk { + guid: [0x0000_00AA, 0x0000_00BB, 0x0000_00CC, 0x0000_00DD], + hash: 0x10, + sha1: [0u8; 20], + group: 0, + window: 10, + file_size: 9, + }, + ] + } + + pub fn sample_files() -> Vec { + let c = sample_chunks(); + vec![ + TestFile { + name: "Game/Binaries/Win64/Game.exe".to_string(), + sha1: [0x51; 20], + tags: vec![], + parts: vec![(c[0].guid, 0, 1_048_576), (c[1].guid, 100, 4000)], + }, + TestFile { + name: "Game\\Content\\Localization\\de.pak".to_string(), + sha1: [0x52; 20], + tags: vec!["German".to_string(), "de-DE".to_string()], + parts: vec![(c[1].guid, 0, 4096)], + }, + TestFile { + name: "Game/Content/Ünïcode.pak".to_string(), + sha1: [0x53; 20], + tags: vec!["French".to_string()], + parts: vec![(c[2].guid, 0, 10), (c[0].guid, 5, 7)], + }, + ] + } +} + +#[cfg(test)] +mod tests { + use super::test_support::*; + use super::*; + + #[test] + fn binary_manifest_round_trips_uncompressed() { + let bytes = build_manifest(&sample_chunks(), &sample_files(), 21, false); + let m = parse_manifest(&bytes).expect("parse"); + assert_eq!(m.chunk_dir, "ChunksV4"); + assert_eq!(m.unique_chunks.len(), 3); + assert_eq!(m.files.len(), 3); + + let c0 = &m.unique_chunks[0]; + assert_eq!(c0.guid_str(), "00000001000000020000000300000004"); + assert_eq!(c0.hash, 0x0123_4567_89AB_CDEF); + assert_eq!(c0.sha1, Some([0x11; 20])); + assert_eq!(c0.group_num, 5); + assert_eq!(c0.window_size, 1_048_576); + assert_eq!(c0.file_size, 700_000); + assert_eq!( + c0.path("ChunksV4"), + "ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" + ); + + let c1 = &m.unique_chunks[1]; + assert_eq!(c1.guid_str(), "DEADBEEFFFFFFFFF0000000080000000"); + assert_eq!( + c1.path("ChunksV4"), + "ChunksV4/123/FFFFFFFFFFFFFFFE_DEADBEEFFFFFFFFF0000000080000000.chunk" + ); + assert_eq!(c1.credit_bytes(), 1, "Math.max(fileSize, 1)"); + assert!(c1.verifiable_sha1().is_some()); + assert!(m.unique_chunks[2].verifiable_sha1().is_none(), "all-zero SHA-1 → unverifiable"); + + let f0 = &m.files[0]; + assert_eq!(f0.filename, "Game/Binaries/Win64/Game.exe"); + assert!(f0.install_tags.is_empty()); + assert_eq!(f0.sha1, Some([0x51; 20])); + assert_eq!(f0.parts.len(), 2); + assert_eq!(f0.parts[1].offset, 100); + assert_eq!(f0.parts[1].size, 4000); + assert_eq!(f0.file_size(), 1_048_576 + 4000); + + let f1 = &m.files[1]; + assert_eq!(f1.filename, "Game\\Content\\Localization\\de.pak"); + assert_eq!(f1.install_tags, vec!["German".to_string(), "de-DE".to_string()]); + + let f2 = &m.files[2]; + assert_eq!(f2.filename, "Game/Content/Ünïcode.pak", "UTF-16LE fstring"); + assert_eq!(f2.parts[1].guid_str(), c0.guid_str()); + } + + #[test] + fn binary_manifest_round_trips_compressed() { + let bytes = build_manifest(&sample_chunks(), &sample_files(), 17, true); + let m = parse_manifest(&bytes).expect("parse"); + assert_eq!(m.unique_chunks.len(), 3); + assert_eq!(m.files.len(), 3); + assert_eq!(m.files[2].filename, "Game/Content/Ünïcode.pak"); + } + + #[test] + fn chunk_dir_follows_version_thresholds() { + for (v, dir) in [(2, "Chunks"), (3, "ChunksV2"), (6, "ChunksV3"), (15, "ChunksV4")] { + let bytes = build_manifest(&sample_chunks(), &sample_files(), v, false); + assert_eq!(parse_manifest(&bytes).unwrap().chunk_dir, dir, "version {v}"); + } + assert_eq!(chunk_dir_for_json_version(1), "ChunksV4"); + assert_eq!(chunk_dir_for_json_version(4), "ChunksV2"); + } + + #[test] + fn duplicate_guid_keeps_first_position_last_value() { + let mut chunks = sample_chunks(); + let dup = TestChunk { + guid: chunks[0].guid, + hash: 0x99, + sha1: [0x77; 20], + group: 9, + window: 1, + file_size: 1, + }; + chunks.push(dup); + let bytes = build_manifest(&chunks, &sample_files(), 21, false); + let m = parse_manifest(&bytes).unwrap(); + assert_eq!(m.unique_chunks.len(), 3); + assert_eq!(m.unique_chunks[0].hash, 0x99); + assert_eq!(m.unique_chunks[0].group_num, 9); + } + + #[test] + fn truncated_manifest_is_a_parse_failure() { + let bytes = build_manifest(&sample_chunks(), &sample_files(), 21, false); + assert!(parse_manifest(&bytes[..bytes.len() - 40]).is_err()); + assert!(parse_manifest(&bytes[..20]).is_err()); + } + + #[test] + fn bad_uncompressed_size_is_a_parse_failure() { + let good = build_manifest(&sample_chunks(), &sample_files(), 21, true); + let real = i32::from_le_bytes([good[8], good[9], good[10], good[11]]); + // Claim MORE than the body inflates to: Java's single `inflate()` returns fewer bytes + // than `sizeUncompressed` → "Decomp size mismatch" → null. + let mut bigger = good.clone(); + bigger[8..12].copy_from_slice(&(real + 1).to_le_bytes()); + assert!(parse_manifest(&bigger).is_err()); + // Claim LESS: Java fills exactly `sizeUncompressed` bytes (silently truncating), then the + // FileManifestList end position lands past the limit → exception → null. + let mut smaller = good; + smaller[8..12].copy_from_slice(&(real - 1).to_le_bytes()); + assert!(parse_manifest(&smaller).is_err()); + } + + #[test] + fn fstring_decoding_matches_java() { + let mut v = fstring_ascii("abc"); + v.extend_from_slice(&fstring_utf16("dé")); + v.extend_from_slice(&fstring_ascii("")); + v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]); // non-ASCII byte → U+FFFD + let mut c = Cursor::new(&v); + assert_eq!(c.fstring().unwrap(), "abc"); + assert_eq!(c.fstring().unwrap(), "dé"); + assert_eq!(c.fstring().unwrap(), ""); + assert_eq!(c.fstring().unwrap(), "a\u{FFFD}b"); + } + + #[test] + fn json_manifest_parses_like_java() { + let json = r#"{ + "ManifestFileVersion": "013", + "ChunkHashList": { + "0000000100000002000000030000000ff": "0123456789ABCDEF00", + "short": "00", + "DEADBEEFFFFFFFFF0000000080000000": "zz" + }, + "DataGroupList": { "0000000100000002000000030000000ff": "7", "DEADBEEFFFFFFFFF0000000080000000": "x" }, + "ChunkFilesizeList": { "0000000100000002000000030000000ff": "FF" }, + "FileManifestList": [ + { "Filename": "a.bin", "InstallTags": ["German", ""], + "FileChunkParts": [ { "Guid": "0000000100000002000000030000000ff", "Offset": "12", "Size": "34" }, + { "Guid": "tooshort", "Offset": "x", "Size": "-5" } ] }, + { "Filename": "b.bin" } + ] + }"#; + let m = parse_manifest(json.as_bytes()).expect("json parse"); + assert_eq!(m.chunk_dir, "ChunksV3"); + assert_eq!(m.unique_chunks.len(), 2, "keys shorter than 32 chars are skipped"); + let c0 = &m.unique_chunks[0]; + assert_eq!(c0.guid, [1, 2, 3, 0x0000_000F]); + assert_eq!(c0.hash, 0x0123_4567_89AB_CDEF); + assert_eq!(c0.group_num, 7); + assert_eq!(c0.file_size, 0xFF); + assert_eq!(c0.window_size, 0); + assert!(c0.sha1.is_none()); + let c1 = &m.unique_chunks[1]; + assert_eq!(c1.hash, 0, "unparseable hash → 0"); + assert_eq!(c1.group_num, 0, "unparseable group → 0"); + assert_eq!(c1.file_size, 0, "missing size → 0"); + + assert_eq!(m.files.len(), 2); + assert_eq!(m.files[0].install_tags, vec!["German".to_string()]); + assert_eq!(m.files[0].parts.len(), 2); + assert_eq!(m.files[0].parts[0].offset, 12); + assert_eq!(m.files[0].parts[0].size, 34); + assert_eq!(m.files[0].parts[1].guid, [0; 4]); + assert_eq!(m.files[0].parts[1].offset, 0); + assert_eq!(m.files[0].parts[1].size, -5); + assert!(m.files[0].sha1.is_none()); + assert!(m.files[1].parts.is_empty()); + } + + #[test] + fn json_manifest_failures_match_java_nulls() { + assert!(parse_manifest(br#"{"FileManifestList": []}"#).is_err(), "no ChunkHashList"); + assert!(parse_manifest(br#"{"ChunkHashList": {}}"#).is_err(), "no FileManifestList"); + assert!( + parse_manifest(br#"{"ChunkHashList": {"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz": "0"}, "FileManifestList": []}"#) + .is_err(), + "invalid GUID hex throws out of the parse" + ); + assert!(parse_manifest(b"not json at all").is_err()); + } +} diff --git a/app/src/main/rust/stores/src/epic/plan.rs b/app/src/main/rust/stores/src/epic/plan.rs new file mode 100644 index 0000000000..49b70d80f3 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/plan.rs @@ -0,0 +1,272 @@ +//! Chunk plan — the part of `EpicDownloadManager.install` between "manifest parsed" and "submit +//! to the pool", ported 1:1: install-tag file selection (`resolveInstallFiles` / +//! `getRequiredInstallFiles`), the unique-chunk list for a file set (`uniqueChunksForFiles`), +//! the byte total (`Σ max(fileSize, 1)`), chunk cache paths and CDN URLs. +//! +//! At runtime the Java manager passes the INDICES of its `pendingFiles` (post delta/verify), so +//! the file selection functions here are exercised by the unit tests (and available to a future +//! caller) while the driver only runs `unique_chunks_for_files` on Java's pending set — the same +//! function Java runs, on the same manifest, in the same order. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use super::manifest::{ChunkInfo, Manifest}; + +/// `getRequiredInstallFiles`: files with no install tags; if NONE carries a tag, every file. +pub fn required_install_files(m: &Manifest) -> Vec { + let required: Vec = m + .files + .iter() + .enumerate() + .filter(|(_, f)| f.install_tags.is_empty()) + .map(|(i, _)| i) + .collect(); + if required.is_empty() { + (0..m.files.len()).collect() + } else { + required + } +} + +/// `resolveInstallFiles(manifest, selectedTags)`: +/// - `None` → every file (legacy / DLC / overlay path); +/// - `Some([])` → required(base) files only; +/// - otherwise → base files + any file carrying at least one selected tag, falling back to +/// required-only when nothing matched. +pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) -> Vec { + let Some(tags) = selected_tags else { + return (0..m.files.len()).collect(); + }; + if tags.is_empty() { + return required_install_files(m); + } + let want: HashSet<&str> = tags.iter().map(String::as_str).collect(); + let mut out = Vec::new(); + for (i, f) in m.files.iter().enumerate() { + if f.install_tags.is_empty() { + out.push(i); + continue; + } + if f.install_tags.iter().any(|t| want.contains(t.as_str())) { + out.push(i); + } + } + if out.is_empty() { + required_install_files(m) + } else { + out + } +} + +/// `uniqueChunksForFiles(manifest, files)`: chunk indices (into `unique_chunks`) in first-seen +/// order over `files[*].parts`, each GUID once, GUIDs absent from the chunk list dropped. +pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec { + let by_guid: HashMap = m.chunk_index_by_guid(); + let mut seen: HashSet = HashSet::new(); + let mut chunks = Vec::new(); + for &fi in file_indices { + let Some(f) = m.files.get(fi) else { + continue; + }; + for p in &f.parts { + let g = p.guid_str(); + if seen.insert(g.clone()) { + if let Some(&ci) = by_guid.get(&g) { + chunks.push(ci); + } + } + } + } + chunks +} + +/// `totalBytes += Math.max(chunk.fileSize, 1)` over the needed chunks. +pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 { + chunk_indices + .iter() + .filter_map(|&i| m.unique_chunks.get(i)) + .map(ChunkInfo::credit_bytes) + .sum() +} + +/// `/.chunks`. +pub fn chunk_cache_dir(install_dir: &str) -> PathBuf { + Path::new(install_dir).join(super::CHUNK_CACHE_DIR) +} + +/// Final cache file for a chunk: `new File(chunkCacheDir, chunk.guidStr())`. +pub fn cached_chunk_path(cache_dir: &Path, chunk: &ChunkInfo) -> PathBuf { + cache_dir.join(chunk.guid_str()) +} + +/// The Java pool's skip rule: a chunk whose cache file EXISTS (any size) is not fetched. +pub fn is_chunk_cached(cache_dir: &Path, chunk: &ChunkInfo) -> bool { + cached_chunk_path(cache_dir, chunk).exists() +} + +/// One CDN entry as the Java manager passes it: `cdn.baseUrl + cdn.cloudDir` (auth params are +/// never used on chunk URLs). Chunk URL = `prefix + "/" + chunk.getPath(chunkDir)`. +pub fn chunk_url(cdn_prefix: &str, chunk_dir: &str, chunk: &ChunkInfo) -> String { + format!("{}/{}", cdn_prefix, chunk.path(chunk_dir)) +} + +/// Undo the JSON string escaping the Java CDN scanner passes through. `EpicApiClient` re-serializes +/// the manifest API response with Android's `org.json`, which escapes every `/` as `\/`, and +/// `parseCdnUrls` takes the raw substrings — so a base URL arrives as `https:\/\/host\` and a cloud +/// dir as `/Builds\/Org\/…\/default\`. Java's `HttpURLConnection` (OkHttp) treats `\` as `/` and +/// only ever reached the first CDN, so it never noticed; reqwest's URL parser does the same +/// mapping, which leaves empty `//` path segments that Fastly/Akamai tolerate but CloudFront +/// rejects outright (immediate 4xx). Produce the URL Java intended: `\/` → `/`, any other `\` +/// dropped, and no trailing slash (the chunk path is joined with one). +pub fn normalize_prefix(raw: &str) -> String { + let unescaped = raw.replace("\\/", "/").replace('\\', ""); + unescaped.trim_end_matches('/').to_string() +} + +/// Distinct CDN prefixes in first-seen order, normalized (the fetch core wants one host key per +/// distinct endpoint; Java would simply try an identical URL twice, which changes nothing). +pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for p in cdn_prefixes { + let n = normalize_prefix(p); + if n.is_empty() { + continue; + } + if seen.insert(n.clone()) { + out.push(n); + } + } + out +} + +/// Improvements round 1: split the window across the distinct CDNs so the whole ceiling is +/// reachable whatever the host count — `max(6, ceil(max_workers / hosts))` (3 Epic CDNs at a +/// ceiling of 32 → 11 each; a single host gets all 32). +pub fn per_host_cap(max_workers: usize, host_count: usize) -> usize { + let hosts = host_count.max(1); + let workers = max_workers.max(1); + let split = (workers + hosts - 1) / hosts; + split.max(6) +} + +#[cfg(test)] +mod tests { + use super::super::manifest::test_support::*; + use super::super::manifest::parse_manifest; + use super::*; + + fn manifest() -> Manifest { + parse_manifest(&build_manifest(&sample_chunks(), &sample_files(), 21, false)).unwrap() + } + + #[test] + fn install_tag_rules_match_java() { + let m = manifest(); + // null → all + assert_eq!(resolve_install_files(&m, None), vec![0, 1, 2]); + // empty → required only (file 0 has no tags) + assert_eq!(resolve_install_files(&m, Some(&[])), vec![0]); + // German → base + the German file + let de = vec!["German".to_string(), "de-DE".to_string()]; + assert_eq!(resolve_install_files(&m, Some(&de)), vec![0, 1]); + // unknown tag → nothing extra matched, but base files matched, so base only + let xx = vec!["Klingon".to_string()]; + assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0]); + } + + #[test] + fn manifest_without_tags_treats_every_file_as_required() { + let mut files = sample_files(); + for f in files.iter_mut() { + f.tags.clear(); + } + let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); + assert_eq!(required_install_files(&m), vec![0, 1, 2]); + assert_eq!(resolve_install_files(&m, Some(&[])), vec![0, 1, 2]); + } + + #[test] + fn manifest_where_every_file_is_tagged_falls_back_to_all() { + let mut files = sample_files(); + files[0].tags = vec!["English".to_string()]; + let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); + // No untagged file, unknown tag selected → matched nothing → required → (none) → all. + let xx = vec!["Klingon".to_string()]; + assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0, 1, 2]); + } + + #[test] + fn unique_chunks_follow_first_seen_part_order() { + let m = manifest(); + // file 2 first: chunk c[2] then c[0]; then file 0: c[0] (dup), c[1]. + assert_eq!(unique_chunks_for_files(&m, &[2, 0]), vec![2, 0, 1]); + assert_eq!(unique_chunks_for_files(&m, &[1]), vec![1]); + assert!(unique_chunks_for_files(&m, &[]).is_empty()); + assert!(unique_chunks_for_files(&m, &[99]).is_empty(), "out-of-range index ignored"); + // total = 700000 + max(0,1) + 9 + assert_eq!(total_credit_bytes(&m, &[0, 1, 2]), 700_000 + 1 + 9); + } + + #[test] + fn parts_referencing_unknown_guids_are_dropped() { + let mut files = sample_files(); + files[0] + .parts + .push(([0x1234, 0x5678, 0x9ABC, 0xDEF0], 0, 1)); + let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); + assert_eq!(unique_chunks_for_files(&m, &[0]), vec![0, 1]); + } + + #[test] + fn urls_and_cache_paths_match_java() { + let m = manifest(); + let c = &m.unique_chunks[0]; + assert_eq!( + chunk_url("https://fastly-download.epicgames.com/Builds/Org/o-x/y/default", "ChunksV4", c), + "https://fastly-download.epicgames.com/Builds/Org/o-x/y/default/ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" + ); + let cache = chunk_cache_dir("/data/x/imagefs/epic_games/Game"); + assert_eq!(cache, PathBuf::from("/data/x/imagefs/epic_games/Game/.chunks")); + assert_eq!( + cached_chunk_path(&cache, c), + PathBuf::from("/data/x/imagefs/epic_games/Game/.chunks/00000001000000020000000300000004") + ); + let prefixes = vec!["a".to_string(), "b".to_string(), "a".to_string()]; + assert_eq!(distinct_prefixes(&prefixes), vec!["a".to_string(), "b".to_string()]); + } + + #[test] + fn prefixes_are_unescaped_from_the_java_json_scan() { + // Exactly what Bannerlator's Epic debug log shows its Java manager extracting for a real title. + let raw = "https:\\/\\/egdownload.fastly-edge.com\\/Builds\\/Org\\/o-83e8\\/04b7\\/default\\"; + assert_eq!( + normalize_prefix(raw), + "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default" + ); + // Already-clean input is untouched (minus a trailing slash). + assert_eq!( + normalize_prefix("https://download.epicgames.com/Builds/x/default/"), + "https://download.epicgames.com/Builds/x/default" + ); + let m = manifest(); + let c = &m.unique_chunks[0]; + assert_eq!( + chunk_url(&normalize_prefix(raw), "ChunksV4", c), + "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default/ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" + ); + let two = vec![raw.to_string(), "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default".to_string(), String::new()]; + assert_eq!(distinct_prefixes(&two).len(), 1, "escaped and clean forms collapse; empty dropped"); + } + + #[test] + fn per_host_cap_splits_the_window_with_a_floor_of_six() { + assert_eq!(per_host_cap(32, 3), 11); + assert_eq!(per_host_cap(32, 1), 32); + assert_eq!(per_host_cap(32, 2), 16); + assert_eq!(per_host_cap(8, 3), 6, "floor"); + assert_eq!(per_host_cap(8, 1), 8); + assert_eq!(per_host_cap(0, 0), 6, "degenerate inputs"); + } +} diff --git a/app/src/main/rust/stores/src/fetch_core.rs b/app/src/main/rust/stores/src/fetch_core.rs new file mode 100644 index 0000000000..4cdf4a7278 --- /dev/null +++ b/app/src/main/rust/stores/src/fetch_core.rs @@ -0,0 +1,2221 @@ +//! Shared adaptive HTTP fetch core for the store download engines (Epic / GOG / Amazon). +//! +//! This is the generic half of the fetch layer Bannerlator's Steam depot writer grew: an +//! adaptive in-flight window (slow-start → hold at plateau → shrink on REAL errors only), a +//! speed-ranked per-host-capped scheduler, an in-flight RAW byte budget sized from the window +//! ceiling, per-item retry with exponential back-off + host rotation, one current-thread tokio +//! runtime driving every request through a [`FuturesUnordered`], and an mpsc hand-off to a sync +//! process pool that calls the adapter's [`FetchSink::process`] (inflate + hash + write). +//! +//! The tunables come from [`crate::fetch_tuning`], copied number for number from Bannerlator's +//! depot writer, so a `fetch-window` line from either app reads against the same tuning notes. +//! +//! What the core does NOT know: manifests, resume/skip rules, output layout, hashing. The adapter +//! builds the item list (already excluding present+verified items, exactly as its Java manager +//! does), maps host index → URL, and does everything with the body inside its sink. +//! +//! Log grammar (all lines go through the `log` callback; `label` is adapter-chosen, e.g. +//! `epic app=Fortnite`): +//! - `fetch-start label=… items=… bytes_reserved=… hosts=… ceiling=… budget=…MiB tier_max=… per_host_cap=… distinct_hosts=… window=… mode=body|stream` +//! - `fetch-window label=… window=N (min=… max=…) in_flight=… last=…MB/s ewma=…MB/s best=…MB/s reason=… cooldown=…ms err_rate=…% phase=… rtt=…ms budget_stalls=… host_stalls=…` +//! (same fields as the Steam engine's `fetch-window depot=…` line, so the two are A/B-comparable) +//! - `throughput label=… overall=…MB/s total=…MB elapsed=…s used=x/y servers: [host …MB/s …MB] …` every 5 s +//! - `fetch-end label=… items_ok=… bytes=… credited=… elapsed_ms=… avg_mbps=… peak_mbps=… result=ok|cancelled|error [error=…]` +//! +//! `MB/s` here means MiB/s (1024²), matching the Steam engine's lines. + +use crate::fetch_tuning::{ + budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError, + FetchFailKind, BOOTSTRAP_WINDOW, MAX_CHUNK_ATTEMPTS, NOMINAL_CHUNK_RESERVE_BYTES, + RATE_LIMIT_COOLDOWN_MS, SERVER_EXPLORE_EVERY, USER_AGENT, WINDOW_BPS_EWMA_ALPHA, + WINDOW_COOLDOWN_MS, WINDOW_DECLINE_EPS, WINDOW_ERR_BURST_IMMEDIATE, WINDOW_ERR_RATE_HIGH, + WINDOW_ERR_RATE_LOW, WINDOW_HARD_CAP, WINDOW_IMPROVE_EPS, WINDOW_LOG_EVERY_PROBES, + WINDOW_MIN_FLOOR, WINDOW_PLATEAU_PATIENCE, WINDOW_PLATEAU_REARM_MAX_MS, + WINDOW_PLATEAU_REARM_MS, WINDOW_PROBE_INTERVAL_MS, WINDOW_SHRINK_FACTOR, + WINDOW_SLOW_START_FACTOR, WINDOW_STEP_UP, +}; +use futures_util::stream::FuturesUnordered; +use futures_util::StreamExt; +use std::collections::{HashMap, VecDeque}; +use std::fs; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{mpsc as std_mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; +use tokio::sync::Semaphore; + +/// Maximum attempts per item (fetch failures AND `SinkError::Retry` both count). Same as Steam. +pub const MAX_ITEM_ATTEMPTS: u32 = MAX_CHUNK_ATTEMPTS; +/// How often the throughput reporter samples (and logs) — the samples feed `peak_mbps`. +pub const THROUGHPUT_SAMPLE_INTERVAL_MS: u64 = 5_000; +/// Upper bound on how long the driver sits in one `await` before re-checking cancel/error/feedback. +const DRIVER_POLL_MS: u64 = 250; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Public contract +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// One unit of work: a body to fetch from one of the hosts and hand to the sink. +#[derive(Clone, Debug)] +pub struct FetchItem { + /// Adapter-defined (index into the adapter's own table). Only echoed in errors / logs. + pub id: u64, + /// `len == hosts.len()` → `urls[host_idx]`; `len == 1` → the same URL on every host. Any other + /// non-zero length falls back to `urls.get(host_idx)` then `urls[0]`. Empty = fatal. + pub urls: Vec, + /// Expected RAW (wire) bytes, for the in-flight byte budget; `0` → nominal 1 MiB. Ignored in + /// stream mode (streams are budgeted piece by piece as they arrive). + pub reserve: u64, + /// Inclusive HTTP `Range` (`bytes=a-b`); `None` = whole body. + pub range: Option<(u64, u64)>, +} + +#[derive(Clone, Debug)] +pub struct FetchOptions { + /// Window ceiling (tier); the core clamps it to `hosts × per_host_cap` and the hard cap. + pub max_workers: usize, + /// Max concurrent requests per distinct host (6 unless the store needs otherwise). Honoured + /// as given: 1 host × `per_host_cap` 8 = a ceiling of 8. + pub per_host_cap: usize, + /// Whole-body mode: the deadline for the entire request (connect + headers + body). Stream + /// mode: the deadline for the response headers, then an IDLE deadline per received piece. + pub timeout: Duration, + /// Sent on EVERY request (applied on the request builder, so they override the client-level + /// defaults — e.g. a store's own `User-Agent`, or `Authorization: Bearer …`). Validated once + /// when the client is built. + pub headers: Vec<(String, String)>, + /// PEM bundle path; `""` = the platform roots reqwest/rustls ships with. + pub ca_bundle_path: String, + /// Sync process-pool threads (inflate + hash + write). + pub process_workers: usize, + /// Log prefix, e.g. `"epic app=Fortnite"`. + pub label: String, + /// `false` (default): each item's whole body is buffered and handed to + /// [`FetchSink::process`]. `true`: the body is streamed — pieces go to + /// [`FetchSink::on_chunk`] in order as they arrive, then [`FetchSink::on_finish`]; for + /// multi-GB files (Amazon) that keeps memory at "pieces not yet written", not whole files. + /// Applies to every item of the run. + pub stream: bool, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { + max_workers: BOOTSTRAP_WINDOW, + per_host_cap: crate::fetch_tuning::PER_HOST_CAP, + timeout: Duration::from_secs(60), + headers: Vec::new(), + ca_bundle_path: String::new(), + process_workers: 2, + label: String::new(), + stream: false, + } + } +} + +/// What the sink says about a body it was handed. +#[derive(Clone, Debug)] +pub enum SinkError { + /// Counts as a failed attempt for that item: host cooled + demoted, back-off, re-dispatch on a + /// different host, up to [`MAX_ITEM_ATTEMPTS`]. In stream mode the whole item is re-streamed + /// (`on_chunk` sees `offset == 0` again). + Retry(String), + /// Abort the whole run; this becomes the run's first error. + Fatal(String), +} + +/// Runs on a process-pool thread. Whole-body mode uses [`FetchSink::process`]; stream mode uses +/// [`FetchSink::on_chunk`] + [`FetchSink::on_finish`] (default impls refuse, so a whole-body sink +/// is unaffected by the stream option existing). +pub trait FetchSink: Send + Sync { + /// Whole-body mode: the raw wire body. Do inflate + hash verify + write here. Returns the + /// number of bytes to credit to progress (adapter's choice, e.g. decompressed bytes). + fn process(&self, item: &FetchItem, body: Vec) -> Result; + + /// Stream mode: one piece of the body, delivered in order (`offset` = bytes of this attempt + /// delivered before this piece). `offset == 0` marks the (re)start of an attempt — truncate any + /// partial output from a previous attempt. Each processed piece credits `data.len()` bytes. + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + let _ = (item, offset, data); + Err(SinkError::Fatal("streaming not supported by this sink".to_string())) + } + + /// Stream mode: the body ended after `total_len` bytes. Finalize (rename `.tmp`, verify the + /// hash, …). Returns EXTRA bytes to credit (pieces were already credited as they were + /// written; return 0 for none). + fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result { + let _ = (item, total_len); + Err(SinkError::Fatal("streaming not supported by this sink".to_string())) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FetchOutcome { + /// Sum of what the sink returned for every successfully processed item (stream mode: every + /// piece written + every `on_finish` extra; pieces of an attempt that later failed stay + /// counted). + pub bytes_credited: u64, + /// First error (fetch after max attempts, sink `Fatal`, client build failure). + pub error: Option, + /// `cancel` was observed set. + pub cancelled: bool, + /// Items the sink accepted. + pub items_ok: u64, +} + +impl FetchOutcome { + pub fn ok(&self) -> bool { + self.error.is_none() && !self.cancelled + } +} + +/// Blocking: spawns its own current-thread tokio runtime + process pool and returns when every +/// item is processed, `cancel` is set, or the first fatal error is recorded. `progress` fires from +/// a process-pool thread after each successful `process` / each written piece + `on_finish` (the +/// caller throttles). +pub fn run_fetch( + items: Vec, + hosts: &[String], + opts: &FetchOptions, + sink: &dyn FetchSink, + cancel: &AtomicBool, + progress: &(dyn Fn(u64, u64) + Sync), + log: &(dyn Fn(&str) + Sync), +) -> FetchOutcome { + let started = Instant::now(); + let label = opts.label.as_str(); + let stream = opts.stream; + let hosts: Vec = if hosts.is_empty() { + vec![String::from("default")] + } else { + hosts.to_vec() + }; + let per_host_cap = opts.per_host_cap.max(1); + let distinct_hosts = distinct_host_count(&hosts); + let (bootstrap, win_min, win_max) = + window_bounds(opts.max_workers, distinct_hosts, per_host_cap); + let budget = inflight_budget_bytes(win_max); + let bytes_reserved: u64 = items.iter().map(reserve_bytes).sum(); + + log(&format!( + "fetch-start label={label} items={} bytes_reserved={bytes_reserved} hosts={} ceiling={win_max} \ +budget={}MiB tier_max={} per_host_cap={per_host_cap} distinct_hosts={distinct_hosts} window={bootstrap} \ +mode={}", + items.len(), + hosts.len(), + budget / (1024 * 1024), + opts.max_workers, + if stream { "stream" } else { "body" }, + )); + + let bytes_credited = AtomicU64::new(0); + let items_ok = AtomicU64::new(0); + let in_flight = AtomicU64::new(0); + let error_slot: Mutex> = Mutex::new(None); + let reporter_done = AtomicBool::new(false); + let meter = BandwidthMeter::new(&hosts); + // Stream mode: per-item "attempt N was rejected by the sink mid-stream" flags, so the driver's + // in-flight stream stops reading instead of feeding pieces nobody wants. + let abandon: Vec = if stream { + items.iter().map(|_| AtomicU32::new(NO_ATTEMPT)).collect() + } else { + Vec::new() + }; + + let proc_count = opts.process_workers.max(1).min(items.len().max(1)); + let items = &items; + let meter = &meter; + let abandon = &abandon; + + thread::scope(|scope| { + // Async(fetch) → sync(process): tokio unbounded mpsc drained with `blocking_recv`. Memory is + // bounded by the RAW byte budget enforced at DISPATCH (whole-body) / per piece (stream), not + // by channel capacity. Whole-body mode: ONE channel shared by every worker (any worker takes + // the next body). Stream mode: one channel PER worker and an item's pieces always go to the + // same worker (`idx % workers`), so pieces are written in order. + let channel_count = if stream { proc_count } else { 1 }; + let mut txs: Vec> = + Vec::with_capacity(channel_count); + let mut rxs: Vec>>> = + Vec::with_capacity(channel_count); + for _ in 0..channel_count { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + txs.push(tx); + rxs.push(Arc::new(Mutex::new(rx))); + } + // Sync(process) → async(driver): sink verdicts, polled non-blocking by the driver. + let (fb_tx, fb_rx) = std_mpsc::channel::(); + let bytes_credited = &bytes_credited; + let items_ok = &items_ok; + let in_flight = &in_flight; + let error_slot = &error_slot; + let reporter_done = &reporter_done; + + // Periodic throughput reporter — atomics only, never on the fetch hot path. Its samples are + // also the `peak_mbps` source. + let reporter = scope.spawn(move || loop { + let mut waited = 0u64; + while waited < THROUGHPUT_SAMPLE_INTERVAL_MS { + if reporter_done.load(Ordering::Relaxed) { + return; + } + thread::sleep(Duration::from_millis(100)); + waited += 100; + } + if reporter_done.load(Ordering::Relaxed) { + return; + } + meter.sample(); + log(&meter.summary_line(label)); + }); + + // ── Process pool ── + let mut proc_handles = Vec::with_capacity(proc_count); + for worker in 0..proc_count { + let rx = Arc::clone(&rxs[if stream { worker } else { 0 }]); + let fb = fb_tx.clone(); + proc_handles.push(scope.spawn(move || { + // Stream mode: (item → attempt, message) whose pieces the sink rejected; the rest + // of that attempt's pieces are dropped and its Finish becomes a Retry verdict. + let mut poisoned: HashMap = HashMap::new(); + loop { + if cancel.load(Ordering::Relaxed) { + return; + } + if error_slot.lock().expect("err slot poisoned").is_some() { + return; + } + let msg = { + let mut guard = rx.lock().expect("rx poisoned"); + guard.blocking_recv() + }; + let Some(msg) = msg else { + return; + }; + match msg { + ProcessMsg::Body { + idx, + attempts, + server_idx, + body, + } => { + let raw_len = body.len() as u64; + // A body that was already queued when cancel / the first error landed + // must not be written any more. + if cancel.load(Ordering::Relaxed) + || error_slot.lock().expect("err slot poisoned").is_some() + { + in_flight.fetch_sub(raw_len, Ordering::Relaxed); + return; + } + let item = &items[idx]; + let res = sink.process(item, body); + // Free the budget on PROCESS-COMPLETE (actual raw bytes). + in_flight.fetch_sub(raw_len, Ordering::Relaxed); + let verdict = match res { + Ok(credited) => { + let total = bytes_credited + .fetch_add(credited, Ordering::Relaxed) + + credited; + let ok = items_ok.fetch_add(1, Ordering::Relaxed) + 1; + progress(total, ok); + SinkVerdict::Ok + } + Err(SinkError::Retry(message)) => SinkVerdict::Retry(message), + Err(SinkError::Fatal(message)) => { + record_first_error( + error_slot, + format!("{label}: item {} failed: {message}", item.id), + ); + SinkVerdict::Fatal + } + }; + let _ = fb.send(Feedback { + idx, + attempts, + server_idx, + verdict, + }); + } + ProcessMsg::Piece { + idx, + attempts, + offset, + data, + } => { + let len = data.len() as u64; + in_flight.fetch_sub(len, Ordering::Relaxed); + if poisoned.get(&idx).is_some_and(|(a, _)| *a == attempts) { + continue; // rest of a rejected attempt + } + if cancel.load(Ordering::Relaxed) + || error_slot.lock().expect("err slot poisoned").is_some() + { + return; + } + let item = &items[idx]; + match sink.on_chunk(item, offset, &data) { + Ok(()) => { + let total = + bytes_credited.fetch_add(len, Ordering::Relaxed) + len; + progress(total, items_ok.load(Ordering::Relaxed)); + } + Err(SinkError::Retry(message)) => { + poisoned.insert(idx, (attempts, message)); + if let Some(flag) = abandon.get(idx) { + flag.store(attempts, Ordering::Relaxed); + } + } + Err(SinkError::Fatal(message)) => { + record_first_error( + error_slot, + format!("{label}: item {} failed: {message}", item.id), + ); + if let Some(flag) = abandon.get(idx) { + flag.store(attempts, Ordering::Relaxed); + } + return; + } + } + } + ProcessMsg::Finish { + idx, + attempts, + server_idx, + total_len, + } => { + let item = &items[idx]; + let verdict = if poisoned + .get(&idx) + .is_some_and(|(a, _)| *a == attempts) + { + let (_, message) = poisoned.remove(&idx).expect("poisoned"); + SinkVerdict::Retry(message) + } else { + match sink.on_finish(item, total_len) { + Ok(extra) => { + let total = bytes_credited + .fetch_add(extra, Ordering::Relaxed) + + extra; + let ok = items_ok.fetch_add(1, Ordering::Relaxed) + 1; + progress(total, ok); + SinkVerdict::Ok + } + Err(SinkError::Retry(message)) => SinkVerdict::Retry(message), + Err(SinkError::Fatal(message)) => { + record_first_error( + error_slot, + format!( + "{label}: item {} failed: {message}", + item.id + ), + ); + SinkVerdict::Fatal + } + } + }; + let _ = fb.send(Feedback { + idx, + attempts, + server_idx, + verdict, + }); + } + } + } + })); + } + drop(fb_tx); + + // ── Async fetch driver on one current-thread runtime. `txs` are moved in and dropped when + // the driver returns, which closes the channels so the pool's `blocking_recv` yields + // `None`. ── + match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => { + rt.block_on(run_driver(DriverCtx { + items: items.as_slice(), + hosts: hosts.as_slice(), + opts, + label, + cancel, + log, + meter, + in_flight, + error_slot, + abandon: abandon.as_slice(), + txs, + fb_rx, + budget, + bootstrap, + win_min, + win_max, + per_host_cap, + })); + } + Err(err) => { + record_first_error(error_slot, format!("{label}: tokio runtime: {err}")); + drop(txs); + } + } + + for handle in proc_handles { + let _ = handle.join(); + } + reporter_done.store(true, Ordering::Relaxed); + let _ = reporter.join(); + }); + + // Final partial-interval sample so a short run still has a peak. + meter.sample(); + let elapsed = started.elapsed(); + let elapsed_secs = elapsed.as_secs_f64().max(0.001); + let wire_total = meter.total_bytes(); + let avg_mbps = (wire_total as f64) / (1024.0 * 1024.0) / elapsed_secs; + let peak_mbps = (meter.peak_bps() / (1024.0 * 1024.0)).max(avg_mbps); + let cancelled = cancel.load(Ordering::Relaxed); + let error = error_slot.lock().expect("err slot poisoned").take(); + let outcome = FetchOutcome { + bytes_credited: bytes_credited.load(Ordering::Relaxed), + error, + cancelled, + items_ok: items_ok.load(Ordering::Relaxed), + }; + let result = if cancelled { + "cancelled" + } else if outcome.error.is_some() { + "error" + } else { + "ok" + }; + let mut line = format!( + "fetch-end label={label} items_ok={} bytes={wire_total} credited={} elapsed_ms={} \ +avg_mbps={avg_mbps:.2} peak_mbps={peak_mbps:.2} result={result}", + outcome.items_ok, + outcome.bytes_credited, + elapsed.as_millis(), + ); + if let Some(err) = &outcome.error { + line.push_str(" error="); + line.push_str(err); + } + log(&line); + outcome +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Pure helpers (unit-tested) +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// Bytes reserved against the in-flight budget at DISPATCH (nominal when the adapter has none). +pub fn reserve_bytes(item: &FetchItem) -> u64 { + if item.reserve == 0 { + NOMINAL_CHUNK_RESERVE_BYTES + } else { + item.reserve + } +} + +/// Window bounds: `max` = tier ceiling clamped to distinct-hosts × per-host-cap and the hard cap; +/// `min` = the floor; `bootstrap` = the start. Same arithmetic as the Steam depot writer. +pub fn window_bounds( + max_workers: usize, + distinct_hosts: usize, + per_host_cap: usize, +) -> (usize, usize, usize) { + let host_ceiling = distinct_hosts.saturating_mul(per_host_cap).max(1); + let max = max_workers.max(1).min(host_ceiling).min(WINDOW_HARD_CAP); + let min = WINDOW_MIN_FLOOR.min(max).max(1); + let bootstrap = BOOTSTRAP_WINDOW.clamp(min, max); + (bootstrap, min, max) +} + +/// Distinct host count (string equality) — the multiplier for the window's host ceiling. +pub fn distinct_host_count(hosts: &[String]) -> usize { + let mut keys: Vec<&str> = Vec::new(); + for h in hosts { + if !keys.contains(&h.as_str()) { + keys.push(h.as_str()); + } + } + keys.len().max(1) +} + +/// The URL to use for `item` on host `host_idx` (see [`FetchItem::urls`]). `None` when the item +/// carries no URL at all. +pub fn url_for_host(item: &FetchItem, host_idx: usize) -> Option<&str> { + if item.urls.is_empty() { + return None; + } + if item.urls.len() == 1 { + return Some(item.urls[0].as_str()); + } + Some( + item.urls + .get(host_idx) + .unwrap_or(&item.urls[0]) + .as_str(), + ) +} + +/// Byte count an inclusive range must deliver; `None` for an inverted range. +pub fn range_len(range: (u64, u64)) -> Option { + let (a, b) = range; + if b < a { + None + } else { + Some(b - a + 1) + } +} + +/// Validates a received body's length: for a range request it must be exactly `b-a+1`; otherwise +/// it must match `Content-Length` when the server sent one (as the Steam client does). +pub fn validate_body_len( + body_len: u64, + content_length: Option, + range: Option<(u64, u64)>, +) -> Result<(), AsyncFetchError> { + if let Some(range) = range { + let expected = match range_len(range) { + Some(n) => n, + None => { + return Err(AsyncFetchError { + message: format!("invalid range {}-{}", range.0, range.1), + kind: FetchFailKind::Other, + }) + } + }; + if body_len != expected { + return Err(AsyncFetchError { + message: format!("range body length mismatch (got {body_len}, want {expected})"), + kind: FetchFailKind::ServerFault, + }); + } + return Ok(()); + } + if let Some(expected) = content_length { + if body_len != expected { + return Err(AsyncFetchError { + message: format!("body truncated (length mismatch: got {body_len}, want {expected})"), + kind: FetchFailKind::ServerFault, + }); + } + } + Ok(()) +} + +/// Whether an HTTP status is acceptable for this request shape (200 always; 206 only when a range +/// was asked for). +pub fn status_accepted(status: u16, ranged: bool) -> bool { + status == 200 || (ranged && status == 206) +} + +/// Classifies a rejected HTTP status the same way the Steam async client does. +pub fn classify_status(status: u16) -> FetchFailKind { + if status == 429 { + FetchFailKind::RateLimited + } else if (500..600).contains(&status) { + FetchFailKind::ServerFault + } else { + FetchFailKind::Other + } +} + +/// The HTTP status carried by a `send_checked` rejection message, if that is what the error was. +/// A 4xx other than 429 is a CLIENT rejection (bad/expired signed URL, wrong host for this path, +/// 404): it says nothing about congestion, so the driver demotes the host but does not shrink the +/// shared window for it — one CDN that refuses a path must not throttle the healthy ones. +pub fn rejected_status(message: &str) -> Option { + let rest = message.strip_prefix("unexpected HTTP status (")?; + let digits = rest.strip_suffix(')')?; + digits.parse().ok() +} + +pub fn is_client_rejection(message: &str) -> bool { + matches!(rejected_status(message), Some(s) if (400..500).contains(&s) && s != 429) +} + +/// How many individual fetch failures a run logs verbatim before going quiet (the window and +/// throughput lines keep reporting the aggregate). +pub const FETCH_ERROR_LOG_LIMIT: u32 = 12; + +fn record_first_error(slot: &Mutex>, err: String) { + let mut guard = slot.lock().expect("err slot poisoned"); + if guard.is_none() { + *guard = Some(err); + } +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// HTTP client (one pooled reqwest client + per-request headers + optional Range + streaming) +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// The pooled client plus the pre-validated per-request headers. +struct Http { + client: reqwest::Client, + headers: Vec<(reqwest::header::HeaderName, reqwest::header::HeaderValue)>, +} + +fn build_http(opts: &FetchOptions, pool_max_idle_per_host: usize) -> Result { + let mut headers = Vec::with_capacity(opts.headers.len()); + for (name, value) in &opts.headers { + let hname = reqwest::header::HeaderName::from_bytes(name.as_bytes()) + .map_err(|err| format!("bad header name '{name}': {err}"))?; + let hvalue = reqwest::header::HeaderValue::from_str(value) + .map_err(|err| format!("bad header value for '{name}': {err}"))?; + headers.push((hname, hvalue)); + } + let mut builder = reqwest::Client::builder() + .user_agent(USER_AGENT) + .connect_timeout(connect_timeout()) + .pool_max_idle_per_host(pool_max_idle_per_host.max(1)) + .pool_idle_timeout(Duration::from_secs(90)); + if !opts.ca_bundle_path.is_empty() { + let pem = fs::read(&opts.ca_bundle_path).map_err(|err| format!("read CA bundle: {err}"))?; + let certs = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|err| format!("parse CA bundle: {err}"))?; + for cert in certs { + builder = builder.add_root_certificate(cert); + } + } + let client = builder.build().map_err(|err| format!("http client: {err}"))?; + Ok(Http { client, headers }) +} + +impl Http { + /// `GET url` with the run's headers (set per request, so they override the client defaults — + /// a store's own `User-Agent` wins over the Steam one) and the optional `Range`. + fn request(&self, url: &str, range: Option<(u64, u64)>) -> reqwest::RequestBuilder { + let mut request = self.client.get(url); + for (name, value) in &self.headers { + request = request.header(name.clone(), value.clone()); + } + if let Some((a, b)) = range { + request = request.header(reqwest::header::RANGE, format!("bytes={a}-{b}")); + } + request + } +} + +fn classify_reqwest_error(err: &reqwest::Error) -> FetchFailKind { + if err.is_timeout() { + FetchFailKind::Timeout + } else if err.is_connect() { + FetchFailKind::Connect + } else { + FetchFailKind::Other + } +} + +/// Sends the request and checks the status; shared by both modes. +async fn send_checked( + http: &Http, + url: &str, + range: Option<(u64, u64)>, + timeout: Option, +) -> Result { + let mut request = http.request(url, range); + if let Some(timeout) = timeout { + request = request.timeout(timeout); + } + let response = match request.send().await { + Ok(response) => response, + Err(err) => { + return Err(AsyncFetchError { + message: format!("http get: {err}"), + kind: classify_reqwest_error(&err), + }); + } + }; + let status = response.status().as_u16(); + if !status_accepted(status, range.is_some()) { + return Err(AsyncFetchError { + message: format!("unexpected HTTP status ({status})"), + kind: classify_status(status), + }); + } + Ok(response) +} + +/// Whole-body mode: one attempt for one item (the driver owns retry/rotation). +async fn fetch_once( + http: &Http, + url: &str, + range: Option<(u64, u64)>, + timeout: Duration, +) -> Result, AsyncFetchError> { + let response = send_checked(http, url, range, Some(timeout)).await?; + let content_length = response.content_length(); + let body = match response.bytes().await { + Ok(body) => body.to_vec(), + Err(err) => { + let kind = if err.is_timeout() { + FetchFailKind::Timeout + } else { + FetchFailKind::Other + }; + return Err(AsyncFetchError { + message: format!("http body: {err}"), + kind, + }); + } + }; + validate_body_len(body.len() as u64, content_length, range)?; + Ok(body) +} + +/// Stream mode: one attempt for one item. Pieces are pushed to the item's pool worker as they +/// arrive (each one added to the in-flight budget until the pool consumes it); the future waits +/// while the budget is full, and stops early on cancel or when the sink rejected this attempt. +/// Returns the total body length after the last piece was queued. +#[allow(clippy::too_many_arguments)] +async fn fetch_stream( + http: &Http, + url: &str, + range: Option<(u64, u64)>, + timeout: Duration, + idx: usize, + attempts: u32, + tx: &tokio::sync::mpsc::UnboundedSender, + in_flight: &AtomicU64, + budget: u64, + cancel: &AtomicBool, + abandon: Option<&AtomicU32>, +) -> Result { + let mut response = + match tokio::time::timeout(timeout, send_checked(http, url, range, None)).await { + Ok(res) => res?, + Err(_) => { + return Err(AsyncFetchError { + message: "http get: response headers timed out".to_string(), + kind: FetchFailKind::Timeout, + }) + } + }; + let content_length = response.content_length(); + let mut offset = 0u64; + loop { + // Budget gate: don't read ahead of the pool by more than the budget. + while in_flight.load(Ordering::Relaxed) >= budget { + if cancel.load(Ordering::Relaxed) { + return Err(AsyncFetchError { + message: "cancelled".to_string(), + kind: FetchFailKind::Other, + }); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + if abandon.is_some_and(|f| f.load(Ordering::Relaxed) == attempts) { + return Err(AsyncFetchError { + message: "sink rejected a piece of this attempt".to_string(), + kind: FetchFailKind::Other, + }); + } + let piece = match tokio::time::timeout(timeout, response.chunk()).await { + Ok(Ok(Some(piece))) => piece, + Ok(Ok(None)) => break, + Ok(Err(err)) => { + return Err(AsyncFetchError { + message: format!("http body: {err}"), + kind: classify_reqwest_error(&err), + }); + } + Err(_) => { + return Err(AsyncFetchError { + message: format!("http body: idle timeout at offset {offset}"), + kind: FetchFailKind::Timeout, + }); + } + }; + if piece.is_empty() { + continue; + } + let data = piece.to_vec(); + let len = data.len() as u64; + in_flight.fetch_add(len, Ordering::Relaxed); + if tx + .send(ProcessMsg::Piece { + idx, + attempts, + offset, + data, + }) + .is_err() + { + in_flight.fetch_sub(len, Ordering::Relaxed); + return Err(AsyncFetchError { + message: "process pool gone".to_string(), + kind: FetchFailKind::Other, + }); + } + offset += len; + } + validate_body_len(offset, content_length, range)?; + Ok(offset) +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Driver plumbing +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// `abandon` sentinel: no attempt of this item has been rejected by the sink. +const NO_ATTEMPT: u32 = u32::MAX; + +enum ProcessMsg { + /// Whole-body mode: the complete raw body. + Body { + idx: usize, + attempts: u32, + server_idx: usize, + body: Vec, + }, + /// Stream mode: one in-order piece of attempt `attempts` of item `idx`. + Piece { + idx: usize, + attempts: u32, + offset: u64, + data: Vec, + }, + /// Stream mode: the body ended; the pool finalizes and reports a verdict. + Finish { + idx: usize, + attempts: u32, + server_idx: usize, + total_len: u64, + }, +} + +enum SinkVerdict { + Ok, + Retry(String), + Fatal, +} + +struct Feedback { + idx: usize, + attempts: u32, + server_idx: usize, + verdict: SinkVerdict, +} + +/// An item awaiting (re)dispatch: `attempts` already spent; not eligible before `not_before`. +#[derive(Clone, Copy, Debug)] +struct PendingItem { + idx: usize, + attempts: u32, + not_before: Instant, +} + +enum Fetched { + Body(Vec), + Streamed(u64), +} + +/// The result the driver awaits for each dispatched fetch. +struct FetchDone { + idx: usize, + attempts: u32, + reserve: u64, + server_idx: usize, + elapsed: Duration, + res: Result, +} + +struct DriverCtx<'a> { + items: &'a [FetchItem], + hosts: &'a [String], + opts: &'a FetchOptions, + label: &'a str, + cancel: &'a AtomicBool, + log: &'a (dyn Fn(&str) + Sync), + meter: &'a BandwidthMeter, + in_flight: &'a AtomicU64, + error_slot: &'a Mutex>, + abandon: &'a [AtomicU32], + txs: Vec>, + fb_rx: std_mpsc::Receiver, + budget: u64, + bootstrap: usize, + win_min: usize, + win_max: usize, + per_host_cap: usize, +} + +async fn run_driver(ctx: DriverCtx<'_>) { + let DriverCtx { + items, + hosts, + opts, + label, + cancel, + log, + meter, + in_flight, + error_slot, + abandon, + txs, + fb_rx, + budget, + bootstrap, + win_min, + win_max, + per_host_cap, + } = ctx; + let timeout = opts.timeout; + let stream = opts.stream; + + let http = match build_http(opts, per_host_cap) { + Ok(http) => http, + Err(err) => { + record_first_error(error_slot, format!("{label}: {err}")); + drop(txs); + return; + } + }; + + let mut window = AdaptiveWindow::new(bootstrap, win_min, win_max, Instant::now()); + let mut sched = FetchScheduler::new(hosts, per_host_cap); + let mut retry: VecDeque = VecDeque::new(); + let mut next_item = 0usize; + let mut errors_logged: u32 = 0; + // Bodies / finishes handed to the pool whose verdict has not come back yet. + let mut outstanding = 0usize; + // Declared AFTER `txs`/`http` so it is dropped BEFORE them (its futures borrow both). + let mut inflight: FuturesUnordered<_> = FuturesUnordered::new(); + + 'outer: loop { + if cancel.load(Ordering::Relaxed) + || error_slot.lock().expect("err slot poisoned").is_some() + { + break; + } + + // ── Sink verdicts: Retry = counts as a failed attempt (cool + demote host, back-off, + // rotate); Fatal was already recorded in the error slot by the pool. ── + while let Ok(fb) = fb_rx.try_recv() { + outstanding = outstanding.saturating_sub(1); + match fb.verdict { + SinkVerdict::Ok => {} + SinkVerdict::Fatal => break 'outer, + SinkVerdict::Retry(message) => { + let now = Instant::now(); + sched.on_error(fb.server_idx, now, FetchFailKind::Other); + if window.record_err(now, FetchFailKind::Other) { + log(&window.summary_line(label, inflight.len(), now)); + } + if fb.attempts < MAX_ITEM_ATTEMPTS { + retry.push_back(PendingItem { + idx: fb.idx, + attempts: fb.attempts, + not_before: now + Duration::from_millis(retry_backoff_millis(fb.attempts)), + }); + } else { + record_first_error( + error_slot, + format!( + "{label}: item {} failed after {MAX_ITEM_ATTEMPTS} attempts: {message}", + items[fb.idx].id + ), + ); + break 'outer; + } + } + } + } + + let probe_now = Instant::now(); + if window.maybe_probe(probe_now, meter.total_bytes()) { + log(&window.summary_line(label, inflight.len(), probe_now)); + } + + // ── Dispatch up to the current window ── + while inflight.len() < window.current { + if cancel.load(Ordering::Relaxed) + || error_slot.lock().expect("err slot poisoned").is_some() + { + break 'outer; + } + let now = Instant::now(); + + // A due retry first, else the next fresh item. + let use_retry = retry.front().is_some_and(|p| p.not_before <= now); + let (idx, attempts) = if use_retry { + let p = *retry.front().expect("retry nonempty"); + (p.idx, p.attempts) + } else if next_item < items.len() { + (next_item, 0u32) + } else { + break; // nothing dispatchable now (retry not due, or all dispatched) + }; + let item = &items[idx]; + + // Byte-budget gate (hard memory bound): reserve at DISPATCH. Always admits when nothing + // is in flight, so an item bigger than the whole budget can't deadlock. Streams reserve + // nothing here — their pieces are budgeted as they arrive. + let reserve = if stream { 0 } else { reserve_bytes(item) }; + if !budget_admits(in_flight.load(Ordering::Relaxed), reserve, budget) { + window.note_budget_stall(); + break; // wait for a completion to free budget + } + + // Speed-ranked, per-host-capped host pick. + let Some(server_idx) = sched.pick(now) else { + window.note_host_stall(); + break; // every host at cap or cooling → wait for a completion + }; + let permit = match sched.host_sem(server_idx).clone().try_acquire_owned() { + Ok(permit) => permit, + Err(_) => { + window.note_host_stall(); + break; + } + }; + + let Some(url) = url_for_host(item, server_idx) else { + record_first_error(error_slot, format!("{label}: item {} has no url", item.id)); + break 'outer; + }; + let url = url.to_string(); + let range = item.range; + + // Commit: pop the work item, reserve budget, launch the fetch future. + if use_retry { + retry.pop_front(); + } else { + next_item += 1; + } + in_flight.fetch_add(reserve, Ordering::Relaxed); + let http = &http; + let tx = &txs[if stream { idx % txs.len() } else { 0 }]; + let abandon_flag = abandon.get(idx); + let started = Instant::now(); + inflight.push(async move { + let res = if stream { + fetch_stream( + http, + &url, + range, + timeout, + idx, + attempts + 1, + tx, + in_flight, + budget, + cancel, + abandon_flag, + ) + .await + .map(Fetched::Streamed) + } else { + fetch_once(http, &url, range, timeout) + .await + .map(Fetched::Body) + }; + drop(permit); // RAII per-host permit release (also on future drop / cancel-abort) + FetchDone { + idx, + attempts: attempts + 1, + reserve, + server_idx, + elapsed: started.elapsed(), + res, + } + }); + } + + // ── Nothing in flight? finished, or waiting on retry-not-due / host cooldown / the pool. ── + if inflight.is_empty() { + if next_item >= items.len() && retry.is_empty() && outstanding == 0 { + break; // every item fetched and processed + } + tokio::time::sleep(Duration::from_millis(5)).await; + continue; + } + + // ── Await one completion (drives ALL in-flight requests cooperatively), bounded so + // cancel / a pool Fatal / a Retry verdict are noticed promptly even mid-fetch. ── + let done = match tokio::time::timeout(Duration::from_millis(DRIVER_POLL_MS), inflight.next()) + .await + { + Ok(Some(done)) => done, + Ok(None) => continue, + Err(_elapsed) => continue, + }; + let now = Instant::now(); + match done.res { + Ok(Fetched::Body(body)) => { + let raw_len = body.len() as u64; + meter.record(done.server_idx, raw_len); + sched.on_success(done.server_idx, raw_len, done.elapsed); + window.record_ok(done.elapsed.as_secs_f64() * 1000.0); + // Reconcile the dispatch reservation to the actual raw size; the pool subtracts the + // actual size on process-complete, netting this item's budget to zero. + if raw_len >= done.reserve { + in_flight.fetch_add(raw_len - done.reserve, Ordering::Relaxed); + } else { + in_flight.fetch_sub(done.reserve - raw_len, Ordering::Relaxed); + } + outstanding += 1; + if txs[0] + .send(ProcessMsg::Body { + idx: done.idx, + attempts: done.attempts, + server_idx: done.server_idx, + body, + }) + .is_err() + { + break; // process side gone + } + } + Ok(Fetched::Streamed(total_len)) => { + meter.record(done.server_idx, total_len); + sched.on_success(done.server_idx, total_len, done.elapsed); + window.record_ok(done.elapsed.as_secs_f64() * 1000.0); + outstanding += 1; + if txs[done.idx % txs.len()] + .send(ProcessMsg::Finish { + idx: done.idx, + attempts: done.attempts, + server_idx: done.server_idx, + total_len, + }) + .is_err() + { + break; // process side gone + } + } + Err(err) => { + in_flight.fetch_sub(done.reserve, Ordering::Relaxed); + let client_rejection = is_client_rejection(&err.message); + if errors_logged < FETCH_ERROR_LOG_LIMIT { + errors_logged += 1; + let host = hosts.get(done.server_idx).map(String::as_str).unwrap_or("?"); + log(&format!( + "fetch-error label={label} item={} host={host} attempt={} kind={:?} \ +window_shrink={} msg={}", + items[done.idx].id, + done.attempts, + err.kind, + !client_rejection, + err.message + )); + if errors_logged == FETCH_ERROR_LOG_LIMIT { + log(&format!( + "fetch-error label={label} further errors not logged individually" + )); + } + } + sched.on_error(done.server_idx, now, err.kind); + if !client_rejection && window.record_err(now, err.kind) { + log(&window.summary_line(label, inflight.len(), now)); + } + if done.attempts < MAX_ITEM_ATTEMPTS { + let backoff = retry_backoff_millis(done.attempts); + retry.push_back(PendingItem { + idx: done.idx, + attempts: done.attempts, + not_before: now + Duration::from_millis(backoff), + }); + } else { + record_first_error( + error_slot, + format!( + "{label}: item {} failed after {MAX_ITEM_ATTEMPTS} attempts: {}", + items[done.idx].id, err.message + ), + ); + break; + } + } + } + } + + // Dropping `inflight` aborts any still-running requests (the cancel / error / final-failure + // teardown path); dropping the senders then closes the channels so the pool drains and exits. + drop(inflight); + drop(txs); +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Bandwidth meter (per-host bytes + 5-s samples for peak) +// ───────────────────────────────────────────────────────────────────────────────────────────── + +pub struct BandwidthMeter { + start: Instant, + total_bytes: AtomicU64, + per_server_bytes: Vec, + hosts: Vec, + /// (bytes, instant) at the last sample; peak = max sample rate in bytes/s. + last_sample: Mutex<(u64, Instant)>, + peak_bps: Mutex, +} + +impl BandwidthMeter { + pub fn new(hosts: &[String]) -> Self { + let now = Instant::now(); + Self { + start: now, + total_bytes: AtomicU64::new(0), + per_server_bytes: hosts.iter().map(|_| AtomicU64::new(0)).collect(), + hosts: hosts.to_vec(), + last_sample: Mutex::new((0, now)), + peak_bps: Mutex::new(0.0), + } + } + + #[inline] + pub fn record(&self, server_idx: usize, bytes: u64) { + self.total_bytes.fetch_add(bytes, Ordering::Relaxed); + if let Some(counter) = self.per_server_bytes.get(server_idx) { + counter.fetch_add(bytes, Ordering::Relaxed); + } + } + + #[inline] + pub fn total_bytes(&self) -> u64 { + self.total_bytes.load(Ordering::Relaxed) + } + + /// Take a throughput sample since the previous one (≥ 1 s apart to be counted) and fold it + /// into the peak. + pub fn sample(&self) { + let now = Instant::now(); + let total = self.total_bytes(); + let mut last = self.last_sample.lock().expect("meter poisoned"); + let dt = now.duration_since(last.1).as_secs_f64(); + if dt >= 1.0 { + let bps = total.saturating_sub(last.0) as f64 / dt; + let mut peak = self.peak_bps.lock().expect("meter poisoned"); + if bps > *peak { + *peak = bps; + } + *last = (total, now); + } + } + + pub fn peak_bps(&self) -> f64 { + *self.peak_bps.lock().expect("meter poisoned") + } + + pub fn summary_line(&self, label: &str) -> String { + let secs = self.start.elapsed().as_secs_f64().max(0.001); + let total = self.total_bytes(); + let mb = |b: u64| (b as f64) / (1024.0 * 1024.0); + let overall_mbps = mb(total) / secs; + let mut per: Vec<(usize, u64)> = self + .per_server_bytes + .iter() + .enumerate() + .map(|(i, c)| (i, c.load(Ordering::Relaxed))) + .filter(|(_, b)| *b > 0) + .collect(); + per.sort_by(|a, b| b.1.cmp(&a.1)); + let used = per.len(); + let pool = self.per_server_bytes.len(); + let mut servers = String::new(); + for (i, bytes) in per.iter().take(6) { + let host = self.hosts.get(*i).map(String::as_str).unwrap_or("?"); + servers.push_str(&format!( + " [{host} {:.1}MB/s {:.0}MB]", + mb(*bytes) / secs, + mb(*bytes) + )); + } + format!( + "throughput label={label} overall={overall_mbps:.2}MB/s total={:.0}MB elapsed={secs:.0}s used={used}/{pool} servers:{servers}", + mb(total) + ) + } +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Scheduler + adaptive window — a copy of the Steam depot writer's (keyed on host strings). +// ───────────────────────────────────────────────────────────────────────────────────────────── + +#[derive(Clone, Debug, Default)] +struct ServerHealth { + ewma_bps: f64, + samples: u32, + consecutive_errors: u32, + cooldown_until: Option, +} + +/// Speed-ranked, per-host-capped host picker. Cold hosts are probed first, then selection is +/// epsilon-greedy over the measured EWMA so a demoted-but-recovered host returns. Per-host caps key +/// on the host STRING (one `tokio::sync::Semaphore` each) so two entries that share a host still +/// share one cap. +struct FetchScheduler { + health: Vec, + server_host: Vec, + host_sems: Vec>, + dispatch_counter: u64, +} + +impl FetchScheduler { + fn new(hosts: &[String], per_host_cap: usize) -> Self { + let mut host_keys: Vec<&str> = Vec::new(); + let mut server_host: Vec = Vec::with_capacity(hosts.len()); + for h in hosts { + let idx = host_keys + .iter() + .position(|k| *k == h.as_str()) + .unwrap_or_else(|| { + host_keys.push(h.as_str()); + host_keys.len() - 1 + }); + server_host.push(idx); + } + let host_sems = host_keys + .iter() + .map(|_| Arc::new(Semaphore::new(per_host_cap.max(1)))) + .collect(); + Self { + health: hosts.iter().map(|_| ServerHealth::default()).collect(), + server_host, + host_sems, + dispatch_counter: 0, + } + } + + fn host_sem(&self, server_idx: usize) -> &Arc { + &self.host_sems[self.server_host[server_idx]] + } + + fn eligible(&self, server_idx: usize, now: Instant) -> bool { + self.host_sem(server_idx).available_permits() > 0 + && self.health[server_idx] + .cooldown_until + .map_or(true, |t| t <= now) + } + + fn pick(&mut self, now: Instant) -> Option { + let eligible: Vec = (0..self.health.len()) + .filter(|&i| self.eligible(i, now)) + .collect(); + if eligible.is_empty() { + return None; + } + if let Some(&cold) = eligible.iter().find(|&&i| self.health[i].samples == 0) { + return Some(cold); + } + self.dispatch_counter = self.dispatch_counter.wrapping_add(1); + if SERVER_EXPLORE_EVERY > 0 && self.dispatch_counter % SERVER_EXPLORE_EVERY == 0 { + return eligible + .iter() + .copied() + .min_by_key(|&i| self.health[i].samples); + } + eligible.iter().copied().max_by(|&a, &b| { + self.health[a] + .ewma_bps + .partial_cmp(&self.health[b].ewma_bps) + .unwrap_or(std::cmp::Ordering::Equal) + .then( + self.host_sem(a) + .available_permits() + .cmp(&self.host_sem(b).available_permits()), + ) + .then(b.cmp(&a)) + }) + } + + fn on_success(&mut self, server_idx: usize, bytes: u64, elapsed: Duration) { + let secs = elapsed.as_secs_f64().max(0.001); + let bps = bytes as f64 / secs; + let h = &mut self.health[server_idx]; + h.ewma_bps = if h.samples == 0 { + bps + } else { + 0.7 * h.ewma_bps + 0.3 * bps + }; + h.samples = h.samples.saturating_add(1); + h.consecutive_errors = 0; + h.cooldown_until = None; + } + + fn on_error(&mut self, server_idx: usize, now: Instant, kind: FetchFailKind) { + let h = &mut self.health[server_idx]; + h.consecutive_errors = h.consecutive_errors.saturating_add(1); + let mut backoff = retry_backoff_millis(h.consecutive_errors); + if kind == FetchFailKind::RateLimited { + backoff = backoff.max(RATE_LIMIT_COOLDOWN_MS); + } + h.cooldown_until = Some(now + Duration::from_millis(backoff)); + h.ewma_bps *= 0.5; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum WindowReason { + Start, + GrowSlowStart, + GrowThroughputUp, + GrowProbe, + HoldPlateau, + HoldCooldown, + HoldCeiling, + HoldErrors, + HoldThroughputDown, + ShrinkRateLimited, + ShrinkTimeout, + ShrinkReset, + ShrinkServerFault, + ShrinkErrorRate, + ShrinkErrorBurst, +} + +impl WindowReason { + fn code(self) -> &'static str { + match self { + WindowReason::Start => "start", + WindowReason::GrowSlowStart => "grow:slow-start", + WindowReason::GrowThroughputUp => "grow:throughput-up", + WindowReason::GrowProbe => "grow:probe", + WindowReason::HoldPlateau => "hold:plateau", + WindowReason::HoldCooldown => "hold:cooldown", + WindowReason::HoldCeiling => "hold:ceiling", + WindowReason::HoldErrors => "hold:errors", + WindowReason::HoldThroughputDown => "hold:throughput-down", + WindowReason::ShrinkRateLimited => "shrink:429", + WindowReason::ShrinkTimeout => "shrink:timeout", + WindowReason::ShrinkReset => "shrink:reset", + WindowReason::ShrinkServerFault => "shrink:5xx", + WindowReason::ShrinkErrorRate => "shrink:err-rate", + WindowReason::ShrinkErrorBurst => "shrink:err-burst", + } + } + + fn from_fail_kind(kind: FetchFailKind) -> Self { + match kind { + FetchFailKind::RateLimited => WindowReason::ShrinkRateLimited, + FetchFailKind::Timeout => WindowReason::ShrinkTimeout, + FetchFailKind::Connect => WindowReason::ShrinkReset, + FetchFailKind::ServerFault => WindowReason::ShrinkServerFault, + FetchFailKind::Other => WindowReason::ShrinkErrorBurst, + } + } +} + +/// Adaptive in-flight window: slow-start doubles every healthy probe until the first real back-off, +/// then grows additively; holds on plateau; shrinks proportionally only on real errors. +#[derive(Clone, Debug)] +struct AdaptiveWindow { + current: usize, + min: usize, + max: usize, + last_probe: Instant, + last_bytes: u64, + last_bps: f64, + bps_ewma: f64, + best_bps: f64, + slow_start: bool, + plateau_streak: u32, + plateau_since: Option, + plateau_rearm_ms: u64, + ok_count: u32, + err_count: u32, + budget_stalls: u32, + host_stalls: u32, + last_budget_stalls: u32, + last_host_stalls: u32, + latency_ewma_ms: f64, + cooldown_until: Option, + last_reason: WindowReason, + last_err_rate: f64, + probes_since_log: u32, +} + +impl AdaptiveWindow { + fn new(bootstrap: usize, min: usize, max: usize, now: Instant) -> Self { + let max = max.max(1); + let min = min.clamp(1, max); + Self { + current: bootstrap.clamp(min, max), + min, + max, + last_probe: now, + last_bytes: 0, + last_bps: 0.0, + bps_ewma: 0.0, + best_bps: 0.0, + slow_start: true, + plateau_streak: 0, + plateau_since: None, + plateau_rearm_ms: WINDOW_PLATEAU_REARM_MS, + ok_count: 0, + err_count: 0, + budget_stalls: 0, + host_stalls: 0, + last_budget_stalls: 0, + last_host_stalls: 0, + latency_ewma_ms: 0.0, + cooldown_until: None, + last_reason: WindowReason::Start, + last_err_rate: 0.0, + probes_since_log: 0, + } + } + + fn note_budget_stall(&mut self) { + self.budget_stalls = self.budget_stalls.saturating_add(1); + } + + fn note_host_stall(&mut self) { + self.host_stalls = self.host_stalls.saturating_add(1); + } + + fn record_ok(&mut self, latency_ms: f64) { + self.ok_count = self.ok_count.saturating_add(1); + self.latency_ewma_ms = if self.latency_ewma_ms == 0.0 { + latency_ms + } else { + 0.8 * self.latency_ewma_ms + 0.2 * latency_ms + }; + } + + fn shrink(&mut self, now: Instant, reason: WindowReason) { + let scaled = (self.current as f64 * WINDOW_SHRINK_FACTOR).floor() as usize; + let next = scaled.min(self.current.saturating_sub(1)).max(self.min); + self.current = next.max(self.min); + self.cooldown_until = Some(now + Duration::from_millis(WINDOW_COOLDOWN_MS)); + self.slow_start = false; + self.best_bps = self.bps_ewma; + self.plateau_streak = 0; + self.plateau_since = None; + self.plateau_rearm_ms = WINDOW_PLATEAU_REARM_MS; + self.last_reason = reason; + } + + fn grow(&mut self, reason: WindowReason, multiplicative: bool) { + let next = if multiplicative { + self.current.saturating_mul(WINDOW_SLOW_START_FACTOR) + } else { + self.current + WINDOW_STEP_UP + }; + self.current = next.min(self.max).max(self.min); + self.last_reason = reason; + } + + fn record_err(&mut self, now: Instant, kind: FetchFailKind) -> bool { + self.err_count = self.err_count.saturating_add(1); + let immediate = kind == FetchFailKind::RateLimited + || self.err_count >= WINDOW_ERR_BURST_IMMEDIATE; + if !immediate { + return false; + } + let before = self.current; + self.shrink(now, WindowReason::from_fail_kind(kind)); + if before != self.current { + self.probes_since_log = 0; + true + } else { + false + } + } + + fn maybe_probe(&mut self, now: Instant, total_bytes: u64) -> bool { + let dt = now.duration_since(self.last_probe); + if dt < Duration::from_millis(WINDOW_PROBE_INTERVAL_MS) { + return false; + } + let secs = dt.as_secs_f64().max(0.001); + let sample_bps = total_bytes.saturating_sub(self.last_bytes) as f64 / secs; + self.bps_ewma = if self.bps_ewma == 0.0 { + sample_bps + } else { + WINDOW_BPS_EWMA_ALPHA * sample_bps + (1.0 - WINDOW_BPS_EWMA_ALPHA) * self.bps_ewma + }; + let events = self.ok_count + self.err_count; + let err_rate = if events == 0 { + 0.0 + } else { + self.err_count as f64 / events as f64 + }; + let before = self.current; + let cooling = self.cooldown_until.is_some_and(|t| now < t); + + if err_rate > WINDOW_ERR_RATE_HIGH { + self.shrink(now, WindowReason::ShrinkErrorRate); + } else if cooling { + self.last_reason = WindowReason::HoldCooldown; + } else if self.current >= self.max { + self.last_reason = WindowReason::HoldCeiling; + } else if err_rate > WINDOW_ERR_RATE_LOW { + self.last_reason = WindowReason::HoldErrors; + } else { + let improving = self.bps_ewma > self.best_bps * (1.0 + WINDOW_IMPROVE_EPS); + let falling = self.bps_ewma < self.best_bps * (1.0 - WINDOW_DECLINE_EPS); + if improving { + self.best_bps = self.bps_ewma; + self.plateau_streak = 0; + self.plateau_since = None; + self.plateau_rearm_ms = WINDOW_PLATEAU_REARM_MS; + let doubling = self.slow_start; + let reason = if doubling { + WindowReason::GrowSlowStart + } else { + WindowReason::GrowThroughputUp + }; + self.grow(reason, doubling); + } else if falling { + self.last_reason = WindowReason::HoldThroughputDown; + } else if self.plateau_streak < WINDOW_PLATEAU_PATIENCE { + self.plateau_streak += 1; + self.grow(WindowReason::GrowProbe, false); + } else { + let since = *self.plateau_since.get_or_insert(now); + if now.duration_since(since) >= Duration::from_millis(self.plateau_rearm_ms) { + self.plateau_streak = 0; + self.plateau_since = None; + self.plateau_rearm_ms = + (self.plateau_rearm_ms * 2).min(WINDOW_PLATEAU_REARM_MAX_MS); + } + self.last_reason = WindowReason::HoldPlateau; + } + } + + self.last_bytes = total_bytes; + self.last_bps = sample_bps; + self.last_probe = now; + self.last_err_rate = err_rate; + self.ok_count = 0; + self.err_count = 0; + self.last_budget_stalls = self.budget_stalls; + self.last_host_stalls = self.host_stalls; + self.budget_stalls = 0; + self.host_stalls = 0; + self.probes_since_log = self.probes_since_log.saturating_add(1); + let changed = before != self.current; + if changed || self.probes_since_log >= WINDOW_LOG_EVERY_PROBES { + self.probes_since_log = 0; + true + } else { + false + } + } + + fn cooldown_left_ms(&self, now: Instant) -> u64 { + self.cooldown_until + .map(|t| t.saturating_duration_since(now).as_millis() as u64) + .unwrap_or(0) + } + + fn summary_line(&self, label: &str, in_flight_requests: usize, now: Instant) -> String { + let mbps = |bps: f64| bps / (1024.0 * 1024.0); + format!( + "fetch-window label={label} window={} (min={} max={}) in_flight={in_flight_requests} \ +last={:.2}MB/s ewma={:.2}MB/s best={:.2}MB/s reason={} cooldown={}ms err_rate={:.1}% \ +phase={} rtt={:.0}ms budget_stalls={} host_stalls={}", + self.current, + self.min, + self.max, + mbps(self.last_bps), + mbps(self.bps_ewma), + mbps(self.best_bps), + self.last_reason.code(), + self.cooldown_left_ms(now), + self.last_err_rate * 100.0, + if self.slow_start { "slow-start" } else { "steady" }, + self.latency_ewma_ms, + self.last_budget_stalls, + self.last_host_stalls + ) + } +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + #[test] + fn client_rejection_is_4xx_but_not_429() { + assert_eq!(rejected_status("unexpected HTTP status (403)"), Some(403)); + assert!(is_client_rejection("unexpected HTTP status (403)")); + assert!(is_client_rejection("unexpected HTTP status (404)")); + assert!(!is_client_rejection("unexpected HTTP status (429)")); + assert!(!is_client_rejection("unexpected HTTP status (503)")); + assert!(!is_client_rejection("http get: connection reset")); + assert_eq!(rejected_status("http body: timeout"), None); + } + + use super::*; + use crate::fetch_tuning::{ + FETCH_INFLIGHT_BUDGET_FLOOR_BYTES, FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES, PER_HOST_CAP, + }; + use std::collections::HashMap; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::atomic::AtomicUsize; + + fn item(id: u64, urls: &[&str], range: Option<(u64, u64)>) -> FetchItem { + FetchItem { + id, + urls: urls.iter().map(|s| s.to_string()).collect(), + reserve: 0, + range, + } + } + + #[test] + fn window_bounds_clamp_to_hosts_and_hard_cap() { + assert_eq!(window_bounds(64, 7, PER_HOST_CAP), (8, 2, 42)); + assert_eq!(window_bounds(8, 1, PER_HOST_CAP), (6, 2, 6)); + assert_eq!(window_bounds(1, 1, PER_HOST_CAP), (1, 1, 1)); + assert_eq!(window_bounds(10_000, 100, 6), (8, 2, WINDOW_HARD_CAP)); + assert_eq!(window_bounds(0, 0, 0), (1, 1, 1)); + } + + #[test] + fn budget_scales_with_ceiling_between_floor_and_cap() { + assert_eq!(inflight_budget_bytes(1), FETCH_INFLIGHT_BUDGET_FLOOR_BYTES); + assert_eq!(inflight_budget_bytes(16), FETCH_INFLIGHT_BUDGET_FLOOR_BYTES); + assert_eq!(inflight_budget_bytes(42), 63 * 1024 * 1024); + assert_eq!(inflight_budget_bytes(256), FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES); + assert!(budget_admits(0, u64::MAX, 1)); + assert!(!budget_admits(1, 100, 100)); + } + + #[test] + fn reserve_defaults_to_nominal() { + let mut i = item(1, &["u"], None); + assert_eq!(reserve_bytes(&i), NOMINAL_CHUNK_RESERVE_BYTES); + i.reserve = 77; + assert_eq!(reserve_bytes(&i), 77); + } + + #[test] + fn url_per_host_selection() { + let none = item(1, &[], None); + assert_eq!(url_for_host(&none, 0), None); + let one = item(1, &["a"], None); + assert_eq!(url_for_host(&one, 0), Some("a")); + assert_eq!(url_for_host(&one, 5), Some("a")); + let many = item(1, &["a", "b", "c"], None); + assert_eq!(url_for_host(&many, 1), Some("b")); + assert_eq!(url_for_host(&many, 2), Some("c")); + assert_eq!(url_for_host(&many, 9), Some("a")); + } + + #[test] + fn distinct_hosts_dedupe_by_string() { + let h = vec!["a".to_string(), "b".to_string(), "a".to_string()]; + assert_eq!(distinct_host_count(&h), 2); + assert_eq!(distinct_host_count(&[]), 1); + } + + #[test] + fn range_validation() { + assert_eq!(range_len((0, 0)), Some(1)); + assert_eq!(range_len((10, 19)), Some(10)); + assert_eq!(range_len((5, 4)), None); + assert!(validate_body_len(10, None, Some((10, 19))).is_ok()); + assert!(validate_body_len(10, Some(10), Some((10, 19))).is_ok()); + let e = validate_body_len(9, None, Some((10, 19))).unwrap_err(); + assert_eq!(e.kind, FetchFailKind::ServerFault); + assert_eq!( + validate_body_len(1, None, Some((5, 4))).unwrap_err().kind, + FetchFailKind::Other + ); + assert!(validate_body_len(100, None, None).is_ok()); + assert!(validate_body_len(100, Some(100), None).is_ok()); + assert!(validate_body_len(99, Some(100), None).is_err()); + assert!(status_accepted(200, false)); + assert!(status_accepted(200, true)); + assert!(status_accepted(206, true)); + assert!(!status_accepted(206, false)); + assert!(!status_accepted(404, true)); + assert_eq!(classify_status(429), FetchFailKind::RateLimited); + assert_eq!(classify_status(503), FetchFailKind::ServerFault); + assert_eq!(classify_status(404), FetchFailKind::Other); + } + + #[test] + fn scheduler_probes_cold_hosts_then_ranks() { + let hosts = vec!["a".to_string(), "b".to_string()]; + let mut s = FetchScheduler::new(&hosts, 2); + let now = Instant::now(); + assert_eq!(s.pick(now), Some(0)); + s.on_success(0, 1_000_000, Duration::from_millis(100)); + assert_eq!(s.pick(now), Some(1)); + s.on_success(1, 1_000_000, Duration::from_millis(500)); + assert_eq!(s.pick(now), Some(0)); + s.on_error(0, now, FetchFailKind::Timeout); + assert_eq!(s.pick(now), Some(1)); + s.on_error(1, now, FetchFailKind::RateLimited); + assert_eq!(s.pick(now), None); + assert_eq!(s.pick(now + Duration::from_secs(6)), Some(0)); + } + + #[test] + fn window_shrinks_only_on_error_bursts() { + let now = Instant::now(); + let mut w = AdaptiveWindow::new(8, 2, 42, now); + assert!(!w.record_err(now, FetchFailKind::Timeout)); + assert!(!w.record_err(now, FetchFailKind::Timeout)); + assert!(w.record_err(now, FetchFailKind::Timeout)); + assert_eq!(w.current, 6); + assert!(!w.slow_start); + let line = w.summary_line("t", 3, now); + assert!(line.starts_with("fetch-window label=t window=6 (min=2 max=42) in_flight=3 ")); + assert!(line.contains("reason=shrink:timeout")); + assert!(line.contains("phase=steady")); + } + + // ── Minimal HTTP/1.1 stub: one thread per connection, `Connection: close`, Range-aware. ── + struct Stub { + base: String, + hits: Arc>>, + } + + fn start_stub(bodies: HashMap>, flaky_first: &[&str]) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + let hits: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let flaky: Vec = flaky_first.iter().map(|s| s.to_string()).collect(); + let bodies = Arc::new(bodies); + let hits2 = Arc::clone(&hits); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let bodies = Arc::clone(&bodies); + let hits = Arc::clone(&hits2); + let flaky = flaky.clone(); + thread::spawn(move || { + let mut buf = Vec::new(); + let mut tmp = [0u8; 1024]; + loop { + let n = match stream.read(&mut tmp) { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + buf.extend_from_slice(&tmp[..n]); + if buf.windows(4).any(|w| w == b"\r\n\r\n") { + break; + } + } + let text = String::from_utf8_lossy(&buf).to_string(); + let mut lines = text.lines(); + let request = lines.next().unwrap_or(""); + let path = request.split(' ').nth(1).unwrap_or("/").to_string(); + let mut range: Option<(u64, u64)> = None; + let mut auth = String::new(); + let mut ua = String::new(); + for line in lines { + let lower = line.to_ascii_lowercase(); + if let Some(v) = lower.strip_prefix("range: bytes=") { + let mut it = v.trim().split('-'); + let a = it.next().and_then(|s| s.parse().ok()); + let b = it.next().and_then(|s| s.parse().ok()); + if let (Some(a), Some(b)) = (a, b) { + range = Some((a, b)); + } + } + if let Some(v) = line.strip_prefix("Authorization: ") { + auth = v.trim().to_string(); + } else if let Some(v) = line.strip_prefix("authorization: ") { + auth = v.trim().to_string(); + } else if let Some(v) = line.strip_prefix("User-Agent: ") { + ua = v.trim().to_string(); + } else if let Some(v) = line.strip_prefix("user-agent: ") { + ua = v.trim().to_string(); + } + } + let count = { + let mut h = hits.lock().unwrap(); + let c = h.entry(path.clone()).or_insert(0); + *c += 1; + *c + }; + let response: Vec = if path == "/auth" || path == "/ua" { + let body = if path == "/auth" { auth } else { ua }.into_bytes(); + let mut r = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + r.extend_from_slice(&body); + r + } else if flaky.contains(&path) && count == 1 { + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec() + } else if let Some(full) = bodies.get(&path) { + match range { + Some((a, b)) if (b as usize) < full.len() && a <= b => { + let slice = &full[a as usize..=b as usize]; + let mut r = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {a}-{b}/{}\r\nConnection: close\r\n\r\n", + slice.len(), + full.len() + ) + .into_bytes(); + r.extend_from_slice(slice); + r + } + _ => { + let mut r = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + full.len() + ) + .into_bytes(); + r.extend_from_slice(full); + r + } + } + } else { + b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec() + }; + let _ = stream.write_all(&response); + let _ = stream.flush(); + }); + } + }); + Stub { + base: format!("http://127.0.0.1:{port}"), + hits, + } + } + + struct CollectSink { + got: Mutex>>, + retry_once_id: Option, + retried: AtomicUsize, + } + + impl FetchSink for CollectSink { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + if self.retry_once_id == Some(item.id) && self.retried.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(SinkError::Retry("simulated hash mismatch".into())); + } + let n = body.len() as u64; + self.got.lock().unwrap().insert(item.id, body); + Ok(n * 2) + } + } + + fn opts(label: &str) -> FetchOptions { + FetchOptions { + max_workers: 8, + per_host_cap: 4, + timeout: Duration::from_secs(10), + headers: vec![("Authorization".to_string(), "Bearer test-token".to_string())], + ca_bundle_path: String::new(), + process_workers: 2, + label: label.to_string(), + stream: false, + } + } + + /// Stream-mode sink: appends pieces per item (offset 0 restarts), finalizes by length check. + struct StreamSink { + got: Mutex>>, + finished: Mutex>, + /// Reject the first piece of this item once (exercises the mid-stream Retry path). + retry_once_id: Option, + retried: AtomicUsize, + } + + impl FetchSink for StreamSink { + fn process(&self, _item: &FetchItem, _body: Vec) -> Result { + Err(SinkError::Fatal("whole-body mode not expected".into())) + } + + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + let mut got = self.got.lock().unwrap(); + let buf = got.entry(item.id).or_default(); + if offset == 0 { + buf.clear(); + if self.retry_once_id == Some(item.id) + && self.retried.fetch_add(1, Ordering::SeqCst) == 0 + { + return Err(SinkError::Retry("simulated bad first piece".into())); + } + } + assert_eq!(buf.len() as u64, offset, "pieces must arrive in order"); + buf.extend_from_slice(data); + Ok(()) + } + + fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result { + let got = self.got.lock().unwrap(); + let len = got.get(&item.id).map(|b| b.len() as u64).unwrap_or(0); + if len != total_len { + return Err(SinkError::Retry(format!("length {len} != {total_len}"))); + } + self.finished.lock().unwrap().insert(item.id, total_len); + Ok(7) + } + } + + #[test] + fn end_to_end_stream_mode_against_local_stub() { + let mut bodies = HashMap::new(); + bodies.insert("/big".to_string(), (0..=255u8).cycle().take(300_000).collect::>()); + bodies.insert("/small".to_string(), b"tiny".to_vec()); + bodies.insert("/flaky".to_string(), vec![9u8; 20_000]); + let stub = start_stub(bodies, &["/flaky"]); + let hosts = vec!["127.0.0.1".to_string()]; + let items = vec![ + item(1, &[format!("{}/big", stub.base).as_str()], None), + item(2, &[format!("{}/small", stub.base).as_str()], None), + item(3, &[format!("{}/flaky", stub.base).as_str()], None), + item(4, &[format!("{}/big", stub.base).as_str()], Some((100, 1099))), + item(5, &[format!("{}/ua", stub.base).as_str()], None), + ]; + let sink = StreamSink { + got: Mutex::new(HashMap::new()), + finished: Mutex::new(HashMap::new()), + retry_once_id: Some(1), + retried: AtomicUsize::new(0), + }; + let mut options = opts("stream app=stub"); + options.stream = true; + options.per_host_cap = 8; + options.headers.push(("User-Agent".to_string(), "nile/0.1 Amazon".to_string())); + let cancel = AtomicBool::new(false); + let lines: Mutex> = Mutex::new(Vec::new()); + let outcome = run_fetch( + items, + &hosts, + &options, + &sink, + &cancel, + &|_: u64, _: u64| {}, + &|line: &str| lines.lock().unwrap().push(line.to_string()), + ); + let lines = lines.lock().unwrap(); + assert!(outcome.ok(), "outcome={outcome:?} lines={lines:?}"); + assert_eq!(outcome.items_ok, 5); + let got = sink.got.lock().unwrap(); + assert_eq!(got[&1].len(), 300_000); + assert_eq!(got[&1][1000], (1000 % 256) as u8); + assert_eq!(got[&2], b"tiny".to_vec()); + assert_eq!(got[&3], vec![9u8; 20_000]); + assert_eq!(got[&4].len(), 1000); + assert_eq!(got[&4][0], 100u8); + assert_eq!(got[&5], b"nile/0.1 Amazon".to_vec()); + let finished = sink.finished.lock().unwrap(); + assert_eq!(finished.len(), 5); + // Pieces credited as written + 7 extra per on_finish; the rejected attempt of /big + // contributed nothing (its first piece was refused before any write). + let written: u64 = got.values().map(|b| b.len() as u64).sum(); + assert_eq!(outcome.bytes_credited, written + 5 * 7); + let hits = stub.hits.lock().unwrap(); + assert_eq!(hits["/flaky"], 2); + assert!(hits["/big"] >= 3, "big={} (2 items + 1 retry)", hits["/big"]); + assert!(lines[0].contains(" mode=stream"), "{}", lines[0]); + assert!(lines[0].contains(" ceiling=8 "), "{}", lines[0]); + assert!(lines.last().unwrap().ends_with("result=ok")); + } + + #[test] + fn end_to_end_against_local_stub() { + let mut bodies = HashMap::new(); + bodies.insert("/a".to_string(), vec![1u8; 5000]); + bodies.insert("/b".to_string(), (0..=255u8).cycle().take(70_000).collect::>()); + bodies.insert("/c".to_string(), b"hello world".to_vec()); + let stub = start_stub(bodies, &["/c"]); + let hosts = vec!["127.0.0.1".to_string()]; + let items = vec![ + item(1, &[format!("{}/a", stub.base).as_str()], None), + item(2, &[format!("{}/b", stub.base).as_str()], Some((1000, 1999))), + item(3, &[format!("{}/c", stub.base).as_str()], None), + item(4, &[format!("{}/auth", stub.base).as_str()], None), + ]; + let sink = CollectSink { + got: Mutex::new(HashMap::new()), + retry_once_id: Some(1), + retried: AtomicUsize::new(0), + }; + let cancel = AtomicBool::new(false); + let lines: Mutex> = Mutex::new(Vec::new()); + let last_progress: Mutex<(u64, u64)> = Mutex::new((0, 0)); + let outcome = run_fetch( + items, + &hosts, + &opts("test app=stub"), + &sink, + &cancel, + &|bytes: u64, ok: u64| { + // Pool threads race on the callback order; keep the high-water mark. + let mut cur = last_progress.lock().unwrap(); + if bytes >= cur.0 { + *cur = (bytes, ok); + } + }, + &|line: &str| lines.lock().unwrap().push(line.to_string()), + ); + let lines = lines.lock().unwrap(); + assert!(outcome.ok(), "outcome={outcome:?} lines={lines:?}"); + assert_eq!(outcome.items_ok, 4); + let got = sink.got.lock().unwrap(); + assert_eq!(got[&1], vec![1u8; 5000]); + assert_eq!(got[&2].len(), 1000); + assert_eq!(got[&2][0], (1000 % 256) as u8); + assert_eq!(got[&3], b"hello world".to_vec()); + assert_eq!(got[&4], b"Bearer test-token".to_vec()); + let expected_credit: u64 = got.values().map(|b| b.len() as u64 * 2).sum(); + assert_eq!(outcome.bytes_credited, expected_credit); + assert_eq!(*last_progress.lock().unwrap(), (expected_credit, 4)); + // /c was served 503 once, then 200; /a was re-fetched once after the sink's Retry. + let hits = stub.hits.lock().unwrap(); + assert_eq!(hits["/c"], 2); + assert_eq!(hits["/a"], 2); + assert!(lines[0].starts_with("fetch-start label=test app=stub items=4 ")); + assert!(lines.last().unwrap().starts_with("fetch-end label=test app=stub items_ok=4 ")); + assert!(lines.last().unwrap().ends_with("result=ok")); + } + + #[test] + fn end_to_end_gives_up_after_max_attempts() { + let stub = start_stub(HashMap::new(), &[]); + let hosts = vec!["127.0.0.1".to_string()]; + let items = vec![item(9, &[format!("{}/missing", stub.base).as_str()], None)]; + let sink = CollectSink { + got: Mutex::new(HashMap::new()), + retry_once_id: None, + retried: AtomicUsize::new(0), + }; + let cancel = AtomicBool::new(false); + let outcome = run_fetch( + items, + &hosts, + &opts("t"), + &sink, + &cancel, + &|_: u64, _: u64| {}, + &|_: &str| {}, + ); + assert!(!outcome.ok()); + assert_eq!(outcome.items_ok, 0); + let err = outcome.error.unwrap(); + assert!(err.starts_with("t: item 9 failed after 5 attempts: "), "{err}"); + assert_eq!(stub.hits.lock().unwrap()["/missing"], 5); + } + + #[test] + fn cancel_before_start_returns_cancelled() { + let hosts = vec!["h".to_string()]; + let items = vec![item(1, &["http://127.0.0.1:9/x"], None)]; + let sink = CollectSink { + got: Mutex::new(HashMap::new()), + retry_once_id: None, + retried: AtomicUsize::new(0), + }; + let cancel = AtomicBool::new(true); + let outcome = run_fetch( + items, + &hosts, + &opts("t"), + &sink, + &cancel, + &|_: u64, _: u64| {}, + &|_: &str| {}, + ); + assert!(outcome.cancelled); + assert_eq!(outcome.items_ok, 0); + } +} diff --git a/app/src/main/rust/stores/src/fetch_tuning.rs b/app/src/main/rust/stores/src/fetch_tuning.rs new file mode 100644 index 0000000000..abbe2d4925 --- /dev/null +++ b/app/src/main/rust/stores/src/fetch_tuning.rs @@ -0,0 +1,211 @@ +//! The numbers and the two small types [`fetch_core`](crate::fetch_core) shares with the Steam +//! depot writer it was carved out of. +//! +//! In Bannerlator these live in `depot_writer.rs` and `cdn_client.rs`, next to the Steam CM and +//! depot code, so that the Steam engine and the store engines run on ONE set of tunables and their +//! `fetch-window` log lines stay A/B-comparable. DroidDeck ships no Steam depot engine (the Steam +//! client runs natively in the Linux runtime), so only this slice came across: the values are +//! copied verbatim and keep their names, which is what lets a GOG/Epic/Amazon log from either app +//! be read against the same tuning notes. +//! +//! The important behaviours are structural (slow-start ramp -> hold at plateau -> back off on REAL +//! errors, per-host cap, hard byte budget), not the exact numbers. The r3/r4 notes below record +//! why each number is what it is, because every one of them was arrived at by a device run that +//! went wrong the other way. + +use std::time::Duration; + +/// Default `User-Agent` of the pooled client. Every store engine overrides it per request with +/// the agent its own launcher sends (`FetchOptions::headers`), so this value is only what a +/// request without an explicit header would carry. +pub const USER_AGENT: &str = "Valve/Steam HTTP Client 1.0"; + +/// TCP connect deadline for every request the pooled client makes. +pub fn connect_timeout() -> Duration { + Duration::from_secs(15) +} + +/// Classified fetch failure so the fetch driver can back off proportionally (429/5xx/timeout/reset +/// shrink the window + cool the host harder than a one-off protocol error). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FetchFailKind { + Timeout, + RateLimited, + ServerFault, + Connect, + Other, +} + +#[derive(Clone, Debug)] +pub struct AsyncFetchError { + pub message: String, + pub kind: FetchFailKind, +} + +/// Attempts per item before the run gives up on it (fetch failures and sink retries both count). +pub const MAX_CHUNK_ATTEMPTS: u32 = 5; + +/// Floor for the in-flight byte budget — the pre-r4 fixed value, so no run ever gets *less* +/// buffered memory than it had before the budget started scaling. See [`inflight_budget_bytes`]. +pub const FETCH_INFLIGHT_BUDGET_FLOOR_BYTES: u64 = 24 * 1024 * 1024; + +/// Hard ceiling for the in-flight byte budget. This is THE OOM guard: whatever the tier, the host +/// count or the chunk size, the RAW bytes in memory for one run can never exceed this. 96 MiB = 4× +/// the old fixed budget and still a small fraction of the heap an Android game installer runs in, +/// and it is only ever reached by a fast-tier run whose CDN pool is large enough to support a 64+ +/// window. Raising this is the one change in the fetch layer that can reintroduce OOM, so it stays +/// fixed and small. +pub const FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES: u64 = 96 * 1024 * 1024; + +/// Assumed RAW (on-the-wire, still-compressed) size of one chunk when sizing the budget. Store +/// chunks are ~1 MiB; this is the same nominal [`NOMINAL_CHUNK_RESERVE_BYTES`] uses at dispatch. +pub const TYPICAL_CHUNK_BYTES: u64 = NOMINAL_CHUNK_RESERVE_BYTES; + +/// Headroom the budget carries over "one chunk per window slot" (3/2 = 1.5×), as a rational so the +/// arithmetic stays integer. The window is a count of concurrent *requests*; a slot's bytes are held +/// from DISPATCH until its WRITE completes, so at any instant some slots hold a fetched-but-not-yet- +/// written chunk while others hold a reservation for a fetch still on the wire. 1.5× covers that +/// overlap (and the occasional above-nominal chunk) without doubling the memory bound. +pub const BUDGET_HEADROOM_NUM: u64 = 3; +pub const BUDGET_HEADROOM_DEN: u64 = 2; + +/// Byte budget for the *compressed* bytes buffered in the fetch→process channel, scaled to the +/// window ceiling this run can actually reach. +/// +/// The decouple channel is bounded by BYTES, not chunk count: a count cap makes in-flight memory +/// swing wildly with chunk size (a 4-chunk cap is ~4 MB for 1 MB chunks but ~120 MB for 30 MB +/// chunks). Budgeting bytes keeps the heap bounded regardless of game size while giving the fetch +/// pool bandwidth-delay-product headroom so the socket stays busy while process workers +/// decompress+write. +/// +/// r4: a device run proved a fixed 24 MiB budget becomes the binding constraint the moment the +/// adaptive window works — on a 1 Gbps line the window grew to 42 but the budget admitted only ~24 +/// chunks, so `in_flight` stalled at 23–28 with `budget_stalls` in the hundreds and +/// `host_stalls=0`. A budget that refuses slots the window has already decided are safe is not a +/// memory guard, it is a throttle. So the budget scales WITH the effective ceiling (tier ∧ +/// hosts×per-host-cap ∧ hard cap) and is clamped between the old fixed value and a hard cap. Low +/// tiers are unaffected; a fast link gets a budget that stops binding before the window does. +/// +/// Budget only the RAW bytes actually queued (reserved at DISPATCH, released on WRITE-COMPLETE); the +/// decoded expansion is transient inside a process worker and never accumulates. See +/// [`budget_admits`] for the single-chunk deadlock guard. +pub fn inflight_budget_bytes(effective_ceiling: usize) -> u64 { + (effective_ceiling as u64) + .saturating_mul(TYPICAL_CHUNK_BYTES) + .saturating_mul(BUDGET_HEADROOM_NUM) + .saturating_div(BUDGET_HEADROOM_DEN) + .clamp( + FETCH_INFLIGHT_BUDGET_FLOOR_BYTES, + FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES, + ) +} + +// ─── Adaptive-window tuning ──────────────────────────────────────────────────────────────────── +// r3 retune: the first device run pinned the window at the floor on a 1 Gbps link. The controller +// shrank on throughput dips and latency jitter (normal CDN noise) as well as on errors, shrank a flat +// −6 on every single error, and only grew when throughput rose >10% in a 700 ms sample. The shrinks +// dominated, so the window ratcheted to the floor and stayed there. r3 makes growth eager and makes +// ERRORS the only shrink signal. + +/// Max concurrent requests to any single CDN host. CDNs throttle/reset per host, so a big window +/// is reached by spreading across MORE hosts, never by piling onto one. +pub const PER_HOST_CAP: usize = 6; +/// Initial in-flight window. Slow-start doubles from here, so this only sets how fast the first +/// couple of probes get useful — the ceiling still comes from the tier × hosts clamp. +pub const BOOTSTRAP_WINDOW: usize = 8; +/// Hard safety ceiling on the window regardless of tier (also clamped to distinct-hosts × per-host). +pub const WINDOW_HARD_CAP: usize = 256; +/// The window never shrinks below this (keeps a little pipelining even on a rough link). +pub const WINDOW_MIN_FLOOR: usize = 2; +/// Additive growth step once slow-start has ended (after the first real back-off). +pub const WINDOW_STEP_UP: usize = 4; +/// Slow-start multiplier: until the first back-off the window DOUBLES each healthy probe, so a fast +/// link reaches a useful window in seconds instead of never. +pub const WINDOW_SLOW_START_FACTOR: usize = 2; +/// Proportional shrink on a real error signal (never a flat step: a flat −6 slammed a small window +/// straight to the floor). Always at least −1 while above the floor. +pub const WINDOW_SHRINK_FACTOR: f64 = 0.75; +/// How often the adaptive window re-evaluates. Long enough that one jittery sample cannot drive a +/// decision; decisions use the EWMA, not the raw sample. +pub const WINDOW_PROBE_INTERVAL_MS: u64 = 2_000; +/// After a back-off, hold the window (no growth) for this long — hysteresis against oscillation. +pub const WINDOW_COOLDOWN_MS: u64 = 3_000; +/// EWMA smoothing for the measured throughput samples (higher = more responsive). +pub const WINDOW_BPS_EWMA_ALPHA: f64 = 0.4; +/// Throughput above `best × (1 + this)` counts as a genuine improvement (keeps slow-start climbing). +pub const WINDOW_IMPROVE_EPS: f64 = 0.05; +/// Throughput below `best × (1 − this)` counts as falling — growth STOPS (hold). It never shrinks: +/// only real errors shrink. +pub const WINDOW_DECLINE_EPS: f64 = 0.15; +/// Error-rate thresholds over one probe window: above HIGH shrinks; growth needs at/below LOW +/// (in between = hold). +pub const WINDOW_ERR_RATE_HIGH: f64 = 0.10; +pub const WINDOW_ERR_RATE_LOW: f64 = 0.05; +/// Errors inside ONE probe interval that trigger an immediate (don't-wait-for-the-probe) shrink. A +/// single stray timeout no longer moves the window; a burst does. A 429 always shrinks immediately. +pub const WINDOW_ERR_BURST_IMMEDIATE: u32 = 3; +/// Consecutive non-improving probes still allowed to grow before declaring a plateau — throughput +/// lags a window change by a probe or two, so one flat sample must not stop the ramp. +pub const WINDOW_PLATEAU_PATIENCE: u32 = 2; +/// Once plateaued, re-arm the ramp this long later (doubling, capped) to re-probe for headroom if the +/// link improved. Bounded re-probing, not a creep: a weak link answers with errors and shrinks back. +pub const WINDOW_PLATEAU_REARM_MS: u64 = 30_000; +pub const WINDOW_PLATEAU_REARM_MAX_MS: u64 = 300_000; +/// Even when the window does not change, emit one `fetch-window` line every N probes so a stuck +/// window explains itself in the log (`reason=` says why it is not moving). +pub const WINDOW_LOG_EVERY_PROBES: u32 = 5; +/// Every Nth exploit dispatch is instead an exploration pick (epsilon-greedy ≈ 1/N) so a +/// demoted-but-recovered host is retried rather than starved forever. +pub const SERVER_EXPLORE_EVERY: u64 = 8; +/// Reservation for an item whose compressed size the manifest didn't carry (keeps the byte budget +/// honest at dispatch time). +pub const NOMINAL_CHUNK_RESERVE_BYTES: u64 = 1024 * 1024; +/// Extra host cooldown floor when a host answers 429 (rate-limited) — back off harder than a +/// one-off timeout. +pub const RATE_LIMIT_COOLDOWN_MS: u64 = 5_000; + +/// Exponential back-off between attempts of one item: 0, 300, 600, 1200, 2400 ms … capped at 4 s. +pub fn retry_backoff_millis(attempt: u32) -> u64 { + if attempt == 0 { + return 0; + } + (300u64 << (attempt - 1)).min(4000) +} + +/// Byte-budget admission test for the fetch→process channel. Always admits when nothing is in +/// flight (`in_flight == 0`) so a single item LARGER than the whole budget can never deadlock — it +/// simply waits until the channel drains, then goes through alone. +#[inline] +pub fn budget_admits(in_flight: u64, raw_len: u64, budget: u64) -> bool { + in_flight == 0 || in_flight.saturating_add(raw_len) <= budget +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_is_exponential_and_capped() { + assert_eq!(retry_backoff_millis(0), 0); + assert_eq!(retry_backoff_millis(1), 300); + assert_eq!(retry_backoff_millis(2), 600); + assert_eq!(retry_backoff_millis(4), 2400); + assert_eq!(retry_backoff_millis(5), 4000); + assert_eq!(retry_backoff_millis(12), 4000); + } + + #[test] + fn budget_scales_between_floor_and_cap() { + assert_eq!(inflight_budget_bytes(1), FETCH_INFLIGHT_BUDGET_FLOOR_BYTES); + assert_eq!(inflight_budget_bytes(8), FETCH_INFLIGHT_BUDGET_FLOOR_BYTES); + assert_eq!(inflight_budget_bytes(32), 48 * 1024 * 1024); + assert_eq!(inflight_budget_bytes(1024), FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES); + } + + #[test] + fn single_oversized_item_is_always_admitted() { + assert!(budget_admits(0, u64::MAX, 1)); + assert!(budget_admits(10, 5, 15)); + assert!(!budget_admits(10, 6, 15)); + } +} diff --git a/app/src/main/rust/stores/src/gog.rs b/app/src/main/rust/stores/src/gog.rs new file mode 100644 index 0000000000..465d7346fa --- /dev/null +++ b/app/src/main/rust/stores/src/gog.rs @@ -0,0 +1,121 @@ +//! GOG gen2 chunk download engine — the Rust replacement for the byte-fetching inner loop of +//! `GogDownloadManager.java` (`assembleDepotFile` / `fetchChunkVerified` driven from the +//! per-file thread pool). Everything around that loop — token/builds/manifest head, base-product +//! + language depot selection, secure-link resolution, disk guard, `_gog_manifest.json` marker, +//! prefs, exe resolution, DLC markers, redist orchestration — stays in Java and is untouched. +//! +//! Parity contract: Bannerlator's `docs/RUST_GOG_PARITY.md`. The adapter is fed the SAME inflated depot-manifest +//! JSON strings Java already parsed, the SAME resolved CDN base (secure link) and the SAME +//! install dir, and produces the SAME on-disk layout (`.bhtmp` staging, atomic rename, +//! size+MD5-verified finals) so a download can be resumed by either engine. +//! +//! Layout: +//! - [`plan`] — depot-manifest JSON → file/chunk plan (mirrors `parseDepotManifest`), CDN path +//! and chunk-URL builders, largest-first ordering. +//! - [`engine`] — resume/skip pass (mirrors `fileVerified`), `FetchItem` list, the `FetchSink` +//! (compressed size/MD5 → inflate → decompressed size/MD5 → positioned write → +//! whole-file size/MD5 → rename), summary logging. +//! - [`jni`] — `GogNative` JNI exports (`nativeStart` / `nativeCancel` / `nativeRelease`). + +pub mod engine; +pub mod jni; +pub mod plan; + +use std::fs::File; +use std::io::Read; +use std::path::Path; + +/// Lowercase hex of `bytes` (matches `GogDownloadManager.toHex`). +pub fn to_hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for b in bytes { + out.push(DIGITS[(b >> 4) as usize] as char); + out.push(DIGITS[(b & 0x0f) as usize] as char); + } + out +} + +/// Lowercase hex MD5 of an in-memory buffer (`GogDownloadManager.md5Hex`). +pub fn md5_hex(data: &[u8]) -> String { + let mut hasher = crate::md5_small::Md5::new(); + hasher.update(data); + to_hex(&hasher.finalize()) +} + +/// Streaming lowercase hex MD5 of a file, never buffering the whole file +/// (`GogDownloadManager.md5HexFile`). `None` when the file cannot be read. +pub fn md5_hex_file(path: &Path) -> Option { + let mut file = File::open(path).ok()?; + let mut hasher = crate::md5_small::Md5::new(); + let mut buf = vec![0u8; 128 * 1024]; + loop { + let n = file.read(&mut buf).ok()?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Some(to_hex(&hasher.finalize())) +} + +/// zlib-inflate a chunk body (`GogDownloadManager.inflateZlib`): `None` when the body is not a +/// zlib stream (first byte != 0x78) or fails to inflate — the caller then treats the body as a +/// stored (non-compressed) chunk, exactly like Java's `if (inflated == null) inflated = raw`. +/// A truncated stream yields the partial output (Java's loop breaks on `n == 0` the same way); +/// the decompressed-size check that follows rejects it. +pub fn inflate_zlib(data: &[u8]) -> Option> { + if data.len() < 2 || data[0] != 0x78 { + return None; + } + let mut out = Vec::with_capacity(data.len().saturating_mul(3)); + let mut decoder = flate2::read::ZlibDecoder::new(data); + match decoder.read_to_end(&mut out) { + Ok(_) => Some(out), + Err(_) => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hex_is_lowercase_and_padded() { + assert_eq!(to_hex(&[0x00, 0x0a, 0xff, 0x10]), "000aff10"); + } + + #[test] + fn inflate_roundtrip_and_stored_fallback() { + use flate2::write::ZlibEncoder; + use flate2::Compression; + use std::io::Write; + let payload = b"hello gog chunk payload".repeat(64); + let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&payload).unwrap(); + let compressed = enc.finish().unwrap(); + assert_eq!(compressed[0], 0x78); + assert_eq!(inflate_zlib(&compressed).unwrap(), payload); + // Not zlib → None (stored chunk path). + assert!(inflate_zlib(b"\x1f\x8bnot zlib").is_none()); + assert!(inflate_zlib(b"\x78").is_none()); + // Truncated zlib → partial output, like Java's `while (!inf.finished()) { if (n == 0) break; }` + // loop; the decompressed-size check downstream rejects it either way. + let mut truncated = compressed.clone(); + let mid = truncated.len() / 2; + truncated.truncate(mid); + assert!(inflate_zlib(&truncated).map(|out| out != payload).unwrap_or(true)); + // Corrupt zlib body → None (Java: DataFormatException → null → raw fallback). + let mut corrupt = compressed.clone(); + for b in corrupt.iter_mut().skip(2) { + *b = !*b; + } + assert!(inflate_zlib(&corrupt).is_none() || inflate_zlib(&corrupt).unwrap() != payload); + } + + #[test] + fn md5_hex_matches_rfc1321_vector() { + assert_eq!(md5_hex(b""), "d41d8cd98f00b204e9800998ecf8427e"); + assert_eq!(md5_hex(b"abc"), "900150983cd24fb0d6963f7d28e17f72"); + } +} diff --git a/app/src/main/rust/stores/src/gog/engine.rs b/app/src/main/rust/stores/src/gog/engine.rs new file mode 100644 index 0000000000..0e70fde17f --- /dev/null +++ b/app/src/main/rust/stores/src/gog/engine.rs @@ -0,0 +1,1653 @@ +//! The GOG gen2 fetch loop on top of the shared fetch core. One call = one Java pool run +//! (`runGen2` base loop, `doInstallDlc` loop, or one `assembleDependencyInstaller`). +//! +//! What this mirrors, and where (see Bannerlator's `docs/RUST_GOG_PARITY.md`): +//! - resume/skip: [`file_verified`] = `GogDownloadManager.fileVerified` (exists, non-empty, size +//! when known, MD5 when known) — a verified file is reported as "Verified…" progress with NO +//! bytes credited, exactly like the Java task; +//! - per chunk: compressed size → compressed MD5 → inflate (stored fallback) → decompressed size → +//! decompressed MD5 (`fetchChunkVerified`); a mismatch is a retryable failure (Java: hard fail +//! ≤3 with backoff; the core: ≤5 attempts with backoff); +//! - per file: write into `.bhtmp` (positioned, so chunks may land out of order), then +//! whole-file size + MD5, delete any existing final, rename (`assembleDepotFile`); +//! - failure of any file aborts the run (`anyFailed`), cancel aborts the run; in both cases every +//! unfinished `.bhtmp` is deleted (Java deletes its tmp on the failing/cancelled thread). + +use std::collections::HashSet; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; + +use super::plan::{ + build_cdn_path, build_chunk_url, build_plan, host_key, parse_gen1_manifest, Gen1File, + PlannedFile, +}; +use super::{inflate_zlib, md5_hex, md5_hex_file}; + +/// Per-request timeout on chunk fetches (Java: `TIMEOUT = 30_000` connect + read). +pub const CHUNK_TIMEOUT: Duration = Duration::from_secs(30); +/// Java sends this on every content-system / CDN request (`fetchBytesEx`). +pub const USER_AGENT: &str = "GOG Galaxy"; +/// Java staging suffix (`assembleDepotFile`). +pub const TMP_SUFFIX: &str = ".bhtmp"; +/// Floor for the per-host cap (the core's default); a single-host store gets the whole ceiling. +pub const PER_HOST_CAP_FLOOR: usize = 6; + +/// `max(6, ceil(max_workers / distinct_hosts))` — GOG's one CDN host gets the whole ceiling. +pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize { + let hosts = distinct_hosts.max(1); + let workers = max_workers.max(1); + PER_HOST_CAP_FLOOR.max(workers.div_ceil(hosts)) +} + +/// Which Java loop this run replaces. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum PlanKind { + /// gen2: depot manifests → chunks → `assembleDepotFile` (base install, DLC, dependencies). + #[default] + Gen2Chunks, + /// gen1: build manifest → per-file HTTP Range GET streamed to disk (`runGen1`). + Gen1Ranges, +} + +impl PlanKind { + /// JNI mapping (`GogNative.KIND_*`): 0 = gen2 chunks, 1 = gen1 ranges, anything else → gen2. + pub fn from_i32(v: i32) -> Self { + if v == 1 { + PlanKind::Gen1Ranges + } else { + PlanKind::Gen2Chunks + } + } +} + +/// Inputs — exactly what the Java manager has in hand after manifest + secure-link resolution. +#[derive(Clone, Debug, Default)] +pub struct GogRequest { + pub kind: PlanKind, + /// gen2: inflated depot-manifest JSON strings, in the order Java fetched them (already filtered + /// by base-product / DLC-product and language in Java). gen1: the inflated build manifest. + pub depot_manifests: Vec, + /// gen2: resolved CDN base from `parseCdnUrl` (secure-link query string kept), or the + /// unauthenticated dependency store base. gen1: unused (file URLs live in the manifest). + pub cdn_base: String, + pub install_dir: String, + /// Files already completed by an earlier run of this same download (secure-link refresh + /// re-run): counted as done WITHOUT re-hashing and WITHOUT a progress event. + pub skip_paths: Vec, + pub ca_bundle_path: String, + /// Java pool size: `resolveDownloadThreads` (base), 8 (DLC), 1 (dependency). + pub max_workers: usize, + /// Inflate/hash/write threads; Java did that on the same pool threads, so pass the same count. + pub process_workers: usize, + /// Base install: largest-first (LPT) ordering; DLC / dependency: manifest order. + pub sort_largest_first: bool, + /// Log prefix, e.g. `gog base=1207658930`. + pub label: String, +} + +/// Callbacks out of the engine (JNI listener or a test recorder). Must be callable from the +/// process-pool threads. +pub trait GogEvents: Sync { + /// One file reached its final state. `verified` = resume-skip (no bytes credited); otherwise a + /// freshly assembled + verified file (`file_bytes` credited). `files_done` counts both. + #[allow(clippy::too_many_arguments)] + fn on_file_done( + &self, + rel_path: &str, + file_bytes: u64, + verified: bool, + files_done: u32, + files_total: u32, + bytes_done: u64, + bytes_total: u64, + ); + fn on_log(&self, line: &str); +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct GogRunResult { + pub success: bool, + pub cancelled: bool, + /// The run died on an HTTP code Java treats as an expired secure link (401/403/404/500) — + /// the Java caller refreshes the link (cap 5) and re-runs with `skip_paths` filled in. + pub link_expiry: bool, + pub error: String, + /// Decompressed bytes of files assembled by THIS run (Java `totalBytes`). + pub bytes_written: u64, + /// Files done = verified-skipped + assembled (Java `doneCount`), excluding `skip_paths`. + pub files_done: u32, + pub files_verified: u32, + pub files_total: u32, +} + +/// `GogDownloadManager.fileVerified`: true only if the file exists, is non-empty, matches the +/// expected size when known and the expected MD5 when known. +pub fn file_verified(path: &Path, expected_size: u64, expected_md5: &str) -> bool { + let Ok(meta) = fs::metadata(path) else { + return false; + }; + if !meta.is_file() || meta.len() == 0 { + return false; + } + if expected_size > 0 && meta.len() != expected_size { + return false; + } + if !expected_md5.is_empty() { + return match md5_hex_file(path) { + Some(actual) => actual.eq_ignore_ascii_case(expected_md5), + None => false, + }; + } + true +} + +/// Finds an HTTP status in a fetch-core error string (`non-200 HTTP status (403)`, `status 403`, +/// `HTTP 403`, `http=403`, `code=403`). The link-expiry set is Java's `fetchChunkVerified` list. +pub fn http_status_in(message: &str) -> Option { + let lower = message.to_ascii_lowercase(); + for marker in ["status (", "status ", "http ", "http=", "code=", "code "] { + let mut from = 0; + while let Some(pos) = lower[from..].find(marker) { + let start = from + pos + marker.len(); + let digits: String = lower[start..].chars().take_while(|c| c.is_ascii_digit()).collect(); + if digits.len() == 3 { + if let Ok(code) = digits.parse::() { + if (100..600).contains(&code) { + return Some(code); + } + } + } + from = start; + } + } + None +} + +/// Java: `code == 401 || code == 403 || code == 404 || code == 500` → refresh the secure link. +pub fn is_link_expiry_status(code: u16) -> bool { + matches!(code, 401 | 403 | 404 | 500) +} + +struct FileState { + handle: Option>, + chunks_done: usize, + finalized: bool, + failed: bool, + tmp_path: PathBuf, + out_path: PathBuf, +} + +struct SpeedSampler { + started: Instant, + bucket_secs: u64, + bucket_bytes: u64, + peak_bps: f64, + wire_bytes: u64, +} + +impl SpeedSampler { + fn new() -> Self { + Self { + started: Instant::now(), + bucket_secs: 0, + bucket_bytes: 0, + peak_bps: 0.0, + wire_bytes: 0, + } + } + + fn record(&mut self, wire_len: u64) { + self.wire_bytes = self.wire_bytes.saturating_add(wire_len); + let now_secs = self.started.elapsed().as_secs(); + if now_secs != self.bucket_secs { + let bps = self.bucket_bytes as f64; + if bps > self.peak_bps { + self.peak_bps = bps; + } + self.bucket_secs = now_secs; + self.bucket_bytes = 0; + } + self.bucket_bytes = self.bucket_bytes.saturating_add(wire_len); + } + + fn finish(&mut self) -> (u64, f64, f64, f64) { + let bps = self.bucket_bytes as f64; + if bps > self.peak_bps { + self.peak_bps = bps; + } + let elapsed = self.started.elapsed().as_secs_f64().max(0.001); + let avg_mbps = self.wire_bytes as f64 * 8.0 / 1_000_000.0 / elapsed; + let peak_mbps = self.peak_bps * 8.0 / 1_000_000.0; + (self.wire_bytes, elapsed, avg_mbps, peak_mbps) + } +} + +/// Positioned writer for ONE chunk's decompressed byte range inside the file's `.bhtmp`: hashes +/// everything it is given (the decompressed MD5) and counts it, but only writes bytes inside the +/// chunk's expected range when the size is known — an inflate overshoot then fails the size +/// check instead of clobbering the neighbouring chunk. An I/O error is remembered so the sink can +/// tell "disk failed" (fatal) from "stream failed" (retry). +struct RangeWriter { + file: Arc, + base: u64, + expected: u64, + written: u64, + md5: crate::md5_small::Md5, + io_error: Option, +} + +impl RangeWriter { + fn new(file: Arc, base: u64, expected: u64) -> Self { + Self { + file, + base, + expected, + written: 0, + md5: crate::md5_small::Md5::new(), + io_error: None, + } + } +} + +impl Write for RangeWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.md5.update(buf); + let start = self.written; + let end = start.saturating_add(buf.len() as u64); + let cap = if self.expected > 0 { self.expected } else { u64::MAX }; + if start < cap { + let n = (end.min(cap) - start) as usize; + if let Err(err) = self.file.write_all_at(&buf[..n], self.base.saturating_add(start)) { + self.io_error = Some(err.to_string()); + return Err(err); + } + } + self.written = end; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +enum ChunkMode { + Zlib(flate2::write::ZlibDecoder), + Stored(RangeWriter), +} + +/// Per-item streaming state for one attempt (reset whenever a piece arrives at `offset == 0`). +struct ChunkStream { + comp_md5: crate::md5_small::Md5, + comp_len: u64, + /// Decided on the first non-empty piece (`0x78` → zlib, else stored — Java's `inflateZlib`). + mode: Option, + /// The writer until the mode is decided (an empty body never decides). + writer: Option, +} + +impl ChunkStream { + fn new(file: Arc, base: u64, expected: u64) -> Self { + Self { + comp_md5: crate::md5_small::Md5::new(), + comp_len: 0, + mode: None, + writer: Some(RangeWriter::new(file, base, expected)), + } + } +} + +/// The `FetchSink`. Stream mode (what `run_gen2` uses): pieces of a chunk are inflated as they +/// arrive straight into the chunk's byte range of the file, the compressed MD5 and the +/// decompressed size/MD5 accumulate on the fly, and `on_finish` runs the checks in Java's order +/// (compressed size → compressed MD5 → inflate → size → MD5). Body mode (`process`) is the same +/// pipeline on a whole buffer and stays available for tests / a body-mode fallback. +struct GogSink<'a> { + files: &'a [PlannedFile], + /// item id → (file index, chunk index) + table: Vec<(usize, usize)>, + states: Vec>, + /// item id → in-flight stream attempt (stream mode only) + streams: Vec>>, + files_total: u32, + bytes_total: u64, + files_done: &'a AtomicU32, + bytes_done: &'a AtomicU64, + events: &'a dyn GogEvents, + speed: Mutex, +} + +impl<'a> GogSink<'a> { + fn log(&self, line: &str) { + self.events.on_log(line); + } + + /// `assembleDepotFile` head: `parent.mkdirs(); tmpFile.delete(); new FileOutputStream(tmp)` — + /// once per file, on its first chunk. Returns a shared handle for positioned writes. + fn file_handle(&self, file_idx: usize) -> Result, SinkError> { + let file = &self.files[file_idx]; + let mut st = self + .states + .get(file_idx) + .and_then(|m| m.lock().ok()) + .ok_or_else(|| SinkError::Fatal("file state poisoned".to_string()))?; + if st.failed || st.finalized { + return Err(SinkError::Fatal(format!( + "chunk for a closed file: {}", + file.relative_path + ))); + } + if let Some(h) = st.handle.as_ref() { + return Ok(Arc::clone(h)); + } + if let Some(parent) = st.tmp_path.parent() { + let _ = fs::create_dir_all(parent); + } + let _ = fs::remove_file(&st.tmp_path); + let opened = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&st.tmp_path); + match opened { + Ok(h) => { + let h = Arc::new(h); + st.handle = Some(Arc::clone(&h)); + Ok(h) + } + Err(err) => { + st.failed = true; + let msg = format!("open tmp failed file={} err={err}", file.relative_path); + self.log(&msg); + Err(SinkError::Fatal(msg)) + } + } + } + + /// One verified chunk landed: count it; on the file's last chunk run the whole-file verify + + /// rename and fire the per-file progress event (Java: doneCount++, totalBytes += df.totalSize, + /// "Downloading: "). + fn chunk_done(&self, file_idx: usize) -> Result<(), SinkError> { + let file = &self.files[file_idx]; + let mut st = self + .states + .get(file_idx) + .and_then(|m| m.lock().ok()) + .ok_or_else(|| SinkError::Fatal("file state poisoned".to_string()))?; + if st.failed || st.finalized { + return Err(SinkError::Fatal(format!( + "chunk for a closed file: {}", + file.relative_path + ))); + } + st.chunks_done += 1; + if st.chunks_done < file.chunks.len() { + return Ok(()); + } + if let Err(err) = self.finalize_file(file_idx, &mut st) { + st.failed = true; + return Err(SinkError::Fatal(err)); + } + drop(st); + let done = self.files_done.fetch_add(1, Ordering::Relaxed) + 1; + let bytes = self + .bytes_done + .fetch_add(file.total_size, Ordering::Relaxed) + + file.total_size; + self.events.on_file_done( + &file.relative_path, + file.total_size, + false, + done, + self.files_total, + bytes, + self.bytes_total, + ); + Ok(()) + } + + /// `assembleDepotFile` tail: whole-file size + MD5 on the tmp, delete existing final, rename. + fn finalize_file(&self, file_idx: usize, st: &mut FileState) -> Result<(), String> { + let file = &self.files[file_idx]; + if let Some(handle) = st.handle.take() { + drop(handle); + } + let actual = fs::metadata(&st.tmp_path).map(|m| m.len()).unwrap_or(0); + if file.total_size > 0 && actual != file.total_size { + self.log(&format!( + "FILE size mismatch file={} exp={} got={}", + file.relative_path, file.total_size, actual + )); + let _ = fs::remove_file(&st.tmp_path); + return Err(format!("file size mismatch: {}", file.relative_path)); + } + if !file.md5.is_empty() { + let ok = match md5_hex_file(&st.tmp_path) { + Some(h) => h.eq_ignore_ascii_case(&file.md5), + None => false, + }; + if !ok { + self.log(&format!("FILE md5 mismatch file={}", file.relative_path)); + let _ = fs::remove_file(&st.tmp_path); + return Err(format!("file md5 mismatch: {}", file.relative_path)); + } + } + if st.out_path.exists() { + let _ = fs::remove_file(&st.out_path); + } + if let Err(err) = fs::rename(&st.tmp_path, &st.out_path) { + self.log(&format!("FILE rename failed file={} err={err}", file.relative_path)); + let _ = fs::remove_file(&st.tmp_path); + return Err(format!("file rename failed: {}", file.relative_path)); + } + st.finalized = true; + Ok(()) + } + + /// Delete every unfinished `.bhtmp` (Java: `tmpFile.delete()` on the failing/cancelled task). + fn cleanup_partials(&self) { + for st in &self.states { + let Ok(mut st) = st.lock() else { + continue; + }; + if st.finalized { + continue; + } + if let Some(handle) = st.handle.take() { + drop(handle); + } + if st.chunks_done > 0 || st.tmp_path.exists() { + let _ = fs::remove_file(&st.tmp_path); + } + } + } + + fn unfinalized_pending(&self) -> usize { + self.states + .iter() + .filter(|st| st.lock().map(|s| !s.finalized).unwrap_or(true)) + .count() + } +} + +impl<'a> FetchSink for GogSink<'a> { + /// Body mode: `fetchChunkVerified` on a whole buffer, then a positioned write of the inflated + /// bytes (used by tests and available as a non-stream fallback). + fn process(&self, item: &FetchItem, body: Vec) -> Result { + let Some(&(file_idx, chunk_idx)) = self.table.get(item.id as usize) else { + return Err(SinkError::Fatal(format!("unknown item id {}", item.id))); + }; + let file = &self.files[file_idx]; + let chunk = &file.chunks[chunk_idx]; + if let Ok(mut speed) = self.speed.lock() { + speed.record(body.len() as u64); + } + + // fetchChunkVerified, in order: compressed size → compressed MD5 → inflate → size → MD5. + if chunk.compressed_size > 0 && body.len() as u64 != chunk.compressed_size { + let msg = format!( + "chunk compressed-size mismatch chunk={} exp={} got={}", + chunk.hash, + chunk.compressed_size, + body.len() + ); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + if !chunk.compressed_md5.is_empty() + && !md5_hex(&body).eq_ignore_ascii_case(&chunk.compressed_md5) + { + let msg = format!("chunk compressed-md5 mismatch chunk={}", chunk.hash); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + let inflated = match inflate_zlib(&body) { + Some(out) => out, + None => body, // stored (non-zlib) chunk + }; + if chunk.size > 0 && inflated.len() as u64 != chunk.size { + let msg = format!( + "chunk decompressed-size mismatch chunk={} exp={} got={}", + chunk.hash, + chunk.size, + inflated.len() + ); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + if !chunk.md5.is_empty() && !md5_hex(&inflated).eq_ignore_ascii_case(&chunk.md5) { + let msg = format!("chunk decompressed-md5 mismatch chunk={}", chunk.hash); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + + let handle = self.file_handle(file_idx)?; + if let Err(err) = handle.write_all_at(&inflated, chunk.offset) { + if let Some(mut st) = self.states.get(file_idx).and_then(|m| m.lock().ok()) { + st.failed = true; + let _ = st.handle.take(); + let _ = fs::remove_file(&st.tmp_path); + } + let msg = format!("write failed file={} err={err}", file.relative_path); + self.log(&msg); + return Err(SinkError::Fatal(msg)); + } + drop(handle); + let credited = inflated.len() as u64; + self.chunk_done(file_idx)?; + Ok(credited) + } + + /// Stream mode: hash the compressed piece, inflate it straight into the chunk's byte range + /// (or pass it through for a stored chunk), hashing + counting the decompressed output. + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + let id = item.id as usize; + let Some(&(file_idx, chunk_idx)) = self.table.get(id) else { + return Err(SinkError::Fatal(format!("unknown item id {}", item.id))); + }; + let file = &self.files[file_idx]; + let chunk = &file.chunks[chunk_idx]; + if let Ok(mut speed) = self.speed.lock() { + speed.record(data.len() as u64); + } + let mut slot = self + .streams + .get(id) + .and_then(|m| m.lock().ok()) + .ok_or_else(|| SinkError::Fatal("chunk stream poisoned".to_string()))?; + if offset == 0 || slot.is_none() { + // (Re)start of an attempt: fresh hashers + inflater; positioned writes re-cover the range. + let handle = self.file_handle(file_idx)?; + *slot = Some(ChunkStream::new(handle, chunk.offset, chunk.size)); + } + let Some(stream) = slot.as_mut() else { + return Err(SinkError::Fatal("chunk stream missing".to_string())); + }; + stream.comp_md5.update(data); + stream.comp_len = stream.comp_len.saturating_add(data.len() as u64); + if data.is_empty() { + return Ok(()); + } + if stream.mode.is_none() { + let writer = stream + .writer + .take() + .ok_or_else(|| SinkError::Fatal("chunk writer missing".to_string()))?; + stream.mode = Some(if data[0] == 0x78 { + ChunkMode::Zlib(flate2::write::ZlibDecoder::new(writer)) + } else { + ChunkMode::Stored(writer) // stored (non-zlib) chunk — Java's `inflated = raw` + }); + } + let result = match stream.mode.as_mut() { + Some(ChunkMode::Zlib(dec)) => dec + .write_all(data) + .map_err(|err| (err.to_string(), dec.get_ref().io_error.clone())), + Some(ChunkMode::Stored(w)) => w + .write_all(data) + .map_err(|err| (err.to_string(), w.io_error.clone())), + None => Ok(()), + }; + match result { + Ok(()) => Ok(()), + Err((_, Some(io))) => { + // Disk, not stream: Java's `fos.write` exception → file fails → run aborts. + if let Some(mut st) = self.states.get(file_idx).and_then(|m| m.lock().ok()) { + st.failed = true; + let _ = st.handle.take(); + let _ = fs::remove_file(&st.tmp_path); + } + let msg = format!("write failed file={} err={io}", file.relative_path); + self.log(&msg); + Err(SinkError::Fatal(msg)) + } + Err((err, None)) => { + // Corrupt zlib body. Java: inflate → null → raw → decompressed-size mismatch → retry. + let msg = format!("chunk inflate failed chunk={} err={err}", chunk.hash); + self.log(&msg); + *slot = None; + Err(SinkError::Retry(msg)) + } + } + } + + /// Stream mode: the body ended — run Java's checks in order and count the chunk. + fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result { + let id = item.id as usize; + let Some(&(file_idx, chunk_idx)) = self.table.get(id) else { + return Err(SinkError::Fatal(format!("unknown item id {}", item.id))); + }; + let file = &self.files[file_idx]; + let chunk = &file.chunks[chunk_idx]; + let taken = self + .streams + .get(id) + .and_then(|m| m.lock().ok()) + .ok_or_else(|| SinkError::Fatal("chunk stream poisoned".to_string()))? + .take(); + let stream = match taken { + Some(s) => s, + // Empty body with no pieces: same checks on zero bytes. + None => ChunkStream::new(self.file_handle(file_idx)?, chunk.offset, chunk.size), + }; + + if chunk.compressed_size > 0 && stream.comp_len != chunk.compressed_size { + let msg = format!( + "chunk compressed-size mismatch chunk={} exp={} got={}", + chunk.hash, chunk.compressed_size, stream.comp_len + ); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + if !chunk.compressed_md5.is_empty() + && !super::to_hex(&stream.comp_md5.finalize()).eq_ignore_ascii_case(&chunk.compressed_md5) + { + let msg = format!("chunk compressed-md5 mismatch chunk={}", chunk.hash); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + let writer = match stream.mode { + Some(ChunkMode::Zlib(dec)) => match dec.finish() { + Ok(w) => w, + Err(err) => { + let msg = format!("chunk inflate failed chunk={} err={err}", chunk.hash); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + }, + Some(ChunkMode::Stored(w)) => w, + None => match stream.writer { + Some(w) => w, + None => return Err(SinkError::Fatal("chunk writer missing".to_string())), + }, + }; + if let Some(io) = writer.io_error { + if let Some(mut st) = self.states.get(file_idx).and_then(|m| m.lock().ok()) { + st.failed = true; + let _ = st.handle.take(); + let _ = fs::remove_file(&st.tmp_path); + } + let msg = format!("write failed file={} err={io}", file.relative_path); + self.log(&msg); + return Err(SinkError::Fatal(msg)); + } + if chunk.size > 0 && writer.written != chunk.size { + let msg = format!( + "chunk decompressed-size mismatch chunk={} exp={} got={}", + chunk.hash, chunk.size, writer.written + ); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + if !chunk.md5.is_empty() + && !super::to_hex(&writer.md5.finalize()).eq_ignore_ascii_case(&chunk.md5) + { + let msg = format!("chunk decompressed-md5 mismatch chunk={}", chunk.hash); + self.log(&msg); + return Err(SinkError::Retry(msg)); + } + drop(writer.file); + self.chunk_done(file_idx)?; + Ok(0) + } +} + +/// Runs one download loop. Blocking; returns when every pending file is finalized, on the first +/// fatal file failure, or on cancel. Safe to call again with `skip_paths` extended by the files +/// this run reported done (that is how the Java side re-runs after a secure-link refresh). +pub fn run(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult { + match req.kind { + PlanKind::Gen2Chunks => run_gen2(req, cancel, events), + PlanKind::Gen1Ranges => run_gen1(req, cancel, events), + } +} + +fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult { + let files = build_plan(&req.depot_manifests, req.sort_largest_first); + let files_total = files.len() as u32; + let install_dir = PathBuf::from(&req.install_dir); + let skip: HashSet<&str> = req.skip_paths.iter().map(String::as_str).collect(); + let planned_bytes: u64 = files.iter().map(|f| f.total_size).sum(); + let chunk_count: usize = files.iter().map(|f| f.chunks.len()).sum(); + let max_workers = req.max_workers.max(1); + let process_workers = req.process_workers.max(1); + let host = host_key(&req.cdn_base); + let per_host_cap = per_host_cap_for(max_workers, 1); + + events.on_log(&format!( + "engine=rust kind=gen2 mode=stream label={} files={} chunks={} planned_bytes={} skip_paths={} workers={} per_host_cap={} process_workers={} sort_largest_first={} host={}", + req.label, + files_total, + chunk_count, + planned_bytes, + skip.len(), + max_workers, + per_host_cap, + process_workers, + req.sort_largest_first, + host + )); + + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let files_verified = AtomicU32::new(0); + + if files.is_empty() { + // Java: the caller already returned "no depot files collected" before the loop; reaching + // here means the two parsers disagree — surface it loudly rather than pretend success. + return GogRunResult { + success: false, + error: "no files in plan (manifest parse mismatch)".to_string(), + files_total, + ..Default::default() + }; + } + + // Resume / repair pass — Java does this per task on the pool threads; here it runs up front + // on `process_workers` threads so the fetch item list can exclude verified files. + let pending: Vec = files.iter().map(|_| AtomicBool::new(false)).collect(); + let next = AtomicUsize::new(0); + let verify_threads = process_workers.min(files.len().max(1)); + std::thread::scope(|scope| { + for _ in 0..verify_threads { + scope.spawn(|| loop { + if cancel.load(Ordering::Relaxed) { + break; + } + let idx = next.fetch_add(1, Ordering::Relaxed); + if idx >= files.len() { + break; + } + let file = &files[idx]; + if skip.contains(file.relative_path.as_str()) { + continue; + } + let out_path = install_dir.join(&file.relative_path); + if file_verified(&out_path, file.total_size, &file.md5) { + files_verified.fetch_add(1, Ordering::Relaxed); + let done = files_done.fetch_add(1, Ordering::Relaxed) + 1; + events.on_file_done( + &file.relative_path, + file.total_size, + true, + done, + files_total, + bytes_done.load(Ordering::Relaxed), + planned_bytes, + ); + } else { + pending[idx].store(true, Ordering::Relaxed); + } + }); + } + }); + if cancel.load(Ordering::Relaxed) { + return GogRunResult { + cancelled: true, + error: "cancelled".to_string(), + files_done: files_done.load(Ordering::Relaxed), + files_verified: files_verified.load(Ordering::Relaxed), + files_total, + ..Default::default() + }; + } + + // Fetch items: one per chunk of every pending file, in plan order (chunks in manifest order). + let mut table: Vec<(usize, usize)> = Vec::new(); + let mut items: Vec = Vec::new(); + let mut states: Vec> = Vec::new(); + let mut pending_files = 0u32; + let mut pending_bytes = 0u64; + for (file_idx, file) in files.iter().enumerate() { + let out_path = install_dir.join(&file.relative_path); + let tmp_path = PathBuf::from(format!("{}{}", out_path.display(), TMP_SUFFIX)); + let is_pending = pending[file_idx].load(Ordering::Relaxed); + states.push(Mutex::new(FileState { + handle: None, + chunks_done: 0, + // Files not pending (verified or skip-listed) are already final for this run. + finalized: !is_pending, + failed: false, + tmp_path, + out_path, + })); + if !is_pending { + continue; + } + pending_files += 1; + pending_bytes = pending_bytes.saturating_add(file.total_size); + for (chunk_idx, chunk) in file.chunks.iter().enumerate() { + let url = build_chunk_url(&req.cdn_base, &build_cdn_path(&chunk.hash)); + items.push(FetchItem { + id: table.len() as u64, + urls: vec![url], + reserve: chunk.compressed_size, + range: None, + }); + table.push((file_idx, chunk_idx)); + } + } + events.on_log(&format!( + "plan verified={} pending_files={} pending_chunks={} pending_bytes={}", + files_verified.load(Ordering::Relaxed), + pending_files, + items.len(), + pending_bytes + )); + + if items.is_empty() { + events.on_log("summary bytes=0 elapsed=0.000 avg_mbps=0.00 peak_mbps=0.00 (nothing to fetch)"); + return GogRunResult { + success: true, + bytes_written: 0, + files_done: files_done.load(Ordering::Relaxed), + files_verified: files_verified.load(Ordering::Relaxed), + files_total, + ..Default::default() + }; + } + + let streams: Vec>> = (0..table.len()).map(|_| Mutex::new(None)).collect(); + let sink = GogSink { + files: &files, + table, + states, + streams, + files_total, + bytes_total: planned_bytes, + files_done: &files_done, + bytes_done: &bytes_done, + events, + speed: Mutex::new(SpeedSampler::new()), + }; + let hosts = vec![host]; + let opts = FetchOptions { + max_workers, + // One CDN host gets the whole ceiling (Java has no per-host cap either). + per_host_cap, + timeout: CHUNK_TIMEOUT, + headers: vec![("User-Agent".to_string(), USER_AGENT.to_string())], + ca_bundle_path: req.ca_bundle_path.clone(), + process_workers, + label: req.label.clone(), + // Stream mode: pieces inflate straight to disk; no whole-body byte budget (the 24 MiB + // body budget + largest-first big chunks left 1-3 requests in flight at window 16). + stream: true, + }; + let log = |line: &str| events.on_log(line); + let progress = |_bytes: u64, _items_ok: u64| {}; + let outcome = run_fetch(items, &hosts, &opts, &sink, cancel, &progress, &log); + + let cancelled = outcome.cancelled || cancel.load(Ordering::Relaxed); + let mut error = outcome.error.clone().unwrap_or_default(); + let mut success = !cancelled && error.is_empty(); + if success && sink.unfinalized_pending() > 0 { + success = false; + error = format!( + "engine finished with {} unfinalized file(s)", + sink.unfinalized_pending() + ); + } + if !success { + sink.cleanup_partials(); + } + let link_expiry = !cancelled + && !success + && http_status_in(&error).map(is_link_expiry_status).unwrap_or(false); + + let (wire_bytes, elapsed, avg_mbps, peak_mbps) = sink + .speed + .lock() + .map(|mut s| s.finish()) + .unwrap_or((0, 0.0, 0.0, 0.0)); + events.on_log(&format!( + "summary bytes={} written={} elapsed={:.3} avg_mbps={:.2} peak_mbps={:.2} items_ok={} files_done={} verified={} success={} cancelled={} link_expiry={}{}", + wire_bytes, + bytes_done.load(Ordering::Relaxed), + elapsed, + avg_mbps, + peak_mbps, + outcome.items_ok, + files_done.load(Ordering::Relaxed), + files_verified.load(Ordering::Relaxed), + success, + cancelled, + link_expiry, + if error.is_empty() { String::new() } else { format!(" error={error}") } + )); + + GogRunResult { + success, + cancelled, + link_expiry, + error: if cancelled && error.is_empty() { "cancelled".to_string() } else { error }, + bytes_written: bytes_done.load(Ordering::Relaxed), + files_done: files_done.load(Ordering::Relaxed), + files_verified: files_verified.load(Ordering::Relaxed), + files_total, + } +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// gen1: per-file HTTP Range GET streamed to disk (`runGen1` / `downloadRange`) +// ───────────────────────────────────────────────────────────────────────────────────────────── + +struct Gen1State { + handle: Option, + out_path: PathBuf, + finalized: bool, +} + +/// Stream-mode sink for gen1. Java's `downloadRange`: `outFile.delete()` before EVERY attempt, +/// `new FileOutputStream(out)`, copy the Range body, no size/hash check; success → `doneG1++`, +/// `totalBytesG1 += size`, `"Downloading: "`. Pieces arrive in order with their +/// offset; `offset == 0` marks the (re)start of an attempt, exactly where Java deletes the file. +struct Gen1Sink<'a> { + files: &'a [Gen1File], + states: Vec>, + files_total: u32, + bytes_total: u64, + files_done: &'a AtomicU32, + bytes_done: &'a AtomicU64, + events: &'a dyn GogEvents, + speed: Mutex, +} + +impl<'a> FetchSink for Gen1Sink<'a> { + fn process(&self, item: &FetchItem, _body: Vec) -> Result { + Err(SinkError::Fatal(format!( + "gen1 sink is stream-only (item {})", + item.id + ))) + } + + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + let idx = item.id as usize; + let (Some(file), Some(state)) = (self.files.get(idx), self.states.get(idx)) else { + return Err(SinkError::Fatal(format!("unknown gen1 item {}", item.id))); + }; + if let Ok(mut speed) = self.speed.lock() { + speed.record(data.len() as u64); + } + let mut st = state + .lock() + .map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?; + if offset == 0 { + // New attempt: Java `outFile.delete()` + `new FileOutputStream(out)`. + st.handle = None; + if let Some(parent) = st.out_path.parent() { + let _ = fs::create_dir_all(parent); + } + let _ = fs::remove_file(&st.out_path); + match OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&st.out_path) + { + Ok(h) => st.handle = Some(h), + Err(err) => { + let msg = format!("gen1 open failed file={} err={err}", file.path); + self.events.on_log(&msg); + return Err(SinkError::Retry(msg)); + } + } + } + let Some(handle) = st.handle.as_ref() else { + return Err(SinkError::Retry(format!( + "gen1 piece before start file={}", + file.path + ))); + }; + if let Err(err) = handle.write_all_at(data, offset) { + let msg = format!("gen1 write failed file={} err={err}", file.path); + self.events.on_log(&msg); + st.handle = None; + return Err(SinkError::Retry(msg)); + } + Ok(()) + } + + fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result { + let idx = item.id as usize; + let (Some(file), Some(state)) = (self.files.get(idx), self.states.get(idx)) else { + return Err(SinkError::Fatal(format!("unknown gen1 item {}", item.id))); + }; + { + let mut st = state + .lock() + .map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?; + st.handle = None; // close + st.finalized = true; + } + let done = self.files_done.fetch_add(1, Ordering::Relaxed) + 1; + let bytes = self.bytes_done.fetch_add(file.size, Ordering::Relaxed) + file.size; + self.events.on_file_done( + &file.path, + file.size, + false, + done, + self.files_total, + bytes, + self.bytes_total, + ); + Ok(0) + } +} + +/// gen1 loop. Resume = `exists && length == size` ("Resuming…", no bytes). Items = one Range GET +/// per pending file; retries/back-off by the core (Java: 3 attempts, 1 s / 2 s). Failed or +/// cancelled attempts leave whatever was written, like Java (a wrong-size partial fails the +/// size-match resume and is re-pulled next time; a base-install cancel deletes the whole dir). +fn run_gen1(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult { + let mut files: Vec = Vec::new(); + for manifest in &req.depot_manifests { + parse_gen1_manifest(manifest, &mut files); + } + let files_total = files.len() as u32; + let install_dir = PathBuf::from(&req.install_dir); + let skip: HashSet<&str> = req.skip_paths.iter().map(String::as_str).collect(); + let planned_bytes: u64 = files.iter().map(|f| f.size).sum(); + let max_workers = req.max_workers.max(1); + let process_workers = req.process_workers.max(1); + let host = files + .first() + .map(|f| host_key(&f.url)) + .unwrap_or_else(|| "https://none".to_string()); + + let per_host_cap = per_host_cap_for(max_workers, 1); + events.on_log(&format!( + "engine=rust kind=gen1 mode=stream label={} files={} planned_bytes={} skip_paths={} workers={} per_host_cap={} process_workers={} host={}", + req.label, + files_total, + planned_bytes, + skip.len(), + max_workers, + per_host_cap, + process_workers, + host + )); + + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let mut files_verified = 0u32; + + if files.is_empty() { + return GogRunResult { + success: false, + error: "no files in gen1 plan (manifest parse mismatch)".to_string(), + files_total, + ..Default::default() + }; + } + + let mut items: Vec = Vec::new(); + let mut states: Vec> = Vec::with_capacity(files.len()); + let mut pending_bytes = 0u64; + for (idx, file) in files.iter().enumerate() { + if cancel.load(Ordering::Relaxed) { + return GogRunResult { + cancelled: true, + error: "cancelled".to_string(), + files_done: files_done.load(Ordering::Relaxed), + files_verified, + files_total, + ..Default::default() + }; + } + let out_path = install_dir.join(&file.path); + let skipped = skip.contains(file.path.as_str()); + let resumed = !skipped + && fs::metadata(&out_path) + .map(|m| m.len() == file.size) + .unwrap_or(false); + states.push(Mutex::new(Gen1State { + handle: None, + out_path, + finalized: skipped || resumed, + })); + if skipped { + continue; + } + if resumed { + files_verified += 1; + let done = files_done.fetch_add(1, Ordering::Relaxed) + 1; + events.on_file_done( + &file.path, + file.size, + true, + done, + files_total, + bytes_done.load(Ordering::Relaxed), + planned_bytes, + ); + continue; + } + pending_bytes = pending_bytes.saturating_add(file.size); + items.push(FetchItem { + id: idx as u64, + urls: vec![file.url.clone()], + reserve: file.size, + range: Some((file.offset, file.offset.saturating_add(file.size).saturating_sub(1))), + }); + } + events.on_log(&format!( + "plan resumed={} pending_files={} pending_bytes={}", + files_verified, + items.len(), + pending_bytes + )); + if items.is_empty() { + events.on_log("summary bytes=0 elapsed=0.000 avg_mbps=0.00 peak_mbps=0.00 (nothing to fetch)"); + return GogRunResult { + success: true, + files_done: files_done.load(Ordering::Relaxed), + files_verified, + files_total, + ..Default::default() + }; + } + + let sink = Gen1Sink { + files: &files, + states, + files_total, + bytes_total: planned_bytes, + files_done: &files_done, + bytes_done: &bytes_done, + events, + speed: Mutex::new(SpeedSampler::new()), + }; + let hosts = vec![host]; + let opts = FetchOptions { + max_workers, + per_host_cap, + timeout: CHUNK_TIMEOUT, + headers: vec![("User-Agent".to_string(), USER_AGENT.to_string())], + ca_bundle_path: req.ca_bundle_path.clone(), + process_workers, + label: req.label.clone(), + stream: true, + }; + let log = |line: &str| events.on_log(line); + let progress = |_bytes: u64, _items_ok: u64| {}; + let outcome = run_fetch(items, &hosts, &opts, &sink, cancel, &progress, &log); + + let cancelled = outcome.cancelled || cancel.load(Ordering::Relaxed); + let error = outcome.error.clone().unwrap_or_default(); + let success = !cancelled && error.is_empty(); + let (wire_bytes, elapsed, avg_mbps, peak_mbps) = sink + .speed + .lock() + .map(|mut s| s.finish()) + .unwrap_or((0, 0.0, 0.0, 0.0)); + events.on_log(&format!( + "summary bytes={} written={} elapsed={:.3} avg_mbps={:.2} peak_mbps={:.2} items_ok={} files_done={} resumed={} success={} cancelled={}{}", + wire_bytes, + bytes_done.load(Ordering::Relaxed), + elapsed, + avg_mbps, + peak_mbps, + outcome.items_ok, + files_done.load(Ordering::Relaxed), + files_verified, + success, + cancelled, + if error.is_empty() { String::new() } else { format!(" error={error}") } + )); + GogRunResult { + success, + cancelled, + link_expiry: false, // gen1 has no secure-link refresh in Java + error: if cancelled && error.is_empty() { "cancelled".to_string() } else { error }, + bytes_written: bytes_done.load(Ordering::Relaxed), + files_done: files_done.load(Ordering::Relaxed), + files_verified, + files_total, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use std::sync::Mutex as StdMutex; + + struct Recorder { + events: StdMutex>, + } + + impl GogEvents for Recorder { + fn on_file_done( + &self, + rel_path: &str, + file_bytes: u64, + verified: bool, + files_done: u32, + files_total: u32, + bytes_done: u64, + _bytes_total: u64, + ) { + self.events.lock().unwrap().push(format!( + "done {rel_path} bytes={file_bytes} verified={verified} {files_done}/{files_total} total={bytes_done}" + )); + } + fn on_log(&self, line: &str) { + self.events.lock().unwrap().push(format!("log {line}")); + } + } + + fn temp_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "gog_engine_{tag}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn file_verified_mirrors_java() { + let dir = temp_dir("verify"); + let path = dir.join("f.bin"); + assert!(!file_verified(&path, 3, ""), "missing → false"); + File::create(&path).unwrap(); + assert!(!file_verified(&path, 0, ""), "empty → false even when nothing is known"); + fs::write(&path, b"abc").unwrap(); + assert!(file_verified(&path, 0, ""), "exists+non-empty, nothing known → true"); + assert!(file_verified(&path, 3, "")); + assert!(!file_verified(&path, 4, "")); + assert!(file_verified(&path, 3, "900150983CD24FB0D6963F7D28E17F72"), "case-insensitive md5"); + assert!(!file_verified(&path, 3, "00000000000000000000000000000000")); + assert!(!file_verified(&dir, 0, ""), "a directory is never verified"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn http_status_detection() { + assert_eq!(http_status_in("non-200 HTTP status (403)"), Some(403)); + assert_eq!(http_status_in("item 12 failed after 5 attempts: http get: status 500 x"), Some(500)); + assert_eq!(http_status_in("HTTP 404 not found"), Some(404)); + assert_eq!(http_status_in("chunk=abc403def timed out"), None, "hash digits are not a status"); + assert_eq!(http_status_in("timeout"), None); + assert!(is_link_expiry_status(401)); + assert!(is_link_expiry_status(403)); + assert!(is_link_expiry_status(404)); + assert!(is_link_expiry_status(500)); + assert!(!is_link_expiry_status(429)); + assert!(!is_link_expiry_status(502)); + } + + #[test] + fn speed_sampler_reports_totals() { + let mut s = SpeedSampler::new(); + s.record(1_000_000); + s.record(1_000_000); + let (bytes, elapsed, avg, peak) = s.finish(); + assert_eq!(bytes, 2_000_000); + assert!(elapsed > 0.0); + assert!(avg > 0.0); + assert!(peak >= 8.0, "2 MB across at most two buckets → ≥8 Mbps peak, got {peak}"); + } + + /// Whole-file finalize through the sink with a synthetic body (no network): verifies the + /// chunk → positioned write → whole-file MD5 → rename path and the progress event shape. + #[test] + fn sink_assembles_verifies_and_renames() { + use flate2::write::ZlibEncoder; + use flate2::Compression; + + let dir = temp_dir("sink"); + let part_a = b"AAAAAAAAAA".to_vec(); + let part_b = b"bbbbb".to_vec(); + let whole = [part_a.clone(), part_b.clone()].concat(); + let z = |raw: &[u8]| { + let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); + enc.write_all(raw).unwrap(); + enc.finish().unwrap() + }; + let ca = z(&part_a); + let cb = z(&part_b); + let manifest = format!( + r#"{{"depot":{{"items":[{{"path":"sub\\out.bin","md5":"{}","chunks":[ + {{"compressedMd5":"{}","md5":"{}","compressedSize":{},"size":{}}}, + {{"compressedMd5":"{}","md5":"{}","compressedSize":{},"size":{}}}]}}]}}}}"#, + md5_hex(&whole), + md5_hex(&ca), md5_hex(&part_a), ca.len(), part_a.len(), + md5_hex(&cb), md5_hex(&part_b), cb.len(), part_b.len(), + ); + let files = build_plan(&[manifest], true); + assert_eq!(files.len(), 1); + let out_path = dir.join("sub/out.bin"); + let tmp_path = PathBuf::from(format!("{}{}", out_path.display(), TMP_SUFFIX)); + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let sink = GogSink { + files: &files, + table: vec![(0, 0), (0, 1)], + states: vec![Mutex::new(FileState { + handle: None, + chunks_done: 0, + finalized: false, + failed: false, + tmp_path: tmp_path.clone(), + out_path: out_path.clone(), + })], + streams: vec![Mutex::new(None), Mutex::new(None)], + files_total: 1, + bytes_total: whole.len() as u64, + files_done: &files_done, + bytes_done: &bytes_done, + events: &rec, + speed: Mutex::new(SpeedSampler::new()), + }; + let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None }; + + // Second chunk first: positioned write must still produce the right bytes. + assert_eq!(sink.process(&item(1), cb.clone()).unwrap(), part_b.len() as u64); + assert!(tmp_path.exists() && !out_path.exists(), "still staging after 1 of 2 chunks"); + // A corrupt body is a retryable failure, not fatal. + match sink.process(&item(0), b"\x78garbage".to_vec()) { + Err(SinkError::Retry(msg)) => assert!(msg.contains("mismatch"), "{msg}"), + Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"), + Ok(_) => panic!("expected Retry, got Ok"), + } + assert_eq!(sink.process(&item(0), ca.clone()).unwrap(), part_a.len() as u64); + assert!(out_path.exists() && !tmp_path.exists(), "renamed after the last chunk"); + assert_eq!(fs::read(&out_path).unwrap(), whole); + assert_eq!(files_done.load(Ordering::Relaxed), 1); + assert_eq!(bytes_done.load(Ordering::Relaxed), whole.len() as u64); + let events = rec.events.lock().unwrap(); + assert!(events.iter().any(|e| e == "done sub/out.bin bytes=15 verified=false 1/1 total=15"), "{events:?}"); + assert_eq!(sink.unfinalized_pending(), 0); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn sink_whole_file_md5_mismatch_is_fatal_and_deletes_tmp() { + let dir = temp_dir("sinkbad"); + let raw = b"stored chunk (not zlib)".to_vec(); + let manifest = format!( + r#"{{"depot":{{"items":[{{"path":"f.bin","md5":"00000000000000000000000000000000","chunks":[ + {{"compressedMd5":"{}","md5":"{}","compressedSize":{},"size":{}}}]}}]}}}}"#, + md5_hex(&raw), md5_hex(&raw), raw.len(), raw.len(), + ); + let files = build_plan(&[manifest], false); + let out_path = dir.join("f.bin"); + let tmp_path = PathBuf::from(format!("{}{}", out_path.display(), TMP_SUFFIX)); + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let sink = GogSink { + files: &files, + table: vec![(0, 0)], + states: vec![Mutex::new(FileState { + handle: None, + chunks_done: 0, + finalized: false, + failed: false, + tmp_path: tmp_path.clone(), + out_path: out_path.clone(), + })], + streams: vec![Mutex::new(None)], + files_total: 1, + bytes_total: raw.len() as u64, + files_done: &files_done, + bytes_done: &bytes_done, + events: &rec, + speed: Mutex::new(SpeedSampler::new()), + }; + let item = FetchItem { id: 0, urls: vec!["x".into()], reserve: 0, range: None }; + match sink.process(&item, raw) { + Err(SinkError::Fatal(msg)) => assert!(msg.contains("md5 mismatch"), "{msg}"), + Err(SinkError::Retry(msg)) => panic!("expected Fatal, got Retry({msg})"), + Ok(_) => panic!("expected Fatal, got Ok"), + } + assert!(!tmp_path.exists() && !out_path.exists()); + assert_eq!(files_done.load(Ordering::Relaxed), 0); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn run_with_everything_verified_fetches_nothing() { + let dir = temp_dir("runverified"); + let raw = b"already here".to_vec(); + fs::write(dir.join("a.bin"), &raw).unwrap(); + let manifest = format!( + r#"{{"depot":{{"items":[ + {{"path":"a.bin","md5":"{}","chunks":[{{"compressedMd5":"{}","size":{}}}]}}, + {{"path":"skipped.bin","chunks":[{{"compressedMd5":"deadbeef","size":4}}]}}]}}}}"#, + md5_hex(&raw), md5_hex(&raw), raw.len(), + ); + let req = GogRequest { + depot_manifests: vec![manifest], + cdn_base: "https://example.invalid/store?tok=1".to_string(), + install_dir: dir.to_string_lossy().into_owned(), + skip_paths: vec!["skipped.bin".to_string()], + max_workers: 4, + process_workers: 2, + sort_largest_first: true, + label: "test".to_string(), + ..Default::default() + }; + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let cancel = AtomicBool::new(false); + let res = run(&req, &cancel, &rec); + assert!(res.success, "{res:?}"); + assert_eq!(res.files_done, 1, "skip-listed file is neither hashed nor reported"); + assert_eq!(res.files_verified, 1); + assert_eq!(res.bytes_written, 0, "verified files credit no bytes (Java totalBytes)"); + let events = rec.events.lock().unwrap(); + assert!(events[0].starts_with("log engine=rust kind=gen2 mode=stream label=test files=2 chunks=2"), "{:?}", events[0]); + assert!(events.iter().any(|e| e == "done a.bin bytes=12 verified=true 1/2 total=0"), "{events:?}"); + assert!(!events.iter().any(|e| e.contains("skipped.bin")), "{events:?}"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn per_host_cap_gives_single_host_the_whole_ceiling() { + assert_eq!(per_host_cap_for(32, 1), 32); + assert_eq!(per_host_cap_for(32, 3), 11); + assert_eq!(per_host_cap_for(8, 1), 8); + assert_eq!(per_host_cap_for(4, 1), 6, "floor is the core default"); + assert_eq!(per_host_cap_for(0, 0), 6); + } + + /// Stream mode: zlib chunk split into pieces (with a poisoned first attempt restarting at + /// offset 0), a stored chunk, corrupt data → Retry, then the whole-file finalize. + #[test] + fn stream_sink_inflates_pieces_verifies_and_finalizes() { + use flate2::write::ZlibEncoder; + use flate2::Compression; + + let dir = temp_dir("stream"); + let part_a = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".repeat(50); // 2000 bytes, compresses well + let part_b = b"\x01\x02stored bytes".to_vec(); // first byte != 0x78 → stored + let whole = [part_a.clone(), part_b.clone()].concat(); + let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&part_a).unwrap(); + let ca = enc.finish().unwrap(); + assert_eq!(ca[0], 0x78); + let manifest = format!( + r#"{{"depot":{{"items":[{{"path":"out.bin","md5":"{}","chunks":[ + {{"compressedMd5":"{}","md5":"{}","compressedSize":{},"size":{}}}, + {{"compressedMd5":"{}","md5":"{}","compressedSize":{},"size":{}}}]}}]}}}}"#, + md5_hex(&whole), + md5_hex(&ca), md5_hex(&part_a), ca.len(), part_a.len(), + md5_hex(&part_b), md5_hex(&part_b), part_b.len(), part_b.len(), + ); + let files = build_plan(&[manifest], true); + let out_path = dir.join("out.bin"); + let tmp_path = PathBuf::from(format!("{}{}", out_path.display(), TMP_SUFFIX)); + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let sink = GogSink { + files: &files, + table: vec![(0, 0), (0, 1)], + states: vec![Mutex::new(FileState { + handle: None, + chunks_done: 0, + finalized: false, + failed: false, + tmp_path: tmp_path.clone(), + out_path: out_path.clone(), + })], + streams: vec![Mutex::new(None), Mutex::new(None)], + files_total: 1, + bytes_total: whole.len() as u64, + files_done: &files_done, + bytes_done: &bytes_done, + events: &rec, + speed: Mutex::new(SpeedSampler::new()), + }; + let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None }; + + // Attempt 1 of the zlib chunk: corrupt tail → Retry from on_chunk. + let mut corrupt = ca.clone(); + for b in corrupt.iter_mut().skip(2) { + *b = !*b; + } + let first_bad = sink.on_chunk(&item(0), 0, &corrupt); + match first_bad { + Err(SinkError::Retry(m)) => assert!(m.contains("inflate failed"), "{m}"), + Err(SinkError::Fatal(m)) => panic!("expected Retry, got Fatal({m})"), + Ok(()) => { + // Some corruptions only surface at finish: that must also be a Retry. + match sink.on_finish(&item(0), corrupt.len() as u64) { + Err(SinkError::Retry(_)) => {} + other => panic!("expected Retry at finish, got {}", match other { + Ok(_) => "Ok".to_string(), + Err(SinkError::Fatal(m)) => format!("Fatal({m})"), + Err(SinkError::Retry(m)) => format!("Retry({m})"), + }), + } + } + } + // Attempt 2: restart at offset 0 in three pieces. + let cut1 = ca.len() / 3; + let cut2 = 2 * ca.len() / 3; + sink.on_chunk(&item(0), 0, &ca[..cut1]).unwrap(); + sink.on_chunk(&item(0), cut1 as u64, &ca[cut1..cut2]).unwrap(); + sink.on_chunk(&item(0), cut2 as u64, &ca[cut2..]).unwrap(); + assert_eq!(sink.on_finish(&item(0), ca.len() as u64).unwrap(), 0); + assert!(tmp_path.exists() && !out_path.exists()); + // Stored chunk, one piece; wrong length first (→ Retry), then the real one. + sink.on_chunk(&item(1), 0, &part_b[..3]).unwrap(); + match sink.on_finish(&item(1), 3) { + Err(SinkError::Retry(m)) => assert!(m.contains("compressed-size mismatch"), "{m}"), + _ => panic!("expected Retry on short stored chunk"), + } + sink.on_chunk(&item(1), 0, &part_b).unwrap(); + assert_eq!(sink.on_finish(&item(1), part_b.len() as u64).unwrap(), 0); + assert!(out_path.exists() && !tmp_path.exists(), "renamed after the last chunk"); + assert_eq!(fs::read(&out_path).unwrap(), whole); + assert_eq!(files_done.load(Ordering::Relaxed), 1); + assert_eq!(bytes_done.load(Ordering::Relaxed), whole.len() as u64); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn gen1_sink_streams_pieces_and_restarts_on_offset_zero() { + let dir = temp_dir("gen1sink"); + let files = vec![Gen1File { + path: "sub/blob.bin".into(), + url: "https://example.invalid/blob".into(), + offset: 100, + size: 8, + }]; + let files_done = AtomicU32::new(0); + let bytes_done = AtomicU64::new(0); + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let sink = Gen1Sink { + files: &files, + states: vec![Mutex::new(Gen1State { + handle: None, + out_path: dir.join("sub/blob.bin"), + finalized: false, + })], + files_total: 1, + bytes_total: 8, + files_done: &files_done, + bytes_done: &bytes_done, + events: &rec, + speed: Mutex::new(SpeedSampler::new()), + }; + let item = FetchItem { id: 0, urls: vec!["x".into()], reserve: 8, range: Some((100, 107)) }; + // Attempt 1 writes a partial, attempt 2 restarts from offset 0 and truncates it. + sink.on_chunk(&item, 0, b"ZZZZZZZZZZZZ").unwrap(); + sink.on_chunk(&item, 0, b"abcd").unwrap(); + sink.on_chunk(&item, 4, b"efgh").unwrap(); + assert_eq!(sink.on_finish(&item, 8).unwrap(), 0); + assert_eq!(fs::read(dir.join("sub/blob.bin")).unwrap(), b"abcdefgh"); + assert_eq!(files_done.load(Ordering::Relaxed), 1); + assert_eq!(bytes_done.load(Ordering::Relaxed), 8); + let events = rec.events.lock().unwrap(); + assert!(events.iter().any(|e| e == "done sub/blob.bin bytes=8 verified=false 1/1 total=8"), "{events:?}"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn run_gen1_resumes_on_size_match_only() { + let dir = temp_dir("gen1run"); + fs::write(dir.join("ok.bin"), b"12345").unwrap(); + fs::write(dir.join("short.bin"), b"12").unwrap(); + let manifest = r#"{"depot":[{"files":[ + {"path":"ok.bin","url":"https://example.invalid/b","offset":0,"size":5}, + {"path":"short.bin","url":"https://example.invalid/b","offset":5,"size":5}, + {"path":"skipped.bin","url":"https://example.invalid/b","offset":10,"size":5}]}]}"#; + let req = GogRequest { + kind: PlanKind::Gen1Ranges, + depot_manifests: vec![manifest.to_string()], + install_dir: dir.to_string_lossy().into_owned(), + skip_paths: vec!["skipped.bin".to_string()], + max_workers: 2, + process_workers: 1, + ..Default::default() + }; + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + // Cancel before the fetch so no network is touched: the resume pass still runs. + let cancel = AtomicBool::new(false); + // Use a pre-set cancel via a second request: run the resume pass, then expect the fetch + // to be attempted for short.bin only. We can't reach the network here, so assert on the + // plan log line instead of the outcome. + cancel.store(true, Ordering::Relaxed); + let res = run(&req, &cancel, &rec); + assert!(res.cancelled, "{res:?}"); + let events = rec.events.lock().unwrap(); + assert!(events[0].starts_with("log engine=rust kind=gen1 mode=stream label= files=3"), "{:?}", events[0]); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn run_cancelled_before_fetch_reports_cancelled() { + let dir = temp_dir("runcancel"); + let manifest = r#"{"depot":{"items":[{"path":"a.bin","chunks":[{"compressedMd5":"ab","size":4}]}]}}"#; + let req = GogRequest { + depot_manifests: vec![manifest.to_string()], + cdn_base: "https://example.invalid/store".to_string(), + install_dir: dir.to_string_lossy().into_owned(), + max_workers: 1, + process_workers: 1, + ..Default::default() + }; + let rec = Recorder { events: StdMutex::new(Vec::new()) }; + let cancel = AtomicBool::new(true); + let res = run(&req, &cancel, &rec); + assert!(res.cancelled && !res.success); + assert_eq!(res.error, "cancelled"); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/app/src/main/rust/stores/src/gog/jni.rs b/app/src/main/rust/stores/src/gog/jni.rs new file mode 100644 index 0000000000..86f5ba840c --- /dev/null +++ b/app/src/main/rust/stores/src/gog/jni.rs @@ -0,0 +1,303 @@ +//! JNI facade for the GOG Rust download engine — the native half of +//! `com.droiddeck.launcher.stores.gog.GogNative` (Kotlin). Symbol names are bound to that +//! package/class; the `@JvmStatic external` declarations there are the other side of this file. +//! +//! Shape mirrors the Steam download JNI (`jni.rs` `nativeDownloadApp`): `nativeStart` validates +//! its inputs, takes a `GlobalRef` on the listener, spawns the worker thread and returns a handle +//! immediately; callbacks are delivered from the worker / process-pool threads through a +//! `JavaVM` attach; `nativeCancel` flips an `AtomicBool`; `nativeRelease` drops the Java side's +//! reference to the handle (the worker keeps its own until it exits). +//! +//! Log tag `GogNative`: every engine log line goes to logcat here AND to the listener's `onLog` +//! (which the Kotlin manager folds into its own debug log). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; + +use jni::objects::{GlobalRef, JClass, JObject, JObjectArray, JString, JValue}; +use jni::sys::{jboolean, jint, jlong, JNI_FALSE, JNI_TRUE}; +use jni::{JNIEnv, JavaVM}; + +use super::engine::{self, GogEvents, GogRequest, GogRunResult, PlanKind}; + +pub const LOG_TAG: &str = "GogNative"; + +#[cfg(target_os = "android")] +#[link(name = "log")] +unsafe extern "C" { + fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32; +} + +/// `android.util.Log.i(LOG_TAG, message)`; a no-op on non-Android hosts (cargo test). +pub fn android_log(message: &str) { + #[cfg(target_os = "android")] + { + use std::ffi::CString; + let Ok(tag) = CString::new(LOG_TAG) else { + return; + }; + let sanitized = message.replace('\0', " "); + let Ok(text) = CString::new(sanitized) else { + return; + }; + unsafe { + let _ = __android_log_write(4, tag.as_ptr().cast(), text.as_ptr().cast()); + } + } + #[cfg(not(target_os = "android"))] + { + let _ = message; + } +} + +/// One in-flight run. The Java side owns a `Box>` (the `jlong`); the worker thread owns a +/// second `Arc` so `nativeRelease` is safe at any time after `nativeStart` returned. +pub struct GogRunHandle { + pub cancel: AtomicBool, +} + +struct JniEvents { + vm: JavaVM, + listener: GlobalRef, +} + +impl JniEvents { + fn with_env(&self, f: impl FnOnce(&mut JNIEnv)) { + let Ok(mut env) = self.vm.attach_current_thread_as_daemon() else { + return; + }; + f(&mut env); + } + + fn clear_exception(env: &mut JNIEnv) { + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } + } + + fn complete(&self, result: &GogRunResult) { + self.with_env(|env| { + let Ok(error) = env.new_string(&result.error) else { + return; + }; + let error = JObject::from(error); + let _ = env.call_method( + self.listener.as_obj(), + "onComplete", + "(ZZZLjava/lang/String;JI)V", + &[ + JValue::Bool(if result.success { JNI_TRUE } else { JNI_FALSE }), + JValue::Bool(if result.cancelled { JNI_TRUE } else { JNI_FALSE }), + JValue::Bool(if result.link_expiry { JNI_TRUE } else { JNI_FALSE }), + JValue::Object(&error), + JValue::Long(result.bytes_written as jlong), + JValue::Int(result.files_done as jint), + ], + ); + Self::clear_exception(env); + let _ = env.delete_local_ref(error); + }); + } +} + +impl GogEvents for JniEvents { + fn on_file_done( + &self, + rel_path: &str, + file_bytes: u64, + verified: bool, + files_done: u32, + files_total: u32, + bytes_done: u64, + bytes_total: u64, + ) { + self.with_env(|env| { + let Ok(path) = env.new_string(rel_path) else { + return; + }; + let path = JObject::from(path); + let _ = env.call_method( + self.listener.as_obj(), + "onProgress", + "(JJIILjava/lang/String;JZ)V", + &[ + JValue::Long(bytes_done as jlong), + JValue::Long(bytes_total as jlong), + JValue::Int(files_done as jint), + JValue::Int(files_total as jint), + JValue::Object(&path), + JValue::Long(file_bytes as jlong), + JValue::Bool(if verified { JNI_TRUE } else { JNI_FALSE }), + ], + ); + Self::clear_exception(env); + let _ = env.delete_local_ref(path); + }); + } + + fn on_log(&self, line: &str) { + android_log(line); + self.with_env(|env| { + let Ok(text) = env.new_string(line) else { + return; + }; + let text = JObject::from(text); + let _ = env.call_method( + self.listener.as_obj(), + "onLog", + "(Ljava/lang/String;)V", + &[JValue::Object(&text)], + ); + Self::clear_exception(env); + let _ = env.delete_local_ref(text); + }); + } +} + +fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { + env.get_string(value) + .ok() + .map(|s| s.to_string_lossy().into_owned()) +} + +fn string_array_to_vec(env: &mut JNIEnv, array: &JObjectArray) -> Vec { + if array.is_null() { + return Vec::new(); + } + let len = env.get_array_length(array).unwrap_or(0); + let mut out = Vec::with_capacity(len.max(0) as usize); + for i in 0..len { + let Ok(element) = env.get_object_array_element(array, i) else { + continue; + }; + if element.is_null() { + continue; + } + let text = JString::from(element); + if let Some(s) = jstring_to_string(env, &text) { + out.push(s); + } + let _ = env.delete_local_ref(text); + } + out +} + +unsafe fn handle_ref(handle: jlong) -> Option<&'static Arc> { + let ptr = handle as *const Arc; + if ptr.is_null() { + None + } else { + Some(unsafe { &*ptr }) + } +} + +/// Symbol-binding probe (see `GogNative.isAvailable`). Returns 1. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeProbe( + _env: JNIEnv, + _class: JClass, +) -> jint { + 1 +} + +/// Starts one download loop on a worker thread and returns a handle (0 = could not start; the +/// listener then receives NO callbacks). Inputs are exactly what `GogDownloadManager` holds after +/// manifest + secure-link resolution — see `GogRequest`. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart( + mut env: JNIEnv, + _class: JClass, + kind: jint, + depot_manifests: JObjectArray, + cdn_base: JString, + install_dir: JString, + skip_paths: JObjectArray, + ca_bundle_path: JString, + max_workers: jint, + process_workers: jint, + sort_largest_first: jboolean, + label: JString, + listener: JObject, +) -> jlong { + if listener.is_null() { + android_log("nativeStart: null listener"); + return 0; + } + let Ok(vm) = env.get_java_vm() else { + android_log("nativeStart: JavaVM unavailable"); + return 0; + }; + let Ok(listener) = env.new_global_ref(&listener) else { + android_log("nativeStart: listener ref failed"); + return 0; + }; + let request = GogRequest { + kind: PlanKind::from_i32(kind), + depot_manifests: string_array_to_vec(&mut env, &depot_manifests), + cdn_base: jstring_to_string(&mut env, &cdn_base).unwrap_or_default(), + install_dir: jstring_to_string(&mut env, &install_dir).unwrap_or_default(), + skip_paths: string_array_to_vec(&mut env, &skip_paths), + ca_bundle_path: jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(), + max_workers: max_workers.max(1) as usize, + process_workers: process_workers.max(1) as usize, + sort_largest_first: sort_largest_first != JNI_FALSE, + label: jstring_to_string(&mut env, &label).unwrap_or_else(|| "gog".to_string()), + }; + let needs_base = request.kind == PlanKind::Gen2Chunks; + if (needs_base && request.cdn_base.is_empty()) + || request.install_dir.is_empty() + || request.depot_manifests.is_empty() + { + android_log("nativeStart: invalid request (empty cdn base / install dir / manifests)"); + return 0; + } + + let handle = Arc::new(GogRunHandle { + cancel: AtomicBool::new(false), + }); + let worker = Arc::clone(&handle); + let spawned = thread::Builder::new() + .name("gog-dl".to_string()) + .spawn(move || { + let events = JniEvents { vm, listener }; + let result = engine::run(&request, &worker.cancel, &events); + events.complete(&result); + }); + if spawned.is_err() { + android_log("nativeStart: worker thread spawn failed"); + return 0; + } + Box::into_raw(Box::new(handle)) as jlong +} + +/// Requests cancellation; the run stops at its next scheduling point and `onComplete` fires with +/// `cancelled = true`. Idempotent. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCancel( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + if let Some(run) = unsafe { handle_ref(handle) } { + run.cancel.store(true, Ordering::Relaxed); + } +} + +/// Drops the Java side's reference. Call once, after `onComplete` (or after cancelling — the worker +/// keeps the run alive until it exits either way). +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeRelease( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + let ptr = handle as *mut Arc; + if ptr.is_null() { + return; + } + unsafe { + drop(Box::from_raw(ptr)); + } +} diff --git a/app/src/main/rust/stores/src/gog/plan.rs b/app/src/main/rust/stores/src/gog/plan.rs new file mode 100644 index 0000000000..830106dc12 --- /dev/null +++ b/app/src/main/rust/stores/src/gog/plan.rs @@ -0,0 +1,370 @@ +//! Depot-manifest JSON → download plan. Mirrors `GogDownloadManager.parseDepotManifest` rule for +//! rule (see Bannerlator's `docs/RUST_GOG_PARITY.md` §3): +//! - root `depot.items[]`; `path` with `\` → `/` and one leading `/` stripped; +//! - a file is kept only when `path` is non-empty AND `chunks[]` is non-empty; +//! - per chunk: CDN hash = `compressedMd5` when non-empty, else `md5`; a chunk with neither is +//! dropped; `compressedSize` / `size` default 0; +//! - file `md5` kept when non-empty; `total_size` = Σ chunk `size` (decompressed); +//! - a file whose chunks all dropped is not added. +//! `org.json` coercion (`optString` turns numbers/bools into their text, `optLong` parses numeric +//! strings) is reproduced by [`opt_string`] / [`opt_u64`]. +//! +//! Additionally each chunk carries `offset` = prefix-sum of the preceding chunks' decompressed +//! sizes — Java appends sequentially so it never needs one; the Rust sink writes chunks at that +//! offset as they arrive so the final bytes are identical. + +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChunkRef { + /// CDN-path hash (`compressedMd5` preferred, else `md5`). + pub hash: String, + /// Verify compressed bytes before inflate (may be empty). + pub compressed_md5: String, + /// Verify decompressed bytes after inflate (may be empty). + pub md5: String, + /// Expected compressed size, 0 if absent. + pub compressed_size: u64, + /// Expected decompressed size, 0 if absent. + pub size: u64, + /// Decompressed offset inside the file (prefix-sum of preceding `size`s). + pub offset: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlannedFile { + /// Install-dir-relative path, forward slashes, no leading slash. + pub relative_path: String, + /// Whole-file MD5 from the manifest ("" when absent). + pub md5: String, + /// Σ chunk decompressed sizes (0 if unknown). + pub total_size: u64, + pub chunks: Vec, +} + +/// `JSONObject.optString(key, "")` semantics: strings as-is, numbers/bools rendered, null/absent +/// → "". (`org.json` renders `JSONObject.NULL` as "null"; GOG manifests never carry JSON nulls in +/// these fields, and treating one as absent is the only safe reading.) +pub fn opt_string(obj: &Value, key: &str) -> String { + match obj.get(key) { + Some(Value::String(s)) => s.clone(), + Some(Value::Number(n)) => n.to_string(), + Some(Value::Bool(b)) => b.to_string(), + _ => String::new(), + } +} + +/// `JSONObject.optLong(key, 0)` semantics for a size field (numbers, or numeric strings; a +/// negative/float value that Java would truncate is clamped to 0 — sizes are never negative). +pub fn opt_u64(obj: &Value, key: &str) -> u64 { + match obj.get(key) { + Some(Value::Number(n)) => n + .as_u64() + .or_else(|| n.as_i64().map(|v| v.max(0) as u64)) + .or_else(|| n.as_f64().map(|v| if v > 0.0 { v as u64 } else { 0 })) + .unwrap_or(0), + Some(Value::String(s)) => s.trim().parse::().map(|v| v.max(0) as u64).unwrap_or(0), + _ => 0, + } +} + +/// Parses one inflated gen2 depot manifest and appends its files to `out` +/// (`GogDownloadManager.parseDepotManifest`). Malformed JSON adds nothing, like Java's catch-all. +pub fn parse_depot_manifest(json: &str, out: &mut Vec) { + let Ok(root) = serde_json::from_str::(json) else { + return; + }; + let Some(items) = root + .get("depot") + .and_then(|d| d.get("items")) + .and_then(Value::as_array) + else { + return; + }; + for entry in items { + if !entry.is_object() { + continue; + } + let mut path = opt_string(entry, "path").replace('\\', "/"); + if let Some(stripped) = path.strip_prefix('/') { + path = stripped.to_string(); + } + let chunks = entry.get("chunks").and_then(Value::as_array); + let Some(chunks) = chunks else { + continue; + }; + if path.is_empty() || chunks.is_empty() { + continue; + } + let file_md5 = opt_string(entry, "md5"); + let mut planned = PlannedFile { + relative_path: path, + md5: file_md5, + total_size: 0, + chunks: Vec::with_capacity(chunks.len()), + }; + let mut offset = 0u64; + for chunk in chunks { + if !chunk.is_object() { + continue; + } + let compressed_md5 = opt_string(chunk, "compressedMd5"); + let dec_md5 = opt_string(chunk, "md5"); + let compressed_size = opt_u64(chunk, "compressedSize"); + let dec_size = opt_u64(chunk, "size"); + let hash = if !compressed_md5.is_empty() { + compressed_md5.clone() + } else { + dec_md5.clone() + }; + if hash.is_empty() { + continue; + } + planned.chunks.push(ChunkRef { + hash, + compressed_md5, + md5: dec_md5, + compressed_size, + size: dec_size, + offset, + }); + offset = offset.saturating_add(dec_size); + } + planned.total_size = offset; + if !planned.chunks.is_empty() { + out.push(planned); + } + } +} + +/// One gen1 file: a byte range of a (pre-signed) depot blob (`GogDownloadManager.Gen1File`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Gen1File { + /// Install-dir-relative path, used verbatim (Java's gen1 path does not normalise slashes). + pub path: String, + pub url: String, + pub offset: u64, + pub size: u64, +} + +/// Parses a gen1 build manifest (`runGen1`): every `depot[]` entry that is not `support: true`, +/// its `files[]` → (`path`, `url`, `offset` default 0, `size` default 0); an entry with +/// `size == 0` is dropped (Java's `path == null || url == null` checks never fire — `optString` +/// returns "" — so `size == 0` is the only effective filter, mirrored here). +pub fn parse_gen1_manifest(json: &str, out: &mut Vec) { + let Ok(root) = serde_json::from_str::(json) else { + return; + }; + let Some(depots) = root.get("depot").and_then(Value::as_array) else { + return; + }; + for depot in depots { + if !depot.is_object() { + continue; + } + if depot.get("support").and_then(Value::as_bool).unwrap_or(false) { + continue; + } + let Some(files) = depot.get("files").and_then(Value::as_array) else { + continue; + }; + for f in files { + if !f.is_object() { + continue; + } + let path = opt_string(f, "path"); + let offset = opt_u64(f, "offset"); + let size = opt_u64(f, "size"); + let url = opt_string(f, "url"); + if size == 0 { + continue; + } + out.push(Gen1File { path, url, offset, size }); + } + } +} + +/// Parses every manifest in order (the order Java fetched the depots) and optionally applies the +/// base-install largest-first (LPT) ordering — a STABLE sort by descending `total_size`, exactly +/// `Collections.sort(files, (a, b) -> Long.compare(b.totalSize, a.totalSize))`. The DLC and +/// dependency paths in Java do not sort, so they pass `sort_largest_first = false`. +pub fn build_plan(manifests: &[String], sort_largest_first: bool) -> Vec { + let mut files = Vec::new(); + for manifest in manifests { + parse_depot_manifest(manifest, &mut files); + } + if sort_largest_first { + files.sort_by(|a, b| b.total_size.cmp(&a.total_size)); + } + files +} + +/// `GogDownloadManager.buildCdnPath`: `"ab/cd/abcdef..."`. A hash shorter than 4 chars would +/// throw in Java (and fail that file); here it is returned unchanged and fails at fetch time. +pub fn build_cdn_path(hash: &str) -> String { + if !hash.is_char_boundary(2) || !hash.is_char_boundary(4) || hash.len() < 4 { + return hash.to_string(); + } + format!("{}/{}/{}", &hash[0..2], &hash[2..4], hash) +} + +/// `GogDownloadManager.buildChunkUrl`: appends the chunk path BEFORE any query string so the +/// secure-link token survives. +pub fn build_chunk_url(base: &str, chunk_path: &str) -> String { + match base.find('?') { + Some(q) => format!("{}/{}{}", &base[..q], chunk_path, &base[q..]), + None => format!("{base}/{chunk_path}"), + } +} + +/// Per-host key for the fetch core's host semaphores: `scheme://host[:port]` of `url` (lowercased +/// host). Java has one resolved CDN base per run, so the host list has exactly one entry. +pub fn host_key(url: &str) -> String { + let (scheme, rest) = match url.find("://") { + Some(i) => (&url[..i], &url[i + 3..]), + None => ("https", url), + }; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + format!("{}://{}", scheme.to_ascii_lowercase(), rest[..end].to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MANIFEST: &str = r#"{ + "depot": { "items": [ + { "path": "\\bin\\game.exe", "md5": "AABBCCDDEEFF00112233445566778899", + "chunks": [ + { "compressedMd5": "c1c1c1c1", "md5": "d1d1d1d1", "compressedSize": 100, "size": 250 }, + { "compressedMd5": "c2c2c2c2", "md5": "d2d2d2d2", "compressedSize": "50", "size": 75 } + ] }, + { "path": "/data/pak0.pak", + "chunks": [ + { "md5": "onlydec1", "size": 1000 }, + { "compressedSize": 5, "size": 5 } + ] }, + { "path": "empty.txt", "chunks": [] }, + { "path": "", "chunks": [ { "compressedMd5": "zz", "size": 1 } ] }, + { "path": "nohash.bin", "chunks": [ { "size": 9 } ] }, + { "path": "dir", "type": "DepotDirectory" }, + "not-an-object" + ] } }"#; + + #[test] + fn parse_mirrors_java_rules() { + let mut files = Vec::new(); + parse_depot_manifest(MANIFEST, &mut files); + assert_eq!(files.len(), 2, "empty-chunk, empty-path, no-hash and directory entries drop"); + + let exe = &files[0]; + assert_eq!(exe.relative_path, "bin/game.exe"); + assert_eq!(exe.md5, "AABBCCDDEEFF00112233445566778899"); + assert_eq!(exe.total_size, 325); + assert_eq!(exe.chunks.len(), 2); + assert_eq!(exe.chunks[0].hash, "c1c1c1c1"); + assert_eq!(exe.chunks[0].offset, 0); + assert_eq!(exe.chunks[1].hash, "c2c2c2c2"); + assert_eq!(exe.chunks[1].compressed_size, 50, "numeric string coerces like optLong"); + assert_eq!(exe.chunks[1].offset, 250); + + let pak = &files[1]; + assert_eq!(pak.relative_path, "data/pak0.pak"); + assert_eq!(pak.md5, "", "absent file md5 → empty (no whole-file verify)"); + assert_eq!(pak.chunks.len(), 1, "chunk without any hash is dropped"); + assert_eq!(pak.chunks[0].hash, "onlydec1", "hash falls back to md5"); + assert_eq!(pak.chunks[0].compressed_md5, ""); + assert_eq!(pak.total_size, 1000); + } + + #[test] + fn malformed_manifest_adds_nothing() { + let mut files = Vec::new(); + parse_depot_manifest("{not json", &mut files); + parse_depot_manifest(r#"{"depot": {}}"#, &mut files); + parse_depot_manifest(r#"{"items": []}"#, &mut files); + assert!(files.is_empty()); + } + + #[test] + fn plan_orders_largest_first_stably_only_when_asked() { + let m1 = r#"{"depot":{"items":[ + {"path":"a","chunks":[{"md5":"a1","size":10}]}, + {"path":"b","chunks":[{"md5":"b1","size":30}]}]}}"# + .to_string(); + let m2 = r#"{"depot":{"items":[ + {"path":"c","chunks":[{"md5":"c1","size":30}]}, + {"path":"d","chunks":[{"md5":"d1","size":20}]}]}}"# + .to_string(); + let unsorted = build_plan(&[m1.clone(), m2.clone()], false); + let order: Vec<&str> = unsorted.iter().map(|f| f.relative_path.as_str()).collect(); + assert_eq!(order, ["a", "b", "c", "d"], "depot order, file order within depot"); + + let sorted = build_plan(&[m1, m2], true); + let order: Vec<&str> = sorted.iter().map(|f| f.relative_path.as_str()).collect(); + assert_eq!(order, ["b", "c", "d", "a"], "stable: b (30) before c (30) keeps source order"); + } + + #[test] + fn gen1_parse_mirrors_run_gen1() { + let json = r#"{"installDirectory":"Game","depot":[ + {"support":true,"files":[{"path":"support.bin","url":"u","offset":0,"size":5}]}, + {"files":[ + {"path":"a.bin","url":"https://cdn.gog.com/blob?t=1","offset":10,"size":20}, + {"path":"zero.bin","url":"u","offset":0,"size":0}, + {"path":"b.bin","url":"https://cdn.gog.com/blob?t=1","size":"7"}, + "junk" + ]}, + {"nofiles":true} + ]}"#; + let mut files = Vec::new(); + parse_gen1_manifest(json, &mut files); + assert_eq!( + files, + vec![ + Gen1File { path: "a.bin".into(), url: "https://cdn.gog.com/blob?t=1".into(), offset: 10, size: 20 }, + Gen1File { path: "b.bin".into(), url: "https://cdn.gog.com/blob?t=1".into(), offset: 0, size: 7 }, + ] + ); + let mut none = Vec::new(); + parse_gen1_manifest(r#"{"depots":[]}"#, &mut none); + assert!(none.is_empty()); + } + + #[test] + fn cdn_path_and_chunk_url() { + assert_eq!(build_cdn_path("abcdef0123"), "ab/cd/abcdef0123"); + assert_eq!(build_cdn_path("abc"), "abc"); + assert_eq!( + build_chunk_url("https://cdn.gog.com/content-system/v2/store/1?token=x&y=1", "ab/cd/abcdef"), + "https://cdn.gog.com/content-system/v2/store/1/ab/cd/abcdef?token=x&y=1" + ); + assert_eq!( + build_chunk_url("https://gog-cdn-fastly.gog.com/content-system/v2/dependencies/store", "ab/cd/abcdef"), + "https://gog-cdn-fastly.gog.com/content-system/v2/dependencies/store/ab/cd/abcdef" + ); + } + + #[test] + fn host_key_strips_path_and_query() { + assert_eq!(host_key("https://CDN.gog.com/a/b?c=1"), "https://cdn.gog.com"); + assert_eq!(host_key("http://host:8080/x"), "http://host:8080"); + assert_eq!(host_key("nohost"), "https://nohost"); + } + + #[test] + fn opt_helpers_coerce_like_org_json() { + let v: Value = serde_json::from_str(r#"{"n": 12, "s": "34", "b": true, "z": null, "f": 2.5, "neg": -4}"#).unwrap(); + assert_eq!(opt_string(&v, "n"), "12"); + assert_eq!(opt_string(&v, "s"), "34"); + assert_eq!(opt_string(&v, "b"), "true"); + assert_eq!(opt_string(&v, "z"), ""); + assert_eq!(opt_string(&v, "missing"), ""); + assert_eq!(opt_u64(&v, "n"), 12); + assert_eq!(opt_u64(&v, "s"), 34); + assert_eq!(opt_u64(&v, "f"), 2); + assert_eq!(opt_u64(&v, "neg"), 0); + assert_eq!(opt_u64(&v, "b"), 0); + assert_eq!(opt_u64(&v, "missing"), 0); + } +} diff --git a/app/src/main/rust/stores/src/lib.rs b/app/src/main/rust/stores/src/lib.rs new file mode 100644 index 0000000000..8aeb3a6f42 --- /dev/null +++ b/app/src/main/rust/stores/src/lib.rs @@ -0,0 +1,28 @@ +//! DroidDeck's native store download engines (`libdroiddeckstores.so`). +//! +//! The GOG, Epic Games Store and Amazon Games download loops, ported from Bannerlator's +//! `bl-steam-client` crate (GPL-3.0-or-later, itself derived from WinNative's `wn-steam-client`). +//! Only the store engines and the fetch core they run on came across: DroidDeck's Steam client is +//! the real one, running in the Linux runtime, so none of the Steam CM / depot code is here. +//! +//! Each store module owns its manifest → plan → `FetchItem` mapping, its `FetchSink` (inflate + +//! hash + write in the exact output layout of the Kotlin manager that drives it) and its JNI facade. +//! The JNI contract — classes, method names, signatures, threading — is written down in `JNI.md` +//! next to `Cargo.toml`; the Kotlin side mirrors it exactly. +//! +//! What a run looks like: Kotlin hands over a manifest it already parsed (or, for Amazon, the +//! plan it built from one), the resolved CDN base(s) and the install directory, calls `nativeStart` +//! and gets a handle back at once. Everything after that happens on native threads: the fetch core +//! keeps an adaptive window of HTTP requests in flight on one tokio runtime, hands bodies to a sync +//! process pool that inflates, hashes and writes them, and reports progress through the listener +//! object Kotlin passed in. `nativeCancel` flips a flag; `onComplete` fires exactly once. + +#![allow(clippy::missing_safety_doc, clippy::result_large_err)] + +pub mod amazon; +pub mod epic; +pub mod fetch_core; +pub mod fetch_tuning; +pub mod gog; +pub mod md5_small; +pub mod stores_jni; diff --git a/app/src/main/rust/stores/src/md5_small.rs b/app/src/main/rust/stores/src/md5_small.rs new file mode 100644 index 0000000000..8abd58bcc4 --- /dev/null +++ b/app/src/main/rust/stores/src/md5_small.rs @@ -0,0 +1,197 @@ +//! Minimal MD5 (RFC 1321) for the store download engines (GOG chunk / file verification). +//! +//! In-crate on purpose: the crate is built `--locked` in CI with no way to add a dependency from +//! this environment, and the algorithm is 100 lines. Not for anything security-sensitive — it only +//! mirrors the checksum the Java managers already compare against. + +/// One-shot digest. +pub fn md5(data: &[u8]) -> [u8; 16] { + let mut h = Md5::new(); + h.update(data); + h.finalize() +} + +/// Lower-case hex of a 16-byte digest. +pub fn md5_hex(digest: &[u8; 16]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(32); + for b in digest { + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 0x0f) as usize] as char); + } + out +} + +/// Streaming MD5: `update` any number of times, then `finalize`. +#[derive(Clone, Debug)] +pub struct Md5 { + state: [u32; 4], + buffer: [u8; 64], + buffer_len: usize, + total_len: u64, +} + +impl Default for Md5 { + fn default() -> Self { + Self::new() + } +} + +const S: [u32; 64] = [ + 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, + 14, 20, 5, 9, 14, 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 6, 10, 15, + 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, +]; + +const K: [u32; 64] = [ + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391, +]; + +impl Md5 { + pub fn new() -> Self { + Self { + state: [0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476], + buffer: [0u8; 64], + buffer_len: 0, + total_len: 0, + } + } + + pub fn update(&mut self, mut data: &[u8]) { + self.total_len = self.total_len.wrapping_add(data.len() as u64); + if self.buffer_len > 0 { + let take = (64 - self.buffer_len).min(data.len()); + self.buffer[self.buffer_len..self.buffer_len + take].copy_from_slice(&data[..take]); + self.buffer_len += take; + data = &data[take..]; + if self.buffer_len == 64 { + let block = self.buffer; + self.compress(&block); + self.buffer_len = 0; + } + } + while data.len() >= 64 { + let mut block = [0u8; 64]; + block.copy_from_slice(&data[..64]); + self.compress(&block); + data = &data[64..]; + } + if !data.is_empty() { + self.buffer[..data.len()].copy_from_slice(data); + self.buffer_len = data.len(); + } + } + + pub fn finalize(mut self) -> [u8; 16] { + let bit_len = self.total_len.wrapping_mul(8); + let mut pad = [0u8; 72]; + pad[0] = 0x80; + // Pad to 56 mod 64, then append the 64-bit little-endian bit length. + let pad_len = if self.buffer_len < 56 { + 56 - self.buffer_len + } else { + 120 - self.buffer_len + }; + let total_len_before = self.total_len; + self.update(&pad[..pad_len]); + self.update(&bit_len.to_le_bytes()); + debug_assert_eq!(self.buffer_len, 0); + self.total_len = total_len_before; + let mut out = [0u8; 16]; + for (i, word) in self.state.iter().enumerate() { + out[i * 4..i * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + out + } + + fn compress(&mut self, block: &[u8; 64]) { + let mut m = [0u32; 16]; + for (i, word) in m.iter_mut().enumerate() { + *word = u32::from_le_bytes([ + block[i * 4], + block[i * 4 + 1], + block[i * 4 + 2], + block[i * 4 + 3], + ]); + } + let [mut a, mut b, mut c, mut d] = self.state; + for i in 0..64 { + let (f, g) = match i / 16 { + 0 => ((b & c) | (!b & d), i), + 1 => ((d & b) | (!d & c), (5 * i + 1) % 16), + 2 => (b ^ c ^ d, (3 * i + 5) % 16), + _ => (c ^ (b | !d), (7 * i) % 16), + }; + let f = f.wrapping_add(a).wrapping_add(K[i]).wrapping_add(m[g]); + a = d; + d = c; + c = b; + b = b.wrapping_add(f.rotate_left(S[i])); + } + self.state[0] = self.state[0].wrapping_add(a); + self.state[1] = self.state[1].wrapping_add(b); + self.state[2] = self.state[2].wrapping_add(c); + self.state[3] = self.state[3].wrapping_add(d); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hex(data: &[u8]) -> String { + md5_hex(&md5(data)) + } + + #[test] + fn rfc_1321_vectors() { + assert_eq!(hex(b""), "d41d8cd98f00b204e9800998ecf8427e"); + assert_eq!(hex(b"a"), "0cc175b9c0f1b6a831c399e269772661"); + assert_eq!(hex(b"abc"), "900150983cd24fb0d6963f7d28e17f72"); + assert_eq!(hex(b"message digest"), "f96b697d7cb7938d525a2f31aaf161d0"); + assert_eq!( + hex(b"abcdefghijklmnopqrstuvwxyz"), + "c3fcd3d76192e4007dfb496cca67e13b" + ); + assert_eq!( + hex(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), + "d174ab98d277d9f5a5611c2c9f419d9f" + ); + assert_eq!( + hex(b"12345678901234567890123456789012345678901234567890123456789012345678901234567890"), + "57edf4a22be3c955ac49da2e2107b67a" + ); + } + + #[test] + fn streaming_matches_one_shot_across_block_boundaries() { + let data: Vec = (0..1000u32).map(|i| (i * 7 % 251) as u8).collect(); + let whole = md5(&data); + for split in [0usize, 1, 55, 56, 63, 64, 65, 127, 128, 500, 999, 1000] { + let mut h = Md5::new(); + h.update(&data[..split]); + h.update(&data[split..]); + assert_eq!(h.finalize(), whole, "split={split}"); + } + let mut h = Md5::new(); + for b in &data { + h.update(std::slice::from_ref(b)); + } + assert_eq!(h.finalize(), whole); + } + + #[test] + fn padding_edge_lengths() { + // 55 / 56 / 64 bytes exercise the "pad fits in one block" vs "needs a second block" paths. + assert_eq!(hex(&[b'a'; 55]), "ef1772b6dff9a122358552954ad0df65"); + assert_eq!(hex(&[b'a'; 56]), "3b0c8ac703f828b04c6c197006d17218"); + assert_eq!(hex(&[b'a'; 64]), "014842d480b571495a4a0363793f7367"); + } +} diff --git a/app/src/main/rust/stores/src/stores_jni.rs b/app/src/main/rust/stores/src/stores_jni.rs new file mode 100644 index 0000000000..e69b7f5bd5 --- /dev/null +++ b/app/src/main/rust/stores/src/stores_jni.rs @@ -0,0 +1,26 @@ +//! The one export that is not a store engine: `com.droiddeck.launcher.stores.StoresNative`. +//! +//! `nativeVersion()` exists so the app can prove, at Setup time, that `libdroiddeckstores.so` +//! loaded and its JNI exports bind — before a user is mid-download and an +//! `UnsatisfiedLinkError` would surface as a failed install. It returns the crate version from +//! `Cargo.toml`, which the app can show next to the other component versions. + +use jni::objects::JClass; +use jni::sys::jstring; +use jni::JNIEnv; + +/// The crate version, as `Cargo.toml` declares it. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// `StoresNative.nativeVersion(): String` — the crate version (`"0.1.0"`). A null return means the +/// JVM could not allocate the string, which the caller treats like a failed load. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion( + env: JNIEnv, + _class: JClass, +) -> jstring { + match env.new_string(VERSION) { + Ok(s) => s.into_raw(), + Err(_) => std::ptr::null_mut(), + } +}