From 5ee71b4862b7105bbacbc74000faf1b806837bfe Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 15:34:24 -0600 Subject: [PATCH 001/112] chore(deps): refresh safe dependency bumps serde_json 1.0.154, proptest 1.12, base64 0.23, arboard 3.7, open 5.4.5, winresource 0.1.34, pollster 1.0, miniz_oxide 0.9, sha2 0.11.1, libc 0.2.191. --- Cargo.lock | 261 +++++++++++++------------------------- Cargo.toml | 6 +- apps/wordcraft/Cargo.toml | 2 +- crates/proof/Cargo.toml | 2 +- 4 files changed, 94 insertions(+), 177 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc052fa739..cfcbb47b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "arboard" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +checksum = "aeb9553e62312c8756a5979254fc6d7da8bbeb5986a1e17037fab932f7037917" dependencies = [ "clipboard-win", "image", @@ -235,7 +235,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.60.2", + "windows-sys 0.59.0", "wl-clipboard-rs", "x11rb", ] @@ -489,15 +489,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec 0.8.0", -] - [[package]] name = "bit-set" version = "0.10.0" @@ -508,10 +499,13 @@ dependencies = [ ] [[package]] -name = "bit-vec" -version = "0.8.0" +name = "bit-set" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.1", +] [[package]] name = "bit-vec" @@ -519,6 +513,16 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -580,6 +584,30 @@ dependencies = [ "piper", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "brotli" version = "8.0.4" @@ -1545,7 +1573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2842,9 +2870,9 @@ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -3457,9 +3485,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "open" -version = "5.4.4" +version = "5.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc" +checksum = "8d8c8b8e725ad26404d6896eb2f064294a0a7b4f1d9120210d85544684c4ca95" dependencies = [ "is-wsl", "libc", @@ -3516,7 +3544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3846,15 +3874,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "presser" version = "0.3.1" @@ -3911,16 +3930,17 @@ checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" [[package]] name = "proptest" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" dependencies = [ - "bit-set 0.8.0", - "bit-vec 0.8.0", + "bit-set 0.11.1", + "bit-vec 0.10.1", "bitflags 2.13.2", + "chacha20", + "core_detect", "num-traits", - "rand 0.9.5", - "rand_chacha", + "rand", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -3994,7 +4014,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.3", + "rand", "rand_pcg", "ring", "rustc-hash 2.1.3", @@ -4018,7 +4038,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4042,16 +4062,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha", - "rand_core 0.9.5", -] - [[package]] name = "rand" version = "0.10.3" @@ -4063,31 +4073,12 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - [[package]] name = "rand_core" version = "0.10.1" @@ -4105,11 +4096,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" dependencies = [ - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -4395,7 +4386,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4454,7 +4445,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4612,9 +4603,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -4701,9 +4692,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -4838,7 +4829,7 @@ dependencies = [ "const-oid 0.9.6", "der", "hex", - "rand 0.10.3", + "rand", "reqwest", "rustls-pki-types", "rustls-webpki", @@ -5238,10 +5229,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.5", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6278,7 +6269,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6399,7 +6390,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -6408,16 +6399,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -6435,31 +6417,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -6477,96 +6442,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winit" version = "0.30.13" @@ -6659,9 +6576,9 @@ dependencies = [ [[package]] name = "winresource" -version = "0.1.31" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" +checksum = "a2bc6d36480b829272a5bdabf08e6dca8004761c1da7e35efef800540c66221b" dependencies = [ "toml", "version_check", @@ -6716,7 +6633,7 @@ dependencies = [ "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "open", - "pollster 0.4.0", + "pollster 1.0.1", "rfd", "serde_json", "windows", @@ -6753,7 +6670,7 @@ version = "0.11.0" name = "wordcraft-doc" version = "0.11.0" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "proptest", "serde", "serde_json", @@ -6770,7 +6687,7 @@ dependencies = [ "chrono", "image", "log", - "miniz_oxide 0.8.9", + "miniz_oxide 0.9.1", "proptest", "serde_json", "sha1 0.11.0", @@ -6784,7 +6701,7 @@ name = "wordcraft-docx" version = "0.11.0" dependencies = [ "aes", - "base64 0.22.1", + "base64 0.23.1", "cfb", "chrono", "getrandom 0.3.4", @@ -6794,7 +6711,7 @@ dependencies = [ "proptest", "quick-xml 0.41.0", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "thiserror 2.0.21", "unicode-bidi", "wordcraft-doc", @@ -6939,7 +6856,7 @@ name = "wordcraft-proof" version = "0.11.0" dependencies = [ "log", - "miniz_oxide 0.8.9", + "miniz_oxide 0.9.1", "proptest", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index 80b5bcc1cd..165eeb1bbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0. quick-xml = "0.41" zip = { version = "3", default-features = false, features = ["deflate"] } # zlib streams (compressed metafile pictures in `.doc` drawings); already in the lock as a dependency of other crates. -miniz_oxide = "0.8" +miniz_oxide = "0.9" cfb = "0.15" # Legacy text encodings (GBK/GB18030, Big5, Shift_JIS, EUC-KR, single-byte code pages) for plain # text and CSV recipient lists (#778): encoding_rs ((Apache-2.0 OR MIT) AND BSD-3-Clause) and its @@ -71,7 +71,7 @@ aes = "0.9" sha1 = "0.11" sha2 = "0.11" hmac = "0.13" -base64 = "0.22" +base64 = "0.23" getrandom = "0.3" zeroize = "1" # Unix open flags (O_NONBLOCK for the engine's checked file reads, #3666); already in the lock as a @@ -86,7 +86,7 @@ egui_kittest = "0.37" # A monotonic clock that works on native and in the browser (`performance.now()`). web-time = "1.1" rfd = "0.17" -arboard = { version = "3.6", default-features = false, features = ["wayland-data-control"] } +arboard = { version = "3.7", default-features = false, features = ["wayland-data-control"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/apps/wordcraft/Cargo.toml b/apps/wordcraft/Cargo.toml index f1d27bc589..aee3872697 100644 --- a/apps/wordcraft/Cargo.toml +++ b/apps/wordcraft/Cargo.toml @@ -77,7 +77,7 @@ craft-print = { git = "https://github.com/storytold/craft-libs", rev = "17744430 rfd = { workspace = true } # Waits for an `rfd::AsyncFileDialog` on a background thread (the dialogs don't block the UI, #94); # rfd itself already depends on it. -pollster = "0.4" +pollster = "1" # Pictures and copied files on the system clipboard (#45); egui's paste only carries text. Already # in the build through egui-winit, with the same features. arboard = { workspace = true, features = ["image-data"] } diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 234684a9fe..4c1a5b1121 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.8" +miniz_oxide = "0.9" unicode-normalization = "0.1" [dev-dependencies] From a1aa17fc1421ba7e71e74570045dfd0621254ed7 Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:53:12 -0600 Subject: [PATCH 002/112] fix(deps): keep miniz_oxide at 0.8 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/proof/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfcbb47b57..065c2c8815 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6687,7 +6687,7 @@ dependencies = [ "chrono", "image", "log", - "miniz_oxide 0.9.1", + "miniz_oxide 0.8.9", "proptest", "serde_json", "sha1 0.11.0", @@ -6856,7 +6856,7 @@ name = "wordcraft-proof" version = "0.11.0" dependencies = [ "log", - "miniz_oxide 0.9.1", + "miniz_oxide 0.8.9", "proptest", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index 165eeb1bbd..4e5407a6a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0. quick-xml = "0.41" zip = { version = "3", default-features = false, features = ["deflate"] } # zlib streams (compressed metafile pictures in `.doc` drawings); already in the lock as a dependency of other crates. -miniz_oxide = "0.9" +miniz_oxide = "0.8" cfb = "0.15" # Legacy text encodings (GBK/GB18030, Big5, Shift_JIS, EUC-KR, single-byte code pages) for plain # text and CSV recipient lists (#778): encoding_rs ((Apache-2.0 OR MIT) AND BSD-3-Clause) and its diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 4c1a5b1121..234684a9fe 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.9" +miniz_oxide = "0.8" unicode-normalization = "0.1" [dev-dependencies] From 68407a34cdb7547353afdfca51ad81c577aa7a2d Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:57:11 -0600 Subject: [PATCH 003/112] chore(deps): bump miniz_oxide to 0.9 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/proof/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 065c2c8815..cfcbb47b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6687,7 +6687,7 @@ dependencies = [ "chrono", "image", "log", - "miniz_oxide 0.8.9", + "miniz_oxide 0.9.1", "proptest", "serde_json", "sha1 0.11.0", @@ -6856,7 +6856,7 @@ name = "wordcraft-proof" version = "0.11.0" dependencies = [ "log", - "miniz_oxide 0.8.9", + "miniz_oxide 0.9.1", "proptest", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index 4e5407a6a5..165eeb1bbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0. quick-xml = "0.41" zip = { version = "3", default-features = false, features = ["deflate"] } # zlib streams (compressed metafile pictures in `.doc` drawings); already in the lock as a dependency of other crates. -miniz_oxide = "0.8" +miniz_oxide = "0.9" cfb = "0.15" # Legacy text encodings (GBK/GB18030, Big5, Shift_JIS, EUC-KR, single-byte code pages) for plain # text and CSV recipient lists (#778): encoding_rs ((Apache-2.0 OR MIT) AND BSD-3-Clause) and its diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 234684a9fe..4c1a5b1121 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.8" +miniz_oxide = "0.9" unicode-normalization = "0.1" [dev-dependencies] From b729cf0cb444cb7741dde05ffa19cb4a5faef796 Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:57:14 -0600 Subject: [PATCH 004/112] fix(proof): keep inflate error output out of messages --- crates/proof/src/affix.rs | 4 ++-- crates/proof/src/dict.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c2580bca77..7fb7d968ac 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -877,8 +877,8 @@ impl AffixDict { /// Read data written by [`AffixDict::to_bytes`]. pub fn from_bytes(data: &[u8]) -> Result { - let raw = - miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20) + .map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?; let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; let mut lines = head.lines(); diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 83a54d4437..789fb6a870 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -78,7 +78,7 @@ impl Dictionary { /// Deserialize data written by [`Dictionary::to_bytes`]. pub fn from_bytes(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); From e73ce9c2742a08c37d186f413e0d8744bba100cd Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:15:41 +0100 Subject: [PATCH 005/112] #3642: .doc reader checks the compound file's DIFAT before cfb builds the FAT Finding 63. cfb 0.15 appends a whole sector of FAT entries for every FAT sector the DIFAT lists and validates the FAT only afterwards, so a small file whose DIFAT chain names one sector thousands of times made it allocate ~128x the file size (2048 DIFAT sectors: 133 MB, 370 ms) before failing. docbin now walks the header DIFAT and chain the way cfb does (same trimming, one step per sector) and refuses a file listing more FAT sectors than it has sectors, so the FAT is never larger than the file. Both is_encrypted and read_with_password go through it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/lib.rs | 68 ++++++++++++++++++++++++++++++++++++-- crates/docbin/src/tests.rs | 35 ++++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index b9179c7d28..b4e97a2193 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -88,10 +88,74 @@ pub enum DocbinError { /// Is this `.doc` password-protected (XOR obfuscation, RC4 or RC4 CryptoAPI)? False for /// anything that is not a readable Word 97-2003 file. pub fn is_encrypted(bytes: &[u8]) -> bool { - let Ok(mut comp) = cfb::CompoundFile::open(std::io::Cursor::new(bytes)) else { return false }; + let Ok(mut comp) = open_container(bytes) else { return false }; stream(&mut comp, "WordDocument").is_some_and(|w| fib::peek_encryption(&w).is_some()) } +/// Open the compound file around a `.doc`, after [`cfb_precheck`]. +fn open_container(bytes: &[u8]) -> Result>, DocbinError> { + cfb_precheck(bytes).map_err(DocbinError::Container)?; + cfb::CompoundFile::open(std::io::Cursor::new(bytes)).map_err(|e| DocbinError::Container(e.to_string())) +} + +/// Check a compound file's FAT layout before `cfb` reads it ([MS-CFB] §2.2 header, §2.5 +/// DIFAT). `cfb` appends a whole sector of FAT entries for every FAT sector the DIFAT lists and +/// only validates the FAT once it is built, so a small header naming one sector over and over +/// makes it allocate gigabytes. Here the DIFAT is walked the way `cfb` walks it (header entries, +/// then the DIFAT chain, at most one step per sector; trailing zero and free entries trimmed +/// the same way) and the file is refused when it lists more FAT sectors than the file has +/// sectors, so the FAT `cfb` builds is never larger than the file. Anything else that is wrong +/// is left for `cfb` to report. +fn cfb_precheck(bytes: &[u8]) -> Result<(), String> { + const MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; + const FREE: u32 = 0xFFFF_FFFF; + const END_OF_CHAIN: u32 = 0xFFFF_FFFE; + const HEADER_ENTRIES: usize = 109; + let u16_at = |at: usize| bytes.get(at..at.checked_add(2)?).map(|b| u16::from_le_bytes([b[0], b[1]])); + let u32_at = |at: usize| bytes.get(at..at.checked_add(4)?).map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + if bytes.len() < 512 || bytes.get(..8) != Some(MAGIC.as_slice()) { + return Ok(()); + } + let sector_len: usize = match u16_at(0x1A) { + Some(3) => 512, + Some(4) => 4096, + _ => return Ok(()), + }; + if bytes.len() < sector_len { + return Ok(()); + } + // `cfb` counts the sectors after the header sector, the last one possibly partial. + let num_sectors = bytes.len().div_ceil(sector_len) - 1; + let num_fat_sectors = u32_at(0x2C).unwrap_or(0) as usize; + let mut difat: Vec = (0..HEADER_ENTRIES).filter_map(|i| u32_at(0x4C + i * 4)).collect(); + let mut next = u32_at(0x44).unwrap_or(END_OF_CHAIN); + let mut steps = 0usize; + while next != END_OF_CHAIN && next != FREE { + steps += 1; + // `cfb` refuses a chain sector past the file or seen before; a chain longer than the + // file has sectors must have done one of the two. + if steps > num_sectors || next as usize >= num_sectors { + return Err(format!("the DIFAT chain is longer than the file or leaves it (sector {next})")); + } + let at = (next as usize + 1) * sector_len; + let entries = sector_len / 4 - 1; + difat.extend((0..entries).map(|i| u32_at(at + i * 4).unwrap_or(FREE))); + next = u32_at(at + entries * 4).unwrap_or(END_OF_CHAIN); + } + while difat.len() > HEADER_ENTRIES && difat.len() > num_fat_sectors && difat.last() == Some(&0) { + difat.pop(); + } + while difat.last() == Some(&FREE) { + difat.pop(); + } + // Each listed FAT sector adds a whole sector of entries. A file can't hold more FAT sectors + // than it has sectors, and below that the FAT is no larger than the file. + if difat.len() > num_sectors { + return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", difat.len())); + } + Ok(()) +} + /// Read a `.doc` (Word 97-2003 binary) file into a [`Document`]. A password-protected file /// gives [`DocbinError::PasswordRequired`]; see [`read_with_password`]. pub fn read(bytes: &[u8]) -> Result { @@ -102,7 +166,7 @@ pub fn read(bytes: &[u8]) -> Result { /// verifier ([`DocbinError::WrongPassword`] when it does not match) and then decrypted. The /// password is ignored for an unprotected file. pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result { - let mut comp = cfb::CompoundFile::open(std::io::Cursor::new(bytes)).map_err(|e| DocbinError::Container(e.to_string()))?; + let mut comp = open_container(bytes)?; let mut word = stream(&mut comp, "WordDocument").ok_or_else(|| DocbinError::NotWord("no WordDocument stream".into()))?; if word.len() as u64 > MAX_STREAM { return Err(DocbinError::Limit(format!("WordDocument stream is {} bytes", word.len()))); diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index 0a66c157b3..bbf9199543 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2392,3 +2392,38 @@ fn paragraph_borders_read_from_both_forms() { assert_eq!((top.style, top.width, top.space), (wordcraft_doc::props::BorderStyle::Single, 1.0, 2.0)); assert_eq!(top.color, Some(wordcraft_doc::Rgb(0xFF, 0, 0))); } + +/// #3642: a small compound file whose DIFAT names one FAT sector thousands of times made `cfb` +/// build a FAT of hundreds of megabytes before checking it. The precheck refuses it first. +#[test] +fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() { + const S: usize = 512; + const FREE: u32 = 0xFFFF_FFFF; + let chain = 2048u32; // DIFAT sectors 1..=chain, sector 0 is the "FAT" + let mut f = vec![0u8; S * (chain as usize + 2)]; + f[..8].copy_from_slice(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]); + f[0x18..0x1A].copy_from_slice(&0x3Eu16.to_le_bytes()); + f[0x1A..0x1C].copy_from_slice(&3u16.to_le_bytes()); + f[0x1C..0x1E].copy_from_slice(&0xFFFEu16.to_le_bytes()); + f[0x1E..0x20].copy_from_slice(&9u16.to_le_bytes()); + f[0x20..0x22].copy_from_slice(&6u16.to_le_bytes()); + f[0x2C..0x30].copy_from_slice(&FREE.to_le_bytes()); // no zero trimming + f[0x38..0x3C].copy_from_slice(&4096u32.to_le_bytes()); + f[0x44..0x48].copy_from_slice(&1u32.to_le_bytes()); + f[0x48..0x4C].copy_from_slice(&chain.to_le_bytes()); + for i in 0..109 { + f[0x4C + i * 4..0x50 + i * 4].copy_from_slice(&0u32.to_le_bytes()); + } + for d in 1..=chain { + let at = (d as usize + 1) * S; + let next = if d == chain { 0xFFFF_FFFE } else { d + 1 }; + f[at + S - 4..at + S].copy_from_slice(&next.to_le_bytes()); + } + let t = std::time::Instant::now(); + assert!(super::cfb_precheck(&f).is_err_and(|e| e.contains("FAT sectors in a file of"))); + assert!(matches!(read(&f), Err(super::DocbinError::Container(e)) if e.contains("FAT sectors in a file of"))); + assert!(!super::is_encrypted(&f)); + assert!(t.elapsed().as_secs() < 2, "refused in {:?}", t.elapsed()); + // A file cfb wrote itself passes. + assert!(super::cfb_precheck(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); +} From dc3a999081af0a0cc77358551aca83519f769df6 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:17:46 +0100 Subject: [PATCH 006/112] #3708: .doc hyperlink target copied once per run, charged to the char budget Finding 75. finish_field cloned the HYPERLINK target into every result character's CharProps and ParaBuild::push then compared it per character, so a long target over a long result was quadratic (250k x 250k: 2.6 s). Equal-formatted result characters now extend the current run; each run start copies the target once and charges its length to the document's character budget, past which the rest of the result is kept unlinked. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/lib.rs | 35 +++++++++++++++++++++++++++++------ crates/docbin/src/tests.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index b4e97a2193..484efb08ef 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -429,6 +429,14 @@ impl ParaBuild { } self.text.push(c); } + + /// Append `c` to the current run, whose properties the caller knows it matches. + fn extend(&mut self, c: char) { + if let Some(r) = self.runs.last_mut() { + r.len += c.len_utf8(); + } + self.text.push(c); + } } /// Walk a CP range, splitting paragraphs at 0x0D/0x07 marks, applying the direct character @@ -554,7 +562,7 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) - f.depth -= 1; field = Some(f); } else { - finish_field(&mut pb, f); + finish_field(&mut pb, f, ctx.budget); } } cp = cp.saturating_add(1); @@ -630,17 +638,32 @@ struct FieldBuild { /// Emit a finished field: HYPERLINK keeps its result text as a linked run; every other /// field becomes one inline object with its cached result. -fn finish_field(pb: &mut ParaBuild, f: FieldBuild) { +/// +/// The target is copied once per run of equally formatted result text, not per character, +/// and each copy is charged to the document's character `budget`; once that is spent the +/// rest of the result is kept as plain text (#3708). +fn finish_field(pb: &mut ParaBuild, f: FieldBuild, budget: &std::cell::Cell) { let instr = f.instr.trim().to_string(); let result = f.sep.unwrap_or_default(); - let result_text: String = result.iter().map(|(c, _)| *c).collect(); if let Some(target) = hyperlink_target(&instr) { - for (c, mut props) in result { - props.link = Some(target.clone()); - pb.push(c, props); + let cost = target.len() as u64; + let mut run: Option = None; + for (c, props) in result { + if run.as_ref() == Some(&props) { + pb.extend(c); + continue; + } + let mut linked = props.clone(); + if budget.get() >= cost { + budget.set(budget.get() - cost); + linked.link = Some(target.clone()); + } + pb.push(c, linked); + run = Some(props); } return; } + let result_text: String = result.iter().map(|(c, _)| *c).collect(); pb.push('\u{FFFC}', CharProps::default()); pb.objects.push(InlineObject::Field { instr, result: result_text, locked: false, dirty: false, code: false, form: String::new() }); } diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index bbf9199543..cf42176fe4 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2427,3 +2427,39 @@ fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() { // A file cfb wrote itself passes. assert!(super::cfb_precheck(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); } + +/// #3708: a long HYPERLINK target over a long result was copied (and compared) once per result +/// character. It is now copied once per run. +#[test] +fn a_long_hyperlink_over_a_long_result_is_one_linked_run() { + let n = 250_000; + let target = "x".repeat(n); + let text: &'static str = Box::leak(format!("\u{13}HYPERLINK \"http://{target}\"\u{14}{}\u{15}\r", "y".repeat(n)).into_boxed_str()); + let spec = structured_spec(text, text.chars().count() as u32); + let t = std::time::Instant::now(); + let doc = read(&struct_doc(&spec)).expect("opens"); + // Copying the target per character took ~2.5 s here; once per run takes milliseconds. + assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); + let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para"); + let linked: Vec<_> = p0.runs.iter().filter(|r| r.props.link.is_some()).collect(); + assert_eq!(linked.len(), 1); + assert_eq!(linked.first().map(|r| r.len), Some(n)); +} + +/// #3708: a result whose formatting changes every character pays for a target copy per run +/// out of the character budget; past it the text stays, unlinked. +#[test] +fn hyperlink_target_copies_are_charged_to_the_char_budget() { + use wordcraft_doc::props::CharProps; + let bold = CharProps { bold: Some(true), ..Default::default() }; + let result: Vec<(char, CharProps)> = (0..1000).map(|i| ('a', if i % 2 == 0 { bold.clone() } else { CharProps::default() })).collect(); + let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"{}\"", "u".repeat(100)), sep: Some(result) }; + let mut pb = super::ParaBuild::default(); + let budget = std::cell::Cell::new(10_000); + super::finish_field(&mut pb, f, &budget); + assert_eq!(pb.text.len(), 1000, "all text kept"); + let linked: usize = pb.runs.iter().filter_map(|r| r.props.link.as_ref()).map(|l| l.len()).sum(); + assert!(linked <= 10_000, "linked bytes {linked}"); + assert_eq!(pb.runs.iter().filter(|r| r.props.link.is_some()).count(), 100); + assert_eq!(budget.get(), 0); +} From 36e43fa6723ef06e1754210a6ec36108be7e7fb1 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:20:14 +0100 Subject: [PATCH 007/112] #3692: .doc piece Prm resolved once per formatting change, under a work budget Finding 64. push_formatted re-ran char_props and apply_prm for every character, so a single piece whose Prm names a 63 KB Prc over 100k characters took 15 s. walk now keeps the last resolved CharProps and reuses it while the CHPX slice, the piece's raw Prm and the paragraph style are unchanged. Prm1 grpprl bytes interpreted are charged to a document-wide MAX_PRM_WORK (64 MiB) budget, so files alternating pieces per character can't multiply the work; past it the Prm is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/lib.rs | 67 ++++++++++++++++++++++++++++---------- crates/docbin/src/piece.rs | 6 ++++ crates/docbin/src/tests.rs | 39 ++++++++++++++++++++++ 3 files changed, 94 insertions(+), 18 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 484efb08ef..2ae5a2965a 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -49,6 +49,9 @@ const MAX_PARTS: usize = 50_000; /// headers, notes, text boxes), whatever the file's size. The working budget is tied to the /// WordDocument stream (see [`read`]); this caps even a maximal stream. const MAX_TOTAL_CHARS: u64 = 64 << 20; +/// Bytes of piece Prm grpprls (Prcs, up to 64 KB each) interpreted per document; past it the +/// remaining pieces' Prm formatting is skipped (#3692). +const MAX_PRM_WORK: u64 = 64 << 20; /// `sprmCPicLocation`: the fc of a picture in the Data stream. const C_PIC_LOCATION: u16 = 0x6A03; /// `sprmCSymbol`: the record font and character of a symbol character ([MS-DOC] §2.6.1); the @@ -239,6 +242,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result) -> Result { /// footnotes, text boxes) are hostile and can each span a whole subdocument, so thousands of /// them could otherwise materialise the same text thousands of times; this bounds the total. budget: &'a std::cell::Cell, + /// Prm grpprl bytes still to interpret ([`MAX_PRM_WORK`]). + prm_work: &'a std::cell::Cell, } /// State while assembling one paragraph from per-CP formatting. @@ -448,7 +455,7 @@ impl ParaBuild { /// The range is clamped to the last piece's CP, and unreadable stretches (gaps, truncated /// pieces) are skipped a whole piece at a time, so the work is bounded by the text that /// really exists, not by the CP numbers a file claims. -fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -> Vec { +fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -> Vec { let cp_end = cp_end.min(ctx.pieces.cp_end()); let mut out = Vec::new(); let mut pb = ParaBuild::default(); @@ -457,6 +464,7 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) - // Field assembly state: instruction characters are swallowed, the result is kept until // the end character, then emitted as one object (hyperlinks keep their result text). let mut field: Option = None; + let mut memo: Option> = None; while cp < cp_end { // Bookmark boundaries at this CP (main-document walks only; subdocument walks get // an empty table and `cp` never matches the main-document CPs). @@ -604,16 +612,16 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) - pb.objects.push(InlineObject::NoteRef { kind: *kind, id: *id, custom: String::new() }); } } - '\t' => push_formatted(&mut pb, cp, '\t', ctx, field.as_mut()), - '\u{B}' => push_formatted(&mut pb, cp, '\n', ctx, field.as_mut()), - '\u{C}' => push_formatted(&mut pb, cp, '\u{C}', ctx, field.as_mut()), - '\u{E}' => push_formatted(&mut pb, cp, '\u{E}', ctx, field.as_mut()), - '\u{1E}' => push_formatted(&mut pb, cp, '\u{2011}', ctx, field.as_mut()), - '\u{1F}' => push_formatted(&mut pb, cp, '\u{AD}', ctx, field.as_mut()), + '\t' => push_formatted(&mut pb, cp, '\t', ctx, field.as_mut(), &mut memo), + '\u{B}' => push_formatted(&mut pb, cp, '\n', ctx, field.as_mut(), &mut memo), + '\u{C}' => push_formatted(&mut pb, cp, '\u{C}', ctx, field.as_mut(), &mut memo), + '\u{E}' => push_formatted(&mut pb, cp, '\u{E}', ctx, field.as_mut(), &mut memo), + '\u{1E}' => push_formatted(&mut pb, cp, '\u{2011}', ctx, field.as_mut(), &mut memo), + '\u{1F}' => push_formatted(&mut pb, cp, '\u{AD}', ctx, field.as_mut(), &mut memo), // Object anchors, comment references and other special characters are dropped // (or handled by later passes). c if (c as u32) < 0x20 => {} - c => push_formatted(&mut pb, cp, c, ctx, field.as_mut()), + c => push_formatted(&mut pb, cp, c, ctx, field.as_mut(), &mut memo), } cp = cp.saturating_add(units); } @@ -789,19 +797,28 @@ fn num_of(ilfo: i32) -> Option { /// plus the piece's Prm, over the paragraph style's base character props. Inside a field, /// the character is recorded into the instruction or the cached result instead of (or as /// well as) the visible text. -fn push_formatted(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx, field: Option<&mut FieldBuild>) { +fn push_formatted<'a>(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx<'a>, field: Option<&mut FieldBuild>, memo: &mut Option>) { let fc = ctx.pieces.fc_of_cp(cp).unwrap_or(0); let grpprl = ctx.chpx_bins.chpx(ctx.word, fc); - let default; - let base = match pb.props.style.as_deref().and_then(|id| ctx.sheet.get(id)) { - Some(st) => &st.chr, + let prm = ctx.pieces.prm_of(cp); + let style = pb.props.style.as_deref(); + let mut props = match memo.as_ref().filter(|m| std::ptr::eq(m.chpx, grpprl) && m.prm == prm && m.style.as_deref() == style) { + Some(m) => m.props.clone(), None => { - default = CharProps::default(); - &default + let default; + let base = match style.and_then(|id| ctx.sheet.get(id)) { + Some(st) => &st.chr, + None => { + default = CharProps::default(); + &default + } + }; + let mut props = char_props(grpprl, ctx.fonts, base); + apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base, ctx.prm_work); + *memo = Some(FmtMemo { chpx: grpprl, prm, style: style.map(str::to_owned), props: props.clone() }); + props } }; - let mut props = char_props(grpprl, ctx.fonts, base); - apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base); // A symbol character (Insert › Symbol) is stored as the placeholder U+0028 with // sprmCSymbol carrying the font and the character code ([MS-DOC] §2.6.1). let mut c = c; @@ -826,8 +843,20 @@ fn push_formatted(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx, field: Op } } -/// Apply a piece's Prm (Prm0 short sprm or Prm1 grpprl) to character properties. -fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps) { +/// The last character formatting resolved, reused while the CHPX, the piece's Prm and the +/// paragraph style stay the same, so a Prm grpprl (up to 64 KB) is interpreted once per +/// formatting change, not once per character (#3692). +struct FmtMemo<'a> { + /// The CHPX grpprl, compared by address: it is a slice of the WordDocument stream. + chpx: &'a [u8], + prm: u16, + style: Option, + props: CharProps, +} + +/// Apply a piece's Prm (Prm0 short sprm or Prm1 grpprl) to character properties. A Prm1 +/// grpprl is charged to `work`; once that is spent the grpprl is skipped. +fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps, work: &std::cell::Cell) { match prm { PrmRef::None => {} PrmRef::Sprm0 { isprm, val } => { @@ -840,6 +869,8 @@ fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &Ch } } PrmRef::Grpprl(Some(g)) => { + let Some(left) = work.get().checked_sub(g.len() as u64) else { return }; + work.set(left); for prl in sprm::iter(g) { if prl.sgc() == 2 { fmt::apply_char(props, &prl, base, fonts); diff --git a/crates/docbin/src/piece.rs b/crates/docbin/src/piece.rs index a7274c9d9d..d52f968b7a 100644 --- a/crates/docbin/src/piece.rs +++ b/crates/docbin/src/piece.rs @@ -117,6 +117,12 @@ impl PieceTable { } } + /// The raw Prm of the piece holding `cp` (0, no properties, when there is none): two CPs + /// with the same value get the same [`PrmRef`]. + pub(crate) fn prm_of(&self, cp: u32) -> u16 { + self.piece_of(cp).map_or(0, |p| p.prm) + } + fn piece_of(&self, cp: u32) -> Option<&Pcd> { let i = self.cps.partition_point(|&c| c <= cp).checked_sub(1)?; if i >= self.pieces.len() { diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index cf42176fe4..c211b0b388 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2463,3 +2463,42 @@ fn hyperlink_target_copies_are_charged_to_the_char_budget() { assert_eq!(pb.runs.iter().filter(|r| r.props.link.is_some()).count(), 100); assert_eq!(budget.get(), 0); } + +/// #3692: a piece's Prm grpprl (a Prc of up to 64 KB) was re-interpreted for every character. +/// It is now resolved once per formatting change. +#[test] +fn a_large_prc_is_interpreted_once_per_formatting_change() { + let prc = grpprl(&vec![(0x0835u16, &[0x01u8][..]); 21_000]); + let text: &'static str = Box::leak(format!("{}\r", "a".repeat(100_000)).into_boxed_str()); + let spec = RichSpec { + text, + styles: vec![(1, 0x0FFF, "Normal", Vec::new(), Vec::new())], + fonts: vec!["Times New Roman"], + chpx_runs: vec![(0, Vec::new())], + papx_entries: vec![(0, 0, Vec::new())], + }; + let bytes = rich_doc_with(&spec, &RichExtra { prcs: vec![prc], prm: 0x0001, ..Default::default() }); + let t = std::time::Instant::now(); + let doc = read(&bytes).expect("opens"); + assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); + let p = para0(&doc); + assert_eq!(p.runs.len(), 1); + assert_eq!(p.runs.first().and_then(|r| r.props.bold), Some(true)); +} + +/// #3692: Prm grpprls are charged to a document-wide work budget; once it is spent the Prm +/// is skipped (the text keeps its CHPX formatting). +#[test] +fn prm_grpprls_past_the_work_budget_are_skipped() { + use wordcraft_doc::props::CharProps; + let g = grpprl(&[(0x0835, &[0x01])]); + let work = std::cell::Cell::new(g.len() as u64 + 1); + let base = CharProps::default(); + let mut first = CharProps::default(); + super::apply_prm(&mut first, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work); + assert_eq!(first.bold, Some(true)); + let mut second = CharProps::default(); + super::apply_prm(&mut second, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work); + assert_eq!(second.bold, None); + assert_eq!(work.get(), 1); +} From 878bac84ba2955934ba86df6177767a4519b2519 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:22:09 +0100 Subject: [PATCH 008/112] #3709: .doc inline-picture alt text copies charged to a work budget Finding 68. Pictures::get cloned the cached Inline (alt text up to 4096 UTF-16 units) for every '\x01' anchor, and the char budget allows ~64M anchors, so one picture referenced repeatedly could copy hundreds of GB. get now returns a reference; walk copies the alt text per anchor only while the document-wide work budget lasts, then leaves it empty (the picture itself is still placed). The Prm work budget from #3692 becomes one shared MAX_WORK (64 MiB) for repeated work beyond the text: Prm grpprls interpreted, hyperlink targets (#3708, moved off the char budget so long links in small files are never dropped) and alt text copied. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/lib.rs | 42 ++++++++++++++++++------------ crates/docbin/src/media.rs | 17 ++++++------ crates/docbin/src/tests.rs | 12 ++++----- crates/docbin/src/tests/drawing.rs | 20 ++++++++++++++ 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 2ae5a2965a..2b18591321 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -49,9 +49,10 @@ const MAX_PARTS: usize = 50_000; /// headers, notes, text boxes), whatever the file's size. The working budget is tied to the /// WordDocument stream (see [`read`]); this caps even a maximal stream. const MAX_TOTAL_CHARS: u64 = 64 << 20; -/// Bytes of piece Prm grpprls (Prcs, up to 64 KB each) interpreted per document; past it the -/// remaining pieces' Prm formatting is skipped (#3692). -const MAX_PRM_WORK: u64 = 64 << 20; +/// Bytes of repeated work per document beyond the text itself: piece Prm grpprls (Prcs, up +/// to 64 KB each) interpreted (#3692), hyperlink targets (#3708) and picture alt text (#3709) +/// copied into every run or anchor. Past it Prms are skipped and copies left empty. +const MAX_WORK: u64 = 64 << 20; /// `sprmCPicLocation`: the fc of a picture in the Data stream. const C_PIC_LOCATION: u16 = 0x6A03; /// `sprmCSymbol`: the record font and character of a symbol character ([MS-DOC] §2.6.1); the @@ -242,7 +243,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result) -> Result { /// footnotes, text boxes) are hostile and can each span a whole subdocument, so thousands of /// them could otherwise materialise the same text thousands of times; this bounds the total. budget: &'a std::cell::Cell, - /// Prm grpprl bytes still to interpret ([`MAX_PRM_WORK`]). - prm_work: &'a std::cell::Cell, + /// Repeated-work bytes left ([`MAX_WORK`]). + work: &'a std::cell::Cell, } /// State while assembling one paragraph from per-CP formatting. @@ -570,7 +571,7 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic f.depth -= 1; field = Some(f); } else { - finish_field(&mut pb, f, ctx.budget); + finish_field(&mut pb, f, ctx.work); } } cp = cp.saturating_add(1); @@ -582,12 +583,21 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic let fc_pic = pic_location(ctx.chpx_bins.chpx(ctx.word, fc)); match fc_pic.and_then(|at| pics.get(ctx.data, at)) { Some(p) => { + // Every anchor of a picture gets its own copy of the alt text; the + // copies are charged to the work budget (#3709). + let alt = match ctx.work.get().checked_sub(p.alt.len() as u64) { + Some(left) => { + ctx.work.set(left); + p.alt.clone() + } + None => String::new(), + }; pb.push('\u{FFFC}', CharProps::default()); pb.objects.push(InlineObject::Image { - media: p.key, + media: p.key.clone(), w: p.w, h: p.h, - alt: p.alt, + alt, float: Default::default(), crop: p.crop, ole: None, @@ -648,9 +658,9 @@ struct FieldBuild { /// field becomes one inline object with its cached result. /// /// The target is copied once per run of equally formatted result text, not per character, -/// and each copy is charged to the document's character `budget`; once that is spent the -/// rest of the result is kept as plain text (#3708). -fn finish_field(pb: &mut ParaBuild, f: FieldBuild, budget: &std::cell::Cell) { +/// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the +/// result is kept as plain text (#3708). +fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &std::cell::Cell) { let instr = f.instr.trim().to_string(); let result = f.sep.unwrap_or_default(); if let Some(target) = hyperlink_target(&instr) { @@ -662,8 +672,8 @@ fn finish_field(pb: &mut ParaBuild, f: FieldBuild, budget: &std::cell::Cell continue; } let mut linked = props.clone(); - if budget.get() >= cost { - budget.set(budget.get() - cost); + if let Some(left) = work.get().checked_sub(cost) { + work.set(left); linked.link = Some(target.clone()); } pb.push(c, linked); @@ -814,7 +824,7 @@ fn push_formatted<'a>(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx<'a>, f } }; let mut props = char_props(grpprl, ctx.fonts, base); - apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base, ctx.prm_work); + apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base, ctx.work); *memo = Some(FmtMemo { chpx: grpprl, prm, style: style.map(str::to_owned), props: props.clone() }); props } diff --git a/crates/docbin/src/media.rs b/crates/docbin/src/media.rs index 52e0aa9109..314f70eae5 100644 --- a/crates/docbin/src/media.rs +++ b/crates/docbin/src/media.rs @@ -67,16 +67,15 @@ impl Pictures { /// The inline picture at `fc_pic` in the Data stream, copying its bytes the first time. /// `None` when the header is malformed, the picture can't be read, or a picture /// count/byte cap is reached. - pub(crate) fn get(&mut self, data: &[u8], fc_pic: u32) -> Option { - if let Some(hit) = self.by_fc.get(&fc_pic) { - return hit.clone(); + pub(crate) fn get(&mut self, data: &[u8], fc_pic: u32) -> Option<&Inline> { + if !self.by_fc.contains_key(&fc_pic) { + if self.by_fc.len().saturating_add(self.by_bse.len()) >= MAX_PICTURES { + return None; + } + let pic = self.read(data, fc_pic); + self.by_fc.insert(fc_pic, pic); } - if self.by_fc.len().saturating_add(self.by_bse.len()) >= MAX_PICTURES { - return None; - } - let pic = self.read(data, fc_pic); - self.by_fc.insert(fc_pic, pic.clone()); - pic + self.by_fc.get(&fc_pic)?.as_ref() } /// Read the inline picture at `fc_pic`: nothing is decoded or copied beyond what is left of diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index c211b0b388..afd29c5085 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -1732,9 +1732,9 @@ fn picture_bytes_are_capped_per_document() { let data: Vec = [a.clone(), b].concat(); let mut pics = crate::media::Pictures::default(); pics.limit = 1000; - let first = pics.get(&data, 0).expect("first fits"); + let first = pics.get(&data, 0).cloned().expect("first fits"); // Same location again: no new bytes. - assert_eq!(pics.get(&data, 0).map(|p| p.key), Some(first.key.clone())); + assert_eq!(pics.get(&data, 0).map(|p| p.key.clone()), Some(first.key.clone())); // A second distinct picture would exceed the cap. assert!(pics.get(&data, a.len() as u32).is_none()); assert_eq!(pics.media.len(), 1); @@ -1754,8 +1754,8 @@ fn rejected_inline_pictures_are_not_read_again() { assert!(pics.get(&small, 0).is_none()); // The rejected picture took nothing off the cap, and anchors sharing a picture share it. let at = big.len() as u32; - let fits = pics.get(&data, at).expect("fits"); - assert_eq!(pics.get(&data, at).map(|p| p.key), Some(fits.key)); + let fits = pics.get(&data, at).cloned().expect("fits"); + assert_eq!(pics.get(&data, at).map(|p| p.key.clone()), Some(fits.key)); assert_eq!(pics.media.len(), 1); // Filling the cap exactly works; after that, a place never tried before isn't read. let rest = picf_with_png(700); @@ -2447,9 +2447,9 @@ fn a_long_hyperlink_over_a_long_result_is_one_linked_run() { } /// #3708: a result whose formatting changes every character pays for a target copy per run -/// out of the character budget; past it the text stays, unlinked. +/// out of the work budget; past it the text stays, unlinked. #[test] -fn hyperlink_target_copies_are_charged_to_the_char_budget() { +fn hyperlink_target_copies_are_charged_to_the_work_budget() { use wordcraft_doc::props::CharProps; let bold = CharProps { bold: Some(true), ..Default::default() }; let result: Vec<(char, CharProps)> = (0..1000).map(|i| ('a', if i % 2 == 0 { bold.clone() } else { CharProps::default() })).collect(); diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index 4007408e11..fc453d54cf 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -332,3 +332,23 @@ fn dib_conversion_caps_header_and_encoded_bytes() { assert!(normal.bytes.len() > bmp_size && normal.bytes.len() <= 512); assert_eq!(image::load_from_memory(&normal.bytes).unwrap().width(), 1); } + +/// #3709: every anchor of one inline picture got its own copy of the alt text (up to 4096 +/// UTF-16 units). Copies are charged to the document's work budget; past it alt is empty. +#[test] +fn alt_text_copies_across_many_anchors_are_bounded() { + let alt: String = "€".repeat(4096); // 12 KB as UTF-8 + let shape = sp(&[fsp(75, 1025, 0xA00), fopt(&[(0x0104, 1)], &[(0x0381, wz(&alt))])]); + let data = picf_shape(&[shape, fbse_png(&tiny_png())].concat()); + let text: &'static str = Box::leak(format!("{}\r", "\u{1}".repeat(8000)).into_boxed_str()); + let ccp = text.chars().count() as u32; + let spec = StructSpec { text, data_stream: data, chpx_runs: vec![(0, grpprl(&[(0x6A03, &0u32.to_le_bytes())]))], ..structured_spec(text, ccp) }; + let doc = read(&struct_doc(&spec)).expect("opens"); + let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para"); + let alts: Vec<&str> = p0.objects.iter().filter_map(|o| if let InlineObject::Image { alt, .. } = o { Some(alt.as_str()) } else { None }).collect(); + assert_eq!(alts.len(), 8000, "every anchor keeps its picture"); + assert_eq!(alts.first().copied(), Some(alt.as_str())); + let total: usize = alts.iter().map(|a| a.len()).sum(); + assert!(total <= 64 << 20, "alt bytes {total}"); + assert_eq!(alts.last().copied(), Some("")); +} From 0ecdf2ff2dc28ea503cd843330a84117f08fd31a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:24:22 +0100 Subject: [PATCH 009/112] #3710: .doc floating shapes built once per spid; anchor copies charged to the work budget Finding 49. drawing::anchors built a fresh object for every PlcSpa entry, so one group of 1000 members named by 5000 entries built 5M children (1.4 s, ~1 GB) with the per-group child cap resetting each time. Word anchors each shape once, so a repeated spid is now skipped: every node of the shape tree is built at most once, bounded by the shared OfficeArt record budget. walk also cloned each floating object every time a story holding its anchor was walked; each copy is now charged by size (a group with all its members) to the document-wide MAX_WORK budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/drawing.rs | 6 ++++++ crates/docbin/src/lib.rs | 14 ++++++++++++-- crates/docbin/src/tests/drawing.rs | 24 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/crates/docbin/src/drawing.rs b/crates/docbin/src/drawing.rs index 461d6b372c..f5c02699f3 100644 --- a/crates/docbin/src/drawing.rs +++ b/crates/docbin/src/drawing.rs @@ -81,12 +81,18 @@ pub(crate) fn anchors( ) -> Vec<(u32, InlineObject)> { let Some(pair) = pair else { return Vec::new() }; let mut out = Vec::new(); + let mut seen = std::collections::HashSet::new(); for (cp, spa) in plc(table, pair, CB_SPA) { let i32_at = |o: usize| spa.get(o..o + 4).and_then(|b| b.try_into().ok()).map(i32::from_le_bytes).unwrap_or(0); let spid = i32_at(0) as u32; let [xl, yt, xr, yb] = [4, 8, 12, 16].map(i32_at); let flags = spa.get(20..22).map(|b| u16::from_le_bytes([b[0], b[1]])).unwrap_or(0); let Some(node) = shapes.get(&spid) else { continue }; + // Word anchors each shape once. A repeated spid would build another copy of the shape, + // and of every member of a group, per anchor (#3710). + if !seen.insert(spid) { + continue; + } let tw = |v: i64| (v as f32 / 20.0).clamp(-2.0 * MAX_PT, 2.0 * MAX_PT); let (mut x, mut y) = (tw(xl as i64), tw(yt as i64)); let (mut w, mut h) = (tw(xr as i64 - xl as i64).clamp(0.0, MAX_PT), tw(yb as i64 - yt as i64).clamp(0.0, MAX_PT)); diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 2b18591321..4c6c0c4419 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -50,8 +50,9 @@ const MAX_PARTS: usize = 50_000; /// WordDocument stream (see [`read`]); this caps even a maximal stream. const MAX_TOTAL_CHARS: u64 = 64 << 20; /// Bytes of repeated work per document beyond the text itself: piece Prm grpprls (Prcs, up -/// to 64 KB each) interpreted (#3692), hyperlink targets (#3708) and picture alt text (#3709) -/// copied into every run or anchor. Past it Prms are skipped and copies left empty. +/// to 64 KB each) interpreted (#3692), hyperlink targets (#3708), picture alt text (#3709) +/// and floating objects (#3710) copied into every run or anchor. Past it Prms are skipped, +/// copied strings left empty and floating objects dropped. const MAX_WORK: u64 = 64 << 20; /// `sprmCPicLocation`: the fc of a picture in the Data stream. const C_PIC_LOCATION: u16 = 0x6A03; @@ -609,7 +610,16 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic '\u{8}' => { // Drawn object anchor: the floating shape or picture anchored at this CP. let at = ctx.drawn.partition_point(|(c, _)| *c < cp); + // Each copy (a story can be walked more than once) is charged to the work budget + // by size, a group with all its members (#3710). for (_, obj) in ctx.drawn.get(at..).unwrap_or_default().iter().take_while(|(c, _)| *c == cp) { + let parts = match obj { + InlineObject::Group { children, .. } => children.len().saturating_add(1), + _ => 1, + }; + let cost = (parts as u64).saturating_mul(std::mem::size_of::() as u64); + let Some(left) = ctx.work.get().checked_sub(cost) else { continue }; + ctx.work.set(left); pb.push('\u{FFFC}', CharProps::default()); pb.objects.push(obj.clone()); } diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index fc453d54cf..49851883ba 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -352,3 +352,27 @@ fn alt_text_copies_across_many_anchors_are_bounded() { assert!(total <= 64 << 20, "alt bytes {total}"); assert_eq!(alts.last().copied(), Some("")); } + +/// #3710: every PlcSpa entry naming a group built another copy of all its members. A shape is +/// anchored once; repeated spids are skipped. +#[test] +fn a_group_named_by_many_anchors_is_built_once() { + let rect = |l: i32| rec(0, 0, 0xF00F, &[l, 0, l + 10, 10].iter().flat_map(|v| v.to_le_bytes()).collect::>()); + let head = sp(&[fsp(0, 3000, 0x201), rec(1, 0, 0xF009, &[0i32, 0, 10_000, 10].iter().flat_map(|v| v.to_le_bytes()).collect::>())]); + let members: Vec> = (0..1000).map(|i| sp(&[fsp(1, 4000 + i as u32, 0xA02), rect(i * 10)])).collect(); + let group = container(0xF003, &[vec![head], members].concat()); + let art = art_content(&[], &[group]); + let n = 5000u32; + let cps: Vec = (1..=n + 1).collect(); + let spas: Vec = (0..n).flat_map(|_| spa(3000, [0, 0, 1440, 1440], spa_flags(0, 0, 3, false))).collect(); + let text = "A\u{8}\u{8}\u{8}B\r"; + let ccp = text.chars().count() as u32; + let spec = StructSpec { text, extra_pairs: vec![(40, plc(&cps, &spas)), (50, art)], ..structured_spec(text, ccp) }; + let t = std::time::Instant::now(); + let doc = read(&struct_doc(&spec)).expect("opens"); + assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); + let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para"); + let groups: Vec = + p0.objects.iter().filter_map(|o| if let InlineObject::Group { children, .. } = o { Some(children.len()) } else { None }).collect(); + assert_eq!(groups, [1000]); +} From b2f1197f847fcec3a98a49c65f52aa424066f7c3 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:27:52 +0100 Subject: [PATCH 010/112] #3711: .doc picture decoding has a document-wide work budget; aliased delay-stream BLIPs decode once Finding 60 (also #3712). Metafile inflation and DIB decoding had per-picture caps only: thousands of BLIP store entries aliasing one delay-stream record each inflated and copied it again, failed inflations cost nothing, and an RLE DIB tiny on disk decoded to its full declared w x h x 4 pixels. Now every inflate (its declared size, success or not) and every DIB decode (its declared pixels) is charged to a 512 MB per-document decode budget before it runs; past it metafiles are skipped and DIBs stay BMP placeholders. Store entries naming the same (foDelay, end) record share one decode and one media key. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/media.rs | 38 +++++++++-- crates/docbin/src/officeart.rs | 45 ++++++++++--- crates/docbin/src/tests/drawing.rs | 100 +++++++++++++++++++++++++++-- 3 files changed, 159 insertions(+), 24 deletions(-) diff --git a/crates/docbin/src/media.rs b/crates/docbin/src/media.rs index 314f70eae5..c4314011ea 100644 --- a/crates/docbin/src/media.rs +++ b/crates/docbin/src/media.rs @@ -17,6 +17,10 @@ const OFF_PICMID: usize = 28; const MAX_PICTURES: usize = 5_000; /// Most picture bytes one document copies out of the Data stream, all pictures together. pub(crate) const MAX_MEDIA_BYTES: usize = 256 << 20; +/// Most bytes one document decodes pictures into (metafiles inflated, DIB pixels), charged +/// per attempt whether it succeeds or not: retained media is capped separately, by +/// `MAX_MEDIA_BYTES` (#3711, #3712). +const MAX_DECODE_WORK: usize = 512 << 20; /// The document's pictures, keyed by their location in the Data stream: anchors that share a /// `fc_pic` share one media entry (or one failure), and the total bytes copied are capped, so a @@ -28,11 +32,16 @@ pub(crate) struct Pictures { by_fc: HashMap>, /// BLIP store index → media key (`None`: unreadable, not retried). by_bse: HashMap>, + /// (foDelay, end) of a BLIP record outside the store → media key: store entries that alias + /// one record share one decode (`None`: failed, not retried). + by_delay: HashMap<(usize, usize), Option>, /// The media entries, merged into the document at the end. pub(crate) media: BTreeMap>>, total: usize, /// The byte cap (`MAX_MEDIA_BYTES`; smaller in tests). pub(crate) limit: usize, + /// Decode work left (`MAX_DECODE_WORK`; smaller in tests). + pub(crate) work: usize, /// Anchors whose picture was dropped (only the first few are logged). dropped: usize, /// Pictures kept as placeholders (formats we can't draw). @@ -44,9 +53,11 @@ impl Default for Pictures { Pictures { by_fc: HashMap::new(), by_bse: HashMap::new(), + by_delay: HashMap::new(), media: BTreeMap::new(), total: 0, limit: MAX_MEDIA_BYTES, + work: MAX_DECODE_WORK, dropped: 0, placeholders: 0, } @@ -85,7 +96,7 @@ impl Pictures { if room == 0 { return None; } - match locate(data, fc_pic, room)? { + match locate(data, fc_pic, room, &mut self.work)? { Found::Range(pic) => { let bytes = data.get(pic.range.clone()).filter(|b| b.len() <= room)?.to_vec(); let key = self.insert(bytes, pic.ext, false)?; @@ -108,12 +119,25 @@ impl Pictures { return None; } let fbse = bstore.get((pib as usize).checked_sub(1)?)?; - let room = self.limit.saturating_sub(self.total); - let key = officeart::fbse_blip(fbse, delay, room).and_then(|b| self.insert(b.bytes, b.ext, b.placeholder)); + let key = self.decode_bse(fbse, delay); self.by_bse.insert(pib, key.clone()); key } + /// Decode a store entry's BLIP, once per delay-stream record however many entries name it. + fn decode_bse(&mut self, fbse: &[u8], delay: &[u8]) -> Option { + let (rec, at) = officeart::fbse_record(fbse, delay)?; + if let Some(hit) = at.and_then(|at| self.by_delay.get(&at)) { + return hit.clone(); + } + let room = self.limit.saturating_sub(self.total); + let key = officeart::blip(&rec, room, &mut self.work).and_then(|b| self.insert(b.bytes, b.ext, b.placeholder)); + if let Some(at) = at { + self.by_delay.insert(at, key.clone()); + } + key + } + /// Add picture bytes as a new media entry, within the byte cap. fn insert(&mut self, bytes: Vec, ext: &str, placeholder: bool) -> Option { let total = self.total.checked_add(bytes.len()).filter(|&t| t <= self.limit)?; @@ -160,7 +184,7 @@ const MM_SHAPEFILE: u16 = 0x0066; /// Locate the picture at `fc_pic` in the Data stream ([MS-DOC] §2.9.192 PICFAndOfficeArtData); /// a shape's picture is decoded only up to `room` bytes. -fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option { +fn locate(data: &[u8], fc_pic: u32, room: usize, work: &mut usize) -> Option { let base = fc_pic as usize; let pic = data.get(base..)?; let lcb = u32::from_le_bytes(pic.get(0..4)?.try_into().ok()?) as usize; @@ -179,7 +203,7 @@ fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option { if mm == MM_SHAPEFILE { at = at.checked_add(1 + *pic.get(at)? as usize)?; } - let (blip, props) = inline_shape(pic.get(at..end)?, data, room)?; + let (blip, props) = inline_shape(pic.get(at..end)?, data, room, work)?; // The goal size is before cropping; the crop fractions take their share off it. let [cl, ct, cr, cb] = props.crop; let w = (dxa_goal / 20.0 * (1.0 - cl - cr).max(0.01) * scale(mx)).clamp(1.0, 1584.0); @@ -198,13 +222,13 @@ fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option { /// An `OfficeArtInlineSpContainer`: the picture shape, then the `OfficeArtFBSE` records its /// `pib` indexes (BLIPs embedded, or at `foDelay` in the Data stream). `room` caps the bytes /// the BLIP decodes to. -fn inline_shape(buf: &[u8], data: &[u8], room: usize) -> Option<(officeart::Blip, officeart::Props)> { +fn inline_shape(buf: &[u8], data: &[u8], room: usize, work: &mut usize) -> Option<(officeart::Blip, officeart::Props)> { let mut budget = 10_000usize; let recs = officeart::children(buf, &mut budget); let sp = recs.iter().find(|r| r.typ == officeart::rt::SP_CONTAINER).map(|r| officeart::shape(r.body, &mut budget))?; let fbses: Vec<&[u8]> = recs.iter().filter(|r| r.typ == officeart::rt::FBSE).map(|r| r.body).collect(); let fbse = sp.props.pib.and_then(|p| fbses.get((p as usize).checked_sub(1)?)).or(fbses.first())?; - let blip = officeart::fbse_blip(fbse, data, room)?; + let blip = officeart::fbse_blip(fbse, data, room, work)?; Some((blip, sp.props)) } diff --git a/crates/docbin/src/officeart.rs b/crates/docbin/src/officeart.rs index 12d4a7d2e1..84d68b9d7f 100644 --- a/crates/docbin/src/officeart.rs +++ b/crates/docbin/src/officeart.rs @@ -112,23 +112,32 @@ pub(crate) struct Blip { /// The picture of an `OfficeArtFBSE` body: its embedded BLIP, or the BLIP stored at `foDelay` /// in `delay` (the WordDocument stream for the document's BLIP store, the Data stream for /// inline pictures). `limit` caps the bytes produced. -pub(crate) fn fbse_blip(fbse: &[u8], delay: &[u8], limit: usize) -> Option { +pub(crate) fn fbse_blip(fbse: &[u8], delay: &[u8], limit: usize, work: &mut usize) -> Option { + let (rec, _) = fbse_record(fbse, delay)?; + blip(&rec, limit, work) +} + +/// The BLIP record of an `OfficeArtFBSE`: embedded after it, or in the delay stream, where +/// its (`foDelay`, end) location is returned too (store entries can share one record). +pub(crate) fn fbse_record<'a>(fbse: &'a [u8], delay: &'a [u8]) -> Option<(Rec<'a>, Option<(usize, usize)>)> { // btWin32, btMacOS, rgbUid[16], tag, size, cRef, foDelay, unused, cbName, unused×2. let size = u32_at(fbse, 20)? as usize; let fo_delay = u32_at(fbse, 28)? as usize; let cb_name = *fbse.get(33)? as usize; let at = 36usize.checked_add(cb_name)?; if let Some((rec, _)) = fbse.get(at..).and_then(record) { - return blip(&rec, limit); + return Some((rec, None)); } // Not embedded: the record sits in the delay stream. let end = fo_delay.checked_add(size.max(8))?.min(delay.len()); let (rec, _) = record(delay.get(fo_delay..end)?)?; - blip(&rec, limit) + Some((rec, Some((fo_delay, end)))) } -/// Decode one `OfficeArtBlip*` record ([MS-ODRAW] §2.2.23-§2.2.31). -pub(crate) fn blip(rec: &Rec, limit: usize) -> Option { +/// Decode one `OfficeArtBlip*` record ([MS-ODRAW] §2.2.23-§2.2.31). `work` is the +/// document's decode budget: inflating a metafile or decoding a DIB is charged to it before +/// it runs, whether it succeeds or not, and nothing is decoded once it is spent. +pub(crate) fn blip(rec: &Rec, limit: usize, work: &mut usize) -> Option { // Each type has two instance values; the odd one carries a second 16-byte UID. let uids = if rec.inst & 1 == 1 { 32 } else { 16 }; match rec.typ { @@ -141,7 +150,7 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option { rt::BLIP_PNG => Some(Blip { bytes: data.to_vec(), ext: "png", placeholder: false }), rt::BLIP_JPEG | rt::BLIP_JPEG_CMYK => Some(Blip { bytes: data.to_vec(), ext: "jpeg", placeholder: false }), rt::BLIP_TIFF => Some(Blip { bytes: data.to_vec(), ext: "tiff", placeholder: false }), - _ => dib_to_png(data, limit), + _ => dib_to_png(data, limit, work), } } rt::BLIP_EMF | rt::BLIP_WMF | rt::BLIP_PICT => { @@ -154,7 +163,11 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option { let raw = raw.get(..cb_save.min(raw.len()))?; let cap = limit.min(MAX_METAFILE); let bytes = if compression == 0 { - let want = if cb_size > 0 { cb_size.min(cap) } else { cap }; + let want = (if cb_size > 0 { cb_size.min(cap) } else { cap }).min(*work); + if want == 0 { + return None; + } + *work -= want; miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(raw, want).ok()? } else { if raw.len() > cap { @@ -177,8 +190,9 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option { /// A device-independent bitmap (BITMAPINFOHEADER + palette + pixels) as a PNG: the DIB gets /// the 14-byte BMP file header it lacks, is decoded with size limits and re-encoded. When the -/// bitmap can't be decoded the BMP bytes are kept (and render as a grey placeholder). -fn dib_to_png(dib: &[u8], limit: usize) -> Option { +/// bitmap can't be decoded, or its declared pixels (4 bytes each) don't fit in `work`, the +/// BMP bytes are kept (and render as a grey placeholder). +fn dib_to_png(dib: &[u8], limit: usize, work: &mut usize) -> Option { let header = u32_at(dib, 0)? as usize; if !(12..=124).contains(&header) { return None; @@ -203,6 +217,17 @@ fn dib_to_png(dib: &[u8], limit: usize) -> Option { bmp.extend_from_slice(&[0, 0, 0, 0]); bmp.extend_from_slice(&u32::try_from(off_bits).ok()?.to_le_bytes()); bmp.extend_from_slice(dib); + let placeholder = |bmp| Some(Blip { bytes: bmp, ext: "bmp", placeholder: true }); + // The declared size is what decoding allocates, however small the (RLE) data is. + let (w, h) = if header == 12 { + (u64::from(u16_at(dib, 4)?), u64::from(u16_at(dib, 6)?)) + } else { + (u64::from(u32_at(dib, 4)?), u64::from((u32_at(dib, 8)? as i32).unsigned_abs())) + }; + let Some(left) = w.saturating_mul(h).saturating_mul(4).try_into().ok().and_then(|cost: usize| work.checked_sub(cost)) else { + return placeholder(bmp); + }; + *work = left; let mut reader = image::ImageReader::new(std::io::Cursor::new(&bmp)); reader.set_format(image::ImageFormat::Bmp); let mut limits = image::Limits::default(); @@ -217,7 +242,7 @@ fn dib_to_png(dib: &[u8], limit: usize) -> Option { { Some(Blip { bytes: png.bytes, ext: "png", placeholder: false }) } - _ => Some(Blip { bytes: bmp, ext: "bmp", placeholder: true }), + _ => placeholder(bmp), } } diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index 49851883ba..eeff4651b8 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -247,7 +247,7 @@ fn dib_becomes_png_and_compressed_metafiles_inflate() { dib.extend_from_slice(&[0, 0, 255, 0, 255, 0, 0, 0]); let r = rec(0, 0x7A8, 0xF01F, &[[0u8; 16].as_slice(), &[0xFF], &dib].concat()); let (dib_rec, _) = officeart::record(&r).unwrap(); - let blip = officeart::blip(&dib_rec, 1 << 20).expect("DIB decodes"); + let blip = officeart::blip(&dib_rec, 1 << 20, &mut { usize::MAX }).expect("DIB decodes"); assert_eq!(blip.ext, "png"); let img = image::load_from_memory(&blip.bytes).unwrap().to_rgb8(); assert_eq!((img.width(), img.height(), img.get_pixel(0, 0).0), (2, 1, [255, 0, 0])); @@ -262,7 +262,7 @@ fn dib_becomes_png_and_compressed_metafiles_inflate() { hdr.extend_from_slice(&[0x00, 0xFE]); // compression = deflate, filter let r = rec(0, 0x216, 0xF01B, &[[0u8; 16].as_slice(), &hdr, &packed].concat()); let (wmf_rec, _) = officeart::record(&r).unwrap(); - let blip = officeart::blip(&wmf_rec, 1 << 20).expect("WMF inflates"); + let blip = officeart::blip(&wmf_rec, 1 << 20, &mut { usize::MAX }).expect("WMF inflates"); assert_eq!((blip.ext, blip.bytes), ("wmf", wmf)); } @@ -281,10 +281,10 @@ fn deeply_nested_groups_and_lying_lengths_are_bounded() { hdr.extend_from_slice(&[0, 0]); let r = rec(0, 0x3D4, 0xF01A, &[[0u8; 16].as_slice(), &hdr, &[0x78, 0x9C, 1, 2, 3]].concat()); let (emf, _) = officeart::record(&r).unwrap(); - assert!(officeart::blip(&emf, 1 << 20).is_none()); + assert!(officeart::blip(&emf, 1 << 20, &mut { usize::MAX }).is_none()); let mut fbse = vec![0u8; 36]; fbse[28..32].copy_from_slice(&u32::MAX.to_le_bytes()); - assert!(officeart::fbse_blip(&fbse, &[0u8; 64], 1 << 20).is_none()); + assert!(officeart::fbse_blip(&fbse, &[0u8; 64], 1 << 20, &mut { usize::MAX }).is_none()); } proptest! { @@ -322,12 +322,12 @@ fn dib_conversion_caps_header_and_encoded_bytes() { let record = rec(0, 0x7A8, officeart::rt::BLIP_DIB, &[[0u8; 16].as_slice(), &[0xFF], &dib].concat()); let (record, _) = officeart::record(&record).unwrap(); let bmp_size = 14 + dib.len(); - assert!(officeart::blip(&record, bmp_size - 1).is_none(), "the BMP header also needs room"); - let fallback = officeart::blip(&record, bmp_size).expect("the BMP fallback fits exactly"); + assert!(officeart::blip(&record, bmp_size - 1, &mut { usize::MAX }).is_none(), "the BMP header also needs room"); + let fallback = officeart::blip(&record, bmp_size, &mut { usize::MAX }).expect("the BMP fallback fits exactly"); assert_eq!((fallback.ext, fallback.placeholder, fallback.bytes.len()), ("bmp", true, bmp_size)); let img = image::load_from_memory(&fallback.bytes).unwrap().to_rgb8(); assert_eq!((img.width(), img.height(), img.get_pixel(0, 0).0), (1, 1, [90, 60, 30])); - let normal = officeart::blip(&record, 512).expect("normal conversion fits"); + let normal = officeart::blip(&record, 512, &mut { usize::MAX }).expect("normal conversion fits"); assert_eq!((normal.ext, normal.placeholder), ("png", false)); assert!(normal.bytes.len() > bmp_size && normal.bytes.len() <= 512); assert_eq!(image::load_from_memory(&normal.bytes).unwrap().width(), 1); @@ -376,3 +376,89 @@ fn a_group_named_by_many_anchors_is_built_once() { p0.objects.iter().filter_map(|o| if let InlineObject::Group { children, .. } = o { Some(children.len()) } else { None }).collect(); assert_eq!(groups, [1000]); } + +/// A deflated WMF BLIP record declaring `size` uncompressed bytes. +fn packed_wmf(size: u32, packed: &[u8]) -> Vec { + let mut hdr = size.to_le_bytes().to_vec(); + hdr.extend_from_slice(&[0u8; 24]); // rcBounds, ptSize + hdr.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + hdr.extend_from_slice(&[0x00, 0xFE]); // compression = deflate, filter + rec(0, 0x216, officeart::rt::BLIP_WMF, &[[0u8; 16].as_slice(), &hdr, packed].concat()) +} + +/// An `OfficeArtFBSE` body whose BLIP (of `size` bytes) sits at `fo_delay` in the delay stream. +fn delayed_fbse(size: usize, fo_delay: usize) -> Vec { + let mut fbse = vec![0u8; 36]; + fbse[20..24].copy_from_slice(&(size as u32).to_le_bytes()); + fbse[28..32].copy_from_slice(&(fo_delay as u32).to_le_bytes()); + fbse +} + +/// #3711: store entries aliasing one delay-stream metafile each inflated (and copied) it again. +/// Now the record is decoded once and every entry shares its media key. +#[test] +fn store_entries_aliasing_one_delayed_metafile_decode_it_once() { + let wmf = vec![0u8; 1 << 20]; + let delay = packed_wmf(wmf.len() as u32, &miniz_oxide::deflate::compress_to_vec_zlib(&wmf, 6)); + let fbse = delayed_fbse(delay.len(), 0); + let store: Vec<&[u8]> = vec![&fbse; 5_000]; + let mut pics = crate::media::Pictures::default(); + pics.work = 8 << 20; + let keys: Vec<_> = (1..=store.len() as u32).map(|pib| pics.bse(pib, &store, &delay)).collect(); + assert!(keys.iter().all(|k| k.is_some() && *k == keys[0]), "every entry shares one picture"); + assert_eq!(pics.work, (8 << 20) - wmf.len(), "inflated once"); +} + +/// #3711: distinct metafiles that fail to inflate are charged their declared size anyway, so +/// a document of them stops decoding once the work budget is spent. +#[test] +fn failing_metafile_inflations_spend_the_decode_budget() { + let records: Vec> = (0..10u8).map(|i| packed_wmf(1 << 20, &[0x78, 0x9C, i, 1, 2, 3])).collect(); + let delay = records.concat(); + let mut at = 0; + let fbses: Vec> = records + .iter() + .map(|r| { + at += r.len(); + delayed_fbse(r.len(), at - r.len()) + }) + .collect(); + let store: Vec<&[u8]> = fbses.iter().map(Vec::as_slice).collect(); + let mut pics = crate::media::Pictures::default(); + pics.work = 3 << 20; + for pib in 1..=store.len() as u32 { + assert!(pics.bse(pib, &store, &delay).is_none()); + } + assert_eq!(pics.work, 0, "three attempts spent the budget; the rest weren't tried"); +} + +/// #3712: a DIB's declared pixels are charged before decoding (an RLE bitmap is tiny on disk but +/// decodes to w × h × 4 bytes); past the budget it stays a BMP placeholder, undecoded. +#[test] +fn dib_decoding_is_charged_its_declared_pixels() { + let blip_of = |dib: &[u8]| rec(0, 0x7A8, officeart::rt::BLIP_DIB, &[[0u8; 16].as_slice(), &[0xFF], dib].concat()); + let one = blip_of(&one_pixel_dib()); + let (one, _) = officeart::record(&one).unwrap(); + let mut work = 3; + assert_eq!(officeart::blip(&one, 512, &mut work).map(|b| (b.ext, b.placeholder)), Some(("bmp", true))); + assert_eq!(work, 3, "nothing decoded, nothing charged"); + let mut work = 4; + assert_eq!(officeart::blip(&one, 512, &mut work).map(|b| (b.ext, b.placeholder)), Some(("png", false))); + assert_eq!(work, 0); + + // 8-bit RLE declaring 16384 × 16384 (1 GiB decoded) in a few bytes. + let mut big = [40u32, 16_384, 16_384].iter().flat_map(|v| v.to_le_bytes()).collect::>(); + big.extend_from_slice(&1u16.to_le_bytes()); + big.extend_from_slice(&8u16.to_le_bytes()); + big.extend_from_slice(&1u32.to_le_bytes()); // BI_RLE8 + big.extend_from_slice(&[0u8; 12]); + big.extend_from_slice(&1u32.to_le_bytes()); // one palette colour + big.extend_from_slice(&[0u8; 4]); // clrImportant + big.extend_from_slice(&[0u8; 4]); // the palette colour + big.extend_from_slice(&[0, 0, 0, 1]); // end of line, end of bitmap + let big = blip_of(&big); + let (big, _) = officeart::record(&big).unwrap(); + let mut work = 512 << 20; + assert_eq!(officeart::blip(&big, 1 << 20, &mut work).map(|b| (b.ext, b.placeholder)), Some(("bmp", true))); + assert_eq!(work, 512 << 20); +} From a58a7618be4f87a82e3edf9e41452939655e6863 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:28:17 +0100 Subject: [PATCH 011/112] =?UTF-8?q?#3692:=20docs=20=E2=80=94=20.doc=20read?= =?UTF-8?q?er=20hostile-file=20bounds=20in=20the=20progress=20log=20and=20?= =?UTF-8?q?file-format=20parity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings 49, 60, 63, 64, 68, 75: progress-log row and a note on the .doc parity row. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/file-format-parity.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 41f370a1e4..365d226aad 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -49,7 +49,7 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅ | `.docx` Word Document (ECMA-376 Transitional) | 70% | R/W | ✅ | ✅ | ~65% | 70–110 | `crates/docx/tests` (roundtrip 34, fixtures 29, malformed 8, bidi 5, bidi_runs 12, citations 5); synthetic fixtures only | Word opens our files. Untested on a real-world corpus. Tracked changes inside a field result stay revisions (the field is kept as a range, #1037). Details below | | `.docx` Strict Open XML | 1% | R/W | ✅ (namespace mapping in `xml.rs`) | ❌ | ~60% | 2–4 | `strict_namespace_and_bad_numbers`, `strict_line_heights_with_a_unit_are_read_as_point_multiples` | Writing Strict is rare; unit-suffixed measures (`13.90pt`) read as points (#2329) | | `.docm` / `.dotx` / `.dotm` | 3% | R/W | ✅ | ✅ | ~65% | incl. above | `macro_packages.rs` | Macros and signatures kept on save (#172); macros never run | -| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). | +| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). Hostile files are bounded: a sector table longer than the file is refused before parsing (#3642); formatting and link runs are read once per change, not per character (#3692, #3708); copies of alt text, link targets and drawn objects share a 64 MB budget (#3709, #3710); picture decoding shares a 512 MB budget and aliased store entries decode once (#3711, #3712). | | PDF export | 8% | W | — | ✅ | ~80% | 6–10 | `crates/pdf/src/tests.rs` | krilla: embedded subsets, selectable text incl. field results (#133); Hebrew and Arabic copy and search as whole words (right-to-left glyphs written in visual order, #1383, also with fonts that draw lam-alef as two glyphs at one x, #2310); All Caps and Small Caps text extracts as the capitals shown, as from Word's PDFs, and never changes the case of other text (#1473), links, outline, tagging. Outline destinations follow leading page breaks to the heading text. Fonts it can't subset become outlines (#132). Print Markup (#589): comments in balloons beside each page (scaled to its paper) and tracked changes; HTML/Markdown/text copies end with a comment list. Horizontal and vertical lines inside groups are drawn (#1474). No PDF/A option, no "best for printing/online" choice | | PDF open (PDF Reflow, Word for Windows) | 1% | R | ❌ | — | 0% | 30–50 | — | Converting a PDF into an editable document. A sibling app (PdfCraft) may supply the parser | | `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). East Asian multibyte code pages decoded as runs: 932 Shift-JIS (`\fcharset128`, Japanese), 936 GBK (134, Simplified Chinese), 949 UHC/windows-949 (129, Korean) and 950 Big5 (136, Traditional Chinese) through `\ansicpg`, `\cpg` or the font's charset, escaped or raw, a lead byte and its trail byte together; invalid sequences read as U+FFFD, and a run that starts `EF BB BF` is text, not a BOM (#1653). Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; tracked paragraph marks (`\par` inside `\revised`/`\deleted` groups, including a `\sect` inside a deletion) and character and paragraph formatting revisions (`\crauth`/`\crdate` with `\oldcprops`, `\prauth`/`\prdate` with `\oldpprops`) read and written; section and table formatting revisions (`\oldsprops`, `\oldtprops`) not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export writes every section too (break type, page size and orientation, margins, header/footer distance, columns, page numbering, title page; not headers/footers). Thin: comments, notes, RTL | @@ -268,6 +268,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio | 2026-10-25 | trivial | Repeated DOCX style IDs: repeated save/reopen regression for whole-definition replacement. | | 2026-10-25 | trivial | Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`) | | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | +| 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | | 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) | | 2026-10-23 | trivial | DOCX: TOC styles Word stores under ids in another language (`Sumrio1` named `toc 1`, `CabealhodoSumrio` named `TOC Heading`) read as `TOC1`…`TOC9` and `TOCHeading` unless the file uses those ids, so Update Table builds the entries in the document's own TOC styles (#3891); a TOC whose heading ends a section (entries in columns after a continuous break) is written and read as the engine's TOC, with the page-number tab at the column's edge (#3889) | From bbeb7e22535509cab3f7d998f640554c761d440b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 17:56:58 +0100 Subject: [PATCH 012/112] #3642: compound-file precheck shared in wordcraft-doc; .docx encryption sniff and .xls reader call it too Finding 63 follow-up. #3642 names three cfb call sites: the .doc reader, the encrypted-package sniff and decrypt in docx/crypt.rs (reached for any .docx name), and the .xls reader. The DIFAT precheck moves from docbin to wordcraft_doc::cfb_guard::check (bytes only, no cfb dependency) and runs before every CompoundFile::open in the three crates. The crypt.rs change is only the two precheck calls (open PR #2125 also edits that file). Regression tests use the shared hostile sample (cfb_guard::repeating_difat_sample): docbin a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat, docx tests/malformed.rs and formats sheet::xls a_repeating_difat_is_refused_before_cfb_opens_it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/doc/src/cfb_guard.rs | 100 ++++++++++++++++++++++++++++++ crates/doc/src/lib.rs | 1 + crates/docbin/src/lib.rs | 62 +----------------- crates/docbin/src/tests.rs | 26 +------- crates/docx/src/crypt.rs | 3 +- crates/docx/tests/it/malformed.rs | 9 +++ crates/formats/src/sheet/xls.rs | 12 +++- 7 files changed, 127 insertions(+), 86 deletions(-) create mode 100644 crates/doc/src/cfb_guard.rs diff --git a/crates/doc/src/cfb_guard.rs b/crates/doc/src/cfb_guard.rs new file mode 100644 index 0000000000..c50bd9314d --- /dev/null +++ b/crates/doc/src/cfb_guard.rs @@ -0,0 +1,100 @@ +//! A bytes-only check of an OLE compound file ([MS-CFB]) before the `cfb` crate opens it +//! (#3642). Every reader of `.doc`, `.xls` and encrypted Office packages calls [`check`] first. + +/// Check a compound file's FAT layout before `cfb` reads it ([MS-CFB] §2.2 header, §2.5 +/// DIFAT). `cfb` appends a whole sector of FAT entries for every FAT sector the DIFAT lists and +/// only validates the FAT once it is built, so a small header naming one sector over and over +/// makes it allocate gigabytes. Here the DIFAT is walked the way `cfb` walks it (header entries, +/// then the DIFAT chain, at most one step per sector; trailing zero and free entries trimmed +/// the same way) and the file is refused when it lists more FAT sectors than the file has +/// sectors, so the FAT `cfb` builds is never larger than the file. Anything else that is wrong +/// is left for `cfb` to report. +pub fn check(bytes: &[u8]) -> Result<(), String> { + const MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; + const FREE: u32 = 0xFFFF_FFFF; + const END_OF_CHAIN: u32 = 0xFFFF_FFFE; + const HEADER_ENTRIES: usize = 109; + let u16_at = |at: usize| bytes.get(at..at.checked_add(2)?).map(|b| u16::from_le_bytes([b[0], b[1]])); + let u32_at = |at: usize| bytes.get(at..at.checked_add(4)?).map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + if bytes.len() < 512 || bytes.get(..8) != Some(MAGIC.as_slice()) { + return Ok(()); + } + let sector_len: usize = match u16_at(0x1A) { + Some(3) => 512, + Some(4) => 4096, + _ => return Ok(()), + }; + if bytes.len() < sector_len { + return Ok(()); + } + // `cfb` counts the sectors after the header sector, the last one possibly partial. + let num_sectors = bytes.len().div_ceil(sector_len) - 1; + let num_fat_sectors = u32_at(0x2C).unwrap_or(0) as usize; + let mut difat: Vec = (0..HEADER_ENTRIES).filter_map(|i| u32_at(0x4C + i * 4)).collect(); + let mut next = u32_at(0x44).unwrap_or(END_OF_CHAIN); + let mut steps = 0usize; + while next != END_OF_CHAIN && next != FREE { + steps += 1; + // `cfb` refuses a chain sector past the file or seen before; a chain longer than the + // file has sectors must have done one of the two. + if steps > num_sectors || next as usize >= num_sectors { + return Err(format!("the DIFAT chain is longer than the file or leaves it (sector {next})")); + } + let at = (next as usize + 1) * sector_len; + let entries = sector_len / 4 - 1; + difat.extend((0..entries).map(|i| u32_at(at + i * 4).unwrap_or(FREE))); + next = u32_at(at + entries * 4).unwrap_or(END_OF_CHAIN); + } + while difat.len() > HEADER_ENTRIES && difat.len() > num_fat_sectors && difat.last() == Some(&0) { + difat.pop(); + } + while difat.last() == Some(&FREE) { + difat.pop(); + } + // Each listed FAT sector adds a whole sector of entries. A file can't hold more FAT sectors + // than it has sectors, and below that the FAT is no larger than the file. + if difat.len() > num_sectors { + return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", difat.len())); + } + Ok(()) +} + +/// A compound file of about 1 MB whose DIFAT chain lists one FAT sector 2048 × 127 times: +/// without [`check`], `cfb` builds a FAT of about 130 MB for it. Regression input for the +/// readers' tests. +#[doc(hidden)] +pub fn repeating_difat_sample() -> Vec { + const S: usize = 512; + const CHAIN: u32 = 2048; // DIFAT sectors 1..=CHAIN, sector 0 is the "FAT" + let mut f = vec![0u8; S * (CHAIN as usize + 2)]; + let mut put = |at: usize, v: &[u8]| { + if let Some(dst) = f.get_mut(at..at + v.len()) { + dst.copy_from_slice(v); + } + }; + put(0, &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]); + put(0x18, &0x3Eu16.to_le_bytes()); + put(0x1A, &3u16.to_le_bytes()); + put(0x1C, &0xFFFEu16.to_le_bytes()); + put(0x1E, &9u16.to_le_bytes()); + put(0x20, &6u16.to_le_bytes()); + put(0x2C, &u32::MAX.to_le_bytes()); // no zero trimming + put(0x38, &4096u32.to_le_bytes()); + put(0x44, &1u32.to_le_bytes()); + put(0x48, &CHAIN.to_le_bytes()); + // The 109 header entries are all 0 already; each chain sector's entries too. + for d in 1..=CHAIN { + let next = if d == CHAIN { 0xFFFF_FFFE } else { d + 1 }; + put((d as usize + 2) * S - 4, &next.to_le_bytes()); + } + f +} + +#[cfg(test)] +mod tests { + #[test] + fn a_repeating_difat_is_refused_and_a_small_file_passes() { + assert!(super::check(&super::repeating_difat_sample()).is_err_and(|e| e.contains("FAT sectors in a file of"))); + assert!(super::check(b"not a compound file").is_ok()); + } +} diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index b3ac326aed..75fa27ac47 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -12,6 +12,7 @@ pub mod bidi; mod byte_store; +pub mod cfb_guard; pub mod chart; mod cjk_numbers; pub mod cluster; diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 4c6c0c4419..b3fbc2361e 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -97,70 +97,12 @@ pub fn is_encrypted(bytes: &[u8]) -> bool { stream(&mut comp, "WordDocument").is_some_and(|w| fib::peek_encryption(&w).is_some()) } -/// Open the compound file around a `.doc`, after [`cfb_precheck`]. +/// Open the compound file around a `.doc`, after [`wordcraft_doc::cfb_guard::check`]. fn open_container(bytes: &[u8]) -> Result>, DocbinError> { - cfb_precheck(bytes).map_err(DocbinError::Container)?; + wordcraft_doc::cfb_guard::check(bytes).map_err(DocbinError::Container)?; cfb::CompoundFile::open(std::io::Cursor::new(bytes)).map_err(|e| DocbinError::Container(e.to_string())) } -/// Check a compound file's FAT layout before `cfb` reads it ([MS-CFB] §2.2 header, §2.5 -/// DIFAT). `cfb` appends a whole sector of FAT entries for every FAT sector the DIFAT lists and -/// only validates the FAT once it is built, so a small header naming one sector over and over -/// makes it allocate gigabytes. Here the DIFAT is walked the way `cfb` walks it (header entries, -/// then the DIFAT chain, at most one step per sector; trailing zero and free entries trimmed -/// the same way) and the file is refused when it lists more FAT sectors than the file has -/// sectors, so the FAT `cfb` builds is never larger than the file. Anything else that is wrong -/// is left for `cfb` to report. -fn cfb_precheck(bytes: &[u8]) -> Result<(), String> { - const MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; - const FREE: u32 = 0xFFFF_FFFF; - const END_OF_CHAIN: u32 = 0xFFFF_FFFE; - const HEADER_ENTRIES: usize = 109; - let u16_at = |at: usize| bytes.get(at..at.checked_add(2)?).map(|b| u16::from_le_bytes([b[0], b[1]])); - let u32_at = |at: usize| bytes.get(at..at.checked_add(4)?).map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); - if bytes.len() < 512 || bytes.get(..8) != Some(MAGIC.as_slice()) { - return Ok(()); - } - let sector_len: usize = match u16_at(0x1A) { - Some(3) => 512, - Some(4) => 4096, - _ => return Ok(()), - }; - if bytes.len() < sector_len { - return Ok(()); - } - // `cfb` counts the sectors after the header sector, the last one possibly partial. - let num_sectors = bytes.len().div_ceil(sector_len) - 1; - let num_fat_sectors = u32_at(0x2C).unwrap_or(0) as usize; - let mut difat: Vec = (0..HEADER_ENTRIES).filter_map(|i| u32_at(0x4C + i * 4)).collect(); - let mut next = u32_at(0x44).unwrap_or(END_OF_CHAIN); - let mut steps = 0usize; - while next != END_OF_CHAIN && next != FREE { - steps += 1; - // `cfb` refuses a chain sector past the file or seen before; a chain longer than the - // file has sectors must have done one of the two. - if steps > num_sectors || next as usize >= num_sectors { - return Err(format!("the DIFAT chain is longer than the file or leaves it (sector {next})")); - } - let at = (next as usize + 1) * sector_len; - let entries = sector_len / 4 - 1; - difat.extend((0..entries).map(|i| u32_at(at + i * 4).unwrap_or(FREE))); - next = u32_at(at + entries * 4).unwrap_or(END_OF_CHAIN); - } - while difat.len() > HEADER_ENTRIES && difat.len() > num_fat_sectors && difat.last() == Some(&0) { - difat.pop(); - } - while difat.last() == Some(&FREE) { - difat.pop(); - } - // Each listed FAT sector adds a whole sector of entries. A file can't hold more FAT sectors - // than it has sectors, and below that the FAT is no larger than the file. - if difat.len() > num_sectors { - return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", difat.len())); - } - Ok(()) -} - /// Read a `.doc` (Word 97-2003 binary) file into a [`Document`]. A password-protected file /// gives [`DocbinError::PasswordRequired`]; see [`read_with_password`]. pub fn read(bytes: &[u8]) -> Result { diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index afd29c5085..ba9bd92f1f 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2397,35 +2397,13 @@ fn paragraph_borders_read_from_both_forms() { /// build a FAT of hundreds of megabytes before checking it. The precheck refuses it first. #[test] fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() { - const S: usize = 512; - const FREE: u32 = 0xFFFF_FFFF; - let chain = 2048u32; // DIFAT sectors 1..=chain, sector 0 is the "FAT" - let mut f = vec![0u8; S * (chain as usize + 2)]; - f[..8].copy_from_slice(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]); - f[0x18..0x1A].copy_from_slice(&0x3Eu16.to_le_bytes()); - f[0x1A..0x1C].copy_from_slice(&3u16.to_le_bytes()); - f[0x1C..0x1E].copy_from_slice(&0xFFFEu16.to_le_bytes()); - f[0x1E..0x20].copy_from_slice(&9u16.to_le_bytes()); - f[0x20..0x22].copy_from_slice(&6u16.to_le_bytes()); - f[0x2C..0x30].copy_from_slice(&FREE.to_le_bytes()); // no zero trimming - f[0x38..0x3C].copy_from_slice(&4096u32.to_le_bytes()); - f[0x44..0x48].copy_from_slice(&1u32.to_le_bytes()); - f[0x48..0x4C].copy_from_slice(&chain.to_le_bytes()); - for i in 0..109 { - f[0x4C + i * 4..0x50 + i * 4].copy_from_slice(&0u32.to_le_bytes()); - } - for d in 1..=chain { - let at = (d as usize + 1) * S; - let next = if d == chain { 0xFFFF_FFFE } else { d + 1 }; - f[at + S - 4..at + S].copy_from_slice(&next.to_le_bytes()); - } + let f = wordcraft_doc::cfb_guard::repeating_difat_sample(); let t = std::time::Instant::now(); - assert!(super::cfb_precheck(&f).is_err_and(|e| e.contains("FAT sectors in a file of"))); assert!(matches!(read(&f), Err(super::DocbinError::Container(e)) if e.contains("FAT sectors in a file of"))); assert!(!super::is_encrypted(&f)); assert!(t.elapsed().as_secs() < 2, "refused in {:?}", t.elapsed()); // A file cfb wrote itself passes. - assert!(super::cfb_precheck(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); + assert!(wordcraft_doc::cfb_guard::check(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); } /// #3708: a long HYPERLINK target over a long result was copied (and compared) once per result diff --git a/crates/docx/src/crypt.rs b/crates/docx/src/crypt.rs index 2087d9bda0..091bd1a83d 100644 --- a/crates/docx/src/crypt.rs +++ b/crates/docx/src/crypt.rs @@ -74,7 +74,7 @@ const NS_PASSWORD: &str = "http://schemas.microsoft.com/office/2006/keyEncryptor /// Is this an encrypted Office package (a compound file with `EncryptionInfo` and /// `EncryptedPackage` streams) rather than a zip? pub fn is_encrypted(bytes: &[u8]) -> bool { - if bytes.get(..8) != Some(&CFB_MAGIC[..]) { + if bytes.get(..8) != Some(&CFB_MAGIC[..]) || wordcraft_doc::cfb_guard::check(bytes).is_err() { return false; } match cfb::CompoundFile::open(Cursor::new(bytes)) { @@ -87,6 +87,7 @@ pub fn is_encrypted(bytes: &[u8]) -> bool { /// [`DocxError::PasswordRequired`] once it knows the file is one it can decrypt; a password that /// doesn't fit gives [`DocxError::WrongPassword`]. pub fn decrypt(bytes: &[u8], password: Option<&str>) -> Result, DocxError> { + wordcraft_doc::cfb_guard::check(bytes).map_err(|e| bad(format!("not a readable compound file ({e})")))?; let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)).map_err(|e| bad(format!("not a readable compound file ({e})")))?; let info = stream(&mut comp, "/EncryptionInfo", MAX_INFO)?; let package = stream(&mut comp, "/EncryptedPackage", bytes.len() as u64)?; diff --git a/crates/docx/tests/it/malformed.rs b/crates/docx/tests/it/malformed.rs index 15a4b263c7..67eedf78d5 100644 --- a/crates/docx/tests/it/malformed.rs +++ b/crates/docx/tests/it/malformed.rs @@ -342,3 +342,12 @@ fn long_header_reference_chain_is_cut_off() { assert!(!d.parts.is_empty()); assert!(d.parts.len() <= 8, "followed {} chained headers", d.parts.len()); } + +/// #3642: a compound file whose DIFAT names one FAT sector thousands of times is refused by the +/// encrypted-package sniff (reached for any `.docx` name) before `cfb` builds its FAT. +#[test] +fn a_repeating_difat_is_refused_before_cfb_opens_it() { + let f = wordcraft_doc::cfb_guard::repeating_difat_sample(); + assert!(!wordcraft_docx::is_encrypted(&f)); + assert!(matches!(wordcraft_docx::decrypt(&f, None), Err(e) if e.to_string().contains("FAT sectors in a file of"))); +} diff --git a/crates/formats/src/sheet/xls.rs b/crates/formats/src/sheet/xls.rs index 3a70f5308a..df6deb7669 100644 --- a/crates/formats/src/sheet/xls.rs +++ b/crates/formats/src/sheet/xls.rs @@ -44,7 +44,7 @@ const TABLE: u16 = 0x0236; /// Whether `bytes` is a compound file holding an Excel workbook stream. pub fn is_xls(bytes: &[u8]) -> bool { - if !bytes.starts_with(&MAGIC) { + if !bytes.starts_with(&MAGIC) || wordcraft_doc::cfb_guard::check(bytes).is_err() { return false; } let Ok(comp) = cfb::CompoundFile::open(Cursor::new(bytes)) else { return false }; @@ -68,6 +68,7 @@ pub fn read_rows(bytes: &[u8], pick: super::Pick<'_>) -> Result<(Vec, Ve } fn workbook_stream(bytes: &[u8]) -> Result, String> { + wordcraft_doc::cfb_guard::check(bytes).map_err(|e| format!("not a valid Excel 97–2003 workbook ({e})"))?; let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)).map_err(|e| format!("not a valid Excel 97–2003 workbook ({e})"))?; if !comp.is_stream("/Workbook") { if comp.is_stream("/Book") { @@ -540,6 +541,15 @@ mod tests { r.iter().map(|r| r.iter().map(|c| c.to_string()).collect()).collect() } + /// #3642: a DIFAT naming one FAT sector thousands of times is refused before `cfb` builds + /// a FAT far larger than the file. + #[test] + fn a_repeating_difat_is_refused_before_cfb_opens_it() { + let f = wordcraft_doc::cfb_guard::repeating_difat_sample(); + assert!(!is_xls(&f)); + assert!(sheet_names(&f).is_err_and(|e| e.contains("FAT sectors in a file of"))); + } + #[test] fn reads_a_biff8_workbook() { let book = build(&[ From dd6585f16619475ff47a0cb888d34985f6473836 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:53:16 +0100 Subject: [PATCH 013/112] #3710: .doc floating-object copies charged their alt text and group members' too CodeRabbit review: the anchor copy cost counted only the struct size, so an aliased picture's alt text (up to ~12 KB) was copied per walk uncharged. Recursion is bounded by MAX_GROUP_DEPTH. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docbin/src/lib.rs | 19 +++++++++++++------ crates/docbin/src/tests/drawing.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index b3fbc2361e..12e4f57afd 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -553,13 +553,9 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic // Drawn object anchor: the floating shape or picture anchored at this CP. let at = ctx.drawn.partition_point(|(c, _)| *c < cp); // Each copy (a story can be walked more than once) is charged to the work budget - // by size, a group with all its members (#3710). + // by what a clone copies: a picture's alt text, a group with all its members (#3710). for (_, obj) in ctx.drawn.get(at..).unwrap_or_default().iter().take_while(|(c, _)| *c == cp) { - let parts = match obj { - InlineObject::Group { children, .. } => children.len().saturating_add(1), - _ => 1, - }; - let cost = (parts as u64).saturating_mul(std::mem::size_of::() as u64); + let cost = object_cost(obj); let Some(left) = ctx.work.get().checked_sub(cost) else { continue }; ctx.work.set(left); pb.push('\u{FFFC}', CharProps::default()); @@ -612,6 +608,17 @@ struct FieldBuild { /// The target is copied once per run of equally formatted result text, not per character, /// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the /// result is kept as plain text (#3708). +/// Bytes a clone of `obj` copies: the object itself plus a picture's alt text, a group with all +/// its members (#3710). +fn object_cost(obj: &InlineObject) -> u64 { + let shallow = std::mem::size_of::() as u64; + match obj { + InlineObject::Image { alt, .. } => shallow.saturating_add(alt.len() as u64), + InlineObject::Group { children, .. } => children.iter().fold(shallow, |n, c| n.saturating_add(object_cost(&c.obj))), + _ => shallow, + } +} + fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &std::cell::Cell) { let instr = f.instr.trim().to_string(); let result = f.sep.unwrap_or_default(); diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index eeff4651b8..bbf80a5db6 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -377,6 +377,33 @@ fn a_group_named_by_many_anchors_is_built_once() { assert_eq!(groups, [1000]); } +/// #3710: a floating picture's copy is charged its alt text too, and a group every member's. +#[test] +fn a_floating_copy_is_charged_its_alt_text() { + let image = |alt: String| InlineObject::Image { + media: String::new(), + w: 1.0, + h: 1.0, + alt, + float: Default::default(), + crop: Default::default(), + ole: None, + }; + let bare = crate::object_cost(&image(String::new())); + assert_eq!(crate::object_cost(&image("€".repeat(4096))), bare + 3 * 4096); + let member = |obj| wordcraft_doc::para::GroupChild { x: 0.0, y: 0.0, obj }; + let group = InlineObject::Group { + w: 1.0, + h: 1.0, + float: Default::default(), + ch_w: 1.0, + ch_h: 1.0, + children: vec![member(image("x".repeat(100))), member(image(String::new()))], + canvas: None, + }; + assert_eq!(crate::object_cost(&group), 3 * bare + 100); +} + /// A deflated WMF BLIP record declaring `size` uncompressed bytes. fn packed_wmf(size: u32, packed: &[u8]) -> Vec { let mut hdr = size.to_le_bytes().to_vec(); From 3aadd0a4af8bd3460105c6f2a6da19df3127c68f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 02:45:56 +0100 Subject: [PATCH 014/112] =?UTF-8?q?#3642:=20review=20=E2=80=94=20compound-?= =?UTF-8?q?file=20check=20refuses=20a=20FAT=20sector=20listed=20twice=20or?= =?UTF-8?q?=20more=20FAT=20sectors=20than=20the=20file=20needs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/doc/src/cfb_guard.rs | 46 +++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/crates/doc/src/cfb_guard.rs b/crates/doc/src/cfb_guard.rs index c50bd9314d..5e9befb544 100644 --- a/crates/doc/src/cfb_guard.rs +++ b/crates/doc/src/cfb_guard.rs @@ -6,9 +6,10 @@ /// only validates the FAT once it is built, so a small header naming one sector over and over /// makes it allocate gigabytes. Here the DIFAT is walked the way `cfb` walks it (header entries, /// then the DIFAT chain, at most one step per sector; trailing zero and free entries trimmed -/// the same way) and the file is refused when it lists more FAT sectors than the file has -/// sectors, so the FAT `cfb` builds is never larger than the file. Anything else that is wrong -/// is left for `cfb` to report. +/// the same way) and the file is refused when it lists one FAT sector twice, or more FAT sectors +/// than its sectors need (one FAT sector covers `sector_len / 4` of them, plus one spare), so the +/// FAT `cfb` builds is about the size the file needs. Anything else that is wrong is left for +/// `cfb` to report. pub fn check(bytes: &[u8]) -> Result<(), String> { const MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; const FREE: u32 = 0xFFFF_FFFF; @@ -51,10 +52,17 @@ pub fn check(bytes: &[u8]) -> Result<(), String> { while difat.last() == Some(&FREE) { difat.pop(); } - // Each listed FAT sector adds a whole sector of entries. A file can't hold more FAT sectors - // than it has sectors, and below that the FAT is no larger than the file. - if difat.len() > num_sectors { - return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", difat.len())); + // Each listed FAT sector adds a whole sector of entries. A FAT sector holds the entries of + // `sector_len / 4` sectors, so a file needs no more FAT sectors than that covers (one spare), + // and none of them twice. A free entry in the middle is left for `cfb` to refuse. + let listed: Vec = difat.into_iter().filter(|&s| s != FREE).collect(); + let needed = num_sectors.div_ceil(sector_len / 4).saturating_add(1); + if listed.len() > needed { + return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", listed.len())); + } + let mut seen = std::collections::HashSet::with_capacity(listed.len()); + if let Some(dup) = listed.iter().find(|&&s| !seen.insert(s)) { + return Err(format!("the DIFAT lists FAT sector {dup} twice")); } Ok(()) } @@ -97,4 +105,28 @@ mod tests { assert!(super::check(&super::repeating_difat_sample()).is_err_and(|e| e.contains("FAT sectors in a file of"))); assert!(super::check(b"not a compound file").is_ok()); } + + /// A 512-byte-sector file of `sectors` sectors after the header, whose header DIFAT lists + /// `fat` (the rest free). + fn header_difat(sectors: usize, fat: &[u32]) -> Vec { + let mut f = vec![0u8; 512 * (sectors + 1)]; + f[..8].copy_from_slice(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]); + f[0x1A..0x1C].copy_from_slice(&3u16.to_le_bytes()); + f[0x2C..0x30].copy_from_slice(&(fat.len() as u32).to_le_bytes()); + f[0x44..0x48].copy_from_slice(&0xFFFF_FFFEu32.to_le_bytes()); + for i in 0..109 { + let v = fat.get(i).copied().unwrap_or(u32::MAX); + f[0x4C + i * 4..0x50 + i * 4].copy_from_slice(&v.to_le_bytes()); + } + f + } + + #[test] + fn a_fat_sector_listed_twice_or_more_than_the_file_needs_is_refused() { + // Each list is shorter than the file has sectors, which the guard used to accept. + assert!(super::check(&header_difat(10, &[0, 0])).is_err_and(|e| e.contains("FAT sector 0 twice"))); + assert!(super::check(&header_difat(10, &[0, 1, 2])).is_err_and(|e| e.contains("3 FAT sectors"))); + assert!(super::check(&header_difat(10, &[0])).is_ok()); + assert!(super::check(&header_difat(200, &[0, 1])).is_ok()); + } } From bd3d260837b9a901244c42b86985f4647b3aa089 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 02:45:56 +0100 Subject: [PATCH 015/112] =?UTF-8?q?#3708:=20review=20=E2=80=94=20finish=5F?= =?UTF-8?q?field's=20doc=20comment=20back=20on=20finish=5Ffield?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docbin/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 12e4f57afd..549de5a4de 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -602,12 +602,6 @@ struct FieldBuild { sep: Option>, } -/// Emit a finished field: HYPERLINK keeps its result text as a linked run; every other -/// field becomes one inline object with its cached result. -/// -/// The target is copied once per run of equally formatted result text, not per character, -/// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the -/// result is kept as plain text (#3708). /// Bytes a clone of `obj` copies: the object itself plus a picture's alt text, a group with all /// its members (#3710). fn object_cost(obj: &InlineObject) -> u64 { @@ -619,6 +613,12 @@ fn object_cost(obj: &InlineObject) -> u64 { } } +/// Emit a finished field: HYPERLINK keeps its result text as a linked run; every other +/// field becomes one inline object with its cached result. +/// +/// The target is copied once per run of equally formatted result text, not per character, +/// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the +/// result is kept as plain text (#3708). fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &std::cell::Cell) { let instr = f.instr.trim().to_string(); let result = f.sep.unwrap_or_default(); From 17d19b42c5dc58d8ee39ad5826cbc52dd8e64948 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:43:17 +0100 Subject: [PATCH 016/112] =?UTF-8?q?#3642:=20review=20=E2=80=94=20the=20DIF?= =?UTF-8?q?AT=20sample=20is=20test-only,=20valid=20.doc/.xls=20files=20pas?= =?UTF-8?q?s=20the=20guard,=20and=20budget=20drops=20are=20logged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - repeating_difat_sample is built only for tests: this crate's, and the readers' through a `test-support` feature they turn on as a dev-dependency; it is no longer in the production API. - Tests write valid compound files with the cfb crate in Word's and Excel's shapes (WordDocument + table stream with Data, summary and ObjectPool streams; Workbook with summary streams), in 512- and 4096-byte sectors, plus an 8 MB one whose FAT sectors spill into a DIFAT chain; each passes cfb_guard::check and opens (no Word- or Excel-made files are added). - The .doc work budget notes what it leaves out (floating objects, link targets, alt text, Prm formatting) and logs one warning per kind once the document is read (#3692). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/doc/Cargo.toml | 5 ++ crates/doc/src/cfb_guard.rs | 5 +- crates/docbin/Cargo.toml | 1 + crates/docbin/src/lib.rs | 115 ++++++++++++++++++++++++----- crates/docbin/src/tests.rs | 91 ++++++++++++++++++++++- crates/docbin/src/tests/drawing.rs | 2 + crates/docx/Cargo.toml | 1 + crates/formats/Cargo.toml | 1 + crates/formats/src/sheet/xls.rs | 44 +++++++++++ docs/file-format-parity.md | 5 +- 10 files changed, 242 insertions(+), 28 deletions(-) diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml index 9ab62d34d8..5a6d6622e4 100644 --- a/crates/doc/Cargo.toml +++ b/crates/doc/Cargo.toml @@ -8,6 +8,11 @@ description = "WordCraft document model: stories, paragraphs, runs, tables, sect [lints] workspace = true +[features] +# Test inputs other crates' tests share (`cfb_guard::repeating_difat_sample`); turned on only as a +# dev-dependency, so never built into the app. +test-support = [] + [dependencies] wordcraft-geom = { workspace = true } base64 = { workspace = true } diff --git a/crates/doc/src/cfb_guard.rs b/crates/doc/src/cfb_guard.rs index 5e9befb544..ec32e4a1a8 100644 --- a/crates/doc/src/cfb_guard.rs +++ b/crates/doc/src/cfb_guard.rs @@ -69,8 +69,9 @@ pub fn check(bytes: &[u8]) -> Result<(), String> { /// A compound file of about 1 MB whose DIFAT chain lists one FAT sector 2048 × 127 times: /// without [`check`], `cfb` builds a FAT of about 130 MB for it. Regression input for the -/// readers' tests. -#[doc(hidden)] +/// readers' tests: built only for tests, this crate's and (through the `test-support` feature, +/// which they turn on as a dev-dependency) the readers', never into the app. +#[cfg(any(test, feature = "test-support"))] pub fn repeating_difat_sample() -> Vec { const S: usize = 512; const CHAIN: u32 = 2048; // DIFAT sectors 1..=CHAIN, sector 0 is the "FAT" diff --git a/crates/docbin/Cargo.toml b/crates/docbin/Cargo.toml index 87e9ea1fd5..c819c33885 100644 --- a/crates/docbin/Cargo.toml +++ b/crates/docbin/Cargo.toml @@ -20,5 +20,6 @@ log = { workspace = true } chrono = { workspace = true } [dev-dependencies] +wordcraft-doc = { workspace = true, features = ["test-support"] } proptest = { workspace = true } serde_json = { workspace = true } diff --git a/crates/docbin/src/lib.rs b/crates/docbin/src/lib.rs index 549de5a4de..845171251a 100644 --- a/crates/docbin/src/lib.rs +++ b/crates/docbin/src/lib.rs @@ -52,8 +52,88 @@ const MAX_TOTAL_CHARS: u64 = 64 << 20; /// Bytes of repeated work per document beyond the text itself: piece Prm grpprls (Prcs, up /// to 64 KB each) interpreted (#3692), hyperlink targets (#3708), picture alt text (#3709) /// and floating objects (#3710) copied into every run or anchor. Past it Prms are skipped, -/// copied strings left empty and floating objects dropped. +/// copied strings left empty and floating objects dropped, each kind logged once a document +/// ([`Work::log`]). const MAX_WORK: u64 = 64 << 20; + +/// What the [`MAX_WORK`] budget pays for, and leaves out once it is spent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Cost { + /// A piece's Prm grpprl: past the budget the piece keeps its CHPX formatting only. + Formatting, + /// A hyperlink target copied into a run of its result: past it the text is kept, unlinked. + LinkTarget, + /// A picture's alt text copied into an anchor: past it the picture has none. + AltText, + /// A floating object copied into its anchor: past it the object is left out. + FloatingObject, +} + +impl Cost { + const ALL: [Cost; 4] = [Cost::Formatting, Cost::LinkTarget, Cost::AltText, Cost::FloatingObject]; + + /// What is left out, for the log. + fn left_out(self) -> &'static str { + match self { + Cost::Formatting => "some text keeps only part of its formatting", + Cost::LinkTarget => "some hyperlinks' text is kept without its link", + Cost::AltText => "some pictures are left without their alt text", + Cost::FloatingObject => "some floating shapes and pictures are left out", + } + } +} + +/// One document's [`MAX_WORK`] budget, and what it has left out (each kind logged once when +/// the document has been read, [`Work::log`]). +struct Work { + left: std::cell::Cell, + dropped: std::cell::Cell<[bool; 4]>, +} + +impl Work { + fn new(budget: u64) -> Self { + Work { left: std::cell::Cell::new(budget), dropped: std::cell::Cell::new([false; 4]) } + } + + /// Spends `cost` bytes on `what`: false (and `what` noted as left out) when they aren't left. + fn charge(&self, cost: u64, what: Cost) -> bool { + match self.left.get().checked_sub(cost) { + Some(left) => { + self.left.set(left); + true + } + None => { + let mut dropped = self.dropped.get(); + if let Some(d) = Cost::ALL.iter().position(|c| *c == what).and_then(|i| dropped.get_mut(i)) { + *d = true; + } + self.dropped.set(dropped); + false + } + } + } + + /// The kinds of work left out so far, in [`Cost::ALL`] order. + fn dropped(&self) -> Vec { + Cost::ALL.iter().zip(self.dropped.get()).filter(|(_, d)| *d).map(|(c, _)| *c).collect() + } + + /// Logs one warning for each kind left out. + fn log(&self) { + let dropped = self.dropped(); + for c in &dropped { + log::warn!(".doc: the document repeats more than WordCraft copies ({MAX_WORK} bytes): {}", c.left_out()); + } + #[cfg(test)] + LAST_DROPPED.with(|d| *d.borrow_mut() = dropped); + } +} + +#[cfg(test)] +thread_local! { + /// What the last document read on this thread left out for the work budget ([`Work::log`]). + static LAST_DROPPED: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} /// `sprmCPicLocation`: the fc of a picture in the Data stream. const C_PIC_LOCATION: u16 = 0x6A03; /// `sprmCSymbol`: the record font and character of a symbol character ([MS-DOC] §2.6.1); the @@ -186,7 +266,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result) -> Result { /// footnotes, text boxes) are hostile and can each span a whole subdocument, so thousands of /// them could otherwise materialise the same text thousands of times; this bounds the total. budget: &'a std::cell::Cell, - /// Repeated-work bytes left ([`MAX_WORK`]). - work: &'a std::cell::Cell, + /// Repeated-work bytes left ([`MAX_WORK`]), and what they left out. + work: &'a Work, } /// State while assembling one paragraph from per-CP formatting. @@ -528,13 +609,7 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic Some(p) => { // Every anchor of a picture gets its own copy of the alt text; the // copies are charged to the work budget (#3709). - let alt = match ctx.work.get().checked_sub(p.alt.len() as u64) { - Some(left) => { - ctx.work.set(left); - p.alt.clone() - } - None => String::new(), - }; + let alt = if ctx.work.charge(p.alt.len() as u64, Cost::AltText) { p.alt.clone() } else { String::new() }; pb.push('\u{FFFC}', CharProps::default()); pb.objects.push(InlineObject::Image { media: p.key.clone(), @@ -555,9 +630,9 @@ fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pic // Each copy (a story can be walked more than once) is charged to the work budget // by what a clone copies: a picture's alt text, a group with all its members (#3710). for (_, obj) in ctx.drawn.get(at..).unwrap_or_default().iter().take_while(|(c, _)| *c == cp) { - let cost = object_cost(obj); - let Some(left) = ctx.work.get().checked_sub(cost) else { continue }; - ctx.work.set(left); + if !ctx.work.charge(object_cost(obj), Cost::FloatingObject) { + continue; + } pb.push('\u{FFFC}', CharProps::default()); pb.objects.push(obj.clone()); } @@ -619,7 +694,7 @@ fn object_cost(obj: &InlineObject) -> u64 { /// The target is copied once per run of equally formatted result text, not per character, /// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the /// result is kept as plain text (#3708). -fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &std::cell::Cell) { +fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &Work) { let instr = f.instr.trim().to_string(); let result = f.sep.unwrap_or_default(); if let Some(target) = hyperlink_target(&instr) { @@ -631,8 +706,7 @@ fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &std::cell::Cell) continue; } let mut linked = props.clone(); - if let Some(left) = work.get().checked_sub(cost) { - work.set(left); + if work.charge(cost, Cost::LinkTarget) { linked.link = Some(target.clone()); } pb.push(c, linked); @@ -825,7 +899,7 @@ struct FmtMemo<'a> { /// Apply a piece's Prm (Prm0 short sprm or Prm1 grpprl) to character properties. A Prm1 /// grpprl is charged to `work`; once that is spent the grpprl is skipped. -fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps, work: &std::cell::Cell) { +fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps, work: &Work) { match prm { PrmRef::None => {} PrmRef::Sprm0 { isprm, val } => { @@ -838,8 +912,9 @@ fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &Ch } } PrmRef::Grpprl(Some(g)) => { - let Some(left) = work.get().checked_sub(g.len() as u64) else { return }; - work.set(left); + if !work.charge(g.len() as u64, Cost::Formatting) { + return; + } for prl in sprm::iter(g) { if prl.sgc() == 2 { fmt::apply_char(props, &prl, base, fonts); diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index ba9bd92f1f..568268ce3a 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2406,6 +2406,66 @@ fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() { assert!(wordcraft_doc::cfb_guard::check(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); } +/// A compound file `cfb` writes in `version`: `streams` at its root, then `extra` streams under +/// an `ObjectPool` storage (where Word keeps embedded objects). +fn cfb_file_in(version: cfb::Version, streams: &[(&str, &[u8])], extra: &[(&str, &[u8])]) -> Vec { + let mut comp = cfb::CompoundFile::create_with_version(version, std::io::Cursor::new(Vec::new())).expect("cfb create"); + for (name, data) in streams { + comp.create_stream(format!("/{name}")).expect("cfb stream").write_all(data).expect("cfb write"); + } + comp.create_storage("/ObjectPool").expect("cfb storage"); + for (name, data) in extra { + comp.create_stream(format!("/ObjectPool/{name}")).expect("cfb stream").write_all(data).expect("cfb write"); + } + comp.into_inner().into_inner() +} + +/// #3642 review: valid compound files pass the guard and open, so it refuses only the hostile +/// ones. The repository keeps no files Word made, so these are written by the `cfb` crate in +/// the shape of Word 97–2003 files: the WordDocument and table streams of a .doc these tests +/// build, with the summary, Data and embedded-object streams Word adds, in 512-byte sectors +/// (version 3, Word's own) and 4096-byte ones (version 4); and one with a Data stream of 8 MB, +/// past what the header's 109 FAT sector slots map in 512-byte sectors (109 × 128 sectors, +/// about 7 MB), so its FAT sectors are listed in a DIFAT chain too, which the guard walks. +#[test] +fn valid_doc_shaped_compound_files_pass_the_guard_and_open() { + let doc = struct_doc(&structured_spec("Hello world\r", 12)); + let mut comp = cfb::CompoundFile::open(std::io::Cursor::new(doc.as_slice())).expect("cfb open"); + // The table stream is 1Table or 0Table, as the FIB's fWhichTblStm says: kept under its name. + let table_name = if comp.is_stream("/1Table") { "1Table" } else { "0Table" }; + let mut stream = |name: &str| { + let mut v = Vec::new(); + std::io::Read::read_to_end(&mut comp.open_stream(name).expect("stream"), &mut v).expect("read"); + v + }; + let (word, table) = (stream("/WordDocument"), stream(&format!("/{table_name}"))); + let summary = vec![0xFEu8; 4096]; + let data = |n: usize| (0..n).map(|i| (i % 251) as u8).collect::>(); + let small = data(200_000); + let big = data(8 << 20); + let ole = data(70_000); + let files = [ + ("as built", doc.clone()), + ( + "version 3", + cfb_file_in( + cfb::Version::V3, + &[("WordDocument", &word), (table_name, &table), ("Data", &small), ("\u{5}SummaryInformation", &summary)], + &[("_1234567890", &ole)], + ), + ), + ("version 4", cfb_file_in(cfb::Version::V4, &[("WordDocument", &word), (table_name, &table), ("Data", &small)], &[("_1", &ole)])), + ("with a DIFAT chain", cfb_file_in(cfb::Version::V3, &[("WordDocument", &word), (table_name, &table), ("Data", &big)], &[])), + ]; + let difat_sectors = |f: &[u8]| f.get(0x48..0x4C).map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + assert!(difat_sectors(&files[3].1) > 0, "the 8 MB file lists FAT sectors past the header's"); + for (what, f) in &files { + assert_eq!(wordcraft_doc::cfb_guard::check(f), Ok(()), "{what}"); + let doc = read(f).unwrap_or_else(|e| panic!("{what}: {e}")); + assert_eq!(body_texts(&doc).first().map(String::as_str), Some("Hello world"), "{what}"); + } +} + /// #3708: a long HYPERLINK target over a long result was copied (and compared) once per result /// character. It is now copied once per run. #[test] @@ -2433,13 +2493,35 @@ fn hyperlink_target_copies_are_charged_to_the_work_budget() { let result: Vec<(char, CharProps)> = (0..1000).map(|i| ('a', if i % 2 == 0 { bold.clone() } else { CharProps::default() })).collect(); let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"{}\"", "u".repeat(100)), sep: Some(result) }; let mut pb = super::ParaBuild::default(); - let budget = std::cell::Cell::new(10_000); + let budget = super::Work::new(10_000); super::finish_field(&mut pb, f, &budget); assert_eq!(pb.text.len(), 1000, "all text kept"); let linked: usize = pb.runs.iter().filter_map(|r| r.props.link.as_ref()).map(|l| l.len()).sum(); assert!(linked <= 10_000, "linked bytes {linked}"); assert_eq!(pb.runs.iter().filter(|r| r.props.link.is_some()).count(), 100); - assert_eq!(budget.get(), 0); + assert_eq!(budget.left.get(), 0); + assert_eq!(budget.dropped(), [super::Cost::LinkTarget], "the unlinked text is noted, to be logged"); +} + +/// #3871 review: what the work budget leaves out is noted once per kind, in a fixed order, and +/// logged once per kind when the document has been read; nothing is noted while it lasts. +#[test] +fn the_work_budget_notes_each_kind_it_leaves_out_once() { + use super::Cost::*; + let work = super::Work::new(10); + assert!(work.charge(10, AltText)); + assert!(work.dropped().is_empty(), "spent to the last byte, nothing left out"); + assert!(work.charge(0, LinkTarget), "a free copy is never refused"); + for _ in 0..3 { + assert!(!work.charge(1, FloatingObject)); + } + assert!(!work.charge(1, AltText)); + assert_eq!(work.dropped(), [AltText, FloatingObject]); + work.log(); + assert_eq!(super::LAST_DROPPED.with(|d| d.borrow().clone()), [AltText, FloatingObject]); + // A document within the budget leaves nothing out. + read(&struct_doc(&structured_spec("Hello\r", 6))).expect("opens"); + assert!(super::LAST_DROPPED.with(|d| d.borrow().is_empty())); } /// #3692: a piece's Prm grpprl (a Prc of up to 64 KB) was re-interpreted for every character. @@ -2470,7 +2552,7 @@ fn a_large_prc_is_interpreted_once_per_formatting_change() { fn prm_grpprls_past_the_work_budget_are_skipped() { use wordcraft_doc::props::CharProps; let g = grpprl(&[(0x0835, &[0x01])]); - let work = std::cell::Cell::new(g.len() as u64 + 1); + let work = super::Work::new(g.len() as u64 + 1); let base = CharProps::default(); let mut first = CharProps::default(); super::apply_prm(&mut first, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work); @@ -2478,5 +2560,6 @@ fn prm_grpprls_past_the_work_budget_are_skipped() { let mut second = CharProps::default(); super::apply_prm(&mut second, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work); assert_eq!(second.bold, None); - assert_eq!(work.get(), 1); + assert_eq!(work.left.get(), 1); + assert_eq!(work.dropped(), [super::Cost::Formatting]); } diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index bbf80a5db6..3d4955ad40 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -351,6 +351,8 @@ fn alt_text_copies_across_many_anchors_are_bounded() { let total: usize = alts.iter().map(|a| a.len()).sum(); assert!(total <= 64 << 20, "alt bytes {total}"); assert_eq!(alts.last().copied(), Some("")); + // Leaving it out is logged, once (#3871 review). + assert_eq!(crate::LAST_DROPPED.with(|d| d.borrow().clone()), [crate::Cost::AltText]); } /// #3710: every PlcSpa entry naming a group built another copy of all its members. A shape is diff --git a/crates/docx/Cargo.toml b/crates/docx/Cargo.toml index 4dd1d510c0..5e16d7e4c2 100644 --- a/crates/docx/Cargo.toml +++ b/crates/docx/Cargo.toml @@ -31,6 +31,7 @@ zeroize = { workspace = true } getrandom = { workspace = true, features = ["wasm_js"] } [dev-dependencies] +wordcraft-doc = { workspace = true, features = ["test-support"] } proptest = { workspace = true } image = { workspace = true } zip = { workspace = true } diff --git a/crates/formats/Cargo.toml b/crates/formats/Cargo.toml index bf447f2752..faf6e41583 100644 --- a/crates/formats/Cargo.toml +++ b/crates/formats/Cargo.toml @@ -25,4 +25,5 @@ chrono = { workspace = true } js-sys = "0.3" [dev-dependencies] +wordcraft-doc = { workspace = true, features = ["test-support"] } proptest = { workspace = true } diff --git a/crates/formats/src/sheet/xls.rs b/crates/formats/src/sheet/xls.rs index df6deb7669..9d8dcfb123 100644 --- a/crates/formats/src/sheet/xls.rs +++ b/crates/formats/src/sheet/xls.rs @@ -550,6 +550,50 @@ mod tests { assert!(sheet_names(&f).is_err_and(|e| e.contains("FAT sectors in a file of"))); } + /// #3642 review: valid workbooks pass the guard and open. The repository keeps no files + /// Excel made, so these are written by the `cfb` crate in the shape of Excel 97–2003 ones: + /// the Workbook stream [`build`] writes with the summary streams Excel adds, in 512-byte + /// sectors (version 3, Excel's own) and 4096-byte ones (version 4); and one with an 8 MB + /// stream beside it, past what the header's 109 FAT sector slots map in 512-byte sectors, so + /// its FAT sectors are listed in a DIFAT chain too, which the guard walks. + #[test] + fn valid_xls_shaped_compound_files_pass_the_guard_and_open() { + use std::io::{Read, Write}; + let book = build(&[("Sheet1", rows(&[&["a", "1"], &["b", "2"]]))]).unwrap(); + let mut workbook = Vec::new(); + cfb::CompoundFile::open(Cursor::new(book.as_slice())).unwrap().open_stream("/Workbook").unwrap().read_to_end(&mut workbook).unwrap(); + let file = |version: cfb::Version, streams: &[(&str, &[u8])]| { + let mut comp = cfb::CompoundFile::create_with_version(version, Cursor::new(Vec::new())).unwrap(); + for (name, data) in streams { + comp.create_stream(format!("/{name}")).unwrap().write_all(data).unwrap(); + } + comp.into_inner().into_inner() + }; + let summary = vec![0xFEu8; 4096]; + let big: Vec = (0..8usize << 20).map(|i| (i % 251) as u8).collect(); + let files = [ + ("as built", book.clone()), + ( + "version 3", + file( + cfb::Version::V3, + &[("Workbook", &workbook), ("\u{5}SummaryInformation", &summary), ("\u{5}DocumentSummaryInformation", &summary)], + ), + ), + ("version 4", file(cfb::Version::V4, &[("Workbook", &workbook), ("\u{5}SummaryInformation", &summary)])), + ("with a DIFAT chain", file(cfb::Version::V3, &[("Workbook", &workbook), ("Ctls", &big)])), + ]; + let difat_sectors = |f: &[u8]| f.get(0x48..0x4C).map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])); + assert!(difat_sectors(&files[3].1) > 0, "the 8 MB file lists FAT sectors past the header's"); + for (what, f) in &files { + assert_eq!(wordcraft_doc::cfb_guard::check(f), Ok(()), "{what}"); + assert!(is_xls(f), "{what}"); + assert_eq!(sheet_names(f).unwrap(), vec!["Sheet1"], "{what}"); + let (_, r) = read_rows(f, Pick::Index(0)).unwrap(); + assert_eq!(r, rows(&[&["a", "1"], &["b", "2"]]), "{what}"); + } + } + #[test] fn reads_a_biff8_workbook() { let book = build(&[ diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 365d226aad..c2b352424f 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity with Microsoft Word -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats Every format Word opens or saves, what WordCraft does with it, and how it's tested. Fidelity is the share of a typical real-world file's content and formatting that survives (read: shows correctly; @@ -49,7 +49,7 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅ | `.docx` Word Document (ECMA-376 Transitional) | 70% | R/W | ✅ | ✅ | ~65% | 70–110 | `crates/docx/tests` (roundtrip 34, fixtures 29, malformed 8, bidi 5, bidi_runs 12, citations 5); synthetic fixtures only | Word opens our files. Untested on a real-world corpus. Tracked changes inside a field result stay revisions (the field is kept as a range, #1037). Details below | | `.docx` Strict Open XML | 1% | R/W | ✅ (namespace mapping in `xml.rs`) | ❌ | ~60% | 2–4 | `strict_namespace_and_bad_numbers`, `strict_line_heights_with_a_unit_are_read_as_point_multiples` | Writing Strict is rare; unit-suffixed measures (`13.90pt`) read as points (#2329) | | `.docm` / `.dotx` / `.dotm` | 3% | R/W | ✅ | ✅ | ~65% | incl. above | `macro_packages.rs` | Macros and signatures kept on save (#172); macros never run | -| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). Hostile files are bounded: a sector table longer than the file is refused before parsing (#3642); formatting and link runs are read once per change, not per character (#3692, #3708); copies of alt text, link targets and drawn objects share a 64 MB budget (#3709, #3710); picture decoding shares a 512 MB budget and aliased store entries decode once (#3711, #3712). | +| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). Hostile files are bounded: a sector table longer than the file is refused before parsing (#3642); formatting and link runs are read once per change, not per character (#3692, #3708); copies of alt text, link targets and drawn objects share a 64 MB budget (#3709, #3710), and what it leaves out is logged once a kind per document; picture decoding shares a 512 MB budget and aliased store entries decode once (#3711, #3712). | | PDF export | 8% | W | — | ✅ | ~80% | 6–10 | `crates/pdf/src/tests.rs` | krilla: embedded subsets, selectable text incl. field results (#133); Hebrew and Arabic copy and search as whole words (right-to-left glyphs written in visual order, #1383, also with fonts that draw lam-alef as two glyphs at one x, #2310); All Caps and Small Caps text extracts as the capitals shown, as from Word's PDFs, and never changes the case of other text (#1473), links, outline, tagging. Outline destinations follow leading page breaks to the heading text. Fonts it can't subset become outlines (#132). Print Markup (#589): comments in balloons beside each page (scaled to its paper) and tracked changes; HTML/Markdown/text copies end with a comment list. Horizontal and vertical lines inside groups are drawn (#1474). No PDF/A option, no "best for printing/online" choice | | PDF open (PDF Reflow, Word for Windows) | 1% | R | ❌ | — | 0% | 30–50 | — | Converting a PDF into an editable document. A sibling app (PdfCraft) may supply the parser | | `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). East Asian multibyte code pages decoded as runs: 932 Shift-JIS (`\fcharset128`, Japanese), 936 GBK (134, Simplified Chinese), 949 UHC/windows-949 (129, Korean) and 950 Big5 (136, Traditional Chinese) through `\ansicpg`, `\cpg` or the font's charset, escaped or raw, a lead byte and its trail byte together; invalid sequences read as U+FFFD, and a run that starts `EF BB BF` is text, not a BOM (#1653). Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; tracked paragraph marks (`\par` inside `\revised`/`\deleted` groups, including a `\sect` inside a deletion) and character and paragraph formatting revisions (`\crauth`/`\crdate` with `\oldcprops`, `\prauth`/`\prdate` with `\oldpprops`) read and written; section and table formatting revisions (`\oldsprops`, `\oldtprops`) not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export writes every section too (break type, page size and orientation, margins, header/footer distance, columns, page numbering, title page; not headers/footers). Thin: comments, notes, RTL | @@ -268,6 +268,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio | 2026-10-25 | trivial | Repeated DOCX style IDs: repeated save/reopen regression for whole-definition replacement. | | 2026-10-25 | trivial | Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`) | | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | +| 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) | | 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | | 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) | From 58898d5168c38855ddc5e5abe0684b8b02d5915b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 15:14:06 +0100 Subject: [PATCH 017/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20docbin=20?= =?UTF-8?q?timing=20tests=20get=20a=20margin=20a=20busy=20CI=20machine=20c?= =?UTF-8?q?an't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 1-2 s failed on loaded runners (checked against the pre-fix behaviour, re-enabled in the worktree): - DIFAT precheck: no clock; the precheck's own error is the proof it ran before cfb. - long hyperlink: no clock (the old per-character copy took ~2.5 s, too close to any safe bound); finish_field on the same field charges the work budget exactly one target copy. - large Prc: 1 s -> 10 s; ~60 ms now, interpreting it per character took 88 s. - group named by many anchors: 1 s -> 10 s backstop; ~10 ms now. The old code (6-8 s here) is caught by the group count, [1000; 3] instead of [1000]. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docbin/src/tests.rs | 16 ++++++++++------ crates/docbin/src/tests/drawing.rs | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/docbin/src/tests.rs b/crates/docbin/src/tests.rs index 568268ce3a..e23cd50241 100644 --- a/crates/docbin/src/tests.rs +++ b/crates/docbin/src/tests.rs @@ -2398,10 +2398,9 @@ fn paragraph_borders_read_from_both_forms() { #[test] fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() { let f = wordcraft_doc::cfb_guard::repeating_difat_sample(); - let t = std::time::Instant::now(); + // The precheck's own error is the proof that it ran before cfb (no clock to skew). assert!(matches!(read(&f), Err(super::DocbinError::Container(e)) if e.contains("FAT sectors in a file of"))); assert!(!super::is_encrypted(&f)); - assert!(t.elapsed().as_secs() < 2, "refused in {:?}", t.elapsed()); // A file cfb wrote itself passes. assert!(wordcraft_doc::cfb_guard::check(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok()); } @@ -2474,14 +2473,18 @@ fn a_long_hyperlink_over_a_long_result_is_one_linked_run() { let target = "x".repeat(n); let text: &'static str = Box::leak(format!("\u{13}HYPERLINK \"http://{target}\"\u{14}{}\u{15}\r", "y".repeat(n)).into_boxed_str()); let spec = structured_spec(text, text.chars().count() as u32); - let t = std::time::Instant::now(); let doc = read(&struct_doc(&spec)).expect("opens"); - // Copying the target per character took ~2.5 s here; once per run takes milliseconds. - assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para"); let linked: Vec<_> = p0.runs.iter().filter(|r| r.props.link.is_some()).collect(); assert_eq!(linked.len(), 1); assert_eq!(linked.first().map(|r| r.len), Some(n)); + // One copy of the target for the one run, not one per character (~2.5 s here): counted by + // what the copies charge to the work budget, which a busy machine can't skew. + let result = vec![('y', wordcraft_doc::props::CharProps::default()); n]; + let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"http://{target}\""), sep: Some(result) }; + let work = super::Work::new(u64::MAX); + super::finish_field(&mut super::ParaBuild::default(), f, &work); + assert_eq!(u64::MAX - work.left.get(), ("http://".len() + n) as u64, "one copy of the target"); } /// #3708: a result whose formatting changes every character pays for a target copy per run @@ -2540,7 +2543,8 @@ fn a_large_prc_is_interpreted_once_per_formatting_change() { let bytes = rich_doc_with(&spec, &RichExtra { prcs: vec![prc], prm: 0x0001, ..Default::default() }); let t = std::time::Instant::now(); let doc = read(&bytes).expect("opens"); - assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); + // About 60 ms on a loaded machine; interpreting the Prc per character took 88 s. + assert!(t.elapsed().as_secs() < 10, "opened in {:?}", t.elapsed()); let p = para0(&doc); assert_eq!(p.runs.len(), 1); assert_eq!(p.runs.first().and_then(|r| r.props.bold), Some(true)); diff --git a/crates/docbin/src/tests/drawing.rs b/crates/docbin/src/tests/drawing.rs index 3d4955ad40..041267c16c 100644 --- a/crates/docbin/src/tests/drawing.rs +++ b/crates/docbin/src/tests/drawing.rs @@ -372,7 +372,9 @@ fn a_group_named_by_many_anchors_is_built_once() { let spec = StructSpec { text, extra_pairs: vec![(40, plc(&cps, &spas)), (50, art)], ..structured_spec(text, ccp) }; let t = std::time::Instant::now(); let doc = read(&struct_doc(&spec)).expect("opens"); - assert!(t.elapsed().as_millis() < 1000, "opened in {:?}", t.elapsed()); + // About 10 ms on a loaded machine, so 10 s is a backstop; the old code (a copy per entry, + // 6-8 s here) is caught by the group count below, one group per anchor ([1000; 3]). + assert!(t.elapsed().as_secs() < 10, "opened in {:?}", t.elapsed()); let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para"); let groups: Vec = p0.objects.iter().filter_map(|o| if let InlineObject::Group { children, .. } = o { Some(children.len()) } else { None }).collect(); From 90efee7636c7922a9f7ebe34241fb819f1a1c7ed Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:12:38 +0100 Subject: [PATCH 018/112] #3692: Zotero lists each note's fields once, within a field and code-byte budget Audit finding 12 (CWE-400): fields::list expanded a note's fields again for every body reference to it, so a crafted file with thousands of references to one citation-heavy endnote multiplied into millions of copied fields on the UI thread. Each note part is now enumerated once, at its first reference, and the list stops at MAX_FIELDS fields / MAX_CODE_BYTES of codes. Note references are walked with the paragraph's objects directly instead of object_at per marker. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/fields.rs | 36 +++++++++++++++++---- crates/zotero/src/lib.rs | 2 ++ crates/zotero/src/tests_limits.rs | 52 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 crates/zotero/src/tests_limits.rs diff --git a/crates/zotero/src/fields.rs b/crates/zotero/src/fields.rs index c3776f9a51..21f7d2cc07 100644 --- a/crates/zotero/src/fields.rs +++ b/crates/zotero/src/fields.rs @@ -63,7 +63,14 @@ fn story_fields(doc: &Document, story: StoryRef, note_index: u32) -> Vec .collect() } +/// Most fields [`list`] hands Zotero (no real document comes near it). +pub const MAX_FIELDS: usize = 100_000; +/// Most bytes of field codes [`list`] hands Zotero, all fields together. +pub const MAX_CODE_BYTES: usize = 64 * 1024 * 1024; + /// Every Zotero field in document order: the body's, with each note's fields at its reference. +/// A note the body references more than once (a crafted file) is listed at its first reference +/// only. Past [`MAX_FIELDS`] fields or [`MAX_CODE_BYTES`] of codes the rest are left out. pub fn list(doc: &Document) -> Vec { enum Ev { Field(ZField), @@ -71,10 +78,12 @@ pub fn list(doc: &Document) -> Vec { } let mut evs: Vec<(Pos, Ev)> = story_fields(doc, StoryRef::Body, 0).into_iter().map(|f| (f.start.clone(), Ev::Field(f))).collect(); let (mut foot, mut end) = (0u32, 0u32); + let mut seen = std::collections::HashSet::new(); for path in doc.para_paths(StoryRef::Body) { let Some(p) = doc.para(StoryRef::Body, &path) else { continue }; - for off in p.object_offsets() { - if let Some(InlineObject::NoteRef { kind, id, custom }) = p.object_at(off) { + // The k-th object marker holds the k-th object. + for (off, obj) in p.object_offsets().into_iter().zip(&p.objects) { + if let InlineObject::NoteRef { kind, id, custom } = obj { let n = match kind { NoteKind::Footnote => &mut foot, NoteKind::Endnote => &mut end, @@ -82,16 +91,31 @@ pub fn list(doc: &Document) -> Vec { if custom.is_empty() { *n = n.saturating_add(1); } - evs.push((Pos::new(StoryRef::Body, path.clone(), off), Ev::Note(*id, (*n).max(1)))); + if seen.insert(*id) { + evs.push((Pos::new(StoryRef::Body, path.clone(), off), Ev::Note(*id, (*n).max(1)))); + } } } } evs.sort_by(|a, b| a.0.cmp(&b.0)); let mut out = Vec::new(); + let mut bytes = 0usize; + let mut take = |f: ZField, out: &mut Vec| { + bytes = bytes.saturating_add(f.code.len()); + let fits = out.len() < MAX_FIELDS && bytes <= MAX_CODE_BYTES; + if fits { + out.push(f); + } + fits + }; for (_, ev) in evs { - match ev { - Ev::Field(f) => out.push(f), - Ev::Note(id, n) => out.extend(story_fields(doc, StoryRef::Part(id), n)), + let fits = match ev { + Ev::Field(f) => take(f, &mut out), + Ev::Note(id, n) => story_fields(doc, StoryRef::Part(id), n).into_iter().all(|f| take(f, &mut out)), + }; + if !fits { + log::warn!("Zotero: the document has more citation fields than WordCraft hands Zotero; the rest are left out"); + break; } } out diff --git a/crates/zotero/src/lib.rs b/crates/zotero/src/lib.rs index 40eba55ebe..4c50bab040 100644 --- a/crates/zotero/src/lib.rs +++ b/crates/zotero/src/lib.rs @@ -41,3 +41,5 @@ pub enum ZoteroError { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_limits; diff --git a/crates/zotero/src/tests_limits.rs b/crates/zotero/src/tests_limits.rs new file mode 100644 index 0000000000..93cefc62c9 --- /dev/null +++ b/crates/zotero/src/tests_limits.rs @@ -0,0 +1,52 @@ +//! Hostile documents and calls stay cheap: work grows with the document, not with how often a +//! crafted file or call repeats something (#3692). + +use std::time::{Duration, Instant}; + +use wordcraft_doc::para::{NoteKind, OBJ}; +use wordcraft_doc::{CharProps, Document, InlineObject, Paragraph, PartKind, Run, StoryRef, para_block}; + +use crate::fields; + +const CITE: &str = r#"ADDIN ZOTERO_ITEM CSL_CITATION {"citationID":"k1","citationItems":[{"id":7}]}"#; + +/// A paragraph of `objects`, one marker each, in one plain run. +fn objects_para(objects: Vec) -> Paragraph { + let mut p = Paragraph::new(); + p.text = std::iter::repeat_n(OBJ, objects.len()).collect(); + p.runs = vec![Run { len: p.text.len(), props: CharProps::default() }]; + p.objects = objects; + p +} + +/// `n` empty Zotero citation fields in a row. +fn citations(n: usize) -> Vec { + (0..n).flat_map(|_| [InlineObject::FieldStart { instr: CITE.into(), locked: false, dirty: false }, InlineObject::FieldEnd]).collect() +} + +// Finding 12: one endnote referenced many times is listed once, not once per reference. +#[test] +fn a_note_referenced_many_times_is_listed_once() { + let mut doc = Document::from_text(""); + let note = doc.add_part(PartKind::Endnote, vec![para_block(objects_para(citations(200)))]); + let refs = (0..5_000).map(|_| InlineObject::NoteRef { kind: NoteKind::Endnote, id: note, custom: String::new() }).collect(); + doc.body = vec![para_block(objects_para(refs))]; + let t = Instant::now(); + let list = fields::list(&doc); + assert_eq!(list.len(), 200, "each field once, at the note's first reference"); + assert!(list.iter().all(|f| f.start.story == StoryRef::Part(note) && f.note_index == 1)); + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); +} + +#[test] +fn listing_stops_at_the_field_budget() { + let mut doc = Document::from_text(""); + // Many short paragraphs: the per-paragraph object scan is another finding's (#3614). + doc.body = (0..fields::MAX_FIELDS / 100 + 1).map(|_| para_block(objects_para(citations(100)))).collect(); + assert_eq!(fields::list(&doc).len(), fields::MAX_FIELDS); + // Codes count too: a few huge codes fill the byte budget before the field budget. + let big = format!("{CITE}{}", "x".repeat(fields::MAX_CODE_BYTES / 4 - 1000)); + let huge = (0..5).flat_map(|_| [InlineObject::FieldStart { instr: big.clone(), locked: false, dirty: false }, InlineObject::FieldEnd]).collect(); + doc.body = vec![para_block(objects_para(huge))]; + assert_eq!(fields::list(&doc).len(), 4); +} From 020194ebdb5e1a80963d35e6a41015b959593c91 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:13:00 +0100 Subject: [PATCH 019/112] #3692: Zotero rich text is built in one pass, not re-normalized per run Audit finding 16 (CWE-407): rich::from_doc added each formatting run through Paragraph::insert_text, which splits and normalizes every earlier run, so a compact paragraph of alternating formatting was quadratic on the UI thread (100,000 runs: over 9 minutes; now 0.1 s). Runs are appended, merged only with the run before, and the paragraph is normalized once. A cumulative MAX_RUNS budget keeps the text but stops adding runs past it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/rich.rs | 22 +++++++++++++++++++--- crates/zotero/src/tests_limits.rs | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/zotero/src/rich.rs b/crates/zotero/src/rich.rs index 2fabb62bb8..6285ecf3eb 100644 --- a/crates/zotero/src/rich.rs +++ b/crates/zotero/src/rich.rs @@ -4,10 +4,12 @@ use wordcraft_doc::edit::Fragment; use wordcraft_doc::para::OBJ; -use wordcraft_doc::{Block, CharProps, Document, Paragraph}; +use wordcraft_doc::{Block, CharProps, Document, Paragraph, Run}; /// Most paragraphs taken from one text (a bibliography of thousands of entries is fine). const MAX_PARAS: usize = 100_000; +/// Most formatting runs taken from one text; past it, text keeps the formatting before it. +pub(crate) const MAX_RUNS: usize = 1_000_000; fn style_props(base: &CharProps, src: &CharProps) -> CharProps { let mut p = base.clone(); @@ -40,16 +42,30 @@ fn style_props(base: &CharProps, src: &CharProps) -> CharProps { fn from_doc(d: &Document, base: &CharProps) -> Vec { let mut out = Vec::new(); + let mut runs = 0usize; for b in d.body.iter().take(MAX_PARAS) { let Some(src) = b.as_para() else { continue }; let mut p = Paragraph::new(); p.mark = base.clone(); + // Appended run by run, merging only with the run before; normalized once at the end. for (range, props) in src.run_ranges() { let Some(t) = src.text.get(range) else { continue }; let t: String = t.chars().filter(|c| *c != OBJ).collect(); - let at = p.len(); - let _ = p.insert_text(at, &t, &style_props(base, props)); + if t.is_empty() { + continue; + } + let props = style_props(base, props); + p.text.push_str(&t); + match p.runs.last_mut() { + Some(last) if last.props == props || runs >= MAX_RUNS => last.len += t.len(), + _ => { + p.runs.push(Run { len: t.len(), props }); + runs += 1; + } + } } + p.normalize(); + p.touch(); out.push(p); } out diff --git a/crates/zotero/src/tests_limits.rs b/crates/zotero/src/tests_limits.rs index 93cefc62c9..471d385922 100644 --- a/crates/zotero/src/tests_limits.rs +++ b/crates/zotero/src/tests_limits.rs @@ -50,3 +50,23 @@ fn listing_stops_at_the_field_budget() { doc.body = vec![para_block(objects_para(huge))]; assert_eq!(fields::list(&doc).len(), 4); } + +// Finding 16: rich text with alternating formatting is built in one pass, not run by run with +// every earlier run merged again. +#[test] +fn rich_text_with_many_runs_is_linear() { + let html: String = "ab".repeat(50_000); + let t = Instant::now(); + let frag = crate::rich::fragment(&html, true, &CharProps::default()); + let elapsed = t.elapsed(); + let Some(wordcraft_doc::Block::Para(p)) = frag.blocks.first() else { panic!("no paragraph") }; + assert_eq!(p.text, "ab".repeat(50_000)); + assert_eq!(p.runs.len(), 100_000); + assert_eq!(p.runs.iter().map(|r| r.len).sum::(), p.text.len()); + assert!(elapsed < Duration::from_secs(2), "{elapsed:?}"); + // Neighbours with the same formatting are one run. + let frag = crate::rich::fragment("abc", true, &CharProps::default()); + let Some(wordcraft_doc::Block::Para(p)) = frag.blocks.first() else { panic!("no paragraph") }; + assert_eq!((p.text.as_str(), p.runs.len()), ("abc", 2)); + assert_eq!(p.props_of_char(1).italic, Some(true)); +} From ab26f22c4ac8729c1a72760aafd7c9b1a595f247 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:15:34 +0100 Subject: [PATCH 020/112] #3692: Document_convert checks each distinct field id once against one indexed list Audit finding 32 (CWE-407): Document_convert rebuilt the whole field list for every id it was given, duplicates included, before deciding whether anything had to move, so one valid id repeated 100,000 times took 18 s on the UI thread (now milliseconds). Ids are deduplicated (a repeated id takes its last note type), all of them are checked against one indexed snapshot before anything moves, already-placed citations are skipped from the snapshot, and the index is rebuilt only after a citation actually moves. The id list is capped at fields::MAX_FIELDS, the most fields Zotero is ever handed. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/bridge.rs | 76 +++++++++++++++++++++++++------ crates/zotero/src/tests_limits.rs | 50 +++++++++++++++++++- 2 files changed, 110 insertions(+), 16 deletions(-) diff --git a/crates/zotero/src/bridge.rs b/crates/zotero/src/bridge.rs index b8dfaedac6..36cb2d0b44 100644 --- a/crates/zotero/src/bridge.rs +++ b/crates/zotero/src/bridge.rs @@ -5,6 +5,8 @@ //! Field ids handed to Zotero stay valid for the transaction: they are kept in a list parallel //! to the document-order list of Zotero fields, updated as fields are inserted and removed. +use std::collections::HashMap; + use serde_json::{Value, json}; use wordcraft_doc::edit::Fragment; use wordcraft_doc::para::OBJ; @@ -274,21 +276,8 @@ impl Bridge { // citation moves between the text, footnotes and endnotes (switching styles). "Document_convert" => { let Value::Array(ids) = arg(c, 1)? else { return Err("field ids must be an array".into()) }; - let ids: Vec = ids.iter().take(100_000).cloned().collect(); - let notes = c.args.get(3).cloned().unwrap_or(Value::Null); - for (i, idv) in ids.iter().enumerate() { - let id = match idv { - Value::Number(n) => n.as_u64(), - Value::String(x) => x.trim().parse().ok(), - _ => None, - } - .ok_or("bad field id")?; - let note = match ¬es { - Value::Array(a) => a.get(i).and_then(Value::as_i64).unwrap_or(0), - v => v.as_i64().unwrap_or(0), - }; - self.convert_field(s, id, note)?; - } + let notes = c.args.get(3).unwrap_or(&Value::Null); + self.convert_fields(s, ids, notes)?; Ok(Value::Null) } "Field_convert" => { @@ -484,6 +473,49 @@ impl Bridge { Ok(()) } + /// `Document_convert`: each id once (listed again, its last note type wins), every id + /// checked against one indexed list of the fields before anything moves. Only a citation + /// that actually moves costs a fresh list. + fn convert_fields(&mut self, s: &mut Session, ids: &[Value], notes: &Value) -> Result<(), String> { + let mut todo: Vec<(u64, i64)> = Vec::new(); + let mut slot: HashMap = HashMap::new(); + for (i, idv) in ids.iter().take(fields::MAX_FIELDS).enumerate() { + let id = match idv { + Value::Number(n) => n.as_u64(), + Value::String(x) => x.trim().parse().ok(), + _ => None, + } + .ok_or("bad field id")?; + let note = match notes { + Value::Array(a) => a.get(i).and_then(Value::as_i64).unwrap_or(0), + v => v.as_i64().unwrap_or(0), + }; + match slot.get(&id).and_then(|k| todo.get_mut(*k)) { + Some(t) => t.1 = note, + None => { + slot.insert(id, todo.len()); + todo.push((id, note)); + } + } + } + let index = |ids: &[u64]| -> HashMap { ids.iter().enumerate().map(|(k, id)| (*id, k)).collect() }; + let mut list = self.fields(s); + let mut at = index(&self.ids); + if let Some((id, _)) = todo.iter().find(|(id, _)| !at.contains_key(id)) { + return Err(format!("no field with id {id}")); + } + for (id, note) in todo { + let f = at.get(&id).and_then(|k| list.get(*k)).ok_or_else(|| format!("no field with id {id}"))?; + if !moves(s, f, note) { + continue; + } + self.convert_field(s, id, note)?; + list = self.fields(s); + at = index(&self.ids); + } + Ok(()) + } + fn convert_placeholders(&mut self, s: &mut Session, ph: &[Value], note: i64) -> R { let mut ids = Vec::new(); for v in ph { @@ -519,6 +551,20 @@ impl Bridge { } } +/// Whether moving `f` to the text (`note` 0), a footnote (1) or an endnote (2) changes anything, +/// as [`Bridge::convert_field`] decides it. +fn moves(s: &Session, f: &ZField, note: i64) -> bool { + let current = match f.start.story { + StoryRef::Body => 0, + StoryRef::Part(pid) => match s.doc.parts.get(&pid).map(|p| p.kind) { + Some(PartKind::Footnote) => 1, + Some(PartKind::Endnote) => 2, + _ => return false, + }, + }; + current != if matches!(note, 1 | 2) { note } else { 0 } +} + /// Delete the selection; returns where it collapsed. fn delete_selection(s: &mut Session) -> Result { let (a, b) = s.sel.ordered(); diff --git a/crates/zotero/src/tests_limits.rs b/crates/zotero/src/tests_limits.rs index 471d385922..53d8eaafc1 100644 --- a/crates/zotero/src/tests_limits.rs +++ b/crates/zotero/src/tests_limits.rs @@ -3,10 +3,13 @@ use std::time::{Duration, Instant}; +use serde_json::{Value, json}; use wordcraft_doc::para::{NoteKind, OBJ}; use wordcraft_doc::{CharProps, Document, InlineObject, Paragraph, PartKind, Run, StoryRef, para_block}; +use wordcraft_engine::Session; -use crate::fields; +use crate::wire::Call; +use crate::{Bridge, Headless, fields}; const CITE: &str = r#"ADDIN ZOTERO_ITEM CSL_CITATION {"citationID":"k1","citationItems":[{"id":7}]}"#; @@ -70,3 +73,48 @@ fn rich_text_with_many_runs_is_linear() { assert_eq!((p.text.as_str(), p.runs.len()), ("abc", 2)); assert_eq!(p.props_of_char(1).italic, Some(true)); } + +/// Call `method` on a bridge over `s`. +fn call(b: &mut Bridge, s: &mut Session, method: &str, args: Value) -> Result { + let Value::Array(args) = args else { return Err("args must be an array".into()) }; + b.handle(s, &mut Headless::default(), &Call::new(method, args)) +} + +// Finding 32: Document_convert checks each distinct id once against one list of the fields. +#[test] +fn convert_with_one_id_repeated_is_one_check() { + let mut doc = Document::from_text(""); + doc.body = (0..300).map(|_| para_block(objects_para(citations(1)))).collect(); + let mut s = Session::new(doc); + let mut b = Bridge::new(); + call(&mut b, &mut s, "Application_getActiveDocument", json!([3])).unwrap(); + let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap(); + let first = got[0][0].clone(); + let ids = vec![first.clone(); 100_000]; + let before = format!("{:?}", s.doc.body); + let t = Instant::now(); + call(&mut b, &mut s, "Document_convert", json!([1, ids, 0, 0])).unwrap(); + assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); + assert_eq!(format!("{:?}", s.doc.body), before, "already in the text: nothing to do"); + // An unknown id is refused before anything moves. + let r = call(&mut b, &mut s, "Document_convert", json!([1, [first.clone(), 999_999_999u64], 0, [1, 1]])); + assert!(r.is_err()); + assert!(s.doc.parts.is_empty(), "nothing converted"); +} + +#[test] +fn convert_with_a_repeated_id_takes_its_last_note_type() { + let mut doc = Document::from_text(""); + doc.body = (0..3).map(|_| para_block(objects_para(citations(1)))).collect(); + let mut s = Session::new(doc); + let mut b = Bridge::new(); + call(&mut b, &mut s, "Application_getActiveDocument", json!([3])).unwrap(); + let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap(); + let (a, c) = (got[0][0].clone(), got[0][2].clone()); + call(&mut b, &mut s, "Document_convert", json!([1, [a.clone(), c, a], 0, [2, 1, 1]])).unwrap(); + let kinds: Vec = s.doc.parts.values().map(|p| p.kind).collect(); + assert_eq!(kinds, vec![PartKind::Footnote, PartKind::Footnote]); + // Ids stay valid: the fields are where Zotero now expects them. + let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap(); + assert_eq!(got[2], json!([1, 0, 2])); +} From d8d75e44739370102ff53be42d5fdc857b08bc0d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:21:10 +0100 Subject: [PATCH 021/112] #3692: wordcraft-cli zotero escapes peer text on stderr Audit finding 79 (CWE-117): the CLI printed Zotero's alert text (and, with --trace, the method name and call/reply JSON) to stderr as received, so a peer on Zotero's port could end the "Zotero:" line, forge following output or send terminal escape sequences. escape_peer shows backslashes, control characters (C0, DEL, C1), line separators and bidi overrides as visible escapes (\\, \n, \u{1b}), keeping each record on its own attributable line. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/wordcraft-cli/src/main.rs | 36 +++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/apps/wordcraft-cli/src/main.rs b/apps/wordcraft-cli/src/main.rs index 752f9f68b2..0817d18b34 100644 --- a/apps/wordcraft-cli/src/main.rs +++ b/apps/wordcraft-cli/src/main.rs @@ -657,14 +657,14 @@ fn run>(args: &[S]) -> Result<(), Failure> { let out = wordcraft_zotero::client::run_command(&opts, command, &mut |call| { let r = bridge.handle(&mut s, &mut host, call); if trace { - eprintln!("← {} {}", call.method, clip(&Value::Array(call.args.clone()).to_string())); - eprintln!("→ {}", clip(&String::from_utf8_lossy(&wordcraft_zotero::wire::encode_reply(&r)))); + eprintln!("← {} {}", escape_peer(&call.method), escape_peer(&clip(&Value::Array(call.args.clone()).to_string()))); + eprintln!("→ {}", escape_peer(&clip(&String::from_utf8_lossy(&wordcraft_zotero::wire::encode_reply(&r))))); } r }) .map_err(|e| e.to_string())?; for a in &host.alerts { - eprintln!("Zotero: {a}"); + eprintln!("Zotero: {}", escape_peer(a)); } eprintln!("{} calls, {}", out.calls, if out.completed { "completed" } else { "ended without completing (cancelled?)" }); if let Some(o) = arg_value(&rest, "--save") { @@ -687,6 +687,27 @@ fn clip(s: &str) -> String { } } +/// Text from another program (Zotero, or whatever answered on its port) for one diagnostic +/// line: backslashes, control characters (newline, escape, C1…), line separators and bidi +/// overrides are shown as escapes (`\\`, `\n`, `\u{1b}`), so it can't end its line, drive the +/// terminal or pass for other output. +fn escape_peer(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c.is_control() || matches!(c, '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{2028}'..='\u{202e}' | '\u{2066}'..='\u{2069}') => { + out.extend(c.escape_unicode()) + } + c => out.push(c), + } + } + out +} + fn parity_markdown(p: &Value) -> String { let mut s = String::from( "# WordCraft feature parity\n\nGenerated by `cargo xtask parity` (`wordcraft-cli parity --markdown`): the word-processor feature catalog (`crates/engine/src/catalog.rs`) compared with the live command registry.\n\n", @@ -733,6 +754,15 @@ mod tests { values.iter().map(|value| (*value).into()).collect() } + #[test] + fn peer_text_stays_on_one_visible_line() { + let alert = "Done.\nwrote /etc/passwd\r\u{1b}[2J\u{9b}31m\u{202e}txt.exe\\n\tok é"; + let shown = escape_peer(alert); + assert_eq!(shown, "Done.\\nwrote /etc/passwd\\r\\u{1b}[2J\\u{9b}31m\\u{202e}txt.exe\\\\n\\tok é"); + assert!(!shown.chars().any(|c| c.is_control())); + assert_eq!(escape_peer("Citation (Doe, 2020) — “ok”"), "Citation (Doe, 2020) — “ok”"); + } + #[test] fn validators_accept_only_the_documented_options() { assert!(validate_options("info", &args(&["file.docx"]), PASSWORD_OPTIONS).is_ok()); From a9cc06dc0f326f502a29598114a0e8b3c51ace96 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:27:19 +0100 Subject: [PATCH 022/112] #3692: Zotero client talks only to this computer and, on Linux, only to our own user Audit finding 21 (CWE-306): run_command handed document callback authority to whatever accepted the TCP connection. Zotero's protocol is fixed and has no authentication, so WordCraft now establishes what it can about the peer itself: - the address must be loopback, refused before connecting (ZoteroError::Refused); - on Linux/Android, before the command is sent, the kernel socket table (/proc/net/tcp, tcp6) must show the listener's end of this connection owned by the same UID as ours; another user's listener, or one whose owner can't be established, is refused. No unsafe, no new dependencies. macOS, Windows and the BSDs have no such table without OS calls; there the loopback rule is the guard. The residual risk is documented in docs/zotero.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/client.rs | 3 + crates/zotero/src/lib.rs | 5 + crates/zotero/src/peer.rs | 191 ++++++++++++++++++++++++++++++++++++ docs/zotero.md | 21 ++++ 4 files changed, 220 insertions(+) create mode 100644 crates/zotero/src/peer.rs diff --git a/crates/zotero/src/client.rs b/crates/zotero/src/client.rs index bb6fc98b9e..e7a73354e1 100644 --- a/crates/zotero/src/client.rs +++ b/crates/zotero/src/client.rs @@ -88,7 +88,10 @@ pub struct Outcome { /// Run `command`, answering every call with `answer`. pub fn run_command(opts: &Options, command: Command, answer: &mut dyn FnMut(&Call) -> Result) -> Result { + crate::peer::check_addr(&opts.addr)?; let mut stream = TcpStream::connect_timeout(&opts.addr, opts.connect_timeout).map_err(|e| ZoteroError::NotRunning(e.to_string()))?; + // Before Zotero sees anything: the command, and then the document, go only to our own user. + crate::peer::check(&stream)?; let _ = stream.set_nodelay(true); stream.set_read_timeout(Some(opts.idle_timeout)).map_err(|e| ZoteroError::Io(e.to_string()))?; wire::write_frame(&mut stream, 0, &wire::command_payload(command.wire_name()))?; diff --git a/crates/zotero/src/lib.rs b/crates/zotero/src/lib.rs index 4c50bab040..7dc26eb8e9 100644 --- a/crates/zotero/src/lib.rs +++ b/crates/zotero/src/lib.rs @@ -18,6 +18,7 @@ mod bridge; pub mod client; pub mod fields; +mod peer; mod rich; pub mod wire; @@ -37,6 +38,10 @@ pub enum ZoteroError { /// Zotero sent something that isn't the protocol. #[error("unexpected data from Zotero: {0}")] Protocol(String), + /// What answered on Zotero's port may not be Zotero (another user's program, or another + /// computer); it gets no access to the document. + #[error("WordCraft won't let this program edit the document: {0}")] + Refused(String), } #[cfg(test)] diff --git a/crates/zotero/src/peer.rs b/crates/zotero/src/peer.rs new file mode 100644 index 0000000000..00173c5c4e --- /dev/null +++ b/crates/zotero/src/peer.rs @@ -0,0 +1,191 @@ +//! Who answered on Zotero's port. Zotero's protocol has no handshake that proves the other end +//! is Zotero, and whatever answers gets to read and rewrite the open document. So WordCraft +//! only connects to this computer (a loopback address), and on Linux checks that the program +//! listening there runs as the same user, from the kernel's socket table (`/proc/net/tcp`): +//! another account on a shared machine that took the port while Zotero wasn't running is +//! refused. Other systems have no such table to read without OS calls this crate doesn't make; +//! there the loopback rule is the guard (see `docs/zotero.md`). + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}; + +use crate::ZoteroError; + +/// Whether `addr` is on this computer; anything else is refused before connecting. +pub(crate) fn check_addr(addr: &SocketAddr) -> Result<(), ZoteroError> { + if addr.ip().to_canonical().is_loopback() { + Ok(()) + } else { + Err(ZoteroError::Refused(format!("{addr} is not on this computer; WordCraft only talks to Zotero on 127.0.0.1"))) + } +} + +/// Refuse the program at the other end of `stream` when it may not be ours (see the module +/// documentation). +pub(crate) fn check(stream: &TcpStream) -> Result<(), ZoteroError> { + let io = |e: std::io::Error| ZoteroError::Io(e.to_string()); + let (local, peer) = (stream.local_addr().map_err(io)?, stream.peer_addr().map_err(io)?); + check_addr(&peer)?; + owner_check(local, peer) +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { + use std::io::Read; + /// Longest socket table read (a busy server's table is a few MB). + const MAX_TABLE: u64 = 64 * 1024 * 1024; + let mut table = String::new(); + for path in ["/proc/net/tcp", "/proc/net/tcp6"] { + if let Ok(f) = std::fs::File::open(path) { + let mut part = String::new(); + if f.take(MAX_TABLE).read_to_string(&mut part).is_ok() { + table.push_str(&part); + } + } + } + same_owner(&table, local, peer) +} + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> { + Ok(()) +} + +/// From a `/proc/net/tcp`-format socket table: whether the socket at `peer` connected to +/// `local` (the listener's side) belongs to the same user as ours. A peer that can't be found +/// is refused too: its owner can't be established. +#[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { + let key = |a: SocketAddr| (a.ip().to_canonical(), a.port()); + let (local, peer) = (key(local), key(peer)); + let (mut ours, mut theirs) = (None, None); + for line in table.lines() { + let f: Vec<&str> = line.split_whitespace().take(8).collect(); + let (Some(l), Some(r), Some(uid)) = (f.get(1).and_then(|a| parse_addr(a)), f.get(2).and_then(|a| parse_addr(a)), f.get(7)) else { + continue; + }; + let Ok(uid) = uid.parse::() else { continue }; + let (l, r) = (key(l), key(r)); + if l == local && r == peer { + ours = Some(uid); + } else if l == peer && r == local { + theirs = Some(uid); + } + } + match (ours, theirs) { + (Some(a), Some(b)) if a == b => Ok(()), + (Some(_), Some(b)) => { + Err(ZoteroError::Refused(format!("the program on port {} runs as another user (uid {b}), so it isn't your Zotero", peer.1))) + } + _ => Err(ZoteroError::Refused(format!("WordCraft couldn't tell which user's program is on port {}", peer.1))), + } +} + +/// `0100007F:5A5C` (IPv4) or 32 hex digits and a port (IPv6): each 32-bit word as the kernel +/// prints it, in this machine's byte order. +#[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +fn parse_addr(s: &str) -> Option { + let (ip, port) = s.split_once(':')?; + let port = u16::from_str_radix(port, 16).ok()?; + let word = |k: usize| ip.get(k * 8..k * 8 + 8).and_then(|w| u32::from_str_radix(w, 16).ok()).map(u32::to_ne_bytes); + let ip = match ip.len() { + 8 => IpAddr::V4(Ipv4Addr::from(word(0)?)), + 32 => { + let mut b = [0u8; 16]; + for (k, w) in b.as_chunks_mut::<4>().0.iter_mut().enumerate() { + *w = word(k)?; + } + IpAddr::V6(Ipv6Addr::from(b)) + } + _ => return None, + }; + Some(SocketAddr::new(ip, port)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `a` the way the kernel's socket table prints it. + fn hex(a: SocketAddr) -> String { + let ip = match a.ip() { + IpAddr::V4(v4) => format!("{:08X}", u32::from_ne_bytes(v4.octets())), + IpAddr::V6(v6) => v6.octets().as_chunks::<4>().0.iter().map(|w| format!("{:08X}", u32::from_ne_bytes(*w))).collect(), + }; + format!("{ip}:{:04X}", a.port()) + } + + fn row(l: SocketAddr, r: SocketAddr, uid: u32) -> String { + format!(" 0: {} {} 01 00000000:00000000 00:00000000 00000000 {uid:>4} 0 12345 1 0000000000000000 20 4 30 10 -1\n", hex(l), hex(r)) + } + + const HEADER: &str = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"; + + fn addrs() -> (SocketAddr, SocketAddr) { + ("127.0.0.1:41234".parse().unwrap(), "127.0.0.1:23116".parse().unwrap()) + } + + #[test] + fn addresses_parse_as_the_kernel_prints_them() { + for a in ["127.0.0.1:23116", "[::1]:23116", "[::ffff:127.0.0.1]:80"] { + let a: SocketAddr = a.parse().unwrap(); + assert_eq!(parse_addr(&hex(a)), Some(a)); + } + for bad in ["", "0100007F", "0100007F:", "XYZ00000:0050", "0100007F:12345", "0100:0050"] { + assert_eq!(parse_addr(bad), None, "{bad}"); + } + } + + #[test] + fn a_listener_of_another_user_is_refused() { + let (local, peer) = addrs(); + let same = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1000)); + assert_eq!(same_owner(&same, local, peer), Ok(())); + let other = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1001)); + assert!(matches!(same_owner(&other, local, peer), Err(ZoteroError::Refused(e)) if e.contains("another user"))); + // Not found (or the table couldn't be read): the owner can't be established. + let missing = format!("{HEADER}{}", row(local, peer, 1000)); + assert!(matches!(same_owner(&missing, local, peer), Err(ZoteroError::Refused(_)))); + assert!(matches!(same_owner("", local, peer), Err(ZoteroError::Refused(_)))); + // Junk lines are skipped. + let junk = format!("{HEADER}garbage\n 1: zz:zz yy:yy 01 x x x x\n{}{}", row(local, peer, 7), row(peer, local, 7)); + assert_eq!(same_owner(&junk, local, peer), Ok(())); + } + + #[test] + fn a_dual_stack_listener_is_matched_through_its_mapped_address() { + let (local, peer) = addrs(); + let mapped = |a: SocketAddr| match a.ip() { + IpAddr::V4(v4) => SocketAddr::new(IpAddr::V6(v4.to_ipv6_mapped()), a.port()), + IpAddr::V6(_) => a, + }; + let table = format!("{HEADER}{}{}", row(local, peer, 1000), row(mapped(peer), mapped(local), 1001)); + assert!(matches!(same_owner(&table, local, peer), Err(ZoteroError::Refused(e)) if e.contains("another user"))); + } + + #[test] + fn only_this_computer_is_contacted() { + for ok in ["127.0.0.1:23116", "127.9.9.9:1", "[::1]:23116", "[::ffff:127.0.0.1]:23116"] { + assert_eq!(check_addr(&ok.parse().unwrap()), Ok(()), "{ok}"); + } + for bad in ["192.0.2.1:23116", "0.0.0.0:23116", "[2001:db8::1]:23116", "[::]:23116"] { + assert!(matches!(check_addr(&bad.parse().unwrap()), Err(ZoteroError::Refused(_))), "{bad}"); + } + } + + #[test] + fn another_computer_is_refused_before_connecting() { + let opts = crate::client::Options { addr: "192.0.2.1:23116".parse().unwrap(), ..Default::default() }; + let r = crate::client::run_command(&opts, crate::Command::Refresh, &mut |_| Ok(serde_json::Value::Null)); + assert!(matches!(r, Err(ZoteroError::Refused(_))), "{r:?}"); + } + + /// A listener this process opened runs as us: the real socket table agrees. + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn our_own_listener_passes_the_owner_check() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let stream = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let _accepted = listener.accept().unwrap(); + assert_eq!(check(&stream), Ok(())); + } +} diff --git a/docs/zotero.md b/docs/zotero.md index f05b62c6e2..ed486b4ff7 100644 --- a/docs/zotero.md +++ b/docs/zotero.md @@ -61,6 +61,27 @@ Word's "Field" in the file.) - Zotero's document transfer (Export/Import Document) isn't needed and isn't supported. - The web version can't reach Zotero (browsers can't open the socket). +## Who can drive the document + +Zotero's protocol has no handshake that proves the program on port 23116 is Zotero, and that +program can read the citations and edit the open document. WordCraft limits who that can be: + +- It only connects to this computer (a loopback address such as `127.0.0.1`); any other address + is refused before connecting. +- On Linux it checks, in the kernel's socket table (`/proc/net/tcp`, `/proc/net/tcp6`), that the + program listening on the port runs as the same user as WordCraft, before sending anything. If + another account on a shared machine took the port while Zotero wasn't running, or the owner + can't be established (the table can't be read), WordCraft refuses with an error and the + document isn't touched. +- Remaining risk: macOS, Windows and the BSDs have no such table to read without OS calls + WordCraft doesn't make (it uses no `unsafe` code), so there only the loopback rule applies: + on a computer shared with other accounts, a program another user leaves on port 23116 while + Zotero is closed would be taken for Zotero. A program running as you is trusted on every + system; it could change your documents anyway. +- `wordcraft-cli zotero` prints Zotero's messages (and `--trace` lines) with control + characters, line breaks and backslashes shown as escapes (`\n`, `\u{1b}`, `\\`), so a peer + can't forge other output or drive the terminal. + ## How it works `crates/zotero` implements Zotero's published word-processor wire protocol (port 23116): From 7ee101a5b99f2f5e25b31a3aecf80d609b63da2b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 22:05:54 +0100 Subject: [PATCH 023/112] #3692: Zotero owner check refuses only another user's listener; unlisted falls back to loopback Ultrareview finding 3: on Linux, when /proc/net/tcp(6) doesn't list the listener on Zotero's port (WSL2 mirrored networking with Zotero on Windows, a local proxy, a sandbox), every connection was refused, so Zotero couldn't be used at all there. Per the owner's decision, refuse only when both sockets are found and the listener's uid differs from ours; otherwise log at debug and apply the plain loopback rule, as on macOS/Windows/BSD. Test: an_unlisted_listener_falls_back_to_the_loopback_rule. docs/zotero.md and the progress-log row say so. Non-Linux owner check stays with #3831. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/peer.rs | 31 +++++++++++++++++++++---------- docs/zotero.md | 14 ++++++++------ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/crates/zotero/src/peer.rs b/crates/zotero/src/peer.rs index 00173c5c4e..a9c2ccbe4f 100644 --- a/crates/zotero/src/peer.rs +++ b/crates/zotero/src/peer.rs @@ -3,8 +3,9 @@ //! only connects to this computer (a loopback address), and on Linux checks that the program //! listening there runs as the same user, from the kernel's socket table (`/proc/net/tcp`): //! another account on a shared machine that took the port while Zotero wasn't running is -//! refused. Other systems have no such table to read without OS calls this crate doesn't make; -//! there the loopback rule is the guard (see `docs/zotero.md`). +//! refused. A listener the table doesn't show (WSL2 mirrored networking with Zotero on Windows, +//! a local proxy, a sandbox) leaves only the loopback rule, as on other systems, which have no +//! such table to read without OS calls this crate doesn't make (see `docs/zotero.md`). use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}; @@ -50,9 +51,9 @@ fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> Ok(()) } -/// From a `/proc/net/tcp`-format socket table: whether the socket at `peer` connected to -/// `local` (the listener's side) belongs to the same user as ours. A peer that can't be found -/// is refused too: its owner can't be established. +/// From a `/proc/net/tcp`-format socket table: refuse when the socket at `peer` connected to +/// `local` (the listener's side) belongs to another user than ours. When either socket isn't +/// listed (or the table couldn't be read) the owner is unknown and the loopback rule decides. #[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { let key = |a: SocketAddr| (a.ip().to_canonical(), a.port()); @@ -76,7 +77,10 @@ pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr) -> Re (Some(_), Some(b)) => { Err(ZoteroError::Refused(format!("the program on port {} runs as another user (uid {b}), so it isn't your Zotero", peer.1))) } - _ => Err(ZoteroError::Refused(format!("WordCraft couldn't tell which user's program is on port {}", peer.1))), + _ => { + log::debug!("Zotero: port {} isn't in this system's socket table; trusting the loopback address alone", peer.1); + Ok(()) + } } } @@ -142,15 +146,22 @@ mod tests { assert_eq!(same_owner(&same, local, peer), Ok(())); let other = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1001)); assert!(matches!(same_owner(&other, local, peer), Err(ZoteroError::Refused(e)) if e.contains("another user"))); - // Not found (or the table couldn't be read): the owner can't be established. - let missing = format!("{HEADER}{}", row(local, peer, 1000)); - assert!(matches!(same_owner(&missing, local, peer), Err(ZoteroError::Refused(_)))); - assert!(matches!(same_owner("", local, peer), Err(ZoteroError::Refused(_)))); // Junk lines are skipped. let junk = format!("{HEADER}garbage\n 1: zz:zz yy:yy 01 x x x x\n{}{}", row(local, peer, 7), row(peer, local, 7)); assert_eq!(same_owner(&junk, local, peer), Ok(())); } + /// WSL2 mirrored networking (Zotero on Windows), a local proxy or a sandbox: the listener + /// isn't in our table, so the loopback rule decides, as on other systems. + #[test] + fn an_unlisted_listener_falls_back_to_the_loopback_rule() { + let (local, peer) = addrs(); + let missing = format!("{HEADER}{}", row(local, peer, 1000)); + assert_eq!(same_owner(&missing, local, peer), Ok(())); + // The table couldn't be read. + assert_eq!(same_owner("", local, peer), Ok(())); + } + #[test] fn a_dual_stack_listener_is_matched_through_its_mapped_address() { let (local, peer) = addrs(); diff --git a/docs/zotero.md b/docs/zotero.md index ed486b4ff7..601749d598 100644 --- a/docs/zotero.md +++ b/docs/zotero.md @@ -70,13 +70,15 @@ program can read the citations and edit the open document. WordCraft limits who is refused before connecting. - On Linux it checks, in the kernel's socket table (`/proc/net/tcp`, `/proc/net/tcp6`), that the program listening on the port runs as the same user as WordCraft, before sending anything. If - another account on a shared machine took the port while Zotero wasn't running, or the owner - can't be established (the table can't be read), WordCraft refuses with an error and the - document isn't touched. + another account on a shared machine took the port while Zotero wasn't running, WordCraft + refuses with an error and the document isn't touched. When the table doesn't show the + listener (WSL2 mirrored networking with Zotero running on Windows, a local proxy, a sandbox) + or can't be read, its owner is unknown and only the loopback rule applies, as below. - Remaining risk: macOS, Windows and the BSDs have no such table to read without OS calls - WordCraft doesn't make (it uses no `unsafe` code), so there only the loopback rule applies: - on a computer shared with other accounts, a program another user leaves on port 23116 while - Zotero is closed would be taken for Zotero. A program running as you is trusted on every + WordCraft doesn't make (it uses no `unsafe` code), so there only the loopback rule applies + (and on Linux when the listener isn't in the table): on a computer shared with other + accounts, a program another user leaves on port 23116 while Zotero is closed would be taken + for Zotero. A program running as you is trusted on every system; it could change your documents anyway. - `wordcraft-cli zotero` prints Zotero's messages (and `--trace` lines) with control characters, line breaks and backslashes shown as escapes (`\n`, `\u{1b}`, `\\`), so a peer From 65f3c3418f494dd64e47157ac5e27ca5e0a7c878 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 22:27:01 +0100 Subject: [PATCH 024/112] #3692: Zotero owner check falls back to this process's uid when our socket row is missing Ultrareview finding 3 follow-up (owner's call): with our own socket's row missing from /proc/net/tcp(6), a listed listener of another user was let through on the loopback rule. Our uid now falls back to the owner of /proc/self, so a listed listener with a different uid is refused either way. An unlisted listener still falls back to the loopback rule. Test: a_listed_listener_is_checked_against_our_process_uid. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/zotero/src/peer.rs | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/crates/zotero/src/peer.rs b/crates/zotero/src/peer.rs index a9c2ccbe4f..e808f03d2d 100644 --- a/crates/zotero/src/peer.rs +++ b/crates/zotero/src/peer.rs @@ -43,7 +43,9 @@ fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { } } } - same_owner(&table, local, peer) + // Our uid when our own socket's row is missing: `/proc/self` belongs to this process's user. + let me = std::fs::metadata("/proc/self").ok().map(|m| std::os::unix::fs::MetadataExt::uid(&m)); + same_owner(&table, local, peer, me) } #[cfg(not(any(target_os = "linux", target_os = "android")))] @@ -52,10 +54,11 @@ fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> } /// From a `/proc/net/tcp`-format socket table: refuse when the socket at `peer` connected to -/// `local` (the listener's side) belongs to another user than ours. When either socket isn't -/// listed (or the table couldn't be read) the owner is unknown and the loopback rule decides. +/// `local` (the listener's side) belongs to another user than ours. Our uid is our own socket's +/// row, else `me` (this process's). When the listener isn't listed (or the table couldn't be +/// read, or our uid is unknown) its owner is unknown and the loopback rule decides. #[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] -pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { +pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr, me: Option) -> Result<(), ZoteroError> { let key = |a: SocketAddr| (a.ip().to_canonical(), a.port()); let (local, peer) = (key(local), key(peer)); let (mut ours, mut theirs) = (None, None); @@ -72,13 +75,16 @@ pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr) -> Re theirs = Some(uid); } } - match (ours, theirs) { + match (ours.or(me), theirs) { (Some(a), Some(b)) if a == b => Ok(()), (Some(_), Some(b)) => { Err(ZoteroError::Refused(format!("the program on port {} runs as another user (uid {b}), so it isn't your Zotero", peer.1))) } _ => { - log::debug!("Zotero: port {} isn't in this system's socket table; trusting the loopback address alone", peer.1); + log::debug!( + "Zotero: couldn't tell which user runs the program on port {} (not in the socket table); trusting the loopback address alone", + peer.1 + ); Ok(()) } } @@ -143,12 +149,12 @@ mod tests { fn a_listener_of_another_user_is_refused() { let (local, peer) = addrs(); let same = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1000)); - assert_eq!(same_owner(&same, local, peer), Ok(())); + assert_eq!(same_owner(&same, local, peer, None), Ok(())); let other = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1001)); - assert!(matches!(same_owner(&other, local, peer), Err(ZoteroError::Refused(e)) if e.contains("another user"))); + assert!(matches!(same_owner(&other, local, peer, None), Err(ZoteroError::Refused(e)) if e.contains("another user"))); // Junk lines are skipped. let junk = format!("{HEADER}garbage\n 1: zz:zz yy:yy 01 x x x x\n{}{}", row(local, peer, 7), row(peer, local, 7)); - assert_eq!(same_owner(&junk, local, peer), Ok(())); + assert_eq!(same_owner(&junk, local, peer, None), Ok(())); } /// WSL2 mirrored networking (Zotero on Windows), a local proxy or a sandbox: the listener @@ -157,9 +163,25 @@ mod tests { fn an_unlisted_listener_falls_back_to_the_loopback_rule() { let (local, peer) = addrs(); let missing = format!("{HEADER}{}", row(local, peer, 1000)); - assert_eq!(same_owner(&missing, local, peer), Ok(())); + assert_eq!(same_owner(&missing, local, peer, None), Ok(())); // The table couldn't be read. - assert_eq!(same_owner("", local, peer), Ok(())); + assert_eq!(same_owner("", local, peer, None), Ok(())); + assert_eq!(same_owner("", local, peer, Some(1000)), Ok(())); + } + + /// Our own socket's row missing doesn't hide another user's listener: our uid comes from + /// the process itself. + #[test] + fn a_listed_listener_is_checked_against_our_process_uid() { + let (local, peer) = addrs(); + let only_theirs = format!("{HEADER}{}", row(peer, local, 1001)); + assert!(matches!(same_owner(&only_theirs, local, peer, Some(1000)), Err(ZoteroError::Refused(e)) if e.contains("another user"))); + assert_eq!(same_owner(&only_theirs, local, peer, Some(1001)), Ok(())); + // Neither our row nor our process uid: unknown, so the loopback rule decides. + assert_eq!(same_owner(&only_theirs, local, peer, None), Ok(())); + // Our socket's row is ours by definition and wins over the process uid. + let both = format!("{HEADER}{}{}", row(local, peer, 1001), row(peer, local, 1001)); + assert_eq!(same_owner(&both, local, peer, Some(1000)), Ok(())); } #[test] @@ -170,7 +192,7 @@ mod tests { IpAddr::V6(_) => a, }; let table = format!("{HEADER}{}{}", row(local, peer, 1000), row(mapped(peer), mapped(local), 1001)); - assert!(matches!(same_owner(&table, local, peer), Err(ZoteroError::Refused(e)) if e.contains("another user"))); + assert!(matches!(same_owner(&table, local, peer, None), Err(ZoteroError::Refused(e)) if e.contains("another user"))); } #[test] From e251c960d9745eb791452543129b3fa173a1279e Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:57:00 +0100 Subject: [PATCH 025/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20doc=20com?= =?UTF-8?q?ments=20on=20the=20functions=20the=20Zotero=20hardening=20added?= =?UTF-8?q?=20or=20changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- apps/wordcraft-cli/src/main.rs | 2 ++ crates/zotero/src/bridge.rs | 1 + crates/zotero/src/peer.rs | 4 ++++ crates/zotero/src/rich.rs | 3 +++ 4 files changed, 10 insertions(+) diff --git a/apps/wordcraft-cli/src/main.rs b/apps/wordcraft-cli/src/main.rs index 0817d18b34..c8e46c93cc 100644 --- a/apps/wordcraft-cli/src/main.rs +++ b/apps/wordcraft-cli/src/main.rs @@ -387,6 +387,8 @@ fn valid_connect_addr(addr: &str) -> bool { !host.is_empty() && port.parse::().is_ok() } +/// Run the subcommand named by `args` (the command line after the program name). Files named +/// there are opened by their real names, which may not be valid Unicode. fn run>(args: &[S]) -> Result<(), Failure> { let script_path = script_operand(args); let (args, file) = text_args(args)?; diff --git a/crates/zotero/src/bridge.rs b/crates/zotero/src/bridge.rs index 36cb2d0b44..50ea5fcd5f 100644 --- a/crates/zotero/src/bridge.rs +++ b/crates/zotero/src/bridge.rs @@ -194,6 +194,7 @@ impl Bridge { } } + /// Answer one call by its method name; [`Bridge::handle`] wraps it so a panic becomes an error. fn dispatch(&mut self, s: &mut Session, host: &mut dyn Host, c: &Call) -> R { match c.method.as_str() { "Application_getActiveDocument" => { diff --git a/crates/zotero/src/peer.rs b/crates/zotero/src/peer.rs index e808f03d2d..8e6ea6c4e4 100644 --- a/crates/zotero/src/peer.rs +++ b/crates/zotero/src/peer.rs @@ -29,6 +29,7 @@ pub(crate) fn check(stream: &TcpStream) -> Result<(), ZoteroError> { owner_check(local, peer) } +/// Refuse a listener at `peer` that the kernel's socket tables say belongs to another user. #[cfg(any(target_os = "linux", target_os = "android"))] fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { use std::io::Read; @@ -48,6 +49,7 @@ fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { same_owner(&table, local, peer, me) } +/// No socket table to read here: the loopback rule alone decides. #[cfg(not(any(target_os = "linux", target_os = "android")))] fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> { Ok(()) @@ -124,12 +126,14 @@ mod tests { format!("{ip}:{:04X}", a.port()) } + /// A socket table row for the connection from `l` to `r`, owned by `uid`. fn row(l: SocketAddr, r: SocketAddr, uid: u32) -> String { format!(" 0: {} {} 01 00000000:00000000 00:00000000 00000000 {uid:>4} 0 12345 1 0000000000000000 20 4 30 10 -1\n", hex(l), hex(r)) } const HEADER: &str = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"; + /// Our side and Zotero's listening side of a loopback connection. fn addrs() -> (SocketAddr, SocketAddr) { ("127.0.0.1:41234".parse().unwrap(), "127.0.0.1:23116".parse().unwrap()) } diff --git a/crates/zotero/src/rich.rs b/crates/zotero/src/rich.rs index 6285ecf3eb..7ff3377f1c 100644 --- a/crates/zotero/src/rich.rs +++ b/crates/zotero/src/rich.rs @@ -40,6 +40,9 @@ fn style_props(base: &CharProps, src: &CharProps) -> CharProps { p } +/// The body paragraphs of `d` (Zotero's RTF or HTML, read), at most [`MAX_PARAS`], formatted on +/// top of `base` with the run styling they carry (past [`MAX_RUNS`] runs, text keeps the +/// formatting before it). Objects are dropped. fn from_doc(d: &Document, base: &CharProps) -> Vec { let mut out = Vec::new(); let mut runs = 0usize; From bf2a379ec6fd0b08144f79d343c1a4ede6be72db Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:13:33 +0100 Subject: [PATCH 026/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20say=20pla?= =?UTF-8?q?inly=20the=20same-user=20Zotero=20check=20is=20Linux=20only;=20?= =?UTF-8?q?Windows=20and=20macOS=20rely=20on=20loopback=20alone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/zotero/src/client.rs | 3 ++- crates/zotero/src/lib.rs | 4 ++-- crates/zotero/src/peer.rs | 22 ++++++++++++++-------- docs/zotero.md | 23 +++++++++++++---------- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/crates/zotero/src/client.rs b/crates/zotero/src/client.rs index e7a73354e1..f6e31c83aa 100644 --- a/crates/zotero/src/client.rs +++ b/crates/zotero/src/client.rs @@ -90,7 +90,8 @@ pub struct Outcome { pub fn run_command(opts: &Options, command: Command, answer: &mut dyn FnMut(&Call) -> Result) -> Result { crate::peer::check_addr(&opts.addr)?; let mut stream = TcpStream::connect_timeout(&opts.addr, opts.connect_timeout).map_err(|e| ZoteroError::NotRunning(e.to_string()))?; - // Before Zotero sees anything: the command, and then the document, go only to our own user. + // Before Zotero sees anything: the command, and then the document, go only to this computer + // and, on Linux only, not to a listener the socket table shows belongs to another user. crate::peer::check(&stream)?; let _ = stream.set_nodelay(true); stream.set_read_timeout(Some(opts.idle_timeout)).map_err(|e| ZoteroError::Io(e.to_string()))?; diff --git a/crates/zotero/src/lib.rs b/crates/zotero/src/lib.rs index 7dc26eb8e9..cf4c934431 100644 --- a/crates/zotero/src/lib.rs +++ b/crates/zotero/src/lib.rs @@ -38,8 +38,8 @@ pub enum ZoteroError { /// Zotero sent something that isn't the protocol. #[error("unexpected data from Zotero: {0}")] Protocol(String), - /// What answered on Zotero's port may not be Zotero (another user's program, or another - /// computer); it gets no access to the document. + /// What answered on Zotero's port may not be Zotero (another computer, on any system; on + /// Linux only, another user's program); it gets no access to the document. #[error("WordCraft won't let this program edit the document: {0}")] Refused(String), } diff --git a/crates/zotero/src/peer.rs b/crates/zotero/src/peer.rs index 8e6ea6c4e4..07df98e79f 100644 --- a/crates/zotero/src/peer.rs +++ b/crates/zotero/src/peer.rs @@ -1,11 +1,16 @@ //! Who answered on Zotero's port. Zotero's protocol has no handshake that proves the other end -//! is Zotero, and whatever answers gets to read and rewrite the open document. So WordCraft -//! only connects to this computer (a loopback address), and on Linux checks that the program -//! listening there runs as the same user, from the kernel's socket table (`/proc/net/tcp`): -//! another account on a shared machine that took the port while Zotero wasn't running is -//! refused. A listener the table doesn't show (WSL2 mirrored networking with Zotero on Windows, -//! a local proxy, a sandbox) leaves only the loopback rule, as on other systems, which have no -//! such table to read without OS calls this crate doesn't make (see `docs/zotero.md`). +//! is Zotero, and whatever answers gets to read and rewrite the open document. Two checks: +//! +//! - **Every system:** WordCraft only connects to this computer (a loopback address). +//! - **Linux only:** the program listening there must not run as another user, from the +//! kernel's socket table (`/proc/net/tcp`): another account on a shared machine that took the +//! port while Zotero wasn't running is refused. A listener the table doesn't show (WSL2 +//! mirrored networking with Zotero on Windows, a local proxy, a sandbox) leaves only the +//! loopback rule. +//! +//! On Windows, macOS and the BSDs there is **no same-user check**: the loopback rule is the +//! only one, so any local account's listener on the port is taken for Zotero. Those systems have +//! no such table to read without OS calls this crate doesn't make (see `docs/zotero.md`, #3831). use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}; @@ -49,7 +54,8 @@ fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> { same_owner(&table, local, peer, me) } -/// No socket table to read here: the loopback rule alone decides. +/// Windows, macOS, the BSDs: no same-user check (no socket table to read without OS calls this +/// crate doesn't make, #3831); the loopback rule alone decides. #[cfg(not(any(target_os = "linux", target_os = "android")))] fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> { Ok(()) diff --git a/docs/zotero.md b/docs/zotero.md index 601749d598..cf5813494c 100644 --- a/docs/zotero.md +++ b/docs/zotero.md @@ -64,22 +64,25 @@ Word's "Field" in the file.) ## Who can drive the document Zotero's protocol has no handshake that proves the program on port 23116 is Zotero, and that -program can read the citations and edit the open document. WordCraft limits who that can be: +program can read the citations and edit the open document. WordCraft limits who that can be. +In short: **every system** gets the loopback rule; **only Linux** also checks that the listener +isn't another user's. On Windows and macOS (and the BSDs) the loopback rule is the only check. -- It only connects to this computer (a loopback address such as `127.0.0.1`); any other address - is refused before connecting. -- On Linux it checks, in the kernel's socket table (`/proc/net/tcp`, `/proc/net/tcp6`), that the +- Every system: it only connects to this computer (a loopback address such as `127.0.0.1`); + any other address is refused before connecting. +- Linux only: it checks, in the kernel's socket table (`/proc/net/tcp`, `/proc/net/tcp6`), that the program listening on the port runs as the same user as WordCraft, before sending anything. If another account on a shared machine took the port while Zotero wasn't running, WordCraft refuses with an error and the document isn't touched. When the table doesn't show the listener (WSL2 mirrored networking with Zotero running on Windows, a local proxy, a sandbox) or can't be read, its owner is unknown and only the loopback rule applies, as below. -- Remaining risk: macOS, Windows and the BSDs have no such table to read without OS calls - WordCraft doesn't make (it uses no `unsafe` code), so there only the loopback rule applies - (and on Linux when the listener isn't in the table): on a computer shared with other - accounts, a program another user leaves on port 23116 while Zotero is closed would be taken - for Zotero. A program running as you is trusted on every - system; it could change your documents anyway. +- Windows, macOS and the BSDs: **no same-user check.** They have no such table to read + without OS calls WordCraft doesn't make (it uses no `unsafe` code), so only the loopback rule + applies there (and on Linux when the listener isn't in the table). Remaining risk: on a + computer shared with other accounts, a program another user leaves on port 23116 while + Zotero is closed would be taken for Zotero. An owner check for these systems is tracked in + #3831. A program running as you is trusted on every system; it could change your documents + anyway. - `wordcraft-cli zotero` prints Zotero's messages (and `--trace` lines) with control characters, line breaks and backslashes shown as escapes (`\n`, `\u{1b}`, `\\`), so a peer can't forge other output or drive the terminal. From d785c05c53583b663937ab55c0d4d0ed4ea590df Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 16:22:50 +0100 Subject: [PATCH 027/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20zotero=20?= =?UTF-8?q?timing=20tests=20get=20a=20margin=20a=20busy=20CI=20machine=20c?= =?UTF-8?q?an't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 2 s failed on loaded runners (checked against the pre-fix code, put back in the worktree): - note referenced many times: no clock; listing the note once per reference gave a million fields, which the count of 200 catches. - rich text with many runs: 2 s -> 10 s; 0.35 s now, merging after every run ran past 9 minutes. - Document_convert with one id repeated: 2 s -> 10 s, 1,000 fields (was 300); 22 ms now, the per-id check took 81 s with 300 fields and grows with the field count. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/zotero/src/tests_limits.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/zotero/src/tests_limits.rs b/crates/zotero/src/tests_limits.rs index 53d8eaafc1..ce6b12c673 100644 --- a/crates/zotero/src/tests_limits.rs +++ b/crates/zotero/src/tests_limits.rs @@ -34,11 +34,10 @@ fn a_note_referenced_many_times_is_listed_once() { let note = doc.add_part(PartKind::Endnote, vec![para_block(objects_para(citations(200)))]); let refs = (0..5_000).map(|_| InlineObject::NoteRef { kind: NoteKind::Endnote, id: note, custom: String::new() }).collect(); doc.body = vec![para_block(objects_para(refs))]; - let t = Instant::now(); + // No clock: listing the note once per reference gave a million fields, which the count catches. let list = fields::list(&doc); assert_eq!(list.len(), 200, "each field once, at the note's first reference"); assert!(list.iter().all(|f| f.start.story == StoryRef::Part(note) && f.note_index == 1)); - assert!(t.elapsed() < Duration::from_secs(2), "{:?}", t.elapsed()); } #[test] @@ -66,7 +65,8 @@ fn rich_text_with_many_runs_is_linear() { assert_eq!(p.text, "ab".repeat(50_000)); assert_eq!(p.runs.len(), 100_000); assert_eq!(p.runs.iter().map(|r| r.len).sum::(), p.text.len()); - assert!(elapsed < Duration::from_secs(2), "{elapsed:?}"); + // Under half a second on a loaded machine; merging again after every run ran past 9 minutes. + assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); // Neighbours with the same formatting are one run. let frag = crate::rich::fragment("abc", true, &CharProps::default()); let Some(wordcraft_doc::Block::Para(p)) = frag.blocks.first() else { panic!("no paragraph") }; @@ -84,7 +84,9 @@ fn call(b: &mut Bridge, s: &mut Session, method: &str, args: Value) -> Result Date: Thu, 22 Oct 2026 15:12:03 +0100 Subject: [PATCH 028/112] =?UTF-8?q?#3613:=20affix=20rules=20=E2=80=94=20ch?= =?UTF-8?q?ecker=20follows=20merged=20flag=20masks,=20forms=20bounded,=20s?= =?UTF-8?q?uffix=20index=20hashed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of #3613 (audit findings 2 and 6). - Checker: `suffixed` and `form_has_flag` OR the flags of a group's matching rules into one mask and recurse once, not once per rule. Same answer (the walk is a disjunction over flags); 16k self-continuing empty-append rules: 6 s -> <1 ms per check (debug). - Finding 2: `forms` expands each (form, continuation) pair once, walks continuation rules through a per-flag rule list, and stops at MAX_FORMS (4096) forms / MAX_FORMS_WORK rule visits, so a dictionary of self-continuing rules can't allocate R^2 strings per stem. - Finding 6: the suffix index finds a rule's (append, strip) group by hash, not by scanning the append's groups: building with many strips per append is linear. Not done here (rest of #3613): a work budget for distinct-strip chains in the checker, rule caps in parse_aff, proofing off the UI thread. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/affix.rs | 164 +++++++++++++++++++++++++++++++++----- 1 file changed, 144 insertions(+), 20 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 7fb7d968ac..fefc7f6ac8 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -120,6 +120,11 @@ fn parse_cond(text: &str) -> Result, String> { const MAX_CHAIN: u8 = 2; /// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. const INNER_CHAIN: u8 = MAX_CHAIN - 1; +/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most). +pub const MAX_FORMS: usize = 4096; +/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross +/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off. +const MAX_FORMS_WORK: usize = 1 << 18; /// One prefix or suffix rule. #[derive(Clone, Debug)] @@ -182,6 +187,8 @@ pub struct AffixDict { suffixes: Vec, /// Appended string → groups of suffix rules by stripped string. sfx_index: FxMap, Vec>, + /// Per flag bit: the suffix rules of that flag, in rule order. + sfx_by_flag: Vec>, /// Longest appended suffix, in chars. max_sfx_chars: usize, try_chars: String, @@ -484,14 +491,27 @@ impl AffixDict { /// Build the suffix index and the stem hash table. fn index(&mut self) { let mut idx: FxMap, Vec> = FxMap::default(); + // Each (appended, stripped) pair's group, found by hash rather than by scanning the + // appended string's groups: a dictionary with many strips per suffix stays linear. + let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); + let mut by_flag: Vec> = vec![Vec::new(); 128]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; - match groups.iter_mut().find(|g| g.strip == r.strip) { + match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) { Some(g) => g.rules.push(i), - None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }), + None => { + group_of.insert((&r.add, &r.strip), groups.len()); + groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); + } + } + if r.bit != 0 + && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) + { + list.push(i); } } + self.sfx_by_flag = by_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; let n = self.len(); @@ -596,10 +616,14 @@ impl AffixDict { None } - /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose - /// continuation classes include `bit` (chained suffixation, depth-bounded)? - fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) { + /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose + /// continuation classes include one (chained suffixation, depth-bounded)? + /// + /// The rules of a group that could lead to `w` are merged into one mask of their flags and + /// followed once, not once per rule: the answer is the same, but thousands of rules of one + /// flag cost one walk, not one each (#3613). + fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { + if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { return true; } if depth == 0 { @@ -614,10 +638,14 @@ impl AffixDict { root.clear(); root.push_str(base); root.push_str(&g.strip); - for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) { - return true; - } + let inner = g + .rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) + .fold(0u128, |m, r| m | r.bit); + if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + return true; } } } @@ -690,9 +718,13 @@ impl AffixDict { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — - // it may be a form that gained the flag from another rule's continuation. - if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) { - return true; + // it may be a form that gained the flag from another rule's continuation. The + // matching rules' flags are looked for together, in one walk. + if need == 0 { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); + if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + return true; + } } } } @@ -725,13 +757,19 @@ impl AffixDict { } } - /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats. + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most + /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and + /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem + /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; let mut out = vec![stem.to_string()]; let mut cross_sfx = Vec::new(); - // Suffix forms queued with the continuation classes they carry, for chained expansion. + // Suffix forms queued with the continuation classes they carry, for chained expansion; + // each (form, classes) pair is expanded once. let mut chain: Vec<(String, u128)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut work = 0usize; for r in &self.suffixes { if f & r.bit == 0 || !r.cond_matches(stem, true) { continue; @@ -741,29 +779,39 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 { + if r.cont != 0 && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); + if out.len() >= MAX_FORMS { + return out; + } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] // suffixes in all — the same bound the checker walks. - for _ in 0..INNER_CHAIN { + 'chain: for _ in 0..INNER_CHAIN { let level = std::mem::take(&mut chain); if level.is_empty() { break; } for (form, flags) in level { - for r in &self.suffixes { - if flags & r.bit == 0 || !r.cond_matches(&form, true) { + for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { + work += 1; + if work > MAX_FORMS_WORK { + break 'chain; + } + if !r.cond_matches(&form, true) { continue; } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 { + if r.cont != 0 && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); + if out.len() >= MAX_FORMS { + return out; + } } } } @@ -775,15 +823,38 @@ impl AffixDict { out.push(format!("{}{rest}", r.add)); if r.cross { for s in &cross_sfx { + work += 1; + if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { + return out; + } if let Some(rest) = s.strip_prefix(&*r.strip) { out.push(format!("{}{rest}", r.add)); } } } + if out.len() >= MAX_FORMS { + return out; + } } out } + /// The suffix rules carrying any flag of `mask`, in rule order. + fn suffixes_with(&self, mask: u128) -> Vec { + let mut rules: Vec = Vec::new(); + let mut lists = 0; + for (bit, list) in self.sfx_by_flag.iter().enumerate() { + if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + rules.extend_from_slice(list); + lists += 1; + } + } + if lists > 1 { + rules.sort_unstable(); + } + rules + } + /// The `TRY` letters, most frequent first (suggestions try these). pub fn try_chars(&self) -> &str { &self.try_chars @@ -1216,6 +1287,59 @@ pub(crate) mod tests { } } + #[test] + fn continuation_classes_merge_the_flags_of_a_group() { + // `s` under B and under C share one group (same append, same strip): a form carrying + // either flag takes it, and the stem carrying neither doesn't. + let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap(); + assert!(d.contains("workables") && d.contains("players")); + assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless")); + assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players")); + } + + /// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition. + fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict { + let mut aff = format!("SET UTF-8\nSFX A Y {n}\n"); + for i in 0..n { + aff.push_str(&format!("SFX A {}\n", rule(i))); + } + AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap() + } + + #[test] + fn self_continuing_rules_check_in_linear_time() { + // Thousands of empty-append rules that continue with their own flag: each check used to + // follow every rule from every rule, millions of steps a word (#3613). + let d = many_rules(16_000, |_| "0 0/A .".into()); + let t = std::time::Instant::now(); + assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + } + + #[test] + fn forms_of_self_continuing_rules_are_bounded() { + // Every rule's form continues with every rule: millions of strings for one stem (#3613). + let same = many_rules(3000, |_| "0 0/A .".into()); + let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); + let t = std::time::Instant::now(); + let (a, b) = (same.forms("kot"), distinct.forms("kot")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); + assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); + } + + #[test] + fn many_strips_per_suffix_index_in_linear_time() { + // One append with thousands of strips: the index used to scan the append's groups for + // each rule. + let t = std::time::Instant::now(); + let d = many_rules(30_000, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(30_000)); + assert!(!d.contains("koty")); + } + #[test] fn compact_format_round_trips_continuations_and_forbidden() { let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; From e4d2dafa9d1e26dd5a26c823ae65ab9d48628a98 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:13:52 +0100 Subject: [PATCH 029/112] =?UTF-8?q?#3661:=20bounded=20dictionary=20reads?= =?UTF-8?q?=20=E2=80=94=20hyphenation=20inflate=20limit,=20capped=20Hunspe?= =?UTF-8?q?ll=20and=20Turkish=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit finding 4, and the rest of #3661. - dict.rs: `Dictionary::from_bytes` inflates with `decompress_to_vec_with_limit` (64 MiB; the bundled en-US one is ~1 MB) and caps the words it keeps (bytes and count): front coding lets a 2-byte line spell out a 255-byte word, so the inflated size alone doesn't bound storage. - hunspell.rs: one `read_capped` reader — regular files only, size checked on the open handle and the read itself bounded by `take(MAX_FILE + 1)`, so no TOCTOU between metadata and read and no endless read from a FIFO or device. - tr: the Turkish loader reads through `read_capped` and decodes with `hunspell::decode` (it read both files whole, with no cap at all). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/dict.rs | 48 +++++++++++++++++++++++++++++-- crates/proof/src/hunspell.rs | 56 ++++++++++++++++++++++++++++++------ crates/proof/src/tr/mod.rs | 23 +++++++-------- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 789fb6a870..81eb0de1b4 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -14,6 +14,11 @@ pub struct Dictionary { } const MAGIC: &[u8] = b"DCHYPH1\n"; +/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one +/// is about 1 MB). +const MAX_BYTES: usize = 64 << 20; +/// Most words read from one dictionary (the bundled one has about 160k). +const MAX_ENTRIES: usize = 4_000_000; impl Dictionary { /// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first @@ -76,9 +81,16 @@ impl Dictionary { miniz_oxide::deflate::compress_to_vec(&raw, 10) } - /// Deserialize data written by [`Dictionary::to_bytes`]. + /// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past + /// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused. pub fn from_bytes(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; + Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES) + } + + /// [`Dictionary::from_bytes`] with the limits given. + fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result { + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes) + .map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); @@ -94,6 +106,11 @@ impl Dictionary { i = end + 1; let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?; let (w, m) = parse_hyphenated(s); + // Front coding lets a short line repeat a long prefix: bound what is kept, not only + // what is read. + if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes { + return Err("hyphenation dictionary: too large".into()); + } d.words.push_str(&w); d.offs.push(d.words.len() as u32); d.masks.push(m); @@ -147,4 +164,31 @@ mod tests { assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string())); assert!(back.get("dog").is_none()); } + + #[test] + fn oversized_data_is_refused() { + const LIMIT: usize = 1 << 16; + let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000); + // Data that inflates past the limit is refused before anything is decoded. + let mut raw = MAGIC.to_vec(); + raw.resize(LIMIT + 1, b'a'); + assert!(within(&raw).unwrap_err().contains("inflate failed")); + // Short lines that each repeat a long prefix: small to inflate, but the words they spell + // out would be ~100x larger. Kept storage is bounded too. + let mut raw = MAGIC.to_vec(); + raw.push(0); + raw.extend_from_slice(&[b'a'; 255]); + raw.push(b'\n'); + for _ in 0..(LIMIT / 255 + 2) { + raw.extend_from_slice(&[255, b'\n']); + } + assert!(raw.len() < LIMIT); + assert!(within(&raw).unwrap_err().contains("too large")); + // So is the number of words. + let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect(); + let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap(); + assert!(within(&raw).unwrap_err().contains("too large")); + // The real limits leave the bundled dictionary whole. + assert!(Dictionary::en_us().len() > 100_000); + } } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 0e4303beb2..a5b41214d0 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -140,16 +140,31 @@ pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { }) } +/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is +/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong +/// size) can't make it read more (#3661). +pub(crate) fn read_capped(p: &Path) -> Result, String> { + use std::io::Read; + let err = |e: std::io::Error| format!("{}: {e}", p.display()); + let f = std::fs::File::open(p).map_err(err)?; + let meta = f.metadata().map_err(err)?; + if !meta.is_file() { + return Err(format!("{}: not a regular file", p.display())); + } + if meta.len() > MAX_FILE { + return Err(format!("{}: too large ({} bytes)", p.display(), meta.len())); + } + let mut buf = Vec::new(); + f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?; + if buf.len() as u64 > MAX_FILE { + return Err(format!("{}: too large", p.display())); + } + Ok(buf) +} + /// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. pub fn load(aff: &Path, dic: &Path) -> Result { - let read = |p: &Path| -> Result, String> { - let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); - if len > MAX_FILE { - return Err(format!("{}: too large ({len} bytes)", p.display())); - } - std::fs::read(p).map_err(|e| format!("{}: {e}", p.display())) - }; - let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?; + let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; let source = format!("Hunspell dictionary {}", dic.display()); AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) } @@ -283,4 +298,29 @@ mod tests { assert!(load(&dir.join("missing.aff"), &dic).is_err()); let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn reads_are_bounded_and_regular_files_only() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let aff = dir.join("xx.aff"); + std::fs::write(&aff, "SET UTF-8\n").unwrap(); + // One byte over the cap (sparse: nothing is written) is refused without reading it. + let big = dir.join("big.dic"); + std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); + let t = std::time::Instant::now(); + assert!(load(&aff, &big).unwrap_err().contains("too large")); + // A directory, and on Unix an endless device, are not dictionary files. + assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + #[cfg(unix)] + assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); + assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + // Turkish reads through the same reader. + let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); + std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap(); + assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar")); + assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index bcab1546bf..25ee424c42 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -44,18 +44,17 @@ fn load_installed() -> TrDict { log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); return TrDict::default(); }; - let read = |p: &std::path::Path| std::fs::read_to_string(p); - match (read(&aff), read(&dic)) { - (Ok(aff_text), Ok(dic_text)) => crate::hunspell::note_load(NAMES[0], &dic, TrDict::from_hunspell(&aff_text, &dic_text)).unwrap_or_else(|e| { - log::warn!("Turkish spelling: {e}"); - TrDict::default() - }), - (a, d) => { - let reason = a.err().or(d.err()).map_or_else(String::new, |e| e.to_string()); - let _ = crate::hunspell::note_load::<()>(NAMES[0], &dic, Err(reason)); - TrDict::default() - } - } + crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("Turkish spelling: {e}"); + TrDict::default() + }) +} + +/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line, +/// like the other languages' dictionaries (#3661). +pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?; + TrDict::from_hunspell(&aff_text, &dic_text) } /// Use this Turkish dictionary from now on. From 2cff091c1a63a56990f85579db38e47bd48e1840 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:15:27 +0100 Subject: [PATCH 030/112] #3657: Russian hyphenation and hyphenated output linear in word length Audit findings 10 (#3657) and 15 (#3692). - ru::hyphen::points: the cuts come in increasing order, so a repeat can only be the last one; compare with `out.last()` instead of `out.contains` per syllable (O(V^2) on a long Cyrillic word, during layout). - hyphenate_word_in: walk the sorted break points alongside the chars instead of searching the point list for every char. Same output. Test: russian_hyphenation_and_hyphenated_output_are_linear (8x the letters costs ~8x; the old code measured 31x and 30x). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/hyphen.rs | 30 +++++++++++++++++++++++++++--- crates/proof/src/ru/hyphen.rs | 4 +++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index ed9f77f9de..b203ff1a2a 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -320,10 +320,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String { /// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics). pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String { - let pts = hyphen_points_in(word, lim, lang); - let mut s = String::with_capacity(word.len() + pts.len()); + let mut pts = hyphen_points_in(word, lim, lang); + pts.sort_unstable(); + // Walk the points alongside the chars rather than searching them for each char. + let mut next = pts.into_iter().peekable(); + let mut s = String::with_capacity(word.len() + 8); for (i, ch) in word.chars().enumerate() { - if pts.contains(&i) { + if next.next_if_eq(&i).is_some() { + while next.next_if_eq(&i).is_some() {} s.push('-'); } s.push(ch); @@ -706,6 +710,26 @@ mod tests { } } + #[test] + fn russian_hyphenation_and_hyphenated_output_are_linear() { + // A Russian word breaks at every syllable: each break used to be checked against all the + // earlier ones (#3657), and the hyphenated output searched every break for each char. + let word = |k: usize| "мама".repeat(k); + assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]); + assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма"); + // The breaks' rescan is a cheap one per break: it takes longer words to stand out. + for (what, k, repeat, f) in [ + ("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)), + ("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))), + ] { + let (small, big) = (word(k), word(8 * k)); + let time = |w: &str| (0..repeat).for_each(|_| f(w)); + let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big)); + eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}"); + } + } + #[test] fn heuristic_fallback() { let c: Vec = "happen".chars().collect(); diff --git a/crates/proof/src/ru/hyphen.rs b/crates/proof/src/ru/hyphen.rs index c02aa12728..33d19b7712 100644 --- a/crates/proof/src/ru/hyphen.rs +++ b/crates/proof/src/ru/hyphen.rs @@ -57,7 +57,9 @@ pub fn points(c: &[char]) -> Vec { while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end { cut += 1; } - if cut >= 2 && n - cut >= 2 && !out.contains(&cut) { + // Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat + // can only be the last one: no scan of the whole list per syllable (#3657). + if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) { out.push(cut); } } From c187515ba103a431a5f3b1d11e64aab9de1a9a07 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:15:56 +0100 Subject: [PATCH 031/112] #3625: Turkish suggestions and lookups have a word-length cap Audit finding 20. - tr::suggest_with refuses words over 64 letters (and words with spaces), like the other languages' suggesters (40 there; Turkish agglutinated words run past 40). Each candidate edit copies the whole word, so a long word cost O(n^2) per suggestion request on the UI thread. - TrDict::contains returns false past MAX_WORD_BYTES (120, the affix engine's bound) instead of collecting the char indices of the whole word. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/tr/dict.rs | 3 ++- crates/proof/src/tr/suggest.rs | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index ed57cbca7b..4b59ccc297 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -87,7 +87,8 @@ impl TrDict { /// Check if word is correct (stem or affixed). pub fn contains(&self, word: &str) -> bool { - if self.is_empty() || word.is_empty() { + // Longer input is never a dictionary word, and isn't scanned (#3625). + if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES { return false; } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index dedc1bb266..4f4f3d66d4 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -46,6 +46,12 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ ("deyil", "değil"), ]; +/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so +/// the cost grows with the square of its length (#3625); Turkish words run longer than most +/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other +/// languages' 40. +const MAX_LETTERS: usize = 64; + /// Produce spelling suggestions for a misspelled Turkish word. pub fn suggest(word: &str, max: usize) -> Vec { let dict = super::dictionary(); @@ -54,7 +60,8 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// Suggest with a given dictionary. pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { - if word.is_empty() || max == 0 { + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || word.contains(char::is_whitespace) { return Vec::new(); } @@ -277,6 +284,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + #[test] + fn overlong_words_get_no_suggestions_promptly() { + let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); + assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string())); + assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string())); + let t = std::time::Instant::now(); + let long = "a".repeat(100_000); + assert!(suggest_with(&d, &long, 8).is_empty()); + assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); + assert!(!super::super::is_correct_with(&d, &long)); + assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + } + #[test] fn turkish_to_lower_handles_i() { assert_eq!(turkish_to_lower("IŞIK"), "ışık"); From 5cd68ee632ff9b112e33911fb23d01f917c54061 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:16:30 +0100 Subject: [PATCH 032/112] #3692: progress log and gap 3 note for the proofing robustness fixes Audit findings 2, 4, 6, 10, 15, 20 (#3613 part, #3661, #3657, #3625). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/gaps.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/gaps.md b/docs/gaps.md index e1af69a465..8e424edc86 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -35,6 +35,8 @@ Document renaming: desktop renaming requires a filesystem that supports hard lin Comment command safety (#1690, finding 3): Delete, Resolve, Reply and Edit reject explicit fractional, negative or wrong-type IDs before editing, rather than truncating floats or using the caret. Synthetic tests cover refusal without document or history changes, valid operations with undo/redo and repeated DOCX saves. This narrows the data-path risks in gap #3; the other audit findings and broader stability gate are outside this change. +Proofing robustness (security audit, #3613 part, #3661, #3657, #3625): installed Hunspell dictionaries with thousands of self-continuing rules check each word in linear time and list a bounded number of forms; dictionary files are read with a bounded read (Turkish included) and the hyphenation dictionary inflates under a 64 MiB limit; Russian hyphenation and Turkish suggestions no longer grow with the square of the word's length. This narrows the freeze risks in gap #3; distinct-strip rule chains and proofing off the UI thread remain (#3613). + Font-menu reliability: clearing a query with no matches now shows the recent section from its first row, including its heading, in a scrollable list. Fixed synthetic-family widget and full-ribbon tests cover this case and matching-query controls (#2068). Additive paragraph spacing retains the full before-space after a section page break; ordinary overlapping spacing is unchanged. Other pagination gaps remain open. @@ -158,6 +160,7 @@ Small follow-ups from the 2026-10-25 field, table and AutoFormat batch (#4232): | 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged | | 2026-10-24 | trivial | Exact-spacing table contact uses the specified line height in row selection and word skipping; large/small mark regressions in modes 11/12/14/15 (#3977); shape estimates unchanged | | 2026-10-24 | trivial | Floating tables count exact line-bottom contact; shapes keep strict overlap (#3977); original boundary/skip regressions in modes 11/12/14/15, expected Word rule from cyril265; readiness estimates unchanged | +| 2026-10-22 | trivial | Gap 3: proofing robustness — linear affix rule walks, bounded dictionary reads, linear Russian hyphenation, Turkish word cap (#3613 part, #3661, #3657, #3625); readiness estimates unchanged | | 2026-10-22 | trivial | Gap 10: group connector rerouting preserves rotated/flipped free and unresolved endpoints under nonuniform scaling, with safe degenerate-dimension and display-size-cap handling; routing around shapes and readiness estimates unchanged | | 2026-10-21 | trivial | The ranked gap list held two rows each for #19, #20 and #21; the stale copies are gone, so each gap appears once, and the #20 row carries the measured figure (#3411) | | 2026-10-21 | trivial | #20 narrowed: a document with many relationships opens in the time its part takes (#3411) | From 6a66004a06d0343a84802710b75608743e93eb30 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 02:45:26 +0100 Subject: [PATCH 033/112] =?UTF-8?q?#3625:=20review=20=E2=80=94=20"bir=20ka?= =?UTF-8?q?=C3=A7"=20still=20gets=20"birka=C3=A7":=20the=20replacement=20t?= =?UTF-8?q?able=20is=20checked=20before=20multi-word=20input=20is=20refuse?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/tr/suggest.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 4f4f3d66d4..38020dedb7 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -61,7 +61,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// Suggest with a given dictionary. pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || word.contains(char::is_whitespace) { + if n == 0 || n > MAX_LETTERS || max == 0 { return Vec::new(); } @@ -79,6 +79,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } } + // Only the table takes more than one word (`bir kaç`). + if word.contains(char::is_whitespace) { + return suggestions; + } // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`) if !d.is_empty() { @@ -297,6 +301,14 @@ mod tests { assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); } + #[test] + fn two_words_get_only_the_tables_replacement() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap(); + assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]); + assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]); + assert!(suggest_with(&d, "sorna sorna", 3).is_empty()); + } + #[test] fn turkish_to_lower_handles_i() { assert_eq!(turkish_to_lower("IŞIK"), "ışık"); From 5a5aa591cd84259f4076978de93153fa13c5a0ba Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:27:52 +0100 Subject: [PATCH 034/112] =?UTF-8?q?#3661:=20review=20=E2=80=94=20ISO=20885?= =?UTF-8?q?9-9=20Turkish=20dictionaries=20load=20through=20the=20capped=20?= =?UTF-8?q?reader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asked to confirm a non-UTF-8 Turkish dictionary still loads: before this PR tr_TR was read with read_to_string, so a non-UTF-8 file never loaded; now it goes through read_capped and decode, which didn't know ISO 8859-9 (Latin-5, the encoding of older tr_TR dictionaries) and refused it as unsupported. decode now reads `SET ISO8859-9`, and a test writes an ISO 8859-9 .aff/.dic to disk and finds words with ğ ı ş İ ç ö ü, as stems and with suffixes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/hunspell.rs | 47 ++++++++++++++++++++++++++++++++++- docs/spelling-dictionaries.md | 8 +++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index a5b41214d0..e86c7f6876 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -174,6 +174,23 @@ fn decode_latin1(bytes: &[u8]) -> String { bytes.iter().map(|&b| char::from(b)).collect() } +/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The +/// encoding of older Turkish dictionaries (`SET ISO8859-9`). +fn decode_latin5(bytes: &[u8]) -> String { + bytes + .iter() + .map(|&b| match b { + 0xd0 => 'Ğ', + 0xdd => 'İ', + 0xde => 'Ş', + 0xf0 => 'ğ', + 0xfd => 'ı', + 0xfe => 'ş', + _ => char::from(b), + }) + .collect() +} + /// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; @@ -199,7 +216,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the -/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use). +/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R +/// and Windows-1251 the ones older Russian dictionaries use). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); @@ -210,6 +228,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + "ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))), "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))), "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))), other => Err(format!("unsupported encoding {other}")), @@ -228,6 +247,8 @@ mod tests { assert_eq!(d, "2\nżółw\n"); let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); assert_eq!(d, "żółw"); + let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap(); + assert_eq!(d, "ĞİŞğışÖç"); assert!(decode(b"SET SHIFT_JIS\n", b"").is_err()); assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); } @@ -323,4 +344,28 @@ mod tests { assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); let _ = std::fs::remove_dir_all(&dir); } + + /// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from + /// disk through the capped reader and decoded per its `SET` line: its words with Turkish + /// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the + /// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not. + #[test] + fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap(); + // ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş. + let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n"; + std::fs::write(&dic, words).unwrap(); + assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8"); + let d = crate::tr::load(&aff, &dic).unwrap(); + for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] { + assert!(d.contains(w), "{w}"); + } + // The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those. + assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir")); + assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/docs/spelling-dictionaries.md b/docs/spelling-dictionaries.md index 276f1463f0..70354e7e0c 100644 --- a/docs/spelling-dictionaries.md +++ b/docs/spelling-dictionaries.md @@ -1,6 +1,6 @@ # Installing spelling dictionaries -> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-23 · **Change:** minor (Spanish: `es_ES`, `es_MX`, `es_419` and `es_UY` with their fallbacks, #3360, #3553; previously trivial: every Hunspell language shows (no dictionary), not only Polish and Portuguese, and Indonesian is listed, #3150; previously trivial: ISO 8859-1 dictionaries are read, #2824) +> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-23 · **Change:** minor (ISO 8859-9 Turkish dictionaries are read, #3692; previously minor: Spanish: `es_ES`, `es_MX`, `es_419` and `es_UY` with their fallbacks, #3360, #3553; previously trivial: every Hunspell language shows (no dictionary), not only Polish and Portuguese, and Indonesian is listed, #3150; previously trivial: ISO 8859-1 dictionaries are read, #2824) English spelling works out of the box. Polish, Russian, Portuguese, Czech, Italian, Croatian, Turkish, Indonesian and Spanish spelling use a standard **Hunspell** dictionary (an `.aff` and a `.dic` file) that @@ -44,8 +44,9 @@ Spanish regions of the Americas (Argentina, Colombia, the United States…) the American one found. LibreOffice's `dict-es` has them all. Uruguayan Spanish also accepts the `vos` forms of the dictionary's verbs (`tenés`, `vení`, `decime`, `decímelo`). -The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-1, ISO 8859-2, KOI8-R -and Windows-1251 are read as well; the original VERO `pt_BR` is ISO 8859-1). +The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-1, ISO 8859-2, +ISO 8859-9, KOI8-R and Windows-1251 are read as well; the original VERO `pt_BR` is ISO 8859-1, older +Turkish `tr_TR` dictionaries ISO 8859-9). ## Where WordCraft looks @@ -141,6 +142,7 @@ Never commit dictionary files to this repository: they stay on the user's system |---|---|---| | 2026-10-22 | minor | Spanish (#3360, #3553): the file names for Spain, Mexico, Latin America and Uruguay and their fallbacks | | 2026-10-22 | trivial | Italian, Croatian, Turkish, Russian, Czech and Indonesian without a dictionary now show **(no dictionary)** (or **(dictionary unreadable)**) too, as this page already said; before, only Polish and Portuguese did. Indonesian is listed with its file names (#3150) | +| 2026-10-23 | trivial | ISO 8859-9 (`SET ISO8859-9`) dictionaries are read, so an older Turkish `tr_TR` loads without converting it to UTF-8; the Turkish dictionary is read through the same size-capped reader and `SET` decoding as the others (#3692) | | 2026-10-18 | trivial | ISO 8859-1 (`SET ISO8859-1`) dictionaries are read, so the original VERO `pt_BR` works without converting it to UTF-8 (#2824) | | 2026-10-18 | minor | An installed dictionary that can't be read shows **(dictionary unreadable)** in the status bar, with the file and reason in its tooltip and in File › Options › Proofing (#2830) | | 2026-10-16 | major | First version: the file names per language, the search order and folders on Windows, macOS and Linux, installing `pt_BR` on each, and how to check it works | From 6bcd997be8645578b50fe1c5268ec7151beda662 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 14:20:40 +0100 Subject: [PATCH 035/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20proof=20t?= =?UTF-8?q?iming=20tests=20get=20a=20margin=20a=20busy=20CI=20machine=20ca?= =?UTF-8?q?n't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 1-2 s failed on loaded runners. Each bound is now 10 s, at least 10x the fixed code's time and well under the old code's on the same input (measured by running the pre-fix code): - self-continuing rules: 40,000 rules (was 16,000); ~1 ms now, old checker 14 s at 16,000. - many strips per suffix: 200,000 rules (was 30,000); 0.4 s now, old index 3 s at 30,000 and 11 min at 300,000. - forms of self-continuing rules: the MAX_FORMS count catches the old code (9 million forms); the time bound is a backstop. - overlong Turkish words: ~20 ms now; without the caps it ran past 5 minutes. - capped Hunspell reads: no clock; the "not a regular file" error is the proof (the old reader never returned from /dev/zero). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/affix.rs | 22 ++++++++++++++-------- crates/proof/src/hunspell.rs | 5 ++--- crates/proof/src/tr/suggest.rs | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index fefc7f6ac8..03fc27a6bd 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -1310,11 +1310,13 @@ pub(crate) mod tests { #[test] fn self_continuing_rules_check_in_linear_time() { // Thousands of empty-append rules that continue with their own flag: each check used to - // follow every rule from every rule, millions of steps a word (#3613). - let d = many_rules(16_000, |_| "0 0/A .".into()); + // follow every rule from every rule, millions of steps a word (#3613). The checks take + // about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at + // 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin. + let d = many_rules(40_000, |_| "0 0/A .".into()); let t = std::time::Instant::now(); assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); } #[test] @@ -1324,7 +1326,9 @@ pub(crate) mod tests { let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); let t = std::time::Instant::now(); let (a, b) = (same.forms("kot"), distinct.forms("kot")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + // The cap on the count is what tells the old code (9 million forms each) apart; the + // generous time bound (tens of ms here) only backs up the rule-visit cap. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); } @@ -1332,11 +1336,13 @@ pub(crate) mod tests { #[test] fn many_strips_per_suffix_index_in_linear_time() { // One append with thousands of strips: the index used to scan the append's groups for - // each rule. + // each rule. Building 200,000 rules takes well under a second; the old index took 3 s at + // 30,000 and over 10 minutes at 300,000. + let n = 200_000; let t = std::time::Instant::now(); - let d = many_rules(30_000, |i| format!("k{i} y .")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); - assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(30_000)); + let d = many_rules(n, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n)); assert!(!d.contains("koty")); } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index e86c7f6876..3449c8cab7 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -329,13 +329,12 @@ mod tests { // One byte over the cap (sparse: nothing is written) is refused without reading it. let big = dir.join("big.dic"); std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); - let t = std::time::Instant::now(); assert!(load(&aff, &big).unwrap_err().contains("too large")); - // A directory, and on Unix an endless device, are not dictionary files. + // A directory, and on Unix an endless device, are not dictionary files. (No clock: a + // reader that read `/dev/zero` would never return, so the error itself is the proof.) assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); #[cfg(unix)] assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); - assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); // Turkish reads through the same reader. let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 38020dedb7..c5f4db8959 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -298,7 +298,8 @@ mod tests { assert!(suggest_with(&d, &long, 8).is_empty()); assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); assert!(!super::super::is_correct_with(&d, &long)); - assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + // Tens of milliseconds with the caps; without them this ran for over 5 minutes. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); } #[test] From 7f203bc1c8d472840eaf97b1d900c9bd61b87c46 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 19 Oct 2026 01:49:43 +0100 Subject: [PATCH 036/112] #39: per-user Windows installer that needs no administrator rights Build wordcraft.wxs a second time as a dual-purpose package (Scope=perUserOrMachine: ALLUSERS=2, MSIINSTALLPERUSER=1), shipped as wordcraft--windows-user-.msi. Registry keys move from HKLM to HKMU so the same components land in HKLM per-machine and HKCU per-user. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 14 +++-- ROADMAP.md | 1 + docs/releasing.md | 2 +- packaging/windows/package.ps1 | 31 ++++++---- packaging/windows/wordcraft.wxs | 104 +++++++++++++++++--------------- 5 files changed, 87 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 42340c3133..b52cf3e518 100644 --- a/README.md +++ b/README.md @@ -221,13 +221,15 @@ closer pagination fidelity, a stability sweep and native printing. Details and e ### Windows -| Build | Installer | Portable | -|---|---|---| -| x64 (64-bit Intel/AMD) | `wordcraft--windows-x64.msi` | `wordcraft--windows-x64-portable.zip` | -| arm64 (Snapdragon and other ARM PCs) | `wordcraft--windows-arm64.msi` | `wordcraft--windows-arm64-portable.zip` | -| x86 (32-bit) | `wordcraft--windows-x86.msi` | `wordcraft--windows-x86-portable.zip` | +| Build | Installer (all users) | Installer (just me) | Portable | +|---|---|---|---| +| x64 (64-bit Intel/AMD) | `wordcraft--windows-x64.msi` | `wordcraft--windows-user-x64.msi` | `wordcraft--windows-x64-portable.zip` | +| arm64 (Snapdragon and other ARM PCs) | `wordcraft--windows-arm64.msi` | `wordcraft--windows-user-arm64.msi` | `wordcraft--windows-arm64-portable.zip` | +| x86 (32-bit) | `wordcraft--windows-x86.msi` | `wordcraft--windows-user-x86.msi` | `wordcraft--windows-x86-portable.zip` | -Installers and executables are code-signed. +The all-users installer puts WordCraft in Program Files and needs administrator rights. The +just-me installer needs none: it installs into `%LOCALAPPDATA%\Programs\WordCraft` for your +account only. Installers and executables are code-signed. **If the app doesn't open on Windows:** the desktop app initializes only DirectX 12 by default. Letting wgpu also create an OpenGL instance can crash some graphics drivers (AMD's diff --git a/ROADMAP.md b/ROADMAP.md index 2d0b45ddd2..9072433bdb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -210,6 +210,7 @@ Imported Word 97–2003 `.doc`/`.dot` files now ask for a supported DOCX target | 2026-10-20 | Restrict Editing holds for the file commands too: the privacy options, the properties and the inspector's removals are refused on a protected document, and restoring a stored version keeps the protection and its password instead of the snapshot's stale ones | | 2026-10-17 | Ribbon hints' reasons for a greyed-out control are translated in every interface language; a test fails if a reason it finds in the command sources, or the "Open a document first" hint, has no translation (#2678). | | 2026-10-17 | Ribbon hover hints: a disabled control says why (the command's own reason), split-button arrows, menu buttons and the font, size and outline boxes have hints, and hints show on keyboard focus (#1086). | +| 2026-10-16 | Windows per-user installer (#39): each release also ships `wordcraft--windows-user-.msi`, built from the same WiX source as a dual-purpose package, so it installs into `%LOCALAPPDATA%\Programs` for the current user with no administrator rights; registry keys moved to HKMU so both installers share one set of components | | 2026-10-14 | DOCX row table border exceptions render and survive saving. Explicit cell borders win; a row exception does not change other rows. | | 2026-10-14 | Tab stops edited on the ruler: the corner selector picks Left/Center/Right/Decimal/Bar, a click sets a stop (Alt skips the snap), dragging moves it with a guide line and position tooltip, dragging off the ruler clears it; markers draw per kind and mirror for RTL; one `para.tabs` call and one undo step per gesture | | 2026-10-12 | Fields keep `w:dirty` through DOCX saves (#877): a begin marker flagged by a generator that couldn't compute a result (pandoc-style TOC placeholders) keeps the flag on open/save — as a buffered field or a range field — so Word still refreshes it on open; computing a result (Update Field, Update Table) clears it, like Word's F9 | diff --git a/docs/releasing.md b/docs/releasing.md index 411af7593e..fc2503550a 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -11,7 +11,7 @@ this file has the WordCraft specifics. | Platform | Artifacts | Signing | |---|---|---| | macOS | universal `WordCraft.app` in a `.dmg`, universal CLI zip | Developer ID + notarization (`APPLE_*` secrets) | -| Windows | x64, x86 and arm64 `.msi` + portable `.zip` | Azure Trusted Signing (`AZURE_*` secrets) | +| Windows | x64, x86 and arm64 `.msi` (per-machine, and a per-user `-windows-user-.msi` that needs no admin rights) + portable `.zip` | Azure Trusted Signing (`AZURE_*` secrets) | | Linux | x86_64 and aarch64 `.AppImage` (+ `.zsync` for AppImageUpdate), `.flatpak` bundle, `.deb`, `.rpm`, `.tar.gz`; Flathub manifest | checksums | | Linux riscv64 | `.tar.gz` (cross-compiled on `ubuntu-24.04`, glibc >= 2.39; CLI smoke-tested under QEMU) | checksums | | FreeBSD | x86_64 `.tar.gz` | checksums | diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index 5223527598..e89a25e432 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -5,6 +5,7 @@ .DESCRIPTION Produces, in $env:DIST (default: dist/release): wordcraft--windows-.msi per-machine installer (WiX v5) + wordcraft--windows-user-.msi per-user installer, no admin rights (#39) wordcraft--windows--portable.zip wordcraft.exe + wordcraft-cli.exe The binaries link the C runtime statically (+crt-static), so neither the MSI nor the portable @@ -142,18 +143,26 @@ $rtf = '{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fcharset0 Tahoma;}}\vi ($paragraphs -join "\par`r`n") + "\par`r`n}`r`n" [System.IO.File]::WriteAllText($LicenseRtf, $rtf, [System.Text.Encoding]::ASCII) +# Two MSIs from one source: per-machine (Program Files, needs admin) and per-user (#39: no admin, +# into %LOCALAPPDATA%\Programs; see wordcraft.wxs). The per-user name puts "user" before the arch +# so the per-machine MSI sorts first: the shared windows-arm64 workflow installs the first *.msi +# it finds and checks Program Files. $Msi = Join-Path $Dist "wordcraft-$Version-windows-$Arch.msi" -Invoke-Native 'wix build' { - wix build (Join-Path $PSScriptRoot 'wordcraft.wxs') -arch $Arch ` - -ext WixToolset.UI.wixext -ext WixToolset.Util.wixext ` - -d "Version=$MsiVersion" -d "BinDir=$Stage" -d "IconPath=$(Join-Path $Root 'assets\app-icon\wordcraft.ico')" ` - -d "LicenseRtf=$LicenseRtf" ` - -d "AppNoticesDir=$AppNotices" -d "DocsDir=$Docs" -d "HasCraftFontLicences=$HasCraftFontLicences" ` - -o $Msi +$UserMsi = Join-Path $Dist "wordcraft-$Version-windows-user-$Arch.msi" +foreach ($build in @(@($Msi, 'perMachine'), @($UserMsi, 'perUserOrMachine'))) { + $out = $build[0] + Invoke-Native "wix build ($($build[1]))" { + wix build (Join-Path $PSScriptRoot 'wordcraft.wxs') -arch $Arch ` + -ext WixToolset.UI.wixext -ext WixToolset.Util.wixext ` + -d "Version=$MsiVersion" -d "BinDir=$Stage" -d "IconPath=$(Join-Path $Root 'assets\app-icon\wordcraft.ico')" ` + -d "LicenseRtf=$LicenseRtf" ` + -d "AppNoticesDir=$AppNotices" -d "DocsDir=$Docs" -d "HasCraftFontLicences=$HasCraftFontLicences" -d "Scope=$($build[1])" ` + -o $out + } + # wix writes its debug symbols (.wixpdb) next to the MSI; keep them out of the release assets. + Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($out, '.wixpdb')) + & (Join-Path $PSScriptRoot 'sign.ps1') $out } -# wix writes its debug symbols (.wixpdb) next to the MSI; keep them out of the release assets. -Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($Msi, '.wixpdb')) -& (Join-Path $PSScriptRoot 'sign.ps1') $Msi # ---- portable zip ------------------------------------------------------------------------------ $Portable = Join-Path $TargetDir "windows-package\wordcraft-$Version-windows-$Arch-portable" @@ -182,4 +191,4 @@ if ($Arch -ne 'arm64' -or $HostArch -eq 'arm64') { } else { Write-Output "skipping wordcraft-cli --version: an $Arch build doesn't run on this $HostArch machine" } -Get-Item $Msi, $Zip | Format-Table Name, Length +Get-Item $Msi, $UserMsi, $Zip | Format-Table Name, Length diff --git a/packaging/windows/wordcraft.wxs b/packaging/windows/wordcraft.wxs index 27aa89ca25..5be96705b1 100644 --- a/packaging/windows/wordcraft.wxs +++ b/packaging/windows/wordcraft.wxs @@ -5,16 +5,26 @@ Built by package.ps1: wix build wordcraft.wxs -arch x64 -ext WixToolset.UI.wixext -ext WixToolset.Util.wixext - -d Version=1.2.3 -d BinDir=... -d IconPath=... -d LicenseRtf=... -o out.msi + -d Version=1.2.3 -d BinDir=... -d IconPath=... -d LicenseRtf=... [-d Scope=perUserOrMachine] -o out.msi Per-machine install into Program Files with a Start Menu shortcut. The installer shows the standard WiX dialogs: welcome + license, progress, and a finish page. + Built a second time with -d Scope=perUserOrMachine for people without administrator rights + (#39): a dual-purpose package that WiX gives ALLUSERS=2 and MSIINSTALLPERUSER=1, so it installs + for the current user with no UAC prompt. Windows then redirects Program Files to + %LOCALAPPDATA%\Programs, the Start Menu to the user's own, and HKMU to HKCU, so the same + components work in both packages. Both share the UpgradeCode: each upgrades its own earlier + versions (Windows Installer only matches products installed in the same context). + UpgradeCode and the component GUIDs below belong to WordCraft alone. Never copy them from (or into) a sibling Craft app: Windows Installer treats every package sharing an UpgradeCode as one product, so a shared code makes one app's installer refuse to install ("a newer version is already installed") or silently uninstall the other. packaging-lint.yml checks this. --> + + + @@ -24,7 +34,7 @@ Manufacturer="Learning Machines LLC" Version="$(var.Version)" UpgradeCode="075497F9-E861-45D0-9C7F-A74F8C34E6BA" - Scope="perMachine" + Scope="$(var.Scope)" Compressed="yes" Language="1033"> @@ -88,7 +98,7 @@ Icon="WordcraftIcon.ico" IconIndex="0" /> - + - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - - + + + From c1eb02f23c22106731c5e2d0a5c22fc73effe25c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 19 Oct 2026 01:51:22 +0100 Subject: [PATCH 037/112] #39: README: installing both Windows installers gives two entries Review follow-up (FredApps): the all-users and just-me MSIs install side by side, so pick one. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012WEhZXSxzS8orX7f9J3qpr --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b52cf3e518..b0445a01eb 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,8 @@ closer pagination fidelity, a stability sweep and native printing. Details and e The all-users installer puts WordCraft in Program Files and needs administrator rights. The just-me installer needs none: it installs into `%LOCALAPPDATA%\Programs\WordCraft` for your -account only. Installers and executables are code-signed. +account only. Choose one: installing both gives two WordCraft entries in Settings › Apps and two +in the Start menu. Installers and executables are code-signed. **If the app doesn't open on Windows:** the desktop app initializes only DirectX 12 by default. Letting wgpu also create an OpenGL instance can crash some graphics drivers (AMD's From 6c20b3e609033119f93464c9b7eb2230bad2e390 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:26:43 +0100 Subject: [PATCH 038/112] #3692: proof_text walks the runs once Finding 29: proof_text looked up each character's run with props_of_char, walking the runs from the first for every character (quadratic in a paragraph of many runs). A run cursor advancing with the text gives the same props in one pass. Adds layout::tests_linear (sharing tests.rs's cost_ratio helpers) with proof_text_walks_the_runs_once. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/lib.rs | 2 ++ crates/layout/src/para.rs | 12 ++++++++- crates/layout/src/tests.rs | 6 ++--- crates/layout/src/tests_linear.rs | 45 +++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 crates/layout/src/tests_linear.rs diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index 23f855c5e4..464a7178c4 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -4935,6 +4935,8 @@ mod tests_ligature; #[cfg(test)] mod tests_nbsp; #[cfg(test)] +mod tests_linear; +#[cfg(test)] mod tests_numeral; #[cfg(test)] mod tests_pages; diff --git a/crates/layout/src/para.rs b/crates/layout/src/para.rs index 11e8f95c83..3dff3a8bec 100644 --- a/crates/layout/src/para.rs +++ b/crates/layout/src/para.rs @@ -3319,8 +3319,18 @@ fn proof_issues( /// U+0001 bytes (neither words nor spaces). pub fn proof_text(p: &wordcraft_doc::Paragraph) -> String { let mut out = String::with_capacity(p.text.len()); + // The run holding each character ([`Paragraph::props_of_char`]), found by walking the runs + // along with the text rather than from the first run for every character. + let mut runs = p.runs.iter(); + let mut run = runs.next(); + let mut end = run.map_or(0, |r| r.len); for (i, c) in p.text.char_indices() { - let deleted = c == wordcraft_doc::para::OBJ || p.props_of_char(i).del.is_some(); + while run.is_some() && i >= end { + run = runs.next(); + end = end.saturating_add(run.map_or(0, |r| r.len)); + } + let props = run.map_or(&p.mark, |r| &r.props); + let deleted = c == wordcraft_doc::para::OBJ || props.del.is_some(); if deleted { for _ in 0..c.len_utf8() { out.push('\u{1}'); diff --git a/crates/layout/src/tests.rs b/crates/layout/src/tests.rs index 7ed459c39d..6c63582ea3 100644 --- a/crates/layout/src/tests.rs +++ b/crates/layout/src/tests.rs @@ -738,7 +738,7 @@ fn hidden_text_adds_no_line_break_opportunity_when_not_shown() { /// How much bigger the second document is in the linear-time tests. At 8x a linear layout costs /// about 8x as much and a quadratic one about 64x, so a limit of 24x stays far from both. -const SCALE: usize = 8; +pub(crate) const SCALE: usize = 8; /// The CPU time `f` takes on this thread, in seconds. On Linux it is the scheduler's count of the /// time the thread actually ran, so the time it waits while other work on a busy machine has the @@ -761,7 +761,7 @@ fn cpu_cost(f: &mut impl FnMut()) -> f64 { /// cheapest of several runs after a warm-up, the runs alternating so a busy spell slows both sizes /// alike; `small` is timed `scale` times over per run, so both timings are long against the clock's /// resolution. -fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) { +pub(crate) fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) { small(); big(); let mut small_scaled = || (0..scale).for_each(|_| small()); @@ -774,7 +774,7 @@ fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> ( } /// [`cost_ratio`] of laying out `doc(SCALE * n)` against `doc(n)`. -fn layout_time_ratio(doc: impl Fn(usize) -> Document, n: usize, opts: &LayoutOptions) -> (f64, f64, f64) { +pub(crate) fn layout_time_ratio(doc: impl Fn(usize) -> Document, n: usize, opts: &LayoutOptions) -> (f64, f64, f64) { let (small, big) = (doc(n), doc(SCALE * n)); cost_ratio( SCALE, diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs new file mode 100644 index 0000000000..1e6ca61a9d --- /dev/null +++ b/crates/layout/src/tests_linear.rs @@ -0,0 +1,45 @@ +//! Lookups that once rescanned the page, the paragraph or the document for every item they +//! looked up (#3692, #3706): an input 8x the size must cost about 8x as much, not 64x. + +use super::tests::{SCALE, cost_ratio, layout_time_ratio}; +use super::*; +use wordcraft_doc::para::Run; +use wordcraft_doc::{Pos, Table}; + +fn lay(d: &Document, opts: &LayoutOptions) -> DocLayout { + layout(d, &mut LayoutCache::new(), opts) +} + +fn assert_linear(what: &str, (t1, tk, ratio): (f64, f64, f64)) { + eprintln!("{what}: t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "{what}: 8x the input took {ratio:.1}x the time ({t1:.4} s → {tk:.4} s): not linear"); +} + +/// [`cost_ratio`] of `f(big)` against `f(small)`, each repeated `reps` times so the timings are +/// long against the clock's tick. +fn ratio_of(reps: usize, small: &T, big: &T, f: impl Fn(&T)) -> (f64, f64, f64) { + cost_ratio(SCALE, || (0..reps).for_each(|_| f(small)), || (0..reps).for_each(|_| f(big))) +} + +/// The first laid-out body paragraph on page 0. +fn first_para(l: &DocLayout) -> Option<&ParaLayout> { + l.pages.first()?.items.iter().find_map(|it| match it { + Placed::Lines { para, story: StoryRef::Body, .. } => Some(para.as_ref()), + _ => None, + }) +} + +/// Finding 29: proofing text found each character's run from the paragraph's first run. +#[test] +fn proof_text_walks_the_runs_once() { + // One run per character, every other one deleted. + let para = |n: usize| { + let mut p = Paragraph::with_text(&"a".repeat(n), CharProps::default()); + p.runs = (0..n).map(|i| Run { len: 1, props: CharProps { del: (i % 2 == 1).then_some(0), ..Default::default() } }).collect(); + p + }; + assert_eq!(para::proof_text(¶(5)), "a\u{1}a\u{1}a"); + let (small, big) = (para(50_000), para(SCALE * 50_000)); + assert_linear("proof_text", ratio_of(8, &small, &big, |p| drop(std::hint::black_box(para::proof_text(p))))); +} + From 7d31c01f6f549192bc154b6ce38b7c2b8413c63d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:00 +0100 Subject: [PATCH 039/112] #3692: bidi clicks and caret steps walk a line's carets once Finding 5: off_at_x and visual_step on a bidirectional line called x_of for every caret offset, each searching the clusters from the line's start (quadratic in a line of many zero-width offsets, e.g. hidden text). ParaLayout::line_carets walks them with a cursor; the first matching cluster never moves back for non-decreasing offsets, so the x's are the same. Tests: bidi_clicks_and_caret_steps_take_linear_time, line_carets_match_x_of. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/hit.rs | 4 +-- crates/layout/src/para.rs | 43 ++++++++++++++++++++------- crates/layout/src/tests_linear.rs | 49 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/crates/layout/src/hit.rs b/crates/layout/src/hit.rs index 8a4840479c..63c7b41c79 100644 --- a/crates/layout/src/hit.rs +++ b/crates/layout/src/hit.rs @@ -722,8 +722,8 @@ impl DocLayout { let l = para.lines.get(li)?; let here = para.x_of(li, pos.off)?; let mut best: Option<(usize, f32)> = None; - for off in para.line_offsets(li) { - let Some(x) = para.x_of(li, off) else { continue }; + for (off, x) in para.line_carets(li) { + let Some(x) = x else { continue }; let ahead = if left { x < here - 0.01 } else { x > here + 0.01 }; let d = (x - here).abs(); if ahead && best.is_none_or(|b| d < b.1) { diff --git a/crates/layout/src/para.rs b/crates/layout/src/para.rs index 3dff3a8bec..7218079977 100644 --- a/crates/layout/src/para.rs +++ b/crates/layout/src/para.rs @@ -3107,20 +3107,42 @@ impl ParaLayout { let k = self.clusters.partition_point(|c| c.start < off); self.clusters.get(..k).into_iter().flatten().rev().find(|c| self.is_edit_boundary(c.start)).map_or(0, |c| c.start) }; + let mut from = l.c0; + self.x_from(l, off, &mut from) + } + /// [`Self::x_of`] on line `l`, looking for the cluster at `off` from cluster `*from` on (the + /// clusters before it must all end at or before `off`: so they do when `from` was left there + /// by an offset no greater), and leaving `from` at the cluster found. Walks a line's carets + /// in order in one pass over its clusters. + fn x_from(&self, l: &Line, off: usize, from: &mut usize) -> Option { if !l.vis.is_empty() { - return self.x_of_bidi(l, off); + return self.x_of_bidi(l, off, from); } - for k in l.c0..l.c1 { + for k in *from..l.c1 { let c = self.clusters.get(k)?; - if off <= c.start { - return l.xs.get(k - l.c0).copied(); - } - if off < c.end { + if off <= c.start || off < c.end { + *from = k; return l.xs.get(k - l.c0).copied(); } } + *from = l.c1; l.xs.last().copied() } + /// Each caret offset on line `li` ([`Self::line_offsets`]) with its x ([`Self::x_of`]). + pub fn line_carets(&self, li: usize) -> Vec<(usize, Option)> { + let Some(l) = self.lines.get(li) else { return Vec::new() }; + let (mut from, mut last) = (l.c0, 0); + self.line_offsets(li) + .into_iter() + .map(|off| { + if off < last { + from = l.c0; + } + last = off; + (off, self.x_from(l, off, &mut from)) + }) + .collect() + } /// The farthest right a caret goes on left-to-right line `li`: spaces at the end of a line /// hang past its right edge, but the caret after them stays at the edge, as in Word (#2146); /// text that overflows the line still carries it. `None` on bidirectional lines. @@ -3155,8 +3177,9 @@ impl ParaLayout { /// edge of right-to-left text), except where `off` leaves an embedded run (the cluster before /// it has the higher level), which keeps the caret at that run's trailing edge, so each /// visual boundary of the common mixed cases has its own offset. - fn x_of_bidi(&self, l: &Line, off: usize) -> Option { - let k = (l.c0..l.c1).find(|&k| self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start)); + fn x_of_bidi(&self, l: &Line, off: usize, from: &mut usize) -> Option { + let k = (*from..l.c1).find(|&k| self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start)); + *from = k.unwrap_or(l.c1); let Some(k) = k else { return Some(l.end_x()) }; let c = self.clusters.get(k)?; let here = l.vis.get(k - l.c0)?; @@ -3204,8 +3227,8 @@ impl ParaLayout { let Some(l) = self.lines.get(li) else { return self.text_len }; if !l.vis.is_empty() { let mut best = (l.start, f32::MAX); - for off in self.line_offsets(li) { - let d = self.x_of(li, off).map_or(f32::MAX, |cx| (cx - x).abs()); + for (off, cx) in self.line_carets(li) { + let d = cx.map_or(f32::MAX, |cx| (cx - x).abs()); if d < best.1 { best = (off, d); } diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 1e6ca61a9d..fa26b2200d 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -43,3 +43,52 @@ fn proof_text_walks_the_runs_once() { assert_linear("proof_text", ratio_of(8, &small, &big, |p| drop(std::hint::black_box(para::proof_text(p))))); } +/// A right-to-left paragraph whose one line holds `n` hidden letters: caret offsets with no width. +fn bidi_hidden(n: usize) -> Document { + let head = "سلام "; + let mut d = Document::from_text(&format!("{head}{} world", "x".repeat(n))); + let at = Pos::body(0, 0); + d.format_paragraphs(&at, &at, &|p| p.bidi = Some(true)).unwrap(); + d.format_range(&Pos::body(0, head.len()), &Pos::body(0, head.len() + n), &|c| c.hidden = Some(true)).unwrap(); + d +} + +/// Finding 5: a click on a bidirectional line and a visual caret step measured every caret +/// offset from the line's first cluster. +#[test] +fn bidi_clicks_and_caret_steps_take_linear_time() { + let opts = LayoutOptions { show_hidden: false, ..Default::default() }; + let (ds, db) = (bidi_hidden(5000), bidi_hidden(SCALE * 5000)); + let (ls, lb) = (lay(&ds, &opts), lay(&db, &opts)); + let (Some(ps), Some(pb)) = (first_para(&ls), first_para(&lb)) else { panic!("no paragraph") }; + assert_eq!(pb.lines.len(), 1, "the hidden letters stay on one line"); + let ratio = ratio_of(32, &(ps, &ls), &(pb, &lb), |(p, l)| { + std::hint::black_box(p.off_at_x(0, 1.0)); + std::hint::black_box(l.visual_step(&Pos::body(0, 0), true, 0)); + }); + assert_linear("bidi hit test", ratio); +} + +/// The carets of a line walked in one pass are where `x_of` puts each of them, on bidirectional +/// and left-to-right lines alike. +#[test] +fn line_carets_match_x_of() { + let docs = [bidi_hidden(20), Document::from_text("Plain left-to-right text, long enough to wrap. ".repeat(6).as_str()), { + let mut d = Document::from_text("سلام world 123 دنیا (مرحبا) end"); + let at = Pos::body(0, 0); + d.format_paragraphs(&at, &at, &|p| p.bidi = Some(true)).unwrap(); + d + }]; + for d in &docs { + for show_hidden in [false, true] { + let l = lay(d, &LayoutOptions { show_hidden, ..Default::default() }); + let Some(p) = first_para(&l) else { panic!("no paragraph") }; + for li in 0..p.lines.len() { + let want: Vec<_> = p.line_offsets(li).into_iter().map(|off| (off, p.x_of(li, off))).collect(); + assert!(!want.is_empty()); + assert_eq!(p.line_carets(li), want, "line {li}"); + } + } + } +} + From 4d027de4022aecb9bc79abd357b9fbcb71ac3595 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:00 +0100 Subject: [PATCH 040/112] #3692: math text runs find character boundaries in one walk Finding 23: Ctx::glyphs found each character's first glyph by scanning the shaped glyphs from the start (quadratic in a long run). The first glyph at or past a boundary never moves back as the boundary grows, so one walk gives the same cuts. Test: math::tests::a_long_text_run_lays_out_in_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/math.rs | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/crates/layout/src/math.rs b/crates/layout/src/math.rs index 7291e89dd6..a581635bc4 100644 --- a/crates/layout/src/math.rs +++ b/crates/layout/src/math.rs @@ -631,19 +631,18 @@ impl Ctx { last_gid = Some(g.gid); } b.w = x; - // Character boundaries: where the first glyph of each character starts. + // Character boundaries: where the first glyph of each character starts. The glyph for a + // boundary is never before the previous boundary's, so one walk of the glyphs finds all. let mut cuts = Vec::with_capacity(text.len() + 1); + let (mut j, mut gx) = (0usize, 0.0f32); for (cb, _) in text.char_indices() { - let mut gx = 0.0f32; - let mut found = None; - for g in &shaped { - if g.cluster >= cb { - found = Some(gx); - break; - } + while let Some(g) = shaped.get(j) + && g.cluster < cb + { gx += g.x_advance as f32 * k; + j += 1; } - cuts.push(found.unwrap_or(x)); + cuts.push(if j < shaped.len() { gx } else { x }); } cuts.push(x); b.cuts = cuts; @@ -2374,4 +2373,19 @@ mod tests { } } } + + /// Finding 23: each character's boundary in a run of text was found by walking the run's + /// glyphs from the first. + #[test] + fn a_long_text_run_lays_out_in_linear_time() { + use crate::tests::{SCALE, cost_ratio}; + use wordcraft_doc::math::*; + let eq = |n: usize| Math { nodes: vec![MNode::Run(MRun { nor: true, ..MRun::new("a".repeat(n)) })], ..Default::default() }; + let (small, big) = (eq(400), eq(SCALE * 400)); + // Each timed 16 times over, so the timings are long against the clock's tick. + let lay_16 = |m: &Math| (0..16).for_each(|_| drop(std::hint::black_box(lay_in(m, "", false, 1e6, &MathProps::default())))); + let (t1, tk, ratio) = cost_ratio(SCALE, || lay_16(&small), || lay_16(&big)); + eprintln!("t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "8x the text took {ratio:.1}x the time ({t1:.4} s → {tk:.4} s): not linear"); + } } From 629711582fd550babfd39b9520ed66ca134c1f49 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:01 +0100 Subject: [PATCH 041/112] #3692: autofit counts the grid's columns once Finding 3: column_content called Table::cols (a walk of every row) for each spanning cell, and there can be one in every row. Hoisted out of the loop; the end is also added with saturating_add. Test: autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/table.rs | 4 +++- crates/layout/src/tests_linear.rs | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/layout/src/table.rs b/crates/layout/src/table.rs index d31967af22..1a1f5e5b91 100644 --- a/crates/layout/src/table.rs +++ b/crates/layout/src/table.rs @@ -1144,8 +1144,10 @@ fn column_content(ctx: &mut Ctx, t: &Table, style: Option<&TableStyleProps>, dep } spanning.extend(row_gaps(row, g).map(|(g, span, w)| (g, span, (w, w)))); } + // The grid's width once: counting it walks every row, and there can be a spanning cell in each. + let ncols = t.cols(); for (g, span, need) in spanning { - let Some(cs) = cols.get_mut(g..(g + span).min(t.cols())) else { continue }; + let Some(cs) = cols.get_mut(g..g.saturating_add(span).min(ncols)) else { continue }; let n = cs.len().max(1) as f32; let (have_lo, have_hi) = cs.iter().fold((0.0, 0.0), |(l, h), c| (l + c.0, h + c.1)); for c in cs { diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index fa26b2200d..58542956ec 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -92,3 +92,21 @@ fn line_carets_match_x_of() { } } +/// Finding 3: autofitting a table counted its grid's columns again for every spanning cell. +#[test] +fn autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time() { + let doc = |n: usize| { + let mut t = Table::new(n, 3, 468.0); + for row in &mut t.rows { + row.cells.truncate(2); + if let Some(c) = row.cells.first_mut() { + c.props.span = 2; + } + } + let mut d = Document::from_text("x"); + d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap(); + d + }; + assert_linear("spanning cells", layout_time_ratio(doc, 400, &LayoutOptions::default())); +} + From 897add99a89af2e2041d594a11927d7f5bfa22da Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:01 +0100 Subject: [PATCH 042/112] #3692: headers and footers linked through sections once per layout Finding 13: linked() and defining_width() walked back through the sections before each section, per section and again per page (layout_pass, headers_footers, footer_table_wraps, header_table_areas). HfLinks resolves every section's linked header and footer sets incrementally once (each inherits the previous one's for the parts it leaves out) and keeps, per part id, the sections naming it, so the defining section is a binary search. Same sets and widths as before. Test: headers_inherited_through_many_sections_lay_out_in_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/lib.rs | 147 +++++++++++++++++++----------- crates/layout/src/tests_linear.rs | 23 +++++ 2 files changed, 118 insertions(+), 52 deletions(-) diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index 464a7178c4..e5a9d3ef2f 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -743,6 +743,8 @@ struct Ctx<'a> { /// Each body float's latest placement: on which page and with what wrap area, when it is /// above its anchor paragraph with text before it ([`EarlyFloats`]). floats_above: HashMap<(usize, usize), Option<(usize, WrapArea)>>, + /// The document's sections with their headers and footers linked ([`HfLinks`]). + hf: Arc>, } /// A header or footer (`story`) laid out for page `number` of a section (`sect`, `mirror`): its @@ -2340,8 +2342,10 @@ fn layout_pass( footer_wraps: HashMap::new(), pin: None, floats_above: HashMap::new(), + hf: Arc::new(HfLinks::new(doc.sections())), }; - let sections = doc.sections(); + let hf = Arc::clone(&ctx.hf); + let sections = &hf.sections; let web = matches!(opts.view, ViewMode::Web | ViewMode::Draft | ViewMode::Outline); let read = opts.view == ViewMode::Read; let reflow = web || read; @@ -2471,9 +2475,9 @@ fn layout_pass( ctx.fields.section = u32::try_from(si).unwrap_or(u32::MAX).saturating_add(1); ctx.fields.section_pages = totals.section(si); // Headers and footers left out are linked to the previous section's, and take its room. - let (heads, feet) = (linked(§ions, si, |s| s.headers), linked(§ions, si, |s| s.footers)); - let hb = |i: Option| defining_width(§ions, si, |s| s.headers, i); - let fb = |i: Option| defining_width(§ions, si, |s| s.footers, i); + let (heads, feet) = (hf.heads.set(si), hf.feet.set(si)); + let hb = |i: Option| hf.defining_width(false, si, i); + let fb = |i: Option| hf.defining_width(true, si, i); let body_top = if reflow || turn.is_turned() { sect.margin_top } else { @@ -2513,7 +2517,7 @@ fn layout_pass( { pb.sect = prev; pb.sect_idx = si - 1; - let (ph, pf) = (linked(§ions, si - 1, |s| s.headers), linked(§ions, si - 1, |s| s.footers)); + let (ph, pf) = (hf.heads.set(si - 1), hf.feet.set(si - 1)); let top = body_top_for(&mut ctx, prev, ph.default); pb.foot_top = foot_top_for(&mut ctx, prev, pf.default); pb.new_page(block, top); @@ -2627,7 +2631,7 @@ fn layout_pass( } } if !reflow { - let found = Totals::of(&pages, §ions); + let found = Totals::of(&pages, sections); if ctx.uses_totals && may_repeat && found != *totals { return Pass::Again(found); } @@ -2635,13 +2639,13 @@ fn layout_pass( // Page references in headers and footers are filled in after pagination, like their page // totals: they don't make the body lay out again. if headers_footers_refer_to_pages(doc) { - let marks = bookmark_pages(doc, &pages, §ions); + let marks = bookmark_pages(doc, &pages, sections); ctx.fields.bookmarks_key = marks_key(&marks); ctx.fields.bookmark_pages = Arc::new(marks); } - headers_footers(&mut ctx, &mut pages, §ions, &found); + headers_footers(&mut ctx, &mut pages, sections, &found); if doc.settings.mirror_margins { - mirror_even_pages(&mut pages, §ions); + mirror_even_pages(&mut pages, sections); } } // Evict paragraph layouts that weren't used this pass if the cache grew large. @@ -2924,23 +2928,75 @@ fn section_side(start: SectionStart, restart: Option, last: u32, pages: usi } } -/// The text width of the section that defines header or footer `id`, when section `si` inherits -/// it rather than naming it (`None` when it names it). Measured in Word: an inherited part's -/// percentage-width tables are sized by that section, though it's laid out on `si`'s pages. -fn defining_width(sections: &[(usize, &SectionProps)], si: usize, get: fn(&SectionProps) -> HeaderSet, id: Option) -> Option { - let id = id?; - let names = |s: &SectionProps| { - let h = get(s); - [h.default, h.first, h.even].contains(&Some(id)) - }; - let own = sections.get(si)?.1; - if names(own) { - return None; - } - sections.iter().take(si).rev().find(|(_, s)| names(s)).map(|(_, s)| s.text_width()).filter(|w| (w - own.text_width()).abs() > 0.01) +/// The document's sections with their headers and footers (Word's "link to previous"), each +/// section's resolved once for the whole layout in document order rather than by looking back +/// over the sections before it whenever one is needed. +struct HfLinks<'a> { + sections: Vec<(usize, &'a SectionProps)>, + heads: Linked, + feet: Linked, } -/// Lay out with percentage-width tables sized by `base` (see [`defining_width`]). +impl<'a> HfLinks<'a> { + fn new(sections: Vec<(usize, &'a SectionProps)>) -> Self { + let heads = Linked::new(§ions, |s| s.headers); + let feet = Linked::new(§ions, |s| s.footers); + HfLinks { sections, heads, feet } + } + + /// The text width of the section that defines header (or, `footer`, footer) `id`, when + /// section `si` inherits it rather than naming it (`None` when it names it). Measured in + /// Word: an inherited part's percentage-width tables are sized by that section, though it's + /// laid out on `si`'s pages. + fn defining_width(&self, footer: bool, si: usize, id: Option) -> Option { + let id = id?; + let linked = if footer { &self.feet } else { &self.heads }; + let own = self.sections.get(si)?.1; + if linked.own.get(si).is_some_and(|h| [h.default, h.first, h.even].contains(&Some(id))) { + return None; + } + // The last section before `si` naming it. + let namers = linked.namers.get(&id)?; + let k = *namers.get(namers.partition_point(|&k| k < si).checked_sub(1)?)?; + self.sections.get(k).map(|(_, s)| s.text_width()).filter(|w| (w - own.text_width()).abs() > 0.01) + } +} + +/// The headers (or footers) of every section: those it names, and with those it leaves out +/// inherited from the sections before it. +struct Linked { + own: Vec, + linked: Vec, + /// The sections naming each part, in order. + namers: HashMap>, +} + +impl Linked { + fn new(sections: &[(usize, &SectionProps)], get: fn(&SectionProps) -> HeaderSet) -> Self { + let own: Vec = sections.iter().map(|(_, s)| get(s)).collect(); + let mut linked = Vec::with_capacity(own.len()); + let mut namers: HashMap> = HashMap::new(); + let mut prev = HeaderSet::default(); + for (si, o) in own.iter().enumerate() { + prev = HeaderSet { default: o.default.or(prev.default), first: o.first.or(prev.first), even: o.even.or(prev.even) }; + linked.push(prev); + for id in [o.default, o.first, o.even].into_iter().flatten() { + let v = namers.entry(id).or_default(); + if v.last() != Some(&si) { + v.push(si); + } + } + } + Linked { own, linked, namers } + } + + /// Section `si`'s headers (or footers), those it leaves out inherited. + fn set(&self, si: usize) -> HeaderSet { + self.linked.get(si).copied().unwrap_or_default() + } +} + +/// Lay out with percentage-width tables sized by `base` (see [`HfLinks::defining_width`]). fn with_pct_base(ctx: &mut Ctx, base: Option, f: impl FnOnce(&mut Ctx) -> T) -> T { let before = std::mem::replace(&mut ctx.pct_base, base); let out = f(ctx); @@ -3400,11 +3456,12 @@ fn footer_table_wraps(ctx: &mut Ctx, pb: &PageBuilder) -> Vec { } ctx.footer_wraps.insert(key, Vec::new()); let doc = ctx.doc; - let Some((_, sect)) = ctx.sections.get(page.section).copied() else { return Vec::new() }; + let hf = Arc::clone(&ctx.hf); + let Some((_, sect)) = hf.sections.get(page.section) else { return Vec::new() }; if sect.text_direction.is_turned() { return Vec::new(); } - let set = linked(&ctx.sections, page.section, |s| s.footers); + let set = hf.feet.set(page.section); let id = if sect.title_page && first { set.first } else if doc.settings.even_odd_headers && page.number.is_multiple_of(2) { @@ -3416,7 +3473,7 @@ fn footer_table_wraps(ctx: &mut Ctx, pb: &PageBuilder) -> Vec { let Some(part) = doc.parts.get(&id) else { return Vec::new() }; let blocks = part.blocks.clone(); let x = sect.margin_left + sect.gutter; - let base = defining_width(&ctx.sections, page.section, |s| s.footers, Some(id)); + let base = hf.defining_width(true, page.section, Some(id)); let counters = std::mem::take(&mut ctx.counters); let boxes = std::mem::take(&mut ctx.boxes); let chains = std::mem::take(&mut ctx.chains); @@ -4534,11 +4591,12 @@ fn header_table_areas(ctx: &mut Ctx, pb: &PageBuilder) -> Vec { if let Some(areas) = ctx.header_tables.get(&key) { return areas.clone(); } - let Some((_, sect)) = ctx.sections.get(page.section).copied() else { return Vec::new() }; + let hf = Arc::clone(&ctx.hf); + let Some((_, sect)) = hf.sections.get(page.section) else { return Vec::new() }; if sect.text_direction.is_turned() { return Vec::new(); } - let set = linked(&ctx.sections, page.section, |s| s.headers); + let set = hf.heads.set(page.section); let id = if sect.title_page && first { set.first } else if ctx.doc.settings.even_odd_headers && page.number.is_multiple_of(2) { @@ -4551,7 +4609,7 @@ fn header_table_areas(ctx: &mut Ctx, pb: &PageBuilder) -> Vec { let blocks = part.blocks.clone(); let x = sect.margin_left + sect.gutter; let frame = PageFrame { sect, origin: (x, sect.header), number: page.number, mirror: ctx.mirror }; - let base = defining_width(&ctx.sections, page.section, |s| s.headers, Some(id)); + let base = hf.defining_width(false, page.section, Some(id)); let counters = std::mem::take(&mut ctx.counters); let boxes = std::mem::take(&mut ctx.boxes); let chains = std::mem::take(&mut ctx.chains); @@ -4704,22 +4762,6 @@ fn below(excl: &[WrapArea], mut y: f32, x0: f32, x1: f32) -> f32 { y } -/// Section `si`'s headers (or footers, as `get` picks), with those it leaves out inherited from -/// the sections before it (Word's "link to previous"). -fn linked(sections: &[(usize, &SectionProps)], si: usize, get: fn(&SectionProps) -> HeaderSet) -> HeaderSet { - let mut set = sections.get(si).map(|(_, s)| get(s)).unwrap_or_default(); - for (_, s) in sections.iter().take(si).rev() { - if set.default.is_some() && set.first.is_some() && set.even.is_some() { - break; - } - let o = get(s); - set.default = set.default.or(o.default); - set.first = set.first.or(o.first); - set.even = set.even.or(o.even); - } - set -} - /// Whether a header, a footer or a text box shows a `PAGEREF` field, which they resolve from the /// paginated body (see [`bookmark_pages`]). fn headers_footers_refer_to_pages(doc: &Document) -> bool { @@ -4798,6 +4840,7 @@ fn marks_key(marks: &fields::BookmarkPages) -> u64 { fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &SectionProps)], totals: &Totals) { let total = pages.len() as u32; + let hf = Arc::clone(&ctx.hf); let even_odd = ctx.doc.settings.even_odd_headers; let styles = styleref::headers_use_styleref(ctx.doc).then(|| ctx.cache.style_index(ctx.doc)); let mut first_of_section: HashSet = HashSet::new(); @@ -4810,8 +4853,8 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti for (i, page) in pages.iter_mut().enumerate() { let Some((_, sect)) = sections.get(page.section) else { continue }; // Inherit header/footer references from earlier sections (Word's "link to previous"). - let pick = |get: fn(&SectionProps) -> HeaderSet| -> Option { - let set = linked(sections, page.section, get); + let pick = |linked: &Linked| -> Option { + let set = linked.set(page.section); if sect.title_page && first_of_section.contains(&i) { return set.first; } @@ -4820,8 +4863,8 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti } set.default }; - let hid = pick(|s| s.headers); - let fid = pick(|s| s.footers); + let hid = pick(&hf.heads); + let fid = pick(&hf.feet); page.hf_slot = if sect.title_page && first_of_section.contains(&i) { HfSlot::First } else if even_odd && page.number % 2 == 0 { @@ -4849,7 +4892,7 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti { let blocks = part.blocks.clone(); let frame = PageFrame { sect, origin: (x, sect.header), number: page.number, mirror: ctx.mirror }; - let base = defining_width(sections, page.section, |s| s.headers, hid); + let base = hf.defining_width(false, page.section, hid); let (mut items, behind, front) = with_pin(ctx, id, sect, page.number, |ctx| { with_pct_base(ctx, base, |c| layout_box(c, StoryRef::Part(id), &blocks, &[], w, BoxContext::Other, 0, Some(frame)).items) }); @@ -4863,7 +4906,7 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti && let Some(part) = ctx.doc.parts.get(&id) { let blocks = part.blocks.clone(); - let base = defining_width(sections, page.section, |s| s.footers, fid); + let base = hf.defining_width(true, page.section, fid); // The footer's height fixes where it sits, which page-relative floats need. let (h, ..) = with_pin(ctx, id, sect, page.number, |ctx| { with_pct_base(ctx, base, |c| layout_box(c, StoryRef::Part(id), &blocks, &[], w, BoxContext::Other, 0, None).height) diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 58542956ec..46045c6aeb 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -110,3 +110,26 @@ fn autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time() { assert_linear("spanning cells", layout_time_ratio(doc, 400, &LayoutOptions::default())); } +/// Finding 13: each section's headers and footers were found by walking back through the +/// sections before it, for every section and again for every page. +#[test] +fn headers_inherited_through_many_sections_lay_out_in_linear_time() { + let doc = |n: usize| { + let mut d = Document::from_text(&vec!["x"; n].join("\n")); + let head = d.add_part(wordcraft_doc::PartKind::Header, vec![wordcraft_doc::para_block(Paragraph::with_text("Head", CharProps::default()))]); + for i in 0..n.saturating_sub(1) { + let mut s = SectionProps { start: SectionStart::Continuous, ..Default::default() }; + if i == 0 { + s.headers.default = Some(head); + } + d.para_mut(StoryRef::Body, &Path::top(i)).unwrap().section = Some(Box::new(s)); + } + d.last_section.start = SectionStart::Continuous; + d + }; + // Every page shows the first section's header. + let l = lay(&doc(200), &LayoutOptions::default()); + assert!(l.pages.iter().all(|p| !p.header.is_empty()), "the header is inherited"); + assert_linear("sections", layout_time_ratio(doc, 2000, &LayoutOptions::default())); +} + From 5a07c46534326f4cea9c75d33c0be11bffae6d17 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:01 +0100 Subject: [PATCH 043/112] #3692: selections across a header index each page's pieces once Finding 28: column_segments and selection_rects called pieces() per paragraph, and for a header, footer or note each call scanned the whole page (pieces_on). PartPages indexes a page's lines of the story by paragraph the first time the page is looked at; the page order (hint first, continued notes) is unchanged. Test: selecting_a_long_header_takes_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/hit.rs | 62 ++++++++++++++++++++++++++++--- crates/layout/src/tests_linear.rs | 22 +++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/crates/layout/src/hit.rs b/crates/layout/src/hit.rs index 63c7b41c79..188b89f060 100644 --- a/crates/layout/src/hit.rs +++ b/crates/layout/src/hit.rs @@ -1,5 +1,7 @@ //! Hit testing and caret geometry. +use std::collections::HashMap; + use wordcraft_doc::para::Wrap; use wordcraft_doc::props::TextDirection; use wordcraft_doc::{Document, Path, Pos, StoryRef}; @@ -140,6 +142,38 @@ fn text_at(p: &Page, index: usize, x: f32, y: f32) -> Option { }) } +/// The pieces of story `story`'s paragraphs on each page, by paragraph, for the pages looked at +/// so far ([`DocLayout::pieces_in`]). +struct PartPages<'a> { + story: StoryRef, + pages: HashMap>>, +} + +impl<'a> PartPages<'a> { + fn new(story: StoryRef) -> Self { + PartPages { story, pages: HashMap::new() } + } + + /// The pieces of the paragraph at `path` on page `pi` of `lay`, as [`DocLayout::pieces_on`] + /// gives them. + fn on(&mut self, lay: &'a DocLayout, pi: usize, path: &Path) -> Vec<(usize, &'a Placed)> { + let Some(p) = lay.pages.get(pi) else { return Vec::new() }; + let story = self.story; + let index = self.pages.entry(pi).or_insert_with(|| { + let mut index: HashMap<&'a Path, Vec<&'a Placed>> = HashMap::new(); + for it in items_of(p, story) { + if let Placed::Lines { story: s, path, .. } = it + && *s == story + { + index.entry(path).or_default().push(it); + } + } + index + }); + index.get(path).map(|v| v.iter().map(|it| (pi, *it)).collect()).unwrap_or_default() + } +} + fn items_of(page: &Page, story: StoryRef) -> Box + '_> { match story { StoryRef::Body => Box::new(page.items.iter()), @@ -417,6 +451,11 @@ impl DocLayout { .map(|v| v.iter().filter_map(|(pi, ii)| self.pages.get(*pi).and_then(|p| p.items.get(*ii)).map(|it| (*pi, it))).collect()) .unwrap_or_default(); } + self.part_pieces(page_hint, &mut |pi| self.pieces_on(pi, story, path)) + } + + /// Pieces of a paragraph of a story part (see [`Self::pieces`]), `on` giving those on a page. + fn part_pieces<'a>(&'a self, page_hint: usize, on: &mut dyn FnMut(usize) -> Vec<(usize, &'a Placed)>) -> Vec<(usize, &'a Placed)> { // Headers/footers repeat: prefer the hinted page, else the first that shows the part. let mut order: Vec = (0..self.pages.len()).collect(); if page_hint < order.len() { @@ -424,7 +463,7 @@ impl DocLayout { order.insert(0, page_hint); } for pi in order { - let mut found = self.pieces_on(pi, story, path); + let mut found = on(pi); if found.is_empty() { continue; } @@ -439,7 +478,7 @@ impl DocLayout { let mut k = pi; while lo > 0 && k > 0 { k -= 1; - let more = self.pieces_on(k, story, path); + let more = on(k); if more.is_empty() { break; } @@ -449,7 +488,7 @@ impl DocLayout { let mut k = pi; while hi < n && k + 1 < self.pages.len() { k += 1; - let more = self.pieces_on(k, story, path); + let more = on(k); if more.is_empty() { break; } @@ -461,6 +500,17 @@ impl DocLayout { Vec::new() } + /// [`Self::pieces`] for each of many paragraphs of `pages.story`: a part's pieces on a page + /// are looked up in that page's index, made the first time it is looked at, so a selection + /// over many header, footer or text box paragraphs walks each page's items once rather than + /// once for every paragraph. + fn pieces_in<'a>(&'a self, pages: &mut PartPages<'a>, path: &Path, page_hint: usize) -> Vec<(usize, &'a Placed)> { + if pages.story == StoryRef::Body { + return self.pieces(StoryRef::Body, path, page_hint); + } + self.part_pieces(page_hint, &mut |pi| pages.on(self, pi, path)) + } + /// Pieces of a paragraph of story part `story` on page `pi`. fn pieces_on(&self, pi: usize, story: StoryRef, path: &Path) -> Vec<(usize, &Placed)> { let Some(p) = self.pages.get(pi) else { return Vec::new() }; @@ -629,8 +679,9 @@ impl DocLayout { if a.story != b.story || !left.is_finite() || !right.is_finite() { return out; } + let mut pages = PartPages::new(a.story); for path in doc.paths_between(a, b) { - for (_, it) in self.pieces(a.story, &path, page_hint) { + for (_, it) in self.pieces_in(&mut pages, &path, page_hint) { let Placed::Lines { para, l0, l1, x, .. } = it else { continue }; let first = if path == a.path { para.line_of(a.off) } else { 0 }; let last = if path == b.path { para.line_of(b.off) } else { usize::MAX }; @@ -656,8 +707,9 @@ impl DocLayout { pub fn selection_rects(&self, doc: &Document, a: &Pos, b: &Pos, page_hint: usize) -> Vec<(usize, Rect)> { let (a, b) = if a <= b { (a, b) } else { (b, a) }; let mut out = Vec::new(); + let mut pages = PartPages::new(a.story); for path in doc.paths_between(a, b) { - for (pi, it) in self.pieces(a.story, &path, page_hint) { + for (pi, it) in self.pieces_in(&mut pages, &path, page_hint) { let Placed::Lines { para, l0, l1, x, y, turn, .. } = it else { continue }; let Some(first) = para.lines.get(*l0) else { continue }; // Rectangles in the lines' own frame, turned onto the page. diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 46045c6aeb..9e54440e5b 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -133,3 +133,25 @@ fn headers_inherited_through_many_sections_lay_out_in_linear_time() { assert_linear("sections", layout_time_ratio(doc, 2000, &LayoutOptions::default())); } +/// A header of `n` short paragraphs. +fn tall_header(n: usize) -> (Document, u32) { + let mut d = Document::from_text("Body"); + let paras = (0..n).map(|i| wordcraft_doc::para_block(Paragraph::with_text(&format!("h{i}"), CharProps::default()))).collect(); + let id = d.add_part(wordcraft_doc::PartKind::Header, paras); + d.last_section.headers.default = Some(id); + (d, id) +} + +/// Finding 28: selecting across a header looked through the whole page for each paragraph. +#[test] +fn selecting_a_long_header_takes_linear_time() { + let n = 300; + let ((ds, ids), (db, idb)) = (tall_header(n), tall_header(SCALE * n)); + let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); + let ends = |id: u32, n: usize| (Pos::new(StoryRef::Part(id), Path::top(0), 0), Pos::new(StoryRef::Part(id), Path::top(n - 1), 2)); + let ((a, b), (c, e)) = (ends(ids, n), ends(idb, SCALE * n)); + assert!(lb.selection_rects(&db, &c, &e, 0).len() >= SCALE * n, "a rectangle per paragraph"); + let ratio = ratio_of(32, &(&ds, &ls, a, b), &(&db, &lb, c, e), |(d, l, a, b)| drop(std::hint::black_box(l.selection_rects(d, a, b, 0)))); + assert_linear("header selection", ratio); +} + From 342855315eef417a51eafc1edb4aca9a977d5dae Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:11 +0100 Subject: [PATCH 044/112] #3692: clipped cells looked up by path, not scanned per line Finding 1: cell_clip searched every clipped cell on the page for each line drawn (quadratic in a page of many exact-height cells). clipped_cells now indexes them by story and content path, keeping each one's order; a line looks up only its path's prefixes and takes the earliest, the same cell the scan found first. Test: drawing_many_clipped_cells_takes_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/display.rs | 41 +++++++++++++++++++------------ crates/layout/src/tests_linear.rs | 29 ++++++++++++++++++++++ 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/crates/layout/src/display.rs b/crates/layout/src/display.rs index f7f4d510a6..cb41a75034 100644 --- a/crates/layout/src/display.rs +++ b/crates/layout/src/display.rs @@ -1,5 +1,7 @@ //! Page → draw items, shared by the raster renderer, the PDF exporter and thumbnails. +use std::collections::HashMap; + use wordcraft_doc::graphic::{Graphic, GraphicItem, GraphicKind, PathSeg, TextAlign}; use wordcraft_doc::para::{InlineObject, ShapeKind}; use wordcraft_doc::props::{Border, BorderStyle, Rgb, TextColor, TextDirection, Underline}; @@ -617,24 +619,31 @@ pub fn object_display(doc: &Document, page: &Page, o: &InlineObject, rect: Rect, } /// The cells among `items` whose content is cut off at their edge (see [`Placed::Cell`]'s -/// `clip`): their story, the path of their content (table, row, cell) and their area. -fn clipped_cells(items: &[Placed]) -> Vec<(StoryRef, Vec, Rect)> { - items - .iter() - .filter_map(|it| match it { - Placed::Cell { rect, table, row, cell, story, clip: true, .. } => { - let mut prefix = table.0.clone(); - prefix.extend([u32::try_from(*row).ok()?, u32::try_from(*cell).ok()?]); - Some((*story, prefix, *rect)) - } - _ => None, - }) - .collect() +/// `clip`), by story and the path of their content (table, row, cell): their area, and the +/// order they come in (the first of two that hold a line clips it). +type Clips = HashMap, (usize, Rect)>>; + +fn clipped_cells(items: &[Placed]) -> Clips { + let mut clips = Clips::new(); + let cells = items.iter().filter_map(|it| match it { + Placed::Cell { rect, table, row, cell, story, clip: true, .. } => { + let mut prefix = table.0.clone(); + prefix.extend([u32::try_from(*row).ok()?, u32::try_from(*cell).ok()?]); + Some((*story, prefix, *rect)) + } + _ => None, + }); + for (i, (story, prefix, rect)) in cells.enumerate() { + clips.entry(story).or_default().entry(prefix).or_insert((i, rect)); + } + clips } -/// The area to clip a line at `path` of `story` to: its cell's, when that cell is clipped. -fn cell_clip(clips: &[(StoryRef, Vec, Rect)], story: StoryRef, path: &[u32]) -> Option { - clips.iter().find(|(s, prefix, _)| *s == story && path.starts_with(prefix)).map(|(_, _, r)| *r) +/// The area to clip a line at `path` of `story` to: its cell's, when that cell is clipped. Only +/// the cells holding the line are looked up, not every clipped cell on the page. +fn cell_clip(clips: &Clips, story: StoryRef, path: &[u32]) -> Option { + let cells = clips.get(&story)?; + (0..=path.len()).filter_map(|n| cells.get(path.get(..n)?)).min_by_key(|(i, _)| *i).map(|(_, r)| *r) } /// The text boxes among `items` whose text is warped (Text Effects › Transform, drawn bent): their diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 9e54440e5b..b3420a6cad 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -155,3 +155,32 @@ fn selecting_a_long_header_takes_linear_time() { assert_linear("header selection", ratio); } +/// A table of `n` rows of 60 cells, each row exactly half a point high, so every cell clips its +/// (empty) paragraph and the whole table fits on the first page. +fn clipped_table(n: usize) -> Document { + let mut t = Table::new(n, 60, 468.0); + for row in &mut t.rows { + row.props.height = Some(0.5); + row.props.height_rule = HeightRule::Exact; + } + let mut d = Document::from_text("x"); + d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap(); + d +} + +/// Finding 1: drawing a line in a clipped cell looked through every clipped cell on the page. +#[test] +fn drawing_many_clipped_cells_takes_linear_time() { + let (ds, db) = (clipped_table(16), clipped_table(SCALE * 16)); + let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); + let clips = |d: &Document, l: &DocLayout| { + l.pages + .first() + .map_or(0, |p| display::page_display(d, p, &Default::default()).iter().filter(|i| matches!(i, display::Draw::Clip { .. })).count()) + }; + assert_eq!(clips(&db, &lb), SCALE * 16 * 60, "every cell's line is clipped, all on the first page"); + let opts = display::DisplayOptions::default(); + let ratio = ratio_of(4, &(&ds, &ls), &(&db, &lb), |(d, l)| drop(std::hint::black_box(display::page_display(d, &l.pages[0], &opts)))); + assert_linear("clipped cells", ratio); +} + From ee96c4c99de789abeb889baac666f93caf13285a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:12 +0100 Subject: [PATCH 045/112] #3706: author colours ranked once per drawing, not per run or comment Finding 30: author_index rebuilt the list of revision authors for every tracked run and paragraph mark drawn, and balloons rebuilt the comment-author list for every comment (quadratic in revisions or comments). display::author_order ranks authors by first appearance once; page_display, page_layers and object_display build the revision order lazily once per call and pass it down, and balloons builds the comment order once per layout. Same colours. Tests: drawing_tracked_changes_by_many_authors_takes_linear_time, tracked_changes_colour_by_author_in_order_of_appearance, balloons_for_many_authors_lay_out_in_linear_time. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/balloons.rs | 17 ++---- crates/layout/src/display.rs | 87 +++++++++++++++++++------------ crates/layout/src/tests_linear.rs | 61 ++++++++++++++++++++++ 3 files changed, 118 insertions(+), 47 deletions(-) diff --git a/crates/layout/src/balloons.rs b/crates/layout/src/balloons.rs index ec20b0db03..d20c7987a6 100644 --- a/crates/layout/src/balloons.rs +++ b/crates/layout/src/balloons.rs @@ -20,7 +20,7 @@ use wordcraft_doc::props::{Border, BorderStyle, CharProps, Rgb, TextColor}; use wordcraft_doc::{Document, Paragraph, Pos, StoryRef}; use wordcraft_geom::Rect; -use crate::display::revision_color; +use crate::display::{author_order, revision_color}; use crate::{BoxContext, BoxLayout, Ctx, DocLayout, Edges, Placed, fit_box, layout_box, push_para}; /// Width of the markup area beside each page (points): the editor's. @@ -103,17 +103,6 @@ pub fn replies(doc: &Document) -> BTreeMap> { m } -/// The comment authors in order of first appearance (for their colours). -fn author_index(doc: &Document, author: &str) -> u32 { - let mut seen: Vec<&str> = Vec::new(); - for c in doc.comments.values() { - if !seen.contains(&c.author.as_str()) { - seen.push(&c.author); - } - } - seen.iter().position(|a| *a == author).unwrap_or(0) as u32 -} - fn author_name(a: &str) -> &str { if a.trim().is_empty() { "Author" } else { a } } @@ -132,10 +121,12 @@ pub(crate) fn add(ctx: &mut Ctx, lay: &mut DocLayout) { let card_w = MARKUP_AREA - 2.0 * SIDE; let inner = card_w - STRIPE - 2.0 * PAD; let mut cards: Vec = Vec::new(); + // The comment authors in order of first appearance (for their colours), once for every card. + let authors = author_order(doc.comments.values().map(|c| c.author.as_str())); for (id, pos) in anchors { let Some(caret) = lay.caret_on(&pos, 0) else { continue }; let Some(c) = doc.comments.get(&id) else { continue }; - let color = revision_color(author_index(doc, &c.author)); + let color = revision_color(authors.get(c.author.as_str()).copied().unwrap_or(0)); let mut thread = vec![id]; thread.extend(threads.get(&id).into_iter().flatten().copied()); let x = STRIPE + PAD; diff --git a/crates/layout/src/display.rs b/crates/layout/src/display.rs index cb41a75034..6eacf9c2cb 100644 --- a/crates/layout/src/display.rs +++ b/crates/layout/src/display.rs @@ -532,8 +532,9 @@ fn stroke_of(u: Underline) -> Stroke { pub fn page_display(doc: &Document, page: &Page, opts: &DisplayOptions) -> Vec { let mut out = Vec::new(); let (ha, ba) = dimming(opts); - stories_display(doc, &[&page.header, &page.footer], opts, ha, &mut out); - stories_display(doc, &[&page.items], opts, ba, &mut out); + let authors = Authors::new(doc); + stories_display(doc, &[&page.header, &page.footer], opts, &authors, ha, &mut out); + stories_display(doc, &[&page.items], opts, &authors, ba, &mut out); body_change_bars(doc, page, opts, ba, &mut out); out } @@ -544,8 +545,9 @@ pub fn page_display(doc: &Document, page: &Page, opts: &DisplayOptions) -> Vec [(f32, Vec); 2] { let (ha, ba) = dimming(opts); let (mut hf, mut body) = (Vec::new(), Vec::new()); - stories_display(doc, &[&page.header, &page.footer], opts, 1.0, &mut hf); - stories_display(doc, &[&page.items], opts, 1.0, &mut body); + let authors = Authors::new(doc); + stories_display(doc, &[&page.header, &page.footer], opts, &authors, 1.0, &mut hf); + stories_display(doc, &[&page.items], opts, &authors, 1.0, &mut body); body_change_bars(doc, page, opts, 1.0, &mut body); [(ha, hf), (ba, body)] } @@ -563,7 +565,7 @@ fn body_change_bars(doc: &Document, page: &Page, opts: &DisplayOptions, alpha: f } /// Draw items for the placed items of `stories`, faded by `a`. -fn stories_display(doc: &Document, stories: &[&Vec], opts: &DisplayOptions, a: f32, out: &mut Vec) { +fn stories_display(doc: &Document, stories: &[&Vec], opts: &DisplayOptions, authors: &Authors, a: f32, out: &mut Vec) { let hidden = opts.hide_object.as_ref().map(|p| Hidden::new(doc, p)); for items in stories { let warps = warped_boxes(doc, items); @@ -578,7 +580,7 @@ fn stories_display(doc: &Document, stories: &[&Vec], opts: &DisplayOptio && let Some(rect) = cell_clip(&clips, *story, &path.0) { let mut inner = Vec::new(); - item(doc, it, opts, a, &mut inner); + item(doc, it, opts, authors, a, &mut inner); out.push(Draw::Clip { rect, items: inner }); continue; } @@ -592,10 +594,10 @@ fn stories_display(doc: &Document, stories: &[&Vec], opts: &DisplayOptio if let Some((_, rect, warp)) = warps.iter().find(|(w, ..)| w == id) { let lines: Vec<&Placed> = items.iter().filter(|i| matches!(i, Placed::Lines { story: StoryRef::Part(s), .. } if s == id)).collect(); - warped_text(doc, &lines, *rect, warp, opts, a, out); + warped_text(doc, &lines, *rect, warp, opts, authors, a, out); } } - _ => item(doc, it, opts, a, out), + _ => item(doc, it, opts, authors, a, out), } } } @@ -608,11 +610,12 @@ pub fn object_display(doc: &Document, page: &Page, o: &InlineObject, rect: Rect, let mut out = Vec::new(); object_draws(o, rect, Spin::default(), 1.0, opts, &mut out); let Some(id) = text_box else { return out }; + let authors = Authors::new(doc); for items in [&page.items, &page.header, &page.footer] { let lines: Vec<&Placed> = items.iter().filter(|i| matches!(i, Placed::Lines { story: StoryRef::Part(s), .. } if *s == id)).collect(); match warped_boxes(doc, items).into_iter().find(|(w, ..)| *w == id) { - Some((_, rect, warp)) if !lines.is_empty() => warped_text(doc, &lines, rect, &warp, opts, 1.0, &mut out), - _ => lines.into_iter().for_each(|it| item(doc, it, opts, 1.0, &mut out)), + Some((_, rect, warp)) if !lines.is_empty() => warped_text(doc, &lines, rect, &warp, opts, &authors, 1.0, &mut out), + _ => lines.into_iter().for_each(|it| item(doc, it, opts, &authors, 1.0, &mut out)), } } out @@ -670,12 +673,13 @@ fn warped_text( rect: Rect, warp: &wordcraft_doc::wordart::TextWarp, opts: &DisplayOptions, + authors: &Authors, alpha: f32, out: &mut Vec, ) { let mut raw = Vec::new(); for it in lines { - item(doc, it, opts, alpha, &mut raw); + item(doc, it, opts, authors, alpha, &mut raw); } // The letters: runs with effects as they are, plain runs from their glyphs. let mut arts: Vec<(BezPath, Box, Rgb, f32)> = Vec::new(); @@ -797,13 +801,13 @@ fn char_box(b: &Border, x0: f32, x1: f32, top: f32, bottom: f32, alpha: f32, out out.push(rule(r, top, r, bottom, b, alpha)); } -fn item(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, out: &mut Vec) { - item_in(doc, it, opts, alpha, Spin::default(), out) +fn item(doc: &Document, it: &Placed, opts: &DisplayOptions, authors: &Authors, alpha: f32, out: &mut Vec) { + item_in(doc, it, opts, authors, alpha, Spin::default(), out) } /// [`item`] drawn inside the turn `outer` (a turned Drawing Canvas's members): a shape's /// effects follow all the turns it is drawn inside. -fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer: Spin, out: &mut Vec) { +fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, authors: &Authors, alpha: f32, outer: Spin, out: &mut Vec) { match it { Placed::Fill { rect, color } => out.push(Draw::Fill { rect: *rect, color: *color, alpha, text_bg: false }), Placed::Rule { x0, y0, x1, y1, border } => out.push(rule(*x0, *y0, *x1, *y1, border, alpha)), @@ -823,14 +827,14 @@ fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer Placed::Clip { rect, spin, items } => { let mut inner = Vec::new(); for it in items { - item_in(doc, it, opts, alpha, spin.within(outer), &mut inner); + item_in(doc, it, opts, authors, alpha, spin.within(outer), &mut inner); } spun(*spin, *rect, vec![Draw::Clip { rect: *rect, items: inner }], out) } Placed::Cell { .. } | Placed::Object { .. } => {} Placed::Lines { story, path, para, l0, l1, x, y, turn } if turn.is_turned() => { let mut items = Vec::new(); - lines(doc, *story, path, para, *l0, *l1, 0.0, 0.0, opts, alpha, *turn == TextDirection::Down, &mut items); + lines(doc, *story, path, para, *l0, *l1, 0.0, 0.0, opts, authors, alpha, *turn == TextDirection::Down, &mut items); // Link areas are axis-aligned page rectangles: turned text gets none. for d in &mut items { if let Draw::Glyphs { link, .. } = d { @@ -842,7 +846,7 @@ fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer } out.push(Draw::Turned { x: *x, y: *y, turn: *turn, items }); } - Placed::Lines { story, path, para, l0, l1, x, y, .. } => lines(doc, *story, path, para, *l0, *l1, *x, *y, opts, alpha, false, out), + Placed::Lines { story, path, para, l0, l1, x, y, .. } => lines(doc, *story, path, para, *l0, *l1, *x, *y, opts, authors, alpha, false, out), } } @@ -860,6 +864,7 @@ fn lines( x: f32, y: f32, opts: &DisplayOptions, + authors: &Authors, alpha: f32, down: bool, out: &mut Vec, @@ -1087,7 +1092,7 @@ fn lines( let shown = opts.markup && m.insertions_deletions; let del = rc.del.filter(|_| shown); let ins = rc.ins.filter(|_| shown && del.is_none()); - let author = |r: u32| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0)); + let author = |r: u32| authors.color(r); // Moved text: the Moves marks and colours (by default green, double strikethrough // where it was, double underline where it went). let moved_to = doc.move_name(ins).is_some(); @@ -1420,18 +1425,12 @@ fn lines( let mx = if line.rtl { ex - 1.0 - size * 0.6 } else { ex + 1.0 }; // A tracked (inserted or deleted) paragraph mark is drawn in its author's colour. let m = &opts.revisions; - let by_author = |r: u32| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0)); + let by_author = |r: u32| authors.color(r); let color = para.filter(|_| opts.markup && m.insertions_deletions).and_then(|p| match (p.mark.ins, p.mark.del) { (_, Some(r)) if doc.move_name(Some(r)).is_some() => Some(m.move_from_color.unwrap_or_else(|| by_author(r))), (Some(r), None) if doc.move_name(Some(r)).is_some() => Some(m.move_to_color.unwrap_or_else(|| by_author(r))), - (_, Some(r)) => Some( - m.delete_color - .unwrap_or_else(|| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0))), - ), - (Some(r), None) => Some( - m.insert_color - .unwrap_or_else(|| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0))), - ), + (_, Some(r)) => Some(m.delete_color.unwrap_or_else(|| by_author(r))), + (Some(r), None) => Some(m.insert_color.unwrap_or_else(|| by_author(r))), _ => None, }); // The last paragraph of a section ends in a section break instead of a plain ¶. @@ -1860,14 +1859,34 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) -> }) } -fn author_index(doc: &Document, author: &str) -> u32 { - let mut seen: Vec<&str> = Vec::new(); - for r in &doc.revisions { - if !seen.contains(&r.author.as_str()) { - seen.push(&r.author); - } +/// Each author's place among `authors` in order of first appearance (which colour marks their +/// changes and comments), found in one pass. +pub(crate) fn author_order<'a>(authors: impl Iterator) -> HashMap<&'a str, u32> { + let mut seen = HashMap::new(); + for a in authors { + let n = u32::try_from(seen.len()).unwrap_or(u32::MAX); + seen.entry(a).or_insert(n); + } + seen +} + +/// The colours of the document's revision authors ([`author_order`]), worked out once for a +/// drawing, when its first tracked change is drawn, not once for every change. +struct Authors<'a> { + doc: &'a Document, + order: std::cell::OnceCell>, +} + +impl<'a> Authors<'a> { + fn new(doc: &'a Document) -> Self { + Authors { doc, order: std::cell::OnceCell::new() } + } + + /// The colour of revision `r`'s author. + fn color(&self, r: u32) -> Rgb { + let order = self.order.get_or_init(|| author_order(self.doc.revisions.iter().map(|v| v.author.as_str()))); + revision_color(self.doc.revisions.get(r as usize).and_then(|v| order.get(v.author.as_str()).copied()).unwrap_or(0)) } - seen.iter().position(|a| *a == author).unwrap_or(0) as u32 } /// How far a legacy check box's square sits inside the room it takes on its line, on every side. diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index b3420a6cad..3c086c0154 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -184,3 +184,64 @@ fn drawing_many_clipped_cells_takes_linear_time() { assert_linear("clipped cells", ratio); } +/// `n` words, each inserted by its own author, at a size that fits them all on the first page. +fn tracked_words(n: usize) -> Document { + let mut d = Document::from_text("x"); + let p = Paragraph { + text: "word ".repeat(n), + runs: (0..n).map(|i| Run { len: 5, props: CharProps { ins: u32::try_from(i).ok(), size: Some(1.0), ..Default::default() } }).collect(), + ..Paragraph::with_text("", CharProps::default()) + }; + d.body = vec![wordcraft_doc::para_block(p)]; + d.revisions = (0..n).map(|i| wordcraft_doc::Revision::new(wordcraft_doc::RevisionKind::Insert, format!("Author {i}"), "")).collect(); + d +} + +/// Finding 30 (#3706): every tracked run ranked its author among all the document's revisions. +#[test] +fn drawing_tracked_changes_by_many_authors_takes_linear_time() { + let (ds, db) = (tracked_words(1000), tracked_words(SCALE * 1000)); + let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); + let opts = display::DisplayOptions { markup: true, ..Default::default() }; + assert_eq!(lb.pages.len(), 1); + let ratio = ratio_of(4, &(&ds, &ls), &(&db, &lb), |(d, l)| drop(std::hint::black_box(display::page_display(d, &l.pages[0], &opts)))); + assert_linear("tracked runs", ratio); +} + +/// Each author keeps the colour of their first revision's place among the authors. +#[test] +fn tracked_changes_colour_by_author_in_order_of_appearance() { + let mut d = tracked_words(3); + // Words 0 and 2 by Ana, word 1 by Ben: Ana first, Ben second. + for (r, a) in d.revisions.iter_mut().zip(["Ana", "Ben", "Ana"]) { + r.author = a.into(); + } + let l = lay(&d, &LayoutOptions::default()); + let opts = display::DisplayOptions { markup: true, ..Default::default() }; + let colours: Vec<_> = display::page_display(&d, &l.pages[0], &opts) + .into_iter() + .filter_map(|i| if let display::Draw::Glyphs { color, .. } = i { Some(color) } else { None }) + .collect(); + let (ana, ben) = (display::revision_color(0), display::revision_color(1)); + assert_ne!(ana, ben); + assert_eq!(colours, [ana, ben, ana]); +} + +/// Finding 30 (#3706): every comment balloon ranked its author among all the comments. +#[test] +fn balloons_for_many_authors_lay_out_in_linear_time() { + let doc = |n: usize| { + let mut d = Document::from_text(&"word ".repeat(n)); + for i in (0..n).rev() { + let part = d.add_part(wordcraft_doc::PartKind::Comment, vec![wordcraft_doc::para_block(Paragraph::with_text("c", CharProps::default()))]); + let id = u32::try_from(i).unwrap(); + d.comments.insert(id, wordcraft_doc::Comment { author: format!("Author {i}"), part, ..Default::default() }); + let p = CharProps::default(); + d.insert_object(&Pos::body(0, 5 * i + 4), InlineObject::CommentEnd { id }, &p).unwrap(); + d.insert_object(&Pos::body(0, 5 * i), InlineObject::CommentStart { id }, &p).unwrap(); + } + d + }; + let opts = LayoutOptions { comment_balloons: true, ..Default::default() }; + assert_linear("comment balloons", layout_time_ratio(doc, 120, &opts)); +} From 0674573b452090dab446777fa0db29aabfc97eb4 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:27:12 +0100 Subject: [PATCH 046/112] #3692: progress log and layout parity note for linear-time lookups Findings 1, 3, 5, 13, 23, 28, 29 and 30 (#3706): one Progress log row, one Performance row in layout-parity.md, revision-history rows. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/layout-parity.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/layout-parity.md b/docs/layout-parity.md index 0251aba93c..d35ce0a1cd 100644 --- a/docs/layout-parity.md +++ b/docs/layout-parity.md @@ -196,6 +196,7 @@ No overall parity estimate changes from these probes. | Scrolling a 1,096-page document that is one 5,300-row table (#811) | ~2.4 ms idle, ~4.5 ms median scroll frame (was ~600 ms); opening still lays out for ~4–8 s | not measured | measured 2026-10-12, release, headless (`tests_perf::big_table_scroll_bench`) | | A word with no break opportunity in it, laid out beside a floating object (#3412) | 1 MiB: 29.4 s → 0.35 s; 4 MiB: 472 s → 1.4 s | not measured | measured 2026-10-21, on 9b4b948c before the fix and with it (`layout::tests::a_long_unbreakable_word_lays_out_in_linear_time`, `layout::tests::a_megabyte_long_word_lays_out_quickly`) | | The same word, with a font that pulls half its glyphs back so that its advances alternate in sign (#3412) | 8× the word costs 52× the time before the fix, 15× with it | not measured | measured 2026-10-22, release, on `00bc6a4b` before the fix and with it (`layout::tests::a_word_with_alternating_advances_lays_out_in_linear_time`) | +| Pages, sections and paragraphs with thousands of clipped cells, tracked authors, comment authors, sections, header paragraphs, spanning cells, hidden bidi offsets or math characters (#3692, #3706) | linear: each lookup is indexed or walked once (was a rescan per item) | not measured | `layout::tests_linear`, `math::tests::a_long_text_run_lays_out_in_linear_time` | A word with no break opportunity in it takes a line per few characters, and measuring the word beside a floating object walked the whole paragraph for every one of those lines, so the layout @@ -271,6 +272,7 @@ layout suite are pending; this is not a new Word/native-IME acceptance claim. | 2026-10-24 | trivial | Floating tables count exact line-bottom contact; shapes keep strict overlap (#3977); original boundary/skip regressions in modes 11/12/14/15, expected Word rule from cyril265; readiness estimates unchanged | | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | | 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged | +| 2026-10-22 | trivial | Layout lookups linear in clipped cells, authors, sections, header paragraphs and bidi offsets (#3692, #3706) | | 2026-10-22 | trivial | Group connector rerouting preserves displayed free and unresolved endpoints under nonuniform scaling while resolved ends keep their connection-site path; degenerate dimensions and unrepresentable fallback display spans are guarded; synthetic model/engine coverage only, readiness estimates unchanged | | 2026-10-21 | trivial | A word with no break opportunity in it, laid out beside a floating object, no longer lays out quadratically: every word is measured once, before the line is broken (#3412) | | 2026-10-23 | trivial | An empty paragraph whose mark is hidden is laid out as nothing, like a deleted one: no room, no list number (still counted, as in Word) and no shading, in the body and in table cells, headers and text boxes (#3982) | From 74dbec41078ad8360996e78d543a3ba0dbe10aeb Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 20:55:12 +0100 Subject: [PATCH 047/112] #3692: findings 3 and 13 tests fail clearly on the old code Finding 3: the table's rows carry no cells, only a column with a width skipped before and after, so each row adds two spanning entries for next to no layout and counting the grid per entry dominates (old code 75x for 8x the rows, fixed code 4-8x). Finding 13: sections that link to the previous section's headers and footers are laid out against the same sections naming them (title page on, so first-page parts are looked up too); old code 6.1x the time, fixed code 1.0x, limit 2x. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/layout/src/tests_linear.rs | 69 ++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 3c086c0154..8494e0315a 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -95,42 +95,61 @@ fn line_carets_match_x_of() { /// Finding 3: autofitting a table counted its grid's columns again for every spanning cell. #[test] fn autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time() { + // Rows with no cells, only a column skipped before and after them with a width: two + // spanning entries a row for next to no layout, so counting the grid per entry dominates. let doc = |n: usize| { - let mut t = Table::new(n, 3, 468.0); - for row in &mut t.rows { - row.cells.truncate(2); - if let Some(c) = row.cells.first_mut() { - c.props.span = 2; - } + let mut t = Table::new(n, 1, 468.0); + for row in t.rows.iter_mut().skip(1) { + row.cells.clear(); + row.props.grid_before = 1; + row.props.w_before = Some(10.0); + row.props.grid_after = 1; + row.props.w_after = Some(10.0); } let mut d = Document::from_text("x"); d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap(); d }; - assert_linear("spanning cells", layout_time_ratio(doc, 400, &LayoutOptions::default())); + assert_linear("spanning cells", layout_time_ratio(doc, 2000, &LayoutOptions::default())); +} + +/// `n` continuous sections of an empty paragraph each, each with a different first page. The +/// first section names every header and footer; the others name them too, or (`inherit`) name +/// none and link to the previous section's. +fn sections(n: usize, inherit: bool) -> Document { + use wordcraft_doc::PartKind::{Footer, Header}; + let mut d = Document::from_text(&"\n".repeat(n.saturating_sub(1))); + let mut part = |k| d.add_part(k, vec![wordcraft_doc::para_block(Paragraph::with_text("Head", CharProps::default()))]); + let ids = [part(Header), part(Header), part(Header), part(Footer), part(Footer), part(Footer)]; + let named = SectionProps { + start: SectionStart::Continuous, + title_page: true, + headers: HeaderSet { default: Some(ids[0]), first: Some(ids[1]), even: Some(ids[2]) }, + footers: HeaderSet { default: Some(ids[3]), first: Some(ids[4]), even: Some(ids[5]) }, + ..Default::default() + }; + let linked = SectionProps { headers: HeaderSet::default(), footers: HeaderSet::default(), ..named.clone() }; + for i in 0..n.saturating_sub(1) { + let s = if i == 0 || !inherit { named.clone() } else { linked.clone() }; + d.para_mut(StoryRef::Body, &Path::top(i)).unwrap().section = Some(Box::new(s)); + } + d.last_section = if inherit { linked } else { named }; + d } /// Finding 13: each section's headers and footers were found by walking back through the -/// sections before it, for every section and again for every page. +/// sections before it, for every section and again for every page. Linked to the previous +/// section's, they must cost no more to lay out than named in each section. #[test] fn headers_inherited_through_many_sections_lay_out_in_linear_time() { - let doc = |n: usize| { - let mut d = Document::from_text(&vec!["x"; n].join("\n")); - let head = d.add_part(wordcraft_doc::PartKind::Header, vec![wordcraft_doc::para_block(Paragraph::with_text("Head", CharProps::default()))]); - for i in 0..n.saturating_sub(1) { - let mut s = SectionProps { start: SectionStart::Continuous, ..Default::default() }; - if i == 0 { - s.headers.default = Some(head); - } - d.para_mut(StoryRef::Body, &Path::top(i)).unwrap().section = Some(Box::new(s)); - } - d.last_section.start = SectionStart::Continuous; - d - }; - // Every page shows the first section's header. - let l = lay(&doc(200), &LayoutOptions::default()); - assert!(l.pages.iter().all(|p| !p.header.is_empty()), "the header is inherited"); - assert_linear("sections", layout_time_ratio(doc, 2000, &LayoutOptions::default())); + let l = lay(§ions(200, true), &LayoutOptions::default()); + assert!(l.pages.iter().all(|p| !p.header.is_empty() && !p.footer.is_empty()), "the header and footer are inherited"); + let n = 8000; + let (named, inherited) = (sections(n, false), sections(n, true)); + let lay_once = |d: &Document| drop(std::hint::black_box(lay(d, &LayoutOptions::default()))); + let (t_named, t_inherited, ratio) = cost_ratio(1, || lay_once(&named), || lay_once(&inherited)); + eprintln!("{n} sections: named {t_named:.4} s, inherited {t_inherited:.4} s, ratio {ratio:.1}"); + assert!(ratio < 2.0, "{n} inheriting sections took {ratio:.1}x the time of named ones ({t_named:.4} s → {t_inherited:.4} s)"); } /// A header of `n` short paragraphs. From ddb67eaa45d01b9bff7891a34f8f0c27c84384da Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:06:16 +0100 Subject: [PATCH 048/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20linear-ti?= =?UTF-8?q?me=20tests=20count=20lookup=20steps=20instead=20of=20timing=20t?= =?UTF-8?q?hem?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The layout's linear-time tests (#3866) asserted CPU-time ratios, which a busy CI machine can skew. Each walk that once ran again for every item it served (a line's clusters, a page's items, the revision authors, the sections, a table's rows, a run's glyphs, the clipped cells) now counts the elements it looks at in a test-only counter (crate::work, nothing outside tests), and the tests assert 8x the input takes at most 12x the steps (quadratic: ~64x). proof_text, whose walk is the document model's, stays timed, and a ratio over the limit is measured twice more before it fails. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/layout/src/display.rs | 10 ++- crates/layout/src/hit.rs | 9 +++ crates/layout/src/lib.rs | 3 + crates/layout/src/math.rs | 11 +-- crates/layout/src/para.rs | 6 +- crates/layout/src/table.rs | 15 +++- crates/layout/src/tests_linear.rs | 120 +++++++++++++++++++----------- crates/layout/src/work.rs | 29 ++++++++ 8 files changed, 145 insertions(+), 58 deletions(-) create mode 100644 crates/layout/src/work.rs diff --git a/crates/layout/src/display.rs b/crates/layout/src/display.rs index 6eacf9c2cb..ca83e65f82 100644 --- a/crates/layout/src/display.rs +++ b/crates/layout/src/display.rs @@ -637,6 +637,7 @@ fn clipped_cells(items: &[Placed]) -> Clips { _ => None, }); for (i, (story, prefix, rect)) in cells.enumerate() { + crate::work::steps(1); clips.entry(story).or_default().entry(prefix).or_insert((i, rect)); } clips @@ -646,7 +647,13 @@ fn clipped_cells(items: &[Placed]) -> Clips { /// the cells holding the line are looked up, not every clipped cell on the page. fn cell_clip(clips: &Clips, story: StoryRef, path: &[u32]) -> Option { let cells = clips.get(&story)?; - (0..=path.len()).filter_map(|n| cells.get(path.get(..n)?)).min_by_key(|(i, _)| *i).map(|(_, r)| *r) + (0..=path.len()) + .filter_map(|n| { + crate::work::steps(1); + cells.get(path.get(..n)?) + }) + .min_by_key(|(i, _)| *i) + .map(|(_, r)| *r) } /// The text boxes among `items` whose text is warped (Text Effects › Transform, drawn bent): their @@ -1864,6 +1871,7 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) -> pub(crate) fn author_order<'a>(authors: impl Iterator) -> HashMap<&'a str, u32> { let mut seen = HashMap::new(); for a in authors { + crate::work::steps(1); let n = u32::try_from(seen.len()).unwrap_or(u32::MAX); seen.entry(a).or_insert(n); } diff --git a/crates/layout/src/hit.rs b/crates/layout/src/hit.rs index 188b89f060..bb0955ad8b 100644 --- a/crates/layout/src/hit.rs +++ b/crates/layout/src/hit.rs @@ -174,7 +174,16 @@ impl<'a> PartPages<'a> { } } +/// The items of `page` that `story`'s pieces are among (each one looked at counted as a step of +/// [`crate::work`]). fn items_of(page: &Page, story: StoryRef) -> Box + '_> { + let items = story_items(page, story); + #[cfg(test)] + let items: Box + '_> = Box::new(items.inspect(|_| crate::work::steps(1))); + items +} + +fn story_items(page: &Page, story: StoryRef) -> Box + '_> { match story { StoryRef::Body => Box::new(page.items.iter()), // Headers and footers, and the text boxes in them, are drawn with the page's header and diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index e5a9d3ef2f..1ea54b14df 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -22,6 +22,7 @@ mod table; mod textbox; mod vertical; pub mod wordart; +mod work; use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; @@ -2950,6 +2951,7 @@ impl<'a> HfLinks<'a> { /// laid out on `si`'s pages. fn defining_width(&self, footer: bool, si: usize, id: Option) -> Option { let id = id?; + work::steps(1); let linked = if footer { &self.feet } else { &self.heads }; let own = self.sections.get(si)?.1; if linked.own.get(si).is_some_and(|h| [h.default, h.first, h.even].contains(&Some(id))) { @@ -2978,6 +2980,7 @@ impl Linked { let mut namers: HashMap> = HashMap::new(); let mut prev = HeaderSet::default(); for (si, o) in own.iter().enumerate() { + work::steps(1); prev = HeaderSet { default: o.default.or(prev.default), first: o.first.or(prev.first), even: o.even.or(prev.even) }; linked.push(prev); for id in [o.default, o.first, o.even].into_iter().flatten() { diff --git a/crates/layout/src/math.rs b/crates/layout/src/math.rs index a581635bc4..9f1874ad7e 100644 --- a/crates/layout/src/math.rs +++ b/crates/layout/src/math.rs @@ -639,6 +639,7 @@ impl Ctx { while let Some(g) = shaped.get(j) && g.cluster < cb { + crate::work::steps(1); gx += g.x_advance as f32 * k; j += 1; } @@ -2378,14 +2379,10 @@ mod tests { /// glyphs from the first. #[test] fn a_long_text_run_lays_out_in_linear_time() { - use crate::tests::{SCALE, cost_ratio}; + use crate::tests_linear::assert_linear_steps; use wordcraft_doc::math::*; let eq = |n: usize| Math { nodes: vec![MNode::Run(MRun { nor: true, ..MRun::new("a".repeat(n)) })], ..Default::default() }; - let (small, big) = (eq(400), eq(SCALE * 400)); - // Each timed 16 times over, so the timings are long against the clock's tick. - let lay_16 = |m: &Math| (0..16).for_each(|_| drop(std::hint::black_box(lay_in(m, "", false, 1e6, &MathProps::default())))); - let (t1, tk, ratio) = cost_ratio(SCALE, || lay_16(&small), || lay_16(&big)); - eprintln!("t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}"); - assert!(ratio < 24.0, "8x the text took {ratio:.1}x the time ({t1:.4} s → {tk:.4} s): not linear"); + let lay = |m: &Math| drop(std::hint::black_box(lay_in(m, "", false, 1e6, &MathProps::default()))); + assert_linear_steps("math text run", 400, eq, lay); } } diff --git a/crates/layout/src/para.rs b/crates/layout/src/para.rs index 7218079977..abbecc13ae 100644 --- a/crates/layout/src/para.rs +++ b/crates/layout/src/para.rs @@ -3119,6 +3119,7 @@ impl ParaLayout { return self.x_of_bidi(l, off, from); } for k in *from..l.c1 { + crate::work::steps(1); let c = self.clusters.get(k)?; if off <= c.start || off < c.end { *from = k; @@ -3178,7 +3179,10 @@ impl ParaLayout { /// it has the higher level), which keeps the caret at that run's trailing edge, so each /// visual boundary of the common mixed cases has its own offset. fn x_of_bidi(&self, l: &Line, off: usize, from: &mut usize) -> Option { - let k = (*from..l.c1).find(|&k| self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start)); + let k = (*from..l.c1).find(|&k| { + crate::work::steps(1); + self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start) + }); *from = k.unwrap_or(l.c1); let Some(k) = k else { return Some(l.end_x()) }; let c = self.clusters.get(k)?; diff --git a/crates/layout/src/table.rs b/crates/layout/src/table.rs index 1a1f5e5b91..d2e24e8910 100644 --- a/crates/layout/src/table.rs +++ b/crates/layout/src/table.rs @@ -71,6 +71,13 @@ struct Turned { top: f32, } +/// The number of grid columns of `t` ([`Table::cols`]), which walks every row (each row counted +/// as a step of [`crate::work`]). +fn grid_cols(t: &Table) -> usize { + crate::work::steps(t.rows.len()); + t.cols() +} + /// How long the lines laid out in `items` are without the room they didn't use: the length /// turned text needs so that nothing wraps. fn natural_length(items: &[Placed]) -> f32 { @@ -178,7 +185,7 @@ pub fn measure_table_columns(doc: &wordcraft_doc::Document, t: &Table) -> Vec<(f use wordcraft_doc::Block; let style = t.props.style.as_deref().and_then(|id| doc.styles.table_style(id)); let margins_def = default_margins(t, style.as_ref()); - let ncols = t.cols().clamp(1, wordcraft_doc::table::MAX_COLS); + let ncols = grid_cols(t).clamp(1, wordcraft_doc::table::MAX_COLS); let fields = crate::FieldCtx::default(); let mut cols = vec![(0.0f32, 0.0f32); ncols]; // Cells spanning several columns: (first column, span, min, max), applied after the others. @@ -360,7 +367,7 @@ pub fn layout_table(ctx: &mut Ctx, story: StoryRef, t: &Table, path: &[u32], ava // Cell text formatting and looks per set of style parts, built once per table. let mut cell_text: Vec<(Region, CellText)> = Vec::new(); let mut looks: Vec<(Region, RegionLook)> = Vec::new(); - let ncols = t.cols().max(1); + let ncols = grid_cols(t).max(1); // Column widths. let mut grid: Vec = if t.grid.len() == ncols { // Grid units can be narrower than a text box, including omitted row-end columns. @@ -1122,7 +1129,7 @@ fn region_text(st: &TableStyleProps, region: Region) -> CellText { /// with their own. `None` when every cell is empty. fn column_content(ctx: &mut Ctx, t: &Table, style: Option<&TableStyleProps>, depth: usize) -> Option> { let margins_def = default_margins(t, style); - let mut cols = vec![(0.0f32, 0.0f32); t.cols().max(1)]; + let mut cols = vec![(0.0f32, 0.0f32); grid_cols(t).max(1)]; let mut spanning = Vec::new(); let mut any = false; for (ri, row) in t.rows.iter().enumerate() { @@ -1145,7 +1152,7 @@ fn column_content(ctx: &mut Ctx, t: &Table, style: Option<&TableStyleProps>, dep spanning.extend(row_gaps(row, g).map(|(g, span, w)| (g, span, (w, w)))); } // The grid's width once: counting it walks every row, and there can be a spanning cell in each. - let ncols = t.cols(); + let ncols = grid_cols(t); for (g, span, need) in spanning { let Some(cs) = cols.get_mut(g..g.saturating_add(span).min(ncols)) else { continue }; let n = cs.len().max(1) as f32; diff --git a/crates/layout/src/tests_linear.rs b/crates/layout/src/tests_linear.rs index 8494e0315a..e49409693c 100644 --- a/crates/layout/src/tests_linear.rs +++ b/crates/layout/src/tests_linear.rs @@ -1,7 +1,11 @@ //! Lookups that once rescanned the page, the paragraph or the document for every item they //! looked up (#3692, #3706): an input 8x the size must cost about 8x as much, not 64x. +//! +//! The cost is the steps the lookups' walks count ([`crate::work`]), not a timing, so a busy +//! machine can't make a linear lookup look quadratic or the other way round. Only proofing text, +//! whose walk is the document model's, is timed. -use super::tests::{SCALE, cost_ratio, layout_time_ratio}; +use super::tests::{SCALE, cost_ratio}; use super::*; use wordcraft_doc::para::Run; use wordcraft_doc::{Pos, Table}; @@ -10,15 +14,24 @@ fn lay(d: &Document, opts: &LayoutOptions) -> DocLayout { layout(d, &mut LayoutCache::new(), opts) } -fn assert_linear(what: &str, (t1, tk, ratio): (f64, f64, f64)) { - eprintln!("{what}: t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}"); - assert!(ratio < 24.0, "{what}: 8x the input took {ratio:.1}x the time ({t1:.4} s → {tk:.4} s): not linear"); +/// The most steps 8x the input may take, against the steps of the input: linear walks take +/// about 8x (a little more where a fixed cost is shared out), quadratic ones about 64x. +const MAX_STEP_RATIO: f64 = 1.5 * SCALE as f64; + +/// Asserts `f` takes about [`SCALE`] times the steps ([`crate::work`]) on `make(SCALE * n)` as on +/// `make(n)`, not [`SCALE`]² times. Making the inputs isn't counted. +pub(crate) fn assert_linear_steps(what: &str, n: usize, make: impl Fn(usize) -> T, f: impl Fn(&T)) { + let (small, big) = (make(n), make(SCALE * n)); + let (s, b) = (work::count(|| f(&small)), work::count(|| f(&big))); + let ratio = b as f64 / s.max(1) as f64; + eprintln!("{what}: {s} steps for n, {b} for {SCALE}n, ratio {ratio:.1}"); + assert!(s > 0, "{what}: no steps counted, so the walk this test watches no longer counts them (crate::work)"); + assert!(ratio <= MAX_STEP_RATIO, "{what}: {SCALE}x the input took {ratio:.1}x the steps ({s} → {b}): not linear"); } -/// [`cost_ratio`] of `f(big)` against `f(small)`, each repeated `reps` times so the timings are -/// long against the clock's tick. -fn ratio_of(reps: usize, small: &T, big: &T, f: impl Fn(&T)) -> (f64, f64, f64) { - cost_ratio(SCALE, || (0..reps).for_each(|_| f(small)), || (0..reps).for_each(|_| f(big))) +/// [`assert_linear_steps`] for laying out `doc(n)`. +fn assert_layout_linear(what: &str, n: usize, doc: impl Fn(usize) -> Document, opts: &LayoutOptions) { + assert_linear_steps(what, n, doc, |d| drop(std::hint::black_box(lay(d, opts)))); } /// The first laid-out body paragraph on page 0. @@ -39,8 +52,22 @@ fn proof_text_walks_the_runs_once() { p }; assert_eq!(para::proof_text(¶(5)), "a\u{1}a\u{1}a"); + // The runs are the document model's, whose walk counts no steps, so this one is timed: the + // CPU time this thread ran (`cost_ratio`). At 50,000 runs the old walk took 64x as long for + // 8x the runs, and the new one about 8x; a limit of 24x stays far from both, and a ratio over + // it is measured again, twice, before the test fails, so a busy spell can't fail it alone. let (small, big) = (para(50_000), para(SCALE * 50_000)); - assert_linear("proof_text", ratio_of(8, &small, &big, |p| drop(std::hint::black_box(para::proof_text(p))))); + let proof_8 = |p: &Paragraph| (0..8).for_each(|_| drop(std::hint::black_box(para::proof_text(p)))); + let mut ratios = Vec::new(); + for _ in 0..3 { + let (t1, tk, ratio) = cost_ratio(SCALE, || proof_8(&small), || proof_8(&big)); + eprintln!("proof_text: t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}"); + if ratio < 24.0 { + return; + } + ratios.push(ratio); + } + panic!("proof_text: 8x the runs took {ratios:.1?}x the time, three times over: not linear"); } /// A right-to-left paragraph whose one line holds `n` hidden letters: caret offsets with no width. @@ -58,15 +85,14 @@ fn bidi_hidden(n: usize) -> Document { #[test] fn bidi_clicks_and_caret_steps_take_linear_time() { let opts = LayoutOptions { show_hidden: false, ..Default::default() }; - let (ds, db) = (bidi_hidden(5000), bidi_hidden(SCALE * 5000)); - let (ls, lb) = (lay(&ds, &opts), lay(&db, &opts)); - let (Some(ps), Some(pb)) = (first_para(&ls), first_para(&lb)) else { panic!("no paragraph") }; - assert_eq!(pb.lines.len(), 1, "the hidden letters stay on one line"); - let ratio = ratio_of(32, &(ps, &ls), &(pb, &lb), |(p, l)| { + let laid = |n: usize| lay(&bidi_hidden(n), &opts); + let Some(pb) = first_para(&laid(SCALE * 5000)).map(|p| p.lines.len()) else { panic!("no paragraph") }; + assert_eq!(pb, 1, "the hidden letters stay on one line"); + assert_linear_steps("bidi hit test", 5000, laid, |l| { + let Some(p) = first_para(l) else { panic!("no paragraph") }; std::hint::black_box(p.off_at_x(0, 1.0)); std::hint::black_box(l.visual_step(&Pos::body(0, 0), true, 0)); }); - assert_linear("bidi hit test", ratio); } /// The carets of a line walked in one pass are where `x_of` puts each of them, on bidirectional @@ -110,7 +136,7 @@ fn autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time() { d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap(); d }; - assert_linear("spanning cells", layout_time_ratio(doc, 2000, &LayoutOptions::default())); + assert_layout_linear("spanning cells", 2000, doc, &LayoutOptions::default()); } /// `n` continuous sections of an empty paragraph each, each with a different first page. The @@ -144,12 +170,11 @@ fn sections(n: usize, inherit: bool) -> Document { fn headers_inherited_through_many_sections_lay_out_in_linear_time() { let l = lay(§ions(200, true), &LayoutOptions::default()); assert!(l.pages.iter().all(|p| !p.header.is_empty() && !p.footer.is_empty()), "the header and footer are inherited"); - let n = 8000; - let (named, inherited) = (sections(n, false), sections(n, true)); - let lay_once = |d: &Document| drop(std::hint::black_box(lay(d, &LayoutOptions::default()))); - let (t_named, t_inherited, ratio) = cost_ratio(1, || lay_once(&named), || lay_once(&inherited)); - eprintln!("{n} sections: named {t_named:.4} s, inherited {t_inherited:.4} s, ratio {ratio:.1}"); - assert!(ratio < 2.0, "{n} inheriting sections took {ratio:.1}x the time of named ones ({t_named:.4} s → {t_inherited:.4} s)"); + let opts = LayoutOptions::default(); + assert_layout_linear("inherited headers", 1000, |n| sections(n, true), &opts); + let steps = |inherit: bool| work::count(|| drop(std::hint::black_box(lay(§ions(1000, inherit), &opts)))); + let (named, inherited) = (steps(false), steps(true)); + assert!(inherited <= 2 * named, "1000 inheriting sections took {inherited} steps, named ones {named}"); } /// A header of `n` short paragraphs. @@ -164,14 +189,16 @@ fn tall_header(n: usize) -> (Document, u32) { /// Finding 28: selecting across a header looked through the whole page for each paragraph. #[test] fn selecting_a_long_header_takes_linear_time() { - let n = 300; - let ((ds, ids), (db, idb)) = (tall_header(n), tall_header(SCALE * n)); - let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); - let ends = |id: u32, n: usize| (Pos::new(StoryRef::Part(id), Path::top(0), 0), Pos::new(StoryRef::Part(id), Path::top(n - 1), 2)); - let ((a, b), (c, e)) = (ends(ids, n), ends(idb, SCALE * n)); - assert!(lb.selection_rects(&db, &c, &e, 0).len() >= SCALE * n, "a rectangle per paragraph"); - let ratio = ratio_of(32, &(&ds, &ls, a, b), &(&db, &lb, c, e), |(d, l, a, b)| drop(std::hint::black_box(l.selection_rects(d, a, b, 0)))); - assert_linear("header selection", ratio); + // The header laid out, and the selection from its first paragraph to its last. + let laid = |n: usize| { + let (d, id) = tall_header(n); + let l = lay(&d, &LayoutOptions::default()); + let ends = (Pos::new(StoryRef::Part(id), Path::top(0), 0), Pos::new(StoryRef::Part(id), Path::top(n - 1), 2)); + (d, l, ends) + }; + let (d, l, (a, b)) = laid(SCALE * 300); + assert!(l.selection_rects(&d, &a, &b, 0).len() >= SCALE * 300, "a rectangle per paragraph"); + assert_linear_steps("header selection", 300, laid, |(d, l, (a, b))| drop(std::hint::black_box(l.selection_rects(d, a, b, 0)))); } /// A table of `n` rows of 60 cells, each row exactly half a point high, so every cell clips its @@ -190,17 +217,16 @@ fn clipped_table(n: usize) -> Document { /// Finding 1: drawing a line in a clipped cell looked through every clipped cell on the page. #[test] fn drawing_many_clipped_cells_takes_linear_time() { - let (ds, db) = (clipped_table(16), clipped_table(SCALE * 16)); - let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); - let clips = |d: &Document, l: &DocLayout| { - l.pages - .first() - .map_or(0, |p| display::page_display(d, p, &Default::default()).iter().filter(|i| matches!(i, display::Draw::Clip { .. })).count()) + let laid = |n: usize| { + let d = clipped_table(n); + let l = lay(&d, &LayoutOptions::default()); + (d, l) }; - assert_eq!(clips(&db, &lb), SCALE * 16 * 60, "every cell's line is clipped, all on the first page"); let opts = display::DisplayOptions::default(); - let ratio = ratio_of(4, &(&ds, &ls), &(&db, &lb), |(d, l)| drop(std::hint::black_box(display::page_display(d, &l.pages[0], &opts)))); - assert_linear("clipped cells", ratio); + let draw = |(d, l): &(Document, DocLayout)| l.pages.first().map(|p| display::page_display(d, p, &opts)).unwrap_or_default(); + let clips = draw(&laid(SCALE * 16)).iter().filter(|i| matches!(i, display::Draw::Clip { .. })).count(); + assert_eq!(clips, SCALE * 16 * 60, "every cell's line is clipped, all on the first page"); + assert_linear_steps("clipped cells", 16, laid, |dl| drop(std::hint::black_box(draw(dl)))); } /// `n` words, each inserted by its own author, at a size that fits them all on the first page. @@ -219,12 +245,16 @@ fn tracked_words(n: usize) -> Document { /// Finding 30 (#3706): every tracked run ranked its author among all the document's revisions. #[test] fn drawing_tracked_changes_by_many_authors_takes_linear_time() { - let (ds, db) = (tracked_words(1000), tracked_words(SCALE * 1000)); - let (ls, lb) = (lay(&ds, &LayoutOptions::default()), lay(&db, &LayoutOptions::default())); + let laid = |n: usize| { + let d = tracked_words(n); + let l = lay(&d, &LayoutOptions::default()); + (d, l) + }; + assert_eq!(laid(SCALE * 1000).1.pages.len(), 1); let opts = display::DisplayOptions { markup: true, ..Default::default() }; - assert_eq!(lb.pages.len(), 1); - let ratio = ratio_of(4, &(&ds, &ls), &(&db, &lb), |(d, l)| drop(std::hint::black_box(display::page_display(d, &l.pages[0], &opts)))); - assert_linear("tracked runs", ratio); + let draw = |(d, l): &(Document, DocLayout)| l.pages.first().map(|p| display::page_display(d, p, &opts)).unwrap_or_default(); + // Every run is drawn in its author's colour, so the authors are ranked at least once. + assert_linear_steps("tracked runs", 1000, laid, |dl| drop(std::hint::black_box(draw(dl)))); } /// Each author keeps the colour of their first revision's place among the authors. @@ -262,5 +292,5 @@ fn balloons_for_many_authors_lay_out_in_linear_time() { d }; let opts = LayoutOptions { comment_balloons: true, ..Default::default() }; - assert_linear("comment balloons", layout_time_ratio(doc, 120, &opts)); + assert_layout_linear("comment balloons", 120, doc, &opts); } diff --git a/crates/layout/src/work.rs b/crates/layout/src/work.rs new file mode 100644 index 0000000000..b36368e162 --- /dev/null +++ b/crates/layout/src/work.rs @@ -0,0 +1,29 @@ +//! A count of the steps the layout's lookups take, for the linear-time tests (#3692): they check +//! that an input 8x the size costs about 8x the steps, which a busy machine can't skew the way it +//! skews a timing. Each walk that once ran again for every item it served (a line's clusters, a +//! page's items, the revision authors, the sections, a table's rows) counts the elements it +//! looks at, so a caller that walks it once per item again shows up as a quadratic count. Outside +//! tests the count is nothing. + +#[cfg(test)] +thread_local! { + /// The steps counted on this thread (layout runs on the thread that calls it). + static STEPS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Count `n` steps of a walk (nothing outside tests). +#[inline(always)] +pub(crate) fn steps(n: usize) { + #[cfg(test)] + STEPS.with(|s| s.set(s.get().saturating_add(n as u64))); + #[cfg(not(test))] + let _ = n; +} + +/// The steps `f` takes. +#[cfg(test)] +pub(crate) fn count(f: impl FnOnce()) -> u64 { + let before = STEPS.with(std::cell::Cell::get); + f(); + STEPS.with(std::cell::Cell::get).saturating_sub(before) +} From 17e648bde79c6bdc0cb966d3ab4042c425a7bd89 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 25 Oct 2026 23:59:44 +0000 Subject: [PATCH 049/112] =?UTF-8?q?#3692:=20rebase=20onto=20main=20?= =?UTF-8?q?=E2=80=94=20sort=20the=20merged=20test=20module=20declarations?= =?UTF-8?q?=20(rustfmt)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/layout/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index 1ea54b14df..0fd59bfce3 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -4979,10 +4979,10 @@ mod tests_grid; #[cfg(test)] mod tests_ligature; #[cfg(test)] -mod tests_nbsp; -#[cfg(test)] mod tests_linear; #[cfg(test)] +mod tests_nbsp; +#[cfg(test)] mod tests_numeral; #[cfg(test)] mod tests_pages; From 1fa73832641728d0284035fe545ac37df8fc8456 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 26 Oct 2026 00:14:42 +0000 Subject: [PATCH 050/112] =?UTF-8?q?#3692:=20rebase=20onto=20main=20?= =?UTF-8?q?=E2=80=94=20drop=20the=20Ctx=20sections=20list=20the=20linked?= =?UTF-8?q?=20header/footer=20index=20replaces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/layout/src/lib.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index 0fd59bfce3..f639078ec1 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -695,11 +695,6 @@ enum Pass { /// Walk state while paginating. struct Ctx<'a> { doc: &'a Document, - /// The body's sections ([`Document::sections`]) for one pass, with the block index each - /// starts after: header and footer wrap work asks for them per page, and rebuilding the - /// list walked the whole body every time, so a long document paid a body-length scan per - /// page. The document does not change during a pass. - sections: Vec<(usize, &'a SectionProps)>, cache: &'a mut LayoutCache, opts: &'a LayoutOptions, counters: Counters, @@ -2309,7 +2304,6 @@ fn layout_pass( }); let mut ctx = Ctx { doc, - sections: doc.sections(), cache, opts, hide_deleted: opts.hides_deleted(StoryRef::Body), From af9d5c954198cc88e848d9199b85b1fe06686146 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:08:51 +0100 Subject: [PATCH 051/112] #3692: bitmap glyph PNG header probe keeps the glyph's budget Finding 0. decode_png probed the PNG's size with the image crate's default limits, and the PNG decoder inflates an iCCP profile while reading the header: a tiny colour-font glyph could carry a few-KB chunk that inflates to hundreds of MB before the size check. The probe and the decode now share one reader with the glyph's own size and byte budget; past it the profile is dropped and the glyph still draws. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/render/src/color_glyph.rs | 44 +++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/render/src/color_glyph.rs b/crates/render/src/color_glyph.rs index 8b169d9b6c..549fa5ccd6 100644 --- a/crates/render/src/color_glyph.rs +++ b/crates/render/src/color_glyph.rs @@ -102,18 +102,11 @@ fn decode_png(png: &[u8], w: u32, h: u32) -> Option { if w == 0 || h == 0 || w > side || h > side { return None; } - let reader = || image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png); let fits = |(iw, ih): (u32, u32)| iw > 0 && ih > 0 && iw <= w && ih <= h; - if !fits(reader().into_dimensions().ok()?) { + if !fits(glyph_png_reader(png, w, h).into_dimensions().ok()?) { return None; } - let mut limits = image::Limits::default(); - limits.max_image_width = Some(w); - limits.max_image_height = Some(h); - limits.max_alloc = Some(w as u64 * h as u64 * 4 * 2 + 64 * 1024); - let mut r = reader(); - r.limits(limits); - let img = r.decode().ok()?.to_rgba8(); + let img = glyph_png_reader(png, w, h).decode().ok()?.to_rgba8(); let (iw, ih) = img.dimensions(); if !fits((iw, ih)) { return None; @@ -121,6 +114,19 @@ fn decode_png(png: &[u8], w: u32, h: u32) -> Option { crate::premul_pixmap(iw, ih, img.as_raw()) } +/// A reader for a bitmap glyph's PNG with the glyph's budget: at most `w`×`h` and about two RGBA +/// buffers of it. Every read goes through it, the header probe too: reading the header inflates +/// metadata such as an ICC profile, which the default budget would let run to hundreds of MB. +fn glyph_png_reader(png: &[u8], w: u32, h: u32) -> image::ImageReader> { + let mut limits = image::Limits::default(); + limits.max_image_width = Some(w); + limits.max_image_height = Some(h); + limits.max_alloc = Some(u64::from(w).saturating_mul(u64::from(h)).saturating_mul(8).saturating_add(64 * 1024)); + let mut r = image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png); + r.limits(limits); + r +} + /// A pixmap from premultiplied BGRA rows. fn bgra_pixmap(bgra: &[u8], w: u32, h: u32) -> Option { if w == 0 || h == 0 || w > u16::MAX as u32 || h > u16::MAX as u32 { @@ -436,4 +442,24 @@ mod tests { let side = wordcraft_fonts::color::MAX_BITMAP_SIDE; assert!(decode_png(&png(4, 4), side + 1, 4).is_none(), "past the cap"); } + + /// A tiny bitmap glyph whose PNG carries a big ICC profile (it compresses to almost nothing): + /// the header probe already inflates it, so the probe is held to the glyph's budget too, not + /// the decoder's default hundreds of MB. Past the budget the profile is dropped (the glyph is + /// still drawn, without it). + #[test] + fn bitmap_png_header_probe_keeps_the_glyph_budget() { + use image::ImageDecoder; + let mut out = vec![]; + let mut enc = image::codecs::png::PngEncoder::new(&mut out); + image::ImageEncoder::set_icc_profile(&mut enc, vec![0u8; 4 << 20]).unwrap(); + image::ImageEncoder::write_image(enc, &[200, 0, 0, 255], 1, 1, image::ExtendedColorType::Rgba8).unwrap(); + assert!(out.len() < 64 * 1024, "the profile compresses: {} bytes", out.len()); + let profile = |r: image::ImageReader>| r.into_decoder().unwrap().icc_profile().unwrap().map(|p| p.len()); + let unbounded = image::ImageReader::with_format(std::io::Cursor::new(out.as_slice()), image::ImageFormat::Png); + assert_eq!(profile(unbounded), Some(4 << 20), "the default budget inflates the 4 MB profile"); + assert_eq!(profile(glyph_png_reader(&out, 1, 1)), None, "the glyph budget stops it at the header"); + assert_eq!(decode_png(&out, 1, 1).map(|p| (p.width(), p.height())), Some((1, 1))); + assert_eq!(glyph_png_reader(&png(4, 4), 4, 4).into_dimensions().ok(), Some((4, 4)), "a plain glyph reads"); + } } From f82319deb7f79684c2650014db8b4c5303a11f0e Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:08:57 +0100 Subject: [PATCH 052/112] #3692: picture decode budgets charge the decoder's own pixel size Finding 37. decode_fits charged every picture 8 bytes a pixel (RGBA8 decode plus its copy), but a 16-bit or float image decodes to 8-16 bytes a pixel before conversion, so the real peak was up to 2.5x the budget. geom::decode_peak_fits takes the decoder's native bytes per pixel; render's decode_pixmap_within and pdf's load_image_within read it from the decoder (header only, same limits) before decoding. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/geom/src/lib.rs | 2 +- crates/geom/src/pixels.rs | 25 +++++++++++++++++++++++-- crates/pdf/src/lib.rs | 27 ++++++++++++++++++--------- crates/pdf/src/tests.rs | 7 +++++++ crates/render/src/lib.rs | 31 +++++++++++++++++++++++-------- 5 files changed, 72 insertions(+), 20 deletions(-) diff --git a/crates/geom/src/lib.rs b/crates/geom/src/lib.rs index 48998abd32..868ccf025d 100644 --- a/crates/geom/src/lib.rs +++ b/crates/geom/src/lib.rs @@ -13,7 +13,7 @@ pub mod picture; mod region; pub use region::Region; mod pixels; -pub use pixels::{MAX_DECODED_BYTES, MAX_RASTER_PIXELS, decode_fits, fit_pixels, rgba_bytes, rgba_fits}; +pub use pixels::{MAX_DECODED_BYTES, MAX_RASTER_PIXELS, decode_fits, decode_peak_fits, fit_pixels, rgba_bytes, rgba_fits}; /// Points per inch. pub const PT_PER_IN: f32 = 72.0; diff --git a/crates/geom/src/pixels.rs b/crates/geom/src/pixels.rs index 64258a71f8..263cadfce2 100644 --- a/crates/geom/src/pixels.rs +++ b/crates/geom/src/pixels.rs @@ -18,9 +18,18 @@ pub fn rgba_fits(w: u32, h: u32, budget: u64) -> bool { } /// Whether decoding a `w`×`h` picture fits `budget` bytes at its peak: the decoder's buffer and -/// the RGBA conversion (or its premultiplied copy) are alive together, so two RGBA buffers. +/// the RGBA conversion (or its premultiplied copy) are alive together, so two RGBA buffers. For a +/// decoder whose buffer isn't four bytes a pixel, see [`decode_peak_fits`]. pub fn decode_fits(w: u32, h: u32, budget: u64) -> bool { - w > 0 && h > 0 && rgba_bytes(w, h).and_then(|b| b.checked_mul(2)).is_some_and(|b| b <= budget) + decode_peak_fits(w, h, 4, budget) +} + +/// [`decode_fits`] for a decoder that hands back `native` bytes a pixel (8 for 16-bit RGBA, 16 for +/// float RGBA): its buffer lives alongside the RGBA8 conversion, then the conversion alongside its +/// premultiplied copy, so the peak is the larger of `native + 4` and `8` bytes a pixel. +pub fn decode_peak_fits(w: u32, h: u32, native: u64, budget: u64) -> bool { + let per_pixel = native.saturating_add(4).max(8); + w > 0 && h > 0 && (w as u64).checked_mul(h as u64).and_then(|p| p.checked_mul(per_pixel)).is_some_and(|b| b <= budget) } /// `w`×`h` shrunk (aspect kept, each side at least 1) until it holds at most `max_pixels`. @@ -53,6 +62,18 @@ mod tests { assert!(!decode_fits(0, 8, u64::MAX)); } + /// A 16-bit RGBA decode (8 bytes a pixel) lives alongside its 4-byte RGBA8 conversion: 12 bytes + /// a pixel at the peak, not the 8 two RGBA8 buffers take. + #[test] + fn wide_decoder_buffers_are_charged_at_their_size() { + assert!(decode_peak_fits(16, 8, 4, 1024), "RGBA8: two RGBA8 buffers"); + assert!(decode_peak_fits(16, 8, 1, 1024), "grey: the premultiplied copy is the peak"); + assert!(!decode_peak_fits(16, 8, 8, 1024), "RGBA16 needs 1536 bytes"); + assert!(decode_peak_fits(16, 8, 8, 1536)); + assert!(!decode_peak_fits(16, 8, 16, 2559), "RGBA32F: 20 bytes a pixel"); + assert!(!decode_peak_fits(u32::MAX, u32::MAX, u64::MAX, u64::MAX), "overflow is refused"); + } + #[test] fn fit_pixels_keeps_normal_sizes_and_shrinks_huge_ones() { assert_eq!(fit_pixels(612, 792, MAX_RASTER_PIXELS), (612, 792)); diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 8fa8b236de..7b6001219c 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -1903,19 +1903,28 @@ fn load_image(data: &Arc>) -> Option { /// [`load_image`] with an explicit RGBA byte budget (a small one in tests). fn load_image_within(data: &Arc>, budget: u64) -> Option { + use image::ImageDecoder; let fmt = image::guess_format(data).ok()?; - // Header-only size check before anything is allocated (decode buffer + RGBA conversion are - // charged); the side limits and the default 512 MB allocation cap stay as a second guard. - let (hw, hh) = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?.into_dimensions().ok()?; - if !wordcraft_geom::decode_fits(hw, hh, budget) { - return None; - } - let mut reader = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?; let mut limits = image::Limits::default(); limits.max_image_width = Some(MAX_IMAGE_SIDE); limits.max_image_height = Some(MAX_IMAGE_SIDE); - reader.limits(limits); - let dec = reader.decode().ok()?; + let reader = || { + let mut r = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?; + r.limits(limits.clone()); + Some(r) + }; + // Header-only size check before anything is allocated (the decoder's buffer at its own pixel + // size, 8 bytes for 16-bit RGBA, plus the RGBA conversion are charged); the side limits and + // the default 512 MB allocation cap apply to the header read and the decode alike. + let (hw, hh, native) = { + let d = reader()?.into_decoder().ok()?; + let (w, h) = d.dimensions(); + (w, h, u64::from(d.color_type().bytes_per_pixel())) + }; + if !wordcraft_geom::decode_peak_fits(hw, hh, native, budget) { + return None; + } + let dec = reader()?.decode().ok()?; let (w, h) = (dec.width(), dec.height()); if w == 0 || h == 0 || w > MAX_IMAGE_SIDE || h > MAX_IMAGE_SIDE { return None; diff --git a/crates/pdf/src/tests.rs b/crates/pdf/src/tests.rs index 2d572d7ea2..cc2831ad94 100644 --- a/crates/pdf/src/tests.rs +++ b/crates/pdf/src/tests.rs @@ -2140,6 +2140,13 @@ fn image_budget_is_checked_from_the_header() { // The decoded image and its RGBA conversion are charged: two buffers at peak. assert!(load_image_within(&data, 1024).is_some()); assert!(load_image_within(&data, 1023).is_none()); + // 16-bit RGBA decodes to 8 bytes a pixel, alive with its RGBA8 conversion: 12 bytes a pixel. + let mut wide = Vec::new(); + let img16 = image::ImageBuffer::, _>::from_fn(16, 8, |x, y| image::Rgba([x as u16 * 999, y as u16, 9, 65535])); + image::DynamicImage::ImageRgba16(img16).write_to(&mut std::io::Cursor::new(&mut wide), image::ImageFormat::Png).unwrap(); + let wide = Arc::new(wide); + assert!(load_image_within(&wide, 1024).is_none(), "two RGBA8 buffers' worth is not enough for RGBA16"); + assert!(load_image_within(&wide, 1536).is_some()); // A header (valid CRC) claiming 30000x30000 with no pixel data reads fine, then the budget refuses it. let hdr = ihdr_only_png(30_000, 30_000); let dims = image::ImageReader::new(std::io::Cursor::new(hdr.as_slice())).with_guessed_format().unwrap().into_dimensions().unwrap(); diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 78b5ff1107..959a801ea0 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -209,17 +209,26 @@ pub fn decode_pixmap(bytes: &[u8]) -> Option { /// [`decode_pixmap`] with an explicit RGBA byte budget (a small one in tests). fn decode_pixmap_within(bytes: &[u8], budget: u64) -> Option { - // The header gives the size; refuse before the decoder (and the RGBA copy) allocate anything. - let (hw, hh) = image_size(bytes)?; - if !wordcraft_geom::decode_fits(hw, hh, budget) { - return None; - } - let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?; + use image::ImageDecoder; let mut limits = image::Limits::default(); limits.max_image_width = Some(MAX_DECODE_SIDE); limits.max_image_height = Some(MAX_DECODE_SIDE); - reader.limits(limits); - let img = reader.decode().ok()?.into_rgba8(); + let reader = || { + let mut r = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?; + r.limits(limits.clone()); + Some(r) + }; + // The header gives the size and the decoder's pixel format (16-bit RGBA takes 8 bytes a pixel + // before its RGBA8 copy); refuse before the decoder (and the RGBA copy) allocate anything. + let (hw, hh, native) = { + let d = reader()?.into_decoder().ok()?; + let (w, h) = d.dimensions(); + (w, h, u64::from(d.color_type().bytes_per_pixel())) + }; + if !wordcraft_geom::decode_peak_fits(hw, hh, native, budget) { + return None; + } + let img = reader()?.decode().ok()?.into_rgba8(); let (w, h) = img.dimensions(); premul_pixmap(w, h, img.as_raw()) } @@ -1719,6 +1728,12 @@ mod tests { // Decode, the RGBA conversion and the premultiplied copy are charged: two buffers at peak. assert!(decode_pixmap_within(&buf, 1024).is_some()); assert!(decode_pixmap_within(&buf, 1023).is_none()); + // 16-bit RGBA decodes to 8 bytes a pixel, alive with its RGBA8 copy: 12 bytes a pixel. + let mut wide = Vec::new(); + let img16 = image::ImageBuffer::, _>::from_fn(16, 8, |x, y| image::Rgba([x as u16 * 999, y as u16, 9, 65535])); + image::DynamicImage::ImageRgba16(img16).write_to(&mut std::io::Cursor::new(&mut wide), image::ImageFormat::Png).unwrap(); + assert!(decode_pixmap_within(&wide, 1024).is_none(), "two RGBA8 buffers' worth is not enough for RGBA16"); + assert!(decode_pixmap_within(&wide, 1536).is_some()); // A header (valid CRC) claiming 60000x60000 with no pixel data reads fine, then the budget refuses it. let hdr = ihdr_only_png(60_000, 60_000); assert_eq!(image_size(&hdr), Some((60_000, 60_000))); From 0d5a149093d6b0de4c0c2cff730261f3c4aeb7fa Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:04 +0100 Subject: [PATCH 053/112] #3692: PNG/JPEG export refuses pixels that don't match the page size Finding 26. Rendered::to_png resized its pixel buffer to width*height*4 when the lengths differed, so a Rendered claiming a huge size (its fields are public) allocated that much with no budget; to_jpeg asserted inside the encoder on the same mismatch. Both now encode only pixels that are exactly width*height RGBA within the raster cap, and return empty (logged) otherwise. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/render/src/lib.rs | 41 ++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 959a801ea0..88a6bcd534 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -57,15 +57,21 @@ impl Rendered { } out } + /// Whether the pixels are exactly `width`×`height` RGBA, within a page raster's budget + /// ([`wordcraft_geom::MAX_RASTER_PIXELS`]): the fields are public, so an encoder is never asked + /// to pad a short buffer out to a huge claimed size (or, for JPEG, to assert on a mismatch). + fn encodable(&self) -> bool { + let budget = wordcraft_geom::MAX_RASTER_PIXELS.saturating_mul(4); + wordcraft_geom::rgba_fits(self.width, self.height, budget) + && wordcraft_geom::rgba_bytes(self.width, self.height).is_some_and(|b| b == self.pixels.len() as u64) + } /// PNG bytes (empty on failure, logged). pub fn to_png(&self) -> Vec { - let mut px = self.to_straight(); - let Some(len) = wordcraft_geom::rgba_bytes(self.width, self.height).and_then(|b| usize::try_from(b).ok()) else { - log::error!("PNG encode: size {}×{} overflows", self.width, self.height); + if !self.encodable() { + log::error!("PNG encode: {} bytes for {}×{} pixels", self.pixels.len(), self.width, self.height); return Vec::new(); - }; - px.resize(len, 0); - let Some(img) = image::RgbaImage::from_raw(self.width, self.height, px) else { + } + let Some(img) = image::RgbaImage::from_raw(self.width, self.height, self.to_straight()) else { log::error!("PNG encode: bad size {}×{}", self.width, self.height); return Vec::new(); }; @@ -76,7 +82,12 @@ impl Rendered { } buf } + /// JPEG bytes on white (empty on failure, logged). pub fn to_jpeg(&self, quality: u8) -> Vec { + if !self.encodable() { + log::error!("JPEG encode: {} bytes for {}×{} pixels", self.pixels.len(), self.width, self.height); + return Vec::new(); + } let rgba = self.to_straight(); let rgb: Vec = rgba .as_chunks::<4>() @@ -1740,6 +1751,24 @@ mod tests { assert!(decode_pixmap(&hdr).is_none()); } + /// A `Rendered` whose pixels don't match its claimed size isn't padded out to that size: a + /// 60000×60000 claim over no pixels would allocate 14 GB. It, and any mismatch, encodes to + /// nothing (JPEG used to assert on the length). + #[test] + fn encoders_refuse_pixels_that_dont_match_the_size() { + let huge = Rendered { width: 60_000, height: 60_000, pixels: Vec::new() }; + assert!(huge.to_png().is_empty()); + assert!(huge.to_jpeg(80).is_empty()); + let short = Rendered { width: 2, height: 2, pixels: vec![255; 4] }; + assert!(short.to_png().is_empty()); + assert!(short.to_jpeg(80).is_empty()); + let long = Rendered { width: 1, height: 1, pixels: vec![255; 8] }; + assert!(long.to_png().is_empty()); + let ok = Rendered { width: 2, height: 2, pixels: vec![255; 16] }; + assert!(!ok.to_png().is_empty()); + assert!(!ok.to_jpeg(80).is_empty()); + } + #[test] fn page_raster_is_capped_by_total_pixels() { let d = Document::new(); From 05ee2e01269e058f8649a715a4f6ddcde0ec6a19 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:47 +0100 Subject: [PATCH 054/112] #3692: metafile text far larger than the page is left out Finding 27. An EMF/WMF text record's font height is file data; a glyph billions of device pixels tall reached vello's flattening, which subdivides its curves for ever. draw_vector_text now skips a glyph whose device bounds are non-finite, larger than 65536 px a side, or off the raster. Text at normal sizes draws as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/render/src/lib.rs | 50 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 88a6bcd534..567072e186 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -410,13 +410,25 @@ fn draw_vector_text(ctx: &mut RenderContext, t: &wordcraft_metafile::Text, tf: A let k = lay.em / face.upem.max(1.0); ctx.set_paint(rgba_color(t.color)); let db = FontDb::global(); + let canvas = kurbo::Rect::new(0.0, 0.0, f64::from(ctx.width()), f64::from(ctx.height())); for (g, (x, y)) in shaped.iter().zip(&lay.glyphs) { let o = db.outline(&face, g.gid); - ctx.set_transform(tf * Affine::translate((*x, *y)) * Affine::scale(k)); + let m = tf * Affine::translate((*x, *y)) * Affine::scale(k); + // A record's font size is file data: a glyph many canvases tall (or off the canvas) would + // have its outline flattened into millions of segments for nothing, so it is left out. + let b = m.transform_rect_bbox(o.bounding_box()); + let sane = [b.x0, b.y0, b.x1, b.y1].iter().all(|v| v.is_finite()) && b.width() <= MAX_GLYPH_PIXELS && b.height() <= MAX_GLYPH_PIXELS; + if !sane || !b.overlaps(canvas) { + continue; + } + ctx.set_transform(m); ctx.fill_path(&o); } } +/// Largest metafile glyph drawn, in device pixels per side (four times the widest canvas). +const MAX_GLYPH_PIXELS: f64 = 65_536.0; + /// One device pixel in page points under `view` (its mean axis scale); 1 for a degenerate transform. fn device_pixel(view: Affine) -> f64 { let c = view.as_coeffs(); @@ -2379,6 +2391,42 @@ mod tests { assert!(dark > 50, "the text is painted inside the picture: {dark} dark pixels"); } + /// An EMF text run whose anchor sits inside the picture but whose font is billions of units + /// tall: each glyph's curves would span billions of device pixels, which the rasteriser + /// subdivides for ever. Glyphs that big (or off the raster) are left out; the page renders. + #[test] + fn metafile_text_with_a_huge_font_is_left_out() { + let mut font = emf_rec(82, &le32(&[1, -2_000_000_000, 0, 0, 0, 400, 0, 0])); + let mut name: Vec = "Arial".encode_utf16().flat_map(|u| u.to_le_bytes()).collect(); + name.resize(64, 0); + font.extend(name); + font[4..8].copy_from_slice(&104u32.to_le_bytes()); + let mut text = emf_rec(84, &le32(&[0, 0, 100, 100, 1, 0, 0, 5, 40, 4, 76, 0, 0, 0, 0, 0, 0])); + text.extend("WOWS".encode_utf16().flat_map(|u| u.to_le_bytes())); + text[4..8].copy_from_slice(&84u32.to_le_bytes()); + let mut emf = red_right_half_emf()[..88].to_vec(); + emf.extend([font, emf_rec(37, &le32(&[1])), emf_rec(24, &le32(&[0])), text, emf_rec(14, &[0; 12])].concat()); + let mut d = Document::new(); + let media = d.add_media(emf, "emf"); + let mut p = wordcraft_doc::Paragraph::new(); + let obj = wordcraft_doc::para::InlineObject::Image { + media, + w: 70.0, + h: 70.0, + alt: String::new(), + float: Default::default(), + crop: [0.0; 4], + ole: None, + }; + p.insert_object(0, obj, &Default::default()).unwrap(); + d.body = vec![wordcraft_doc::para_block(p)]; + let l = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default()); + let start = std::time::Instant::now(); + let img = render_page(&d, &l.pages[0], 1.0, &RenderOptions::default()); + assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); + assert_eq!(img.width, 612); + } + #[test] fn unreadable_metafile_keeps_the_grey_box() { let mut d = Document::new(); From 2a647661fbc4663cdad73ce98f7bf0ef1aa95b9a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:47 +0100 Subject: [PATCH 055/112] #3692: dash patterns too fine or broken to draw are drawn solid Finding 36. kurbo's dash iterator stops advancing when a dash is so small that subtracting it from the segment length changes nothing (and loops on negative entries), so a document dash pattern like [1e-30, 1e-30] hung the renderer. Before calling kurbo, safe_dashes drops a pattern that is negative, non-finite, zero-length, or would cut the stroke into more than 100 000 pieces for its path's length bound; the stroke is drawn solid. kurbo is unchanged; normal patterns dash as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/render/src/lib.rs | 74 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 567072e186..9c7639f778 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -887,6 +887,44 @@ fn wave_path(x0: f64, y0: f64, x1: f64, y1: f64, amp: f64) -> BezPath { p } +/// Most pieces one dashed stroke may be cut into; past it the stroke is drawn solid. kurbo's dash +/// iterator makes no progress along a segment when a dash is too small next to it (it never ends), +/// and every piece is more stroke to build. +const MAX_DASH_PIECES: f64 = 100_000.0; + +/// `dashes` to stroke a path at most `len` long with, or none (solid) when they can't be drawn: a +/// negative or non-finite length, a pattern that adds up to nothing, or more than +/// [`MAX_DASH_PIECES`] pieces along the path. +fn safe_dashes(dashes: impl IntoIterator, len: f64) -> Vec { + let d: Vec = dashes.into_iter().take(MAX_DASH_PIECES as usize + 1).collect(); + let period: f64 = d.iter().sum(); + let ok = d.iter().all(|v| v.is_finite() && *v >= 0.0) && period > 0.0 && period.is_finite() && len.is_finite(); + // Each repeat of the pattern is up to `d.len()` pieces; a partial repeat at each end counts too. + if ok && (len / period + 2.0) * d.len() as f64 <= MAX_DASH_PIECES { d } else { Vec::new() } +} + +/// An upper bound on `path`'s length: its control polygon's (a curve is no longer than it). +fn path_len_bound(path: &BezPath) -> f64 { + let (mut len, mut at, mut start) = (0.0, kurbo::Point::ZERO, kurbo::Point::ZERO); + for el in path.elements() { + let pts: &[kurbo::Point] = match el { + kurbo::PathEl::MoveTo(p) => { + (at, start) = (*p, *p); + continue; + } + kurbo::PathEl::LineTo(p) => std::slice::from_ref(p), + kurbo::PathEl::QuadTo(a, b) => &[*a, *b], + kurbo::PathEl::CurveTo(a, b, c) => &[*a, *b, *c], + kurbo::PathEl::ClosePath => std::slice::from_ref(&start), + }; + for p in pts { + len += at.distance(*p); + at = *p; + } + } + len +} + /// Rasterize one display item with `view` mapping its current point coordinates to pixels. /// `visible` is a culling area in those point coordinates; nested clips intersect it and turns /// transform it into the child's coordinates. Shape and path strokes apply their stored caps, @@ -915,10 +953,10 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl // every end, blobbing over the neighbouring border at a table's corners (#2892). Wavy // underlines keep round ends, as the PDF draws them. let mut st = kurbo::Stroke::new(w).with_caps(kurbo::Cap::Butt); - if let Some(d) = stroke.dashes() { - st = st.with_dashes(0.0, d.iter().map(|k| f64::from(*k) * w)); - } let (x0, y0, x1, y1) = (*x0 as f64, *y0 as f64, *x1 as f64, *y1 as f64); + if let Some(d) = stroke.dashes() { + st = st.with_dashes(0.0, safe_dashes(d.iter().map(|k| f64::from(*k) * w), (x1 - x0).hypot(y1 - y0))); + } let line = |ctx: &mut RenderContext, ox: f64, oy: f64| { let mut p = BezPath::new(); p.move_to((x0 + ox, y0 + oy)); @@ -1094,7 +1132,7 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl // Width 0 (or none) is a hairline. let w = if stroke_width.is_finite() && *stroke_width > 0.0 { stroke_width.clamp(0.25, 200.0) } else { 0.75 }; ctx.set_paint(color(opts.ink(*s), 1.0)); - ctx.set_stroke(kurbo::Stroke::new(w as f64).with_caps(screen_cap(*cap)).with_dashes(0.0, dash.iter().map(|v| f64::from(*v)))); + ctx.set_stroke(kurbo::Stroke::new(w as f64).with_caps(screen_cap(*cap)).with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path)))); ctx.stroke_path(&path); } } @@ -1185,7 +1223,7 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl ctx.set_stroke( kurbo::Stroke::new(if stroke_width.is_nan() { 0.25 } else { stroke_width.clamp(0.25, 200.0) } as f64) .with_caps(screen_cap(*cap)) - .with_dashes(0.0, dash.iter().map(|v| f64::from(*v))), + .with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path))), ); ctx.stroke_path(&path); } @@ -2579,6 +2617,32 @@ mod tests { assert_eq!(image.pixel(832, 800), [255, 255, 255, 255], "inside the first gap"); } + /// Dash lengths too small to cut the path make kurbo's dash iterator spin without moving along + /// it (and every piece is more stroke to build): such a pattern is drawn solid, as are negative + /// and non-finite ones. A normal dash pattern still dashes. + #[test] + fn hostile_dash_patterns_draw_solid_and_finish() { + use wordcraft_doc::graphic::PathSeg; + let line = |dash: Vec| Draw::Path { + segs: vec![PathSeg::Move(10.0, 20.0), PathSeg::Line(90.0, 20.0)], + fill: None, + stroke: Some(Rgb(0, 0, 0)), + stroke_width: 2.0, + dash, + }; + let start = std::time::Instant::now(); + for dash in [vec![1e-30, 1e-30], vec![1e-30], vec![-1.0, 2.0], vec![f32::NAN, 1.0], vec![f32::INFINITY, 1.0], vec![0.0, 0.0]] { + let img = render_items(&Document::new(), &[line(dash.clone())], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default()); + assert!(img.pixel(50, 20)[3] > 200, "{dash:?} draws solid"); + } + assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); + let dashed = render_items(&Document::new(), &[line(vec![10.0, 10.0])], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default()); + assert!(dashed.pixel(15, 20)[3] > 200 && dashed.pixel(25, 20)[3] == 0, "a normal pattern dashes"); + assert!(safe_dashes([3.0, 1.0], 1000.0).len() == 2); + assert!(safe_dashes([1e-3, 1e-3], 1000.0).is_empty(), "a million pieces"); + assert!(safe_dashes(std::iter::repeat_n(1.0, 200_000), 1.0).is_empty(), "too many entries"); + } + #[test] fn flowchart_or_spokes() { let r = kurbo::Rect::new(0.0, 0.0, 10.0, 10.0); From c98efffc0638581c2ccfb3a5afde765719fcf7dc Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:47 +0100 Subject: [PATCH 056/112] #3692: empty polygon groups in metafiles reserve nothing Finding 33. geom::subpaths reserved pts + counts ops up front, so a poly-polygon record with a long list of zero counts reserved room for ops it never emits, and the canvas charged only ops.len() against MAX_BYTES. subpaths now sizes the vector exactly, and Canvas::push charges what each item holds (capacity) for paths, bitmaps and text. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/metafile/src/canvas.rs | 9 ++++++--- crates/metafile/src/geom.rs | 12 +++++++++++- crates/metafile/src/tests.rs | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/crates/metafile/src/canvas.rs b/crates/metafile/src/canvas.rs index 03a8f34532..bb6ecd3167 100644 --- a/crates/metafile/src/canvas.rs +++ b/crates/metafile/src/canvas.rs @@ -284,9 +284,12 @@ impl Canvas { fn push(&mut self, raw: Raw) { let cost = match &raw { - Raw::Path { ops, .. } => ops.len() * std::mem::size_of::(), - Raw::Bitmap { img, .. } => img.px.len(), - Raw::Text(t) => t.text.len() + t.font.len() + t.dx.len() * std::mem::size_of::(), + // What is held, capacity and all. + Raw::Path { ops, .. } => ops.capacity().saturating_mul(std::mem::size_of::()), + Raw::Bitmap { img, .. } => img.px.capacity(), + Raw::Text(t) => { + t.text.capacity().saturating_add(t.font.capacity()).saturating_add(t.dx.capacity().saturating_mul(std::mem::size_of::())) + } }; if self.items.len() >= MAX_ITEMS || self.used.saturating_add(cost) > MAX_BYTES { self.full = true; diff --git a/crates/metafile/src/geom.rs b/crates/metafile/src/geom.rs index 0547de18ba..0c2b772cf1 100644 --- a/crates/metafile/src/geom.rs +++ b/crates/metafile/src/geom.rs @@ -99,7 +99,17 @@ fn arc(cx: f64, cy: f64, rx: f64, ry: f64, a0: f64, sweep: f64) -> Vec { /// Consecutive subpaths: `counts[i]` points each, taken in order from `pts`. Counts that run past the /// point list are cut short rather than failing the whole record. pub(crate) fn subpaths(pts: &[P], counts: &[usize], close: bool) -> Vec { - let mut ops = Vec::with_capacity(pts.len() + counts.len()); + // The exact size first: empty groups add no ops, so a long count list of zeros (file data) + // mustn't reserve room the canvas budget never sees. + let (mut n, mut at) = (0usize, 0usize); + for &c in counts { + let end = at.saturating_add(c).min(pts.len()); + if end > at { + n = n.saturating_add(end - at + usize::from(close)); + } + at = end; + } + let mut ops = Vec::with_capacity(n); let mut at = 0usize; for &c in counts { let end = at.saturating_add(c).min(pts.len()); diff --git a/crates/metafile/src/tests.rs b/crates/metafile/src/tests.rs index eaa5b16740..a71481ffaa 100644 --- a/crates/metafile/src/tests.rs +++ b/crates/metafile/src/tests.rs @@ -1053,3 +1053,18 @@ fn plain_wmf_target_comes_from_a_window_that_was_read() { let (x0, y0, x1, y1) = rect_extent(&p.items[0]); assert!(close(x0, 0.0) && close(y0, 0.0) && close(x1, 200.0) && close(y1, 100.0), "{x0} {y0} {x1} {y1}"); } + +/// A poly-polygon's group counts are file data: a long list of empty groups adds no ops, so it +/// mustn't reserve room for them either (the canvas charges what a path holds, capacity included). +#[test] +fn empty_poly_groups_reserve_nothing() { + let pts = [(0.0, 0.0), (10.0, 0.0), (10.0, 10.0)]; + let mut counts = vec![0usize; 100_000]; + counts.push(3); + let ops = crate::geom::subpaths(&pts, &counts, true); + assert_eq!(ops.len(), 4, "M L L Z"); + assert!(ops.capacity() <= 8, "capacity {}", ops.capacity()); + // Counts past the points are cut short, and the size still matches. + let ops = crate::geom::subpaths(&pts, &[2, 5, 4], false); + assert_eq!((ops.len(), ops.capacity()), (3, 3)); +} From 0aae9d05db6d5ecba8d7c10cabe07336fd2092a1 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:57 +0100 Subject: [PATCH 057/112] #3692: PDF export looks bookmarks up case-blind through an index Finding 22. A link whose bookmark name differed in case from the bookmark fell back to scanning the whole bookmark map with eq_ignore_ascii_case, per link (bookmarks x links), and the HashMap's order made which of several same-but-for-case bookmarks won arbitrary. The exporter now keeps an exact and an ASCII-folded index, each filled first-in-document-order. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/pdf/src/lib.rs | 15 +++++++++------ crates/pdf/src/tests.rs | 8 ++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 7b6001219c..4b76816eac 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -747,7 +747,9 @@ struct Exporter<'a> { /// Bookmark positions by name (the first of a name wins), gathered once: looking each internal /// link up by walking the whole document made a long document with a linked table of contents /// take seconds to export. - bookmarks: std::cell::OnceCell>, + /// Bookmarks by exact name, and by ASCII-lowercased name for the case-blind fallback; the + /// first in document order wins either way. + bookmarks: std::cell::OnceCell<(HashMap, HashMap)>, } /// What the content being drawn is, for tagging. @@ -1544,14 +1546,15 @@ impl Exporter<'_> { /// Where a bookmark is: (output page, x, y) — the page and line its start lands on, which /// in a paragraph split across pages need not be the paragraph's first page. fn bookmark(&self, name: &str) -> Option<(usize, f32, f32)> { - let marks = self.bookmarks.get_or_init(|| { - let mut m = HashMap::new(); + let (exact, folded) = self.bookmarks.get_or_init(|| { + let (mut exact, mut folded) = (HashMap::new(), HashMap::new()); for (n, pos) in self.doc.bookmarks() { - m.entry(n).or_insert(pos); + folded.entry(n.to_ascii_lowercase()).or_insert_with(|| pos.clone()); + exact.entry(n).or_insert(pos); } - m + (exact, folded) }); - let pos = marks.get(name).or_else(|| marks.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v))?.clone(); + let pos = exact.get(name).or_else(|| folded.get(&name.to_ascii_lowercase()))?.clone(); if let Some(c) = self.lay.caret(&pos) && let Some(out) = self.out_index.get(&c.page) { diff --git a/crates/pdf/src/tests.rs b/crates/pdf/src/tests.rs index cc2831ad94..c5afa3b527 100644 --- a/crates/pdf/src/tests.rs +++ b/crates/pdf/src/tests.rs @@ -1323,6 +1323,7 @@ fn bookmark_destination_is_the_page_of_its_start() { let mut p = Paragraph::with_text("Paragraph start\u{c}Bookmark target", CharProps::default()); let at = "Paragraph start\u{c}".len(); p.insert_object(at, InlineObject::BookmarkStart { name: "target".into() }, &CharProps::default()).unwrap(); + p.insert_object(at, InlineObject::BookmarkStart { name: "First".into() }, &CharProps::default()).unwrap(); p.insert_object(0, InlineObject::BookmarkStart { name: "first".into() }, &CharProps::default()).unwrap(); d.body = vec![para_block(p)]; let lay = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default()); @@ -1354,6 +1355,13 @@ fn bookmark_destination_is_the_page_of_its_start() { let (page_ci, x_ci, y_ci) = ex.bookmark("TARGET").unwrap(); assert_eq!(page_ci, 1); assert_eq!((x_ci, y_ci), (x, y)); + // An exact name wins; names that differ only in case fall back to the first in the document + // (a folded index, not a scan of a hash map in arbitrary order for every link). + assert_eq!(ex.bookmark("First").map(|b| b.0), Some(1)); + assert_eq!(ex.bookmark("FIRST").map(|b| b.0), Some(0)); + assert!(ex.bookmark("absent").is_none()); + let (exact, folded) = ex.bookmarks.get().unwrap(); + assert_eq!((exact.len(), folded.len()), (3, 2)); // Whitespace in ISO date string is parsed successfully assert!(parse_iso(" 2024-05-06T07:08:09Z \n").is_some()); From f4c9c89c642b7094e49aeecb430e41c07a18b249 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:09:57 +0100 Subject: [PATCH 058/112] #3692: PDF export finds cluster ends and clipped cells without rescans Finding 14: each glyph's text end was found by scanning the following glyphs for a larger cluster, quadratic in a run of glyphs sharing one cluster (a letter with many combining marks); cluster_ends does it in one backward pass. Finding 31: every placed line was drawn with a clone of every clipped table cell on its page (lines x cells); the page's clipped cells are now indexed once by story and path and each line takes only its own cell, the first in page order as layout picks it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/pdf/src/lib.rs | 64 +++++++++++++++++++++++++++++++++++------ crates/pdf/src/tests.rs | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 4b76816eac..9781a04662 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -920,8 +920,9 @@ impl Exporter<'_> { placeholders: false, ..Default::default() }; + let clips = clipped_cells(page); for it in page.header.iter().chain(page.footer.iter()) { - let draws = self.draws(page, it, &dopts); + let draws = self.draws(page, it, &clips, &dopts); self.tagged(s, Role::Artifact(ArtifactType::Other), None, |me, s| { for d in &draws { me.draw(s, d); @@ -929,7 +930,7 @@ impl Exporter<'_> { }); } for it in &page.items { - let draws = self.draws(page, it, &dopts); + let draws = self.draws(page, it, &clips, &dopts); match it { Placed::Lines { story, path, para, .. } => { let idx = self.entry(*story, path, para); @@ -992,13 +993,14 @@ impl Exporter<'_> { } /// Display items for one placed item. - fn draws(&self, page: &Page, it: &Placed, dopts: &DisplayOptions) -> Vec { + fn draws(&self, page: &Page, it: &Placed, clips: &CellClips, dopts: &DisplayOptions) -> Vec { let mut items = vec![it.clone()]; // Lines in a cell cut off at its edge are drawn inside its clip: `page_display` finds the - // cell among the items (cells themselves draw nothing). - if matches!(it, Placed::Lines { .. }) { - let clipped = page.header.iter().chain(&page.footer).chain(&page.items).filter(|c| matches!(c, Placed::Cell { clip: true, .. })); - items.extend(clipped.cloned()); + // cell among the items (cells themselves draw nothing). Only that line's cell goes along. + if let Placed::Lines { story, path, .. } = it + && let Some(cell) = clip_cell(clips, *story, &path.0) + { + items.push(cell.clone()); } let one = Page { w: page.w, h: page.h, section: page.section, number: page.number, items, body: page.body, ..Default::default() }; page_display(self.doc, &one, dopts) @@ -1706,11 +1708,12 @@ impl Exporter<'_> { match font { Some(font) => { let upem = face.upem.max(1.0) as f32; + let ends = cluster_ends(&shaped.iter().map(|g| g.cluster).collect::>(), t.text.len()); let kg: Vec = shaped .iter() .enumerate() .map(|(i, g)| { - let end = shaped.iter().skip(i + 1).map(|n| n.cluster).find(|cl| *cl > g.cluster).unwrap_or(t.text.len()); + let end = ends.get(i).copied().unwrap_or(t.text.len()); let start = g.cluster.min(end); let x = lay.glyphs.get(i).map_or(0.0, |p| p.0); let adv = match lay.glyphs.get(i + 1) { @@ -1761,11 +1764,12 @@ impl Exporter<'_> { if font.is_none() && !self.outlined.contains(&face.id()) { return; } + let ends = cluster_ends(&shaped.iter().map(|g| g.cluster).collect::>(), text.len()); let glyphs: Vec = shaped .iter() .enumerate() .map(|(i, g)| { - let end = shaped.iter().skip(i + 1).map(|n| n.cluster).find(|cl| *cl > g.cluster).unwrap_or(text.len()); + let end = ends.get(i).copied().unwrap_or(text.len()); let start = g.cluster.min(end); KrillaGlyph::new( GlyphId::new(g.gid), @@ -1894,6 +1898,48 @@ fn path_from(b: &kurbo::BezPath) -> Option { pb.finish() } +/// A page's cells whose content is cut off at their edge, by story and content path (table, +/// row, cell), with their place in header, footer, items order (the first wins, as in layout). +type CellClips<'a> = HashMap, (usize, &'a Placed)>>; + +fn clipped_cells(page: &Page) -> CellClips<'_> { + let mut clips: CellClips = HashMap::new(); + for (order, it) in page.header.iter().chain(&page.footer).chain(&page.items).enumerate() { + if let Placed::Cell { table, row, cell, story, clip: true, .. } = it + && let (Ok(row), Ok(cell)) = (u32::try_from(*row), u32::try_from(*cell)) + { + let mut prefix = table.0.clone(); + prefix.extend([row, cell]); + clips.entry(*story).or_default().entry(prefix).or_insert((order, it)); + } + } + clips +} + +/// The clipped cell a line at `path` of `story` is in: the first one whose path starts it. A +/// lookup per prefix of the (short) path instead of passing every clipped cell with every line. +fn clip_cell<'a>(clips: &CellClips<'a>, story: StoryRef, path: &[u32]) -> Option<&'a Placed> { + let cells = clips.get(&story)?; + (0..=path.len()).filter_map(|k| cells.get(path.get(..k)?)).min_by_key(|(order, _)| *order).map(|(_, it)| *it) +} + +/// Where each glyph's text ends: the cluster (byte offset) of the first later glyph whose cluster +/// is past the glyph's own, else `len`. One backward pass keeping the later clusters that can +/// still be an answer, so a long run of glyphs sharing a cluster (combining marks) stays linear. +fn cluster_ends(clusters: &[usize], len: usize) -> Vec { + let mut ends = vec![len; clusters.len()]; + // Later clusters, nearest last, each greater than every one nearer. + let mut later: Vec = Vec::new(); + for (end, &c) in ends.iter_mut().zip(clusters).rev() { + while later.last().is_some_and(|&n| n <= c) { + later.pop(); + } + *end = later.last().copied().unwrap_or(len); + later.push(c); + } + ends +} + /// Decode a picture for embedding (validated with the `image` crate first, so a broken file /// can't fail the whole export). fn load_image(data: &Arc>) -> Option { diff --git a/crates/pdf/src/tests.rs b/crates/pdf/src/tests.rs index c5afa3b527..1a8a20b546 100644 --- a/crates/pdf/src/tests.rs +++ b/crates/pdf/src/tests.rs @@ -2426,3 +2426,62 @@ fn only_safe_link_targets_become_annotations() { .collect(); assert_eq!(uris, ["https://example.com/", "mailto:ada@example.com"]); } + +/// A glyph's text runs to the first later glyph with a cluster past its own (the end of the text +/// without one), found in one pass: a long run of glyphs sharing a cluster used to rescan the rest +/// of the run for every glyph. +#[test] +fn cluster_ends_match_the_next_larger_cluster_in_linear_time() { + let brute = |cl: &[usize], len: usize| -> Vec { + (0..cl.len()).map(|i| cl.iter().skip(i + 1).copied().find(|n| *n > cl[i]).unwrap_or(len)).collect::>() + }; + for cl in [vec![], vec![0], vec![0, 0, 3, 3, 5], vec![5, 3, 3, 0], vec![2, 0, 4, 1, 1, 6, 3], vec![0, 7, 2, 2, 9, 1]] { + assert_eq!(cluster_ends(&cl, 10), brute(&cl, 10), "{cl:?}"); + } + // One base letter and 200 000 combining marks in its cluster, then one more letter. + let mut cl = vec![0; 200_001]; + cl.push(400_002); + let start = std::time::Instant::now(); + let ends = cluster_ends(&cl, 400_004); + assert!(start.elapsed() < std::time::Duration::from_secs(1), "{:?}", start.elapsed()); + assert!(ends[..200_001].iter().all(|e| *e == 400_002)); + assert_eq!(ends.last(), Some(&400_004)); +} + +/// A line in a cell cut off at its edge is drawn with that cell's clip, found by path in an index +/// built once per page: every line used to carry a clone of every clipped cell on the page, so a +/// page of exact-height rows cost lines × cells. +#[test] +fn a_line_finds_its_clipped_cell_by_path() { + let cell = |story, table: Vec, row, cell, clip| Placed::Cell { + rect: Rect::default(), + covered: false, + table: DocPath(table), + row, + cell, + story, + clip, + }; + let mut page = Page { items: vec![cell(StoryRef::Body, vec![0], 0, 1, false), cell(StoryRef::Body, vec![2], 1, 0, true)], ..Default::default() }; + // A nested clipped cell inside the first one; the outer one comes first in the page. + page.items.push(cell(StoryRef::Body, vec![2, 1, 0, 3], 0, 0, true)); + page.footer.push(cell(StoryRef::Part(7), vec![0], 0, 0, true)); + let clips = clipped_cells(&page); + let found = |story, path: &[u32]| { + clip_cell(&clips, story, path).and_then(|c| if let Placed::Cell { table, .. } = c { Some(table.0.clone()) } else { None }) + }; + assert_eq!(found(StoryRef::Body, &[2, 1, 0, 0]), Some(vec![2])); + assert_eq!(found(StoryRef::Body, &[2, 1, 0, 3, 0, 0, 1]), Some(vec![2]), "the first cell in page order wins, as in layout"); + assert_eq!(found(StoryRef::Body, &[0, 0, 1, 0]), None, "that cell isn't clipped"); + assert_eq!(found(StoryRef::Body, &[2, 0, 0, 0]), None, "another row"); + assert_eq!(found(StoryRef::Part(7), &[0, 0, 0, 2]), Some(vec![0])); + assert_eq!(found(StoryRef::Body, &[0, 0, 0, 2]), None, "another story"); + + // 20 000 clipped cells and as many lines: a lookup each, not a pass over the cells each. + let big = Page { items: (0..20_000).map(|r| cell(StoryRef::Body, vec![0], r, 0, true)).collect(), ..Default::default() }; + let start = std::time::Instant::now(); + let clips = clipped_cells(&big); + let hits = (0..20_000u32).filter(|r| clip_cell(&clips, StoryRef::Body, &[0, *r, 0, 0]).is_some()).count(); + assert_eq!(hits, 20_000); + assert!(start.elapsed() < std::time::Duration::from_secs(1), "{:?}", start.elapsed()); +} From 1225f561876b2a2c78f6c97873e5103c800caa39 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:28:04 +0100 Subject: [PATCH 059/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20left-out?= =?UTF-8?q?=20metafile=20glyphs=20and=20over-budget=20pictures=20are=20log?= =?UTF-8?q?ged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A metafile text run's glyphs larger than 65,536 device pixels (or of no finite size) were left out silently; the first time any are this run a warning says how many (once a run: a page is drawn on every repaint). A picture refused by decode_peak_fits is logged too, once per picture since the refusal is cached with it. The 12 bytes a pixel charged to 16-bit images is the decoder's true peak: image's decoder_to_vec allocates w×h×8 for Rgba16 and into_rgba8 copies to RGBA8 while it is alive. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/render/src/lib.rs | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 9c7639f778..976fa9f581 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -237,6 +237,10 @@ fn decode_pixmap_within(bytes: &[u8], budget: u64) -> Option { (w, h, u64::from(d.color_type().bytes_per_pixel())) }; if !wordcraft_geom::decode_peak_fits(hw, hh, native, budget) { + // Logged once a picture: its refusal is cached with it (`pixmap_for`). + log::warn!( + "a {hw}×{hh} picture of {native} bytes a pixel needs more than {budget} bytes to decode: it is not drawn (in the text, a grey box takes its place)" + ); return None; } let img = reader()?.decode().ok()?.into_rgba8(); @@ -411,6 +415,7 @@ fn draw_vector_text(ctx: &mut RenderContext, t: &wordcraft_metafile::Text, tf: A ctx.set_paint(rgba_color(t.color)); let db = FontDb::global(); let canvas = kurbo::Rect::new(0.0, 0.0, f64::from(ctx.width()), f64::from(ctx.height())); + let mut oversized = 0usize; for (g, (x, y)) in shaped.iter().zip(&lay.glyphs) { let o = db.outline(&face, g.gid); let m = tf * Affine::translate((*x, *y)) * Affine::scale(k); @@ -418,17 +423,39 @@ fn draw_vector_text(ctx: &mut RenderContext, t: &wordcraft_metafile::Text, tf: A // have its outline flattened into millions of segments for nothing, so it is left out. let b = m.transform_rect_bbox(o.bounding_box()); let sane = [b.x0, b.y0, b.x1, b.y1].iter().all(|v| v.is_finite()) && b.width() <= MAX_GLYPH_PIXELS && b.height() <= MAX_GLYPH_PIXELS; - if !sane || !b.overlaps(canvas) { + if !sane { + oversized = oversized.saturating_add(1); + continue; + } + if !b.overlaps(canvas) { continue; } ctx.set_transform(m); ctx.fill_path(&o); } + note_oversized_glyphs(&OVERSIZED_GLYPHS_NOTED, oversized); } /// Largest metafile glyph drawn, in device pixels per side (four times the widest canvas). const MAX_GLYPH_PIXELS: f64 = 65_536.0; +/// Whether metafile glyphs left out for their size have been logged ([`note_oversized_glyphs`]). +static OVERSIZED_GLYPHS_NOTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Logs that `n` glyphs of a metafile text run were left out for being larger than +/// [`MAX_GLYPH_PIXELS`] (or of no finite size), the first time any are this run (`noted`): a +/// page is drawn again on every repaint, so a warning each time would flood the log. True when +/// this call logged it. +fn note_oversized_glyphs(noted: &std::sync::atomic::AtomicBool, n: usize) -> bool { + if n == 0 || noted.swap(true, std::sync::atomic::Ordering::Relaxed) { + return false; + } + log::warn!( + "a picture's text has {n} letter(s) larger than {MAX_GLYPH_PIXELS} pixels drawn, left out (logged once a run; later ones are left out too)" + ); + true +} + /// One device pixel in page points under `view` (its mean axis scale); 1 for a degenerate transform. fn device_pixel(view: Affine) -> f64 { let c = view.as_coeffs(); @@ -2463,6 +2490,19 @@ mod tests { let img = render_page(&d, &l.pages[0], 1.0, &RenderOptions::default()); assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); assert_eq!(img.width, 612); + // Leaving them out is logged (once a run, so it may have been by another test already). + assert!(OVERSIZED_GLYPHS_NOTED.load(std::sync::atomic::Ordering::Relaxed), "the left-out glyphs are logged"); + } + + /// Glyphs left out for their size are logged the first time only: a page is drawn on every + /// repaint. A run with none left out logs nothing and doesn't use up the warning. + #[test] + fn oversized_glyphs_are_logged_once_a_run() { + let noted = std::sync::atomic::AtomicBool::new(false); + assert!(!note_oversized_glyphs(¬ed, 0), "nothing left out, nothing logged"); + assert!(!noted.load(std::sync::atomic::Ordering::Relaxed)); + assert!(note_oversized_glyphs(¬ed, 4), "the first glyphs left out are logged"); + assert!(!note_oversized_glyphs(¬ed, 2), "later ones aren't"); } #[test] From 77402c4e7e9e4e9998a30c0d14746cbc5cfc8d9d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 16:22:44 +0100 Subject: [PATCH 060/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20pdf=20and?= =?UTF-8?q?=20render=20timing=20tests=20get=20a=20margin=20a=20busy=20CI?= =?UTF-8?q?=20machine=20can't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 1-2 s failed on loaded runners. Each bound is now 10 s, at least 10x the fixed code's time on a loaded machine and well under the old code's on the same input: - cluster ends: a million marks (was 200,000); 30 ms now. The rescan (the test's brute) took 2.9 s at 50,000 and is quadratic: about 20 minutes at a million. - clipped-cell lookups: 100,000 cells and lines (was 20,000); 0.25 s now, a pass over the cells per line would be 10^10 steps. - metafile text with a huge font: ~20 ms now; with the size check off it ran past 4 minutes. - hostile dash patterns: milliseconds now; drawn as dashes they ran past 4 minutes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/pdf/src/tests.rs | 29 +++++++++++++++++------------ crates/render/src/lib.rs | 6 ++++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/pdf/src/tests.rs b/crates/pdf/src/tests.rs index 1a8a20b546..4f81f9e1a9 100644 --- a/crates/pdf/src/tests.rs +++ b/crates/pdf/src/tests.rs @@ -2438,14 +2438,17 @@ fn cluster_ends_match_the_next_larger_cluster_in_linear_time() { for cl in [vec![], vec![0], vec![0, 0, 3, 3, 5], vec![5, 3, 3, 0], vec![2, 0, 4, 1, 1, 6, 3], vec![0, 7, 2, 2, 9, 1]] { assert_eq!(cluster_ends(&cl, 10), brute(&cl, 10), "{cl:?}"); } - // One base letter and 200 000 combining marks in its cluster, then one more letter. - let mut cl = vec![0; 200_001]; - cl.push(400_002); + // One base letter and a million combining marks in its cluster, then one more letter. + // Milliseconds; the rescan (`brute`) took 2.9 s at 50,000 and grows with the square, so + // about 20 minutes here. + let n = 1_000_000; + let mut cl = vec![0; n + 1]; + cl.push(2 * n + 2); let start = std::time::Instant::now(); - let ends = cluster_ends(&cl, 400_004); - assert!(start.elapsed() < std::time::Duration::from_secs(1), "{:?}", start.elapsed()); - assert!(ends[..200_001].iter().all(|e| *e == 400_002)); - assert_eq!(ends.last(), Some(&400_004)); + let ends = cluster_ends(&cl, 2 * n + 4); + assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); + assert!(ends[..=n].iter().all(|e| *e == 2 * n + 2)); + assert_eq!(ends.last(), Some(&(2 * n + 4))); } /// A line in a cell cut off at its edge is drawn with that cell's clip, found by path in an index @@ -2477,11 +2480,13 @@ fn a_line_finds_its_clipped_cell_by_path() { assert_eq!(found(StoryRef::Part(7), &[0, 0, 0, 2]), Some(vec![0])); assert_eq!(found(StoryRef::Body, &[0, 0, 0, 2]), None, "another story"); - // 20 000 clipped cells and as many lines: a lookup each, not a pass over the cells each. - let big = Page { items: (0..20_000).map(|r| cell(StoryRef::Body, vec![0], r, 0, true)).collect(), ..Default::default() }; + // 100 000 clipped cells and as many lines: a lookup each, not a pass over the cells each + // (10^10 steps, minutes; the lookups take well under a second on a loaded machine). + let n = 100_000; + let big = Page { items: (0..n).map(|r| cell(StoryRef::Body, vec![0], r, 0, true)).collect(), ..Default::default() }; let start = std::time::Instant::now(); let clips = clipped_cells(&big); - let hits = (0..20_000u32).filter(|r| clip_cell(&clips, StoryRef::Body, &[0, *r, 0, 0]).is_some()).count(); - assert_eq!(hits, 20_000); - assert!(start.elapsed() < std::time::Duration::from_secs(1), "{:?}", start.elapsed()); + let hits = (0..n as u32).filter(|r| clip_cell(&clips, StoryRef::Body, &[0, *r, 0, 0]).is_some()).count(); + assert_eq!(hits, n); + assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 976fa9f581..ba31ca3b41 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -2488,7 +2488,8 @@ mod tests { let l = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default()); let start = std::time::Instant::now(); let img = render_page(&d, &l.pages[0], 1.0, &RenderOptions::default()); - assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); + // About 20 ms without the huge glyphs; drawing them ran past 4 minutes without finishing. + assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); assert_eq!(img.width, 612); // Leaving them out is logged (once a run, so it may have been by another test already). assert!(OVERSIZED_GLYPHS_NOTED.load(std::sync::atomic::Ordering::Relaxed), "the left-out glyphs are logged"); @@ -2675,7 +2676,8 @@ mod tests { let img = render_items(&Document::new(), &[line(dash.clone())], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default()); assert!(img.pixel(50, 20)[3] > 200, "{dash:?} draws solid"); } - assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); + // Milliseconds drawn solid; drawn as dashes they ran past 4 minutes without finishing. + assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); let dashed = render_items(&Document::new(), &[line(vec![10.0, 10.0])], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default()); assert!(dashed.pixel(15, 20)[3] > 200 && dashed.pixel(25, 20)[3] == 0, "a normal pattern dashes"); assert!(safe_dashes([3.0, 1.0], 1000.0).len() == 2); From b7403f41adeb6387a191afd224640c3a4093f3aa Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 26 Oct 2026 00:01:30 +0000 Subject: [PATCH 061/112] =?UTF-8?q?#3692:=20rebase=20onto=20main=20?= =?UTF-8?q?=E2=80=94=20rustfmt=20after=20the=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/render/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index ba31ca3b41..41f772ae20 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -1159,7 +1159,11 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl // Width 0 (or none) is a hairline. let w = if stroke_width.is_finite() && *stroke_width > 0.0 { stroke_width.clamp(0.25, 200.0) } else { 0.75 }; ctx.set_paint(color(opts.ink(*s), 1.0)); - ctx.set_stroke(kurbo::Stroke::new(w as f64).with_caps(screen_cap(*cap)).with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path)))); + ctx.set_stroke( + kurbo::Stroke::new(w as f64) + .with_caps(screen_cap(*cap)) + .with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path))), + ); ctx.stroke_path(&path); } } From 91c030bc9f8c8cb3a9e43a004c9d0726a425cf99 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 26 Oct 2026 00:25:08 +0000 Subject: [PATCH 062/112] =?UTF-8?q?#3692:=20rebase=20onto=20main=20?= =?UTF-8?q?=E2=80=94=20the=20dash=20test's=20Draw::Path=20takes=20the=20ne?= =?UTF-8?q?w=20cap=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- crates/render/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 41f772ae20..318904fbe3 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -2674,6 +2674,7 @@ mod tests { stroke: Some(Rgb(0, 0, 0)), stroke_width: 2.0, dash, + cap: Default::default(), }; let start = std::time::Instant::now(); for dash in [vec![1e-30, 1e-30], vec![1e-30], vec![-1.0, 2.0], vec![f32::NAN, 1.0], vec![f32::INFINITY, 1.0], vec![0.0, 0.0]] { From 89a64d1ef5e39cfa1a05b45b2906eb178a9ec0cf Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:04:24 +0100 Subject: [PATCH 063/112] =?UTF-8?q?#3692:=20sheet=20import=20=E2=80=94=20l?= =?UTF-8?q?inear=20duplicate-header=20naming;=20ODS=20string=20attributes?= =?UTF-8?q?=20capped=20and=20over-budget=20rows=20refused=20before=20copyi?= =?UTF-8?q?ng?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 40: field_names restarted every repeated header at "Name 2" and compared each candidate with every name so far (cubic). It now keeps the names in an ASCII-lowercased set and remembers each base name's next number; existing headers like "Name 2" stay collision-free and the output is unchanged. Finding 47: an ODS office:string-value bypassed MAX_CELL_CHARS and was copied for each repeated column before the text budget was checked. The attribute is now capped like text, a row stops being built (and is refused) once its text can't fit the remaining budget, and Grid::push measures what fits before cloning. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/sheet.rs | 104 ++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/crates/formats/src/sheet.rs b/crates/formats/src/sheet.rs index 520fca2f80..392b425c06 100644 --- a/crates/formats/src/sheet.rs +++ b/crates/formats/src/sheet.rs @@ -18,7 +18,7 @@ //! all), sheets, rows, columns, cells and string lengths are capped, repeat counts are clamped to //! what is left, and nothing recurses, so XML depth costs nothing. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::{Cursor, Read}; use quick_xml::XmlVersion; @@ -138,18 +138,28 @@ pub fn read_rows(bytes: &[u8], pick: Pick<'_>) -> Result<(Vec, Vec Vec { let mut out: Vec = Vec::with_capacity(row.len()); + // The names given so far, ASCII-lowercased, and per base name the next number to try: a set + // lookup per candidate, and each base resumes where it stopped, instead of rescanning every + // name from `2` for every repeat (cubic on a row of equal headers). + let mut taken: HashSet = HashSet::with_capacity(row.len()); + let mut next: HashMap = HashMap::new(); for (i, h) in row.iter().enumerate() { let base = match h.trim() { "" => format!("Column {}", i + 1), t => t.to_string(), }; - let taken = |n: &str, out: &[String]| out.iter().any(|o| o.eq_ignore_ascii_case(n)); let mut name = base.clone(); - let mut k = 2usize; - while taken(&name, &out) { - name = format!("{base} {k}"); - k = k.saturating_add(1); + if taken.contains(&name.to_ascii_lowercase()) { + let k = next.entry(base.to_ascii_lowercase()).or_insert(2); + loop { + name = format!("{base} {k}"); + *k = k.saturating_add(1); + if !taken.contains(&name.to_ascii_lowercase()) { + break; + } + } } + taken.insert(name.to_ascii_lowercase()); out.push(name); } out @@ -350,14 +360,14 @@ impl Grid { } let mut left = times; while left > 0 && !self.full() { - let room = MAX_CELLS.saturating_sub(self.cells); - let mut r = row.clone(); - r.truncate(room); + // What fits is measured on `row` itself; only a copy that will be kept is made. + let keep = row.len().min(MAX_CELLS.saturating_sub(self.cells)); + let r = row.get(..keep).unwrap_or_default(); if !self.text.take(r.iter().map(String::len).sum()) { return; } self.cells = self.cells.saturating_add(r.len()); - self.rows.push(r); + self.rows.push(r.to_vec()); left -= 1; } } @@ -854,7 +864,9 @@ fn ods_read(xml: &[u8], want: Option) -> Result<(Vec, Vec, u64, u64)> = None; // cells, pending blank cells, repeat + // Cells, pending blank cells, repeat, and the cells' text bytes (`None` once the row is over + // the text budget and so refused). + let mut row: Option<(Vec, u64, u64, Option)> = None; let mut cell: Option = None; let (mut in_p, mut skip) = (0usize, 0usize); walk(xml, |ev| match ev { @@ -875,7 +887,7 @@ fn ods_read(xml: &[u8], want: Option) -> Result<(Vec, Vec { let repeat = attr(e, b"number-rows-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1); - row = Some((Vec::new(), 0, repeat)); + row = Some((Vec::new(), 0, repeat, Some(0))); } b"table-cell" | b"covered-table-cell" if row.is_some() => { let repeat = attr(e, b"number-columns-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1); @@ -959,7 +971,8 @@ fn ods_read(xml: &[u8], want: Option) -> Result<(Vec, Vec) -> Result<(Vec, Vec MAX_TEXT_BYTES.saturating_sub(grid.text.0) { + cells.clear(); + *budget = None; + return; + } cells.resize(cells.len() + blanks, String::new()); *pending = 0; - let times = c.repeat.min(MAX_COLS.saturating_sub(cells.len()) as u64) as usize; cells.extend(std::iter::repeat_n(text, times)); } b"table-row" => { - if let Some((cells, _, repeat)) = row.take() { + if let Some((cells, _, repeat, Some(_))) = row.take() { grid.push(cells, repeat); } } @@ -997,7 +1018,12 @@ fn ods_value(c: &OCell) -> String { ("time", Some(v)) => iso_time(v).unwrap_or_else(|| c.text.clone()), ("boolean", Some(v)) => if v.eq_ignore_ascii_case("true") || v == "1" { "TRUE" } else { "FALSE" }.into(), ("percentage" | "currency", Some(v)) if c.text.trim().is_empty() => v.parse::().map(number).unwrap_or_else(|_| c.text.clone()), - ("string", Some(v)) if c.text.trim().is_empty() => v.to_string(), + ("string", Some(v)) if c.text.trim().is_empty() => { + // Attribute text obeys the same per-cell cap as `` text. + let mut s = String::new(); + push_capped(&mut s, v); + s + } _ => c.text.clone(), } } @@ -1204,4 +1230,50 @@ mod tests { assert_eq!(rows.len(), 1); assert_eq!(rows[0], vec!["TRUE", "World", "Name"]); } + + #[test] + fn repeated_headers_are_numbered_in_linear_time() { + // Every repeat used to restart at `Name 2` and compare each candidate with every name + // so far: cubic in the number of equal headers. + let row = vec!["Name".to_string(); 5_000]; + let t = std::time::Instant::now(); + let names = field_names(&row); + assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + assert_eq!(names.len(), 5_000); + assert_eq!(names[0], "Name"); + assert_eq!(names[4_999], "Name 5000"); + // Names a header already has stay collision-free, ignoring case. + let row: Vec = ["Name", "Name 2", "Name", "name", "", "Column 5", "NAME 2"].iter().map(|s| s.to_string()).collect(); + assert_eq!(field_names(&row), ["Name", "Name 2", "Name 3", "name 4", "Column 5", "Column 5 2", "NAME 2 2"]); + } + + fn ods(rows: &str) -> Vec { + let content = format!( + r#"{rows}"# + ); + zip(&[("mimetype", "application/vnd.oasis.opendocument.spreadsheet"), ("content.xml", &content)]) + } + + #[test] + fn ods_string_attributes_are_capped_before_repeating() { + // An `office:string-value` is a cell's text: it obeys MAX_CELL_CHARS like `` + // does, so repeating it across the columns can't build a huge row first. + let long = "v".repeat(MAX_CELL_CHARS * 4); + let book = ods(&format!( + r#""# + )); + let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].len(), 3); + assert!(rows[0].iter().all(|c| c.chars().count() == MAX_CELL_CHARS)); + + // A row whose repeated text alone is over the text budget is dropped as it is built + // (it was built in full, then cloned, then refused); the rows after it still count. + let wide = "€".repeat(MAX_CELL_CHARS); + let book = ods(&format!( + r#"next"# + )); + let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap(); + assert_eq!(rows, vec![vec!["next".to_string()]]); + } } From 42c254c87a466f05147e1e54fcabf549d01a3a83 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:10:16 +0100 Subject: [PATCH 064/112] #3705: Para::trim_with drops the leading blank spans with one drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 74: trimming removed each leading whitespace-only inline with Vec::remove(0), shifting the rest every time. A paragraph opening with many separately formatted line breaks (HTML

…, LaTeX \\) took quadratic time on open or paste. The removable prefix is now counted and drained once; the first remaining span is trimmed as before. 200,000 pairs ran past 60 s before, now milliseconds. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/html.rs | 12 ++++++++++++ crates/formats/src/model.rs | 38 +++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index fa6ec57997..06510d6b3e 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -2424,4 +2424,16 @@ mod tests { assert!(html.contains("vertical-align:middle")); assert!(html.contains("vertical-align:bottom")); } + + #[test] + fn many_formatted_leading_breaks_trim_quickly() { + // #3705: a paragraph opening with alternating bold and italic `
` was trimmed one + // span at a time with `Vec::remove(0)`. + let html = format!("

{}x

", "

".repeat(100_000)); + let t = std::time::Instant::now(); + let f = parse(&html); + assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + let texts: Vec = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); + assert_eq!(texts, ["x"]); + } } diff --git a/crates/formats/src/model.rs b/crates/formats/src/model.rs index ef050223ea..46adc7fcef 100644 --- a/crates/formats/src/model.rs +++ b/crates/formats/src/model.rs @@ -239,15 +239,13 @@ impl Para { /// Remove the characters in `start` from the start of the paragraph's text and those in /// `end` from its end. pub fn trim_with(&mut self, start: &[char], end: &[char]) { - while let Some(Inline::Text(t, _)) = self.inlines.first_mut() { - let n = t.trim_start_matches(start).len(); - let cut = t.len() - n; + // The leading spans that trim to nothing go in one `drain`: removing them one at a time + // shifted the rest each time, quadratic in a run of separately formatted line breaks. + let k = self.inlines.iter().take_while(|i| matches!(i, Inline::Text(t, _) if t.trim_start_matches(start).is_empty())).count(); + self.inlines.drain(..k); + if let Some(Inline::Text(t, _)) = self.inlines.first_mut() { + let cut = t.len() - t.trim_start_matches(start).len(); t.replace_range(..cut, ""); - if t.is_empty() { - self.inlines.remove(0); - } else { - break; - } } while let Some(Inline::Text(t, _)) = self.inlines.last_mut() { let n = t.trim_end_matches(end).len(); @@ -1558,4 +1556,28 @@ mod tests { assert_eq!(t.rows[1].len(), 2); assert!(t.rows[1][0].covered); } + + #[test] + fn trim_drops_a_long_run_of_blank_spans_in_linear_time() { + // Separately formatted line breaks don't merge; each used to be removed with + // `Vec::remove(0)`, shifting all the rest (quadratic). + let mut p = Para::default(); + let (b, i) = (Fmt { bold: true, ..Default::default() }, Fmt { italic: true, ..Default::default() }); + for _ in 0..200_000 { + p.push_text("\n", &b); + p.push_text(" ", &i); + } + p.push_text(" x ", &Fmt::default()); + let t = std::time::Instant::now(); + p.trim(); + assert!(t.elapsed() < std::time::Duration::from_secs(1), "{:?}", t.elapsed()); + assert_eq!(p.inlines, vec![Inline::Text("x".into(), Fmt::default())]); + // The first span left keeps its formatting and loses only its leading blanks. + let mut p = Para::default(); + p.push_text(" \t", &b); + p.push_text(" a ", &i); + p.push_text("b ", &b); + p.trim(); + assert_eq!(p.inlines, vec![Inline::Text("a ".into(), i), Inline::Text("b".into(), b)]); + } } From f6398e29a90fd725228e0494ea6827163bc1ee83 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:11:05 +0100 Subject: [PATCH 065/112] #3714: Markdown import tracks the text before each delimiter instead of searching back for it Finding 67: when emphasis was flattened, every delimiter run searched backwards for the nearest text node to borrow its formatting, reaching the start of the paragraph when the paragraph was all delimiters (or links): quadratic. The last text node's formatting is now carried forward through the loop, and nothing is looked up for a delimiter with no literal left. The result is the same. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/markdown.rs | 43 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index e483cc928d..8274effe0e 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -1202,18 +1202,16 @@ fn emit(mut nodes: Vec) -> Vec { } out.push(Inline::Text(t.to_string(), f)); }; - let base_of = |nodes: &[Node], k: usize| -> Fmt { - // Delimiter literals take the formatting (link, …) of neighbouring text. - for j in (0..k).rev() { - if let Some(Node::Text(_, f)) = nodes.get(j) { - return Fmt { bold: false, italic: false, strike: false, ..f.clone() }; - } - } - Fmt::default() - }; - for (k, n) in nodes.iter().enumerate() { + // Delimiter literals take the formatting (link, …) of the text before them: the last text + // node, tracked here rather than searched for backwards from every delimiter (quadratic on + // a paragraph of delimiters). + let mut last_text: Option<&Fmt> = None; + for n in &nodes { match n { - Node::Text(t, f) => push(&mut out, t, f, em, strong, strike), + Node::Text(t, f) => { + push(&mut out, t, f, em, strong, strike); + last_text = Some(f); + } Node::Inl(Inline::Text(t, f)) => push(&mut out, t, f, em, strong, strike), Node::Inl(other) => out.push(other.clone()), Node::Delim { ch, left, opens, closes, .. } => { @@ -1224,9 +1222,11 @@ fn emit(mut nodes: Vec) -> Vec { Emph::Strike => strike = strike.saturating_sub(1), } } - let lit: String = std::iter::repeat_n(*ch, *left).collect(); - let f = base_of(&nodes, k); - push(&mut out, &lit, &f, em, strong, strike); + if *left > 0 { + let lit: String = std::iter::repeat_n(*ch, *left).collect(); + let f = last_text.map_or_else(Fmt::default, |f| Fmt { bold: false, italic: false, strike: false, ..f.clone() }); + push(&mut out, &lit, &f, em, strong, strike); + } for e in opens { match e { Emph::Em => em += 1, @@ -2152,4 +2152,19 @@ mod tests { let texts: Vec = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); assert_eq!(texts, ["Hello world!", "After comment."]); } + + #[test] + fn a_paragraph_of_delimiters_parses_in_linear_time() { + // #3714: each delimiter's literal looked back for the text before it, to the start of + // the paragraph when there was none (quadratic). + let src = "*_".repeat(200_000); + let t = std::time::Instant::now(); + let out = inl(&src); + assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + let text: String = out.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect(); + assert!(text.chars().all(|c| matches!(c, '*' | '_')), "only delimiters come out"); + // Literal delimiters still take the link of the text before them. + let s = inl("[a](http://x.org) [b*](http://y.org)"); + assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "b*" && f.link.as_deref() == Some("http://y.org"))), "{s:?}"); + } } From e9fe3d29b6056e53b15a92ef4040bb020c0395ab Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:12:05 +0100 Subject: [PATCH 066/112] #3692: Markdown export escapes link destinations instead of deleting characters Finding 53: link_target removed `<`, `>` and newlines from a destination after is_safe_href had accepted it, so an accepted relative target such as `java` (or `/`, a backslash that a reader would treat as an escape, and `&` that starts an entity reference, so a CommonMark reader gets back exactly the checked target; control characters, which the check ignores, are still left out. The importer now ends a `<...>` destination at the first unescaped `>`, so these round-trip. Ordinary destinations are written as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/markdown.rs | 75 +++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index 8274effe0e..3617af530c 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -799,10 +799,18 @@ fn link_dest(s: &str, at: usize) -> Option<(String, usize)> { i += ws; let r = s.get(i..)?; let (dest, used) = if let Some(inner) = r.strip_prefix('<') { - let end = inner.find(['>', '\n'])?; - if inner.get(end..)?.starts_with('\n') { - return None; - } + // The first `>` not escaped by a backslash ends it; a line break first means no link. + let b = inner.as_bytes(); + let mut k = 0; + let end = loop { + match b.get(k)? { + b'\\' => k += 1, + b'>' => break k, + b'\n' => return None, + _ => {} + } + k += 1; + }; (inner.get(..end)?.to_string(), end + 2) } else { let mut depth = 0i32; @@ -1436,8 +1444,35 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { out } +/// `l` as a Markdown link destination that reads back as exactly `l`, the target +/// `is_safe_href` passed. What Markdown would decode (a backslash escape, an entity reference) +/// or end the destination on (`<`, `>`) is backslash-escaped, never deleted: deleting `<` or `>` +/// could turn an accepted relative target into `javascript:…` or `//host`. Control characters +/// can't be written in a destination and are left out, as the safety check ignores them. fn link_target(l: &str) -> String { - if l.contains([' ', '(', ')', '<', '>']) { format!("<{}>", l.replace(['<', '>', '\n'], "")) } else { l.to_string() } + let l: String = l.chars().filter(|c| !c.is_ascii_control()).collect(); + let mut out = String::with_capacity(l.len() + 2); + for (i, c) in l.char_indices() { + let rest = l.get(i + c.len_utf8()..).unwrap_or(""); + match c { + '<' | '>' => out.push('\\'), + // A backslash is literal unless punctuation (or the closing `)`/`>`) follows it. + '\\' if rest.chars().next().is_none_or(|n| n.is_ascii_punctuation()) => out.push('\\'), + '&' if entity_like(rest) => out.push('\\'), + _ => {} + } + out.push(c); + } + if l.contains([' ', '(', ')', '<', '>']) { format!("<{out}>") } else { out } +} + +/// Whether `rest` (the text after a `&`) starts like an entity reference (`amp;`, `#106;`, +/// `#x6A;`), which a Markdown reader would decode. +fn entity_like(rest: &str) -> bool { + let Some(end) = within(rest, 34).find(';') else { return false }; + let body = rest.get(..end).unwrap_or(""); + let body = body.strip_prefix('#').map_or(body, |n| n.strip_prefix(['x', 'X']).unwrap_or(n)); + !body.is_empty() && body.chars().all(|c| c.is_ascii_alphanumeric()) } fn marker(m: Mark) -> &'static str { @@ -2167,4 +2202,34 @@ mod tests { let s = inl("[a](http://x.org) [b*](http://y.org)"); assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "b*" && f.link.as_deref() == Some("http://y.org"))), "{s:?}"); } + + #[test] + fn exported_link_destinations_read_back_unchanged() { + // Removing `<` and `>` from a destination after the safety check could make a relative + // target a script or network-path one (`java)"); + for l in [ + "javab c", + "javascript:alert(1)", + "javascript:x", + "&", + "javascript\\:x", + "dir\\", + "dir\\(1)\\", + "q?a=1&b=2", + ] { + let out = md(l); + let back = inl(&out); + let got: Vec<_> = back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.clone() } else { None }).collect(); + assert_eq!(got, [l.to_string()], "{l:?} → {out:?}"); + } + // Ordinary destinations are written as before. + assert_eq!(md("https://example.com/a?b=1&c=2"), "[x](https://example.com/a?b=1&c=2)"); + assert_eq!(md("https://example.com/a b"), "[x]()"); + assert_eq!(md("https://e.com/x\ny"), "[x](https://e.com/xy)"); + } } From faa8221bca4b6e9b68b702ea9d6e3729a75d47d4 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:13:40 +0100 Subject: [PATCH 067/112] =?UTF-8?q?#3692:=20LaTeX=20=E2=80=94=20verbatim?= =?UTF-8?q?=20delimiter=20check=20after=20control=20characters=20go;=20mul?= =?UTF-8?q?ticolumn/multirow=20unwrapping=20bounded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 57: Writer::code replaced `\end{verbatim}` in a code line and then removed control characters, so `\end{verbatim}` became a real closing delimiter and the rest of the line was compiled as TeX. Control characters are now removed first and the replacement runs on the final text. Finding 52: the table reader unwrapped every nested \multicolumn / \multirow in a cell, rescanning the remaining braced content each time (quadratic; the nesting limit didn't apply). At most MAX_NEST wrappers are unwrapped now; real tables use one or two, which read as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/latex.rs | 46 +++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/crates/formats/src/latex.rs b/crates/formats/src/latex.rs index 10de892120..663784624e 100644 --- a/crates/formats/src/latex.rs +++ b/crates/formats/src/latex.rs @@ -243,8 +243,10 @@ impl Writer { } self.out.push_str("\\begin{verbatim}\n"); for l in lines { - // The environment ends at the first `\end{verbatim}`, so keep it from appearing. - let l: String = l.replace("\\end{verbatim}", "\\end {verbatim}").chars().filter(|c| *c == '\t' || !c.is_control()).collect(); + // The environment ends at the first `\end{verbatim}`, so keep it from appearing. The + // control characters go first: removing one afterwards could join a delimiter back up. + let l: String = l.chars().filter(|c| *c == '\t' || !c.is_control()).collect(); + let l = l.replace("\\end{verbatim}", "\\end {verbatim}"); self.out.push_str(&l); self.out.push('\n'); } @@ -2671,19 +2673,24 @@ fn table(toks: &[Tok], spec: &str, depth: usize) -> FTable { for cell_toks in r { let mut cell = Cell::default(); let mut content = trim_cell(cell_toks); - // `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`. + // `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`. Each wrapper + // rescans its content, so only MAX_NEST of them are unwrapped (a cell nests one or + // two); further ones stay in the content. let mut sub = Parser::new(content, depth + 1); + let mut wrappers = 0usize; loop { sub.skip_spaces(); match sub.t.get(sub.pos) { - Some(Tok::Cmd(c)) if c == "multicolumn" => { + Some(Tok::Cmd(c)) if c == "multicolumn" && wrappers < MAX_NEST => { + wrappers += 1; sub.pos += 1; cell.colspan = sub.arg_raw().trim().parse::().unwrap_or(1).clamp(1, 63); let _ = sub.arg_raw(); content = sub.arg_tokens(); sub = Parser::new(content, depth + 1); } - Some(Tok::Cmd(c)) if c == "multirow" || c == "multirow*" => { + Some(Tok::Cmd(c)) if (c == "multirow" || c == "multirow*") && wrappers < MAX_NEST => { + wrappers += 1; sub.pos += 1; let _ = sub.opt_arg(); cell.rowspan = sub.arg_raw().trim().parse::().map(|n| n.unsigned_abs().clamp(1, 1000) as u32).unwrap_or(1); @@ -3234,4 +3241,33 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after. let Some(Block::Table(bt_bordered)) = back_bordered.body.first().map(|b| &**b) else { panic!() }; assert!(bt_bordered.props.style.is_some(), "imported table should remain bordered"); } + + #[test] + fn verbatim_end_is_kept_out_after_control_characters_go() { + // A control character inside `\end{verbatim}` was removed after the check for it, so + // the code line closed the environment and the rest ran as TeX. + let mut d = Document::new(); + let line = "\\end{verb\u{E}atim}\\input{/etc/passwd}"; + d.body = vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(line, Default::default()).styled(model::CODE_STYLE))]; + let out = export(&d); + assert_eq!(out.matches("\\end{verbatim}").count(), 1, "{out}"); + assert!(out.contains("\\end {verbatim}\\input"), "{out}"); + } + + #[test] + fn nested_multicolumn_wrappers_are_bounded() { + // Each `\multicolumn` unwrapped rescanned the rest of the cell; nested ones were all + // unwrapped, quadratic in the nesting. + let n = 50_000; + let src = format!("\\begin{{tabular}}{{l}}{}x{}\\\\\\end{{tabular}}", "\\multicolumn{1}{c}{".repeat(n), "}".repeat(n)); + let t = std::time::Instant::now(); + let f = parse(&src); + assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + assert!(f.blocks.iter().any(|b| matches!(b, FBlock::Table(_)))); + // One or two wrappers, as written in real tables, are still read. + let f = parse("\\begin{tabular}{ll}\\multicolumn{2}{c}{\\multirow{2}{*}{Both}}\\\\\\end{tabular}"); + let Some(FBlock::Table(t)) = f.blocks.first() else { panic!("{:?}", f.blocks) }; + assert_eq!((t.rows[0][0].colspan, t.rows[0][0].rowspan), (2, 2)); + assert_eq!(t.rows[0][0].text(), "Both"); + } } From 79e3610cbbe4dd1581f5a0e23a77747ccde4e20b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:24:47 +0100 Subject: [PATCH 068/112] #3713: importers drop overlong font names and links; spans share them; runs' copies budgeted Findings 58 and 65. Every span deep-copied its inherited link and font strings, so one long name or Markdown definition times many spans or references expanded a small file into gigabytes. Fmt now holds both as Arc (spans and Markdown references share one copy), names over 256 bytes and links over 8 KiB are dropped at the importers (RTF, HTML, Markdown, ODT, LaTeX), and the document builder stops giving runs link and font copies past 64 MiB. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/html.rs | 6 +-- crates/formats/src/latex.rs | 15 ++++--- crates/formats/src/markdown.rs | 21 +++++---- crates/formats/src/model.rs | 56 +++++++++++++++++++----- crates/formats/src/odt.rs | 9 ++-- crates/formats/src/rtf.rs | 22 ++++++---- crates/formats/src/tests.rs | 78 ++++++++++++++++++++++++++++++++-- 7 files changed, 162 insertions(+), 45 deletions(-) diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index 06510d6b3e..4970855ed4 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -724,7 +724,7 @@ fn apply_style(css: &str, f: &mut Fmt, align: &mut Option, preserve: &mut if model::is_mono(&fam) || fam.eq_ignore_ascii_case("monospace") { f.code = true; } else if !fam.is_empty() && !matches!(fam.to_ascii_lowercase().as_str(), "serif" | "sans-serif" | "inherit" | "initial") { - f.font = Some(fam); + f.font = model::kept_font(&fam).or(f.font.take()); } } "vertical-align" => { @@ -1226,7 +1226,7 @@ impl<'r> Builder<'r> { if model::is_mono(&fam) { fmt.code = true; } else if !fam.is_empty() { - fmt.font = Some(fam); + fmt.font = model::kept_font(&fam).or(fmt.font.take()); } } if let Some(sz) = attr(attrs, "size") { @@ -1245,7 +1245,7 @@ impl<'r> Builder<'r> { } "a" => { if let Some(h) = attr(attrs, "href").filter(|h| wordcraft_doc::link::is_safe_href(h)) { - fmt.link = Some(h.trim().to_string()); + fmt.link = model::kept_link(h.trim()); } } _ => {} diff --git a/crates/formats/src/latex.rs b/crates/formats/src/latex.rs index 663784624e..18d807fdd1 100644 --- a/crates/formats/src/latex.rs +++ b/crates/formats/src/latex.rs @@ -2335,20 +2335,21 @@ impl<'a> Parser<'a> { } } "href" => { - let url = self.arg_raw(); - self.with_arg(move |f| f.link = Some(url)); + let url = model::kept_link(&self.arg_raw()); + self.with_arg(move |f| f.link = url); } "url" | "nolinkurl" => { let url = self.arg_raw(); let mut f = self.fmt(); if name == "url" { - f.link = Some(url.clone()); + f.link = model::kept_link(&url); } self.para.push_text(&url, &f); } "hyperlink" => { let target = self.arg_raw(); - self.with_arg(move |f| f.link = Some(format!("#{target}"))); + let link = model::kept_link(&format!("#{target}")); + self.with_arg(move |f| f.link = link); } "hypertarget" => { let target = self.arg_raw(); @@ -2362,7 +2363,7 @@ impl<'a> Parser<'a> { "ref" | "eqref" | "pageref" | "autoref" | "cref" | "Cref" | "nameref" => { let l = self.arg_raw(); let mut f = self.fmt(); - f.link = Some(format!("#{l}")); + f.link = model::kept_link(&format!("#{l}")); self.para.push_text(&l, &f); } "cite" | "citep" | "citet" | "parencite" | "textcite" | "autocite" => { @@ -3168,13 +3169,13 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after. let FBlock::Para(p) = &flow.blocks[0] else { panic!() }; assert_eq!( p.inlines[0], - Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() }) + Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() }) ); assert_eq!( p.inlines[2], Inline::Text( "https://example.com/~user/page".to_string(), - Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() } + Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() } ) ); } diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index 3617af530c..e5346cca7e 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -10,6 +10,7 @@ //! Export writes the same constructs; pictures become `![alt](media-name)` placeholders. use std::collections::HashMap; +use std::sync::Arc; use wordcraft_doc::{Align, Document}; @@ -33,7 +34,8 @@ pub fn import_on(text: &str, page: &wordcraft_doc::SectionProps) -> Document { } /// Link reference definitions: the normalised label to its destination. -pub type Refs = HashMap; +/// A destination is shared by every reference to it, not copied into each (#3713). +pub type Refs = HashMap>; /// A label as CommonMark compares them: case-folded, and any run of white space one space /// (§4.7, "matches"). `[FOO bar]` and `[foo BAR]` are the same label. @@ -141,7 +143,7 @@ fn take_definitions(lines: &mut Vec) -> Refs { if starts_block && let Some((label, dest)) = link_definition(line) { // The first definition of a label wins, as CommonMark says, and definitions may // follow one another. - refs.entry(label).or_insert(dest); + refs.entry(label).or_insert(dest.into()); return false; } starts_block = is_blank(line) || atx_heading(t).is_some() || thematic_break(t); @@ -991,7 +993,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { let dest = link_dest(s, close + 1); scan_budget = scan_budget.saturating_sub(dest.as_ref().map_or(s.len(), |d| d.1).saturating_sub(close)); let (dest, end) = dest?; - return Some((text.to_string(), dest, end)); + return Some((text.to_string(), Arc::from(dest), end)); } // A reference link: `[text][label]`, `[text][]` and `[text]` all take // their destination from a definition collected before parsing (#3238). @@ -1028,7 +1030,8 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { Some((inner, dest, end)) if base.link.is_none() => { flush!(); let mut f = base.clone(); - f.link = Some(dest); + // A destination too long to keep leaves its text unlinked. + f.link = (dest.len() <= model::MAX_LINK).then_some(dest); for x in parse_inlines(&inner, &f, depth + 1, refs) { nodes.push(Node::Inl(x)); } @@ -1055,7 +1058,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { if is_auto { flush!(); let mut f = base.clone(); - f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}") } else { inner.to_string() }); + f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}").into() } else { inner.into() }); push_text(&mut nodes, inner, &f); i += inner.len() + 2; text_start = i; @@ -1386,7 +1389,7 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { } let mut out = String::new(); let mut stack: Vec = Vec::new(); - let mut link: Option = None; + let mut link: Option> = None; let close_all = |out: &mut String, stack: &mut Vec| { while let Some(m) = stack.pop() { out.push_str(marker(m)); @@ -1675,7 +1678,7 @@ mod tests { let s = inl("see [the *site*](http://x.org \"T\") and `a*b*` \\*lit\\* & "); let link: Vec<_> = s .iter() - .filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.clone(), f.italic)) } else { None }) + .filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.to_string(), f.italic)) } else { None }) .collect(); assert_eq!(link[0], ("the ".into(), "http://x.org".into(), false)); assert_eq!(link[1], ("site".into(), "http://x.org".into(), true)); @@ -1696,7 +1699,7 @@ mod tests { .inlines .iter() .filter_map(|x| match x { - Inline::Text(t, f) => f.link.clone().map(|l| (t.clone(), l)), + Inline::Text(t, f) => f.link.as_deref().map(|l| (t.clone(), l.to_string())), _ => None, }) .collect(), @@ -2224,7 +2227,7 @@ mod tests { ] { let out = md(l); let back = inl(&out); - let got: Vec<_> = back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.clone() } else { None }).collect(); + let got: Vec<_> = back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); assert_eq!(got, [l.to_string()], "{l:?} → {out:?}"); } // Ordinary destinations are written as before. diff --git a/crates/formats/src/model.rs b/crates/formats/src/model.rs index 46adc7fcef..156269b1be 100644 --- a/crates/formats/src/model.rs +++ b/crates/formats/src/model.rs @@ -19,6 +19,26 @@ pub const MONO_FONT: &str = "Courier New"; pub const CODE_STYLE: &str = "Code"; /// Nesting limit for tables in tables (hostile input). pub const MAX_DEPTH: usize = 8; +/// Longest font name the importers keep, in bytes (as the .docx reader); a longer one (no real +/// font has one) is dropped (#3713). +pub const MAX_FONT_NAME: usize = 256; +/// Longest hyperlink target the importers keep, in bytes; a longer link is dropped and its text +/// kept (#3713). +pub const MAX_LINK: usize = 8 << 10; +/// Link and font bytes the document's runs may carry between them. A flow's spans share one +/// copy of a name ([`Fmt`]), but every run of a document owns its own: past this budget, runs +/// lose their link and font rather than multiply a small file into gigabytes (#3713). +const MAX_RUN_STRINGS: usize = 64 << 20; + +/// `name` as a span's font, unless it is longer than [`MAX_FONT_NAME`]. +pub fn kept_font(name: &str) -> Option> { + (name.len() <= MAX_FONT_NAME).then(|| name.into()) +} + +/// `target` as a span's link, unless it is longer than [`MAX_LINK`]. +pub fn kept_link(target: &str) -> Option> { + (target.len() <= MAX_LINK).then(|| target.into()) +} /// Character formatting the text formats can express. #[derive(Clone, Debug, Default, PartialEq)] @@ -30,11 +50,13 @@ pub struct Fmt { pub code: bool, pub sup: bool, pub sub: bool, - pub link: Option, + /// Shared: every span of a link holds the same target. + pub link: Option>, pub color: Option, /// Points. pub size: Option, - pub font: Option, + /// Shared, as `link`. + pub font: Option>, /// Background (highlight / shading). pub background: Option, /// Tracked insertion of this text (formats that keep revisions: RTF). @@ -429,10 +451,10 @@ pub fn char_props(f: &Fmt) -> CharProps { } else { None }, - link: f.link.clone().filter(|l| !l.is_empty()), + link: f.link.as_deref().filter(|l| !l.is_empty()).map(str::to_string), color: f.color.map(TextColor::Rgb), size: f.size.filter(|s| s.is_finite() && *s >= 1.0 && *s <= 1638.0), - font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.clone().filter(|s| !s.is_empty()) }, + font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.as_deref().filter(|s| !s.is_empty()).map(str::to_string) }, shading: f.background, ..Default::default() } @@ -516,6 +538,8 @@ struct Builder<'a> { doc: &'a mut Document, /// Revisions already in `doc` by (kind, author, date), so each change is added once. revs: std::collections::HashMap<(u8, RevMark), u32>, + /// Link and font bytes given to runs so far ([`MAX_RUN_STRINGS`]). + run_strings: usize, } fn rev_key(kind: RevisionKind) -> u8 { @@ -535,7 +559,7 @@ impl<'a> Builder<'a> { revs.entry((rev_key(r.kind), mark)).or_insert(i as u32); } } - Builder { doc, revs } + Builder { doc, revs, run_strings: 0 } } /// The document revision for a tracked change, added the first time it is seen. @@ -550,6 +574,17 @@ impl<'a> Builder<'a> { i } + /// `cp` as a run may keep it: without its link and font once the runs' copies of them are + /// over budget. + fn budget(&mut self, mut cp: CharProps) -> CharProps { + let n = cp.link.as_ref().map_or(0, String::len).saturating_add(cp.font.as_ref().map_or(0, String::len)); + match self.run_strings.checked_add(n).filter(|&t| t <= MAX_RUN_STRINGS) { + Some(t) => self.run_strings = t, + None => (cp.link, cp.font) = (None, None), + } + cp + } + fn para(&mut self, p: &Para, list_num: Option) -> Paragraph { let mut out = Paragraph::new(); out.props = para_props(p, list_num); @@ -575,11 +610,12 @@ impl<'a> Builder<'a> { match i { Inline::Text(t, f) => { let t = clean_text(t); - let mut cp = char_props(f); + let mut cp = self.budget(char_props(f)); cp.ins = f.ins.as_ref().map(|m| self.revision(RevisionKind::Insert, m)); cp.del = f.del.as_ref().map(|m| self.revision(RevisionKind::Delete, m)); if let Some(c) = &f.fmt_change { - cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), char_props(&c.old)); + let old = self.budget(char_props(&c.old)); + cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), old); } if p.kind == Kind::Code { // Code's font comes from its style, before a change as after it. @@ -614,7 +650,7 @@ impl<'a> Builder<'a> { } Inline::Ruby { base, ruby, fmt } => { let (base, ruby) = (clean_text(base), clean_text(ruby)); - let cp = char_props(fmt); + let cp = self.budget(char_props(fmt)); // A source without a base or without a reading (malformed markup) is just its // text: a guide that annotates nothing is no guide. if base.is_empty() || ruby.is_empty() { @@ -881,13 +917,13 @@ fn fmt_of(doc: &Document, cp: &CharProps, in_code: bool) -> Fmt { code, sup: e.vert_align == Some(VertAlign::Superscript), sub: e.vert_align == Some(VertAlign::Subscript), - link: e.link.clone(), + link: e.link.as_deref().map(Arc::from), color: match e.color { Some(TextColor::Rgb(c)) => Some(c), _ => None, }, size: e.size, - font: if code || in_code { None } else { e.shown_font().map(str::to_string) }, + font: if code || in_code { None } else { e.shown_font().map(Arc::from) }, background: e.highlight.and_then(|h| h.rgb()).or(e.shading), ins: None, del: None, diff --git a/crates/formats/src/odt.rs b/crates/formats/src/odt.rs index c73720819b..6be9608110 100644 --- a/crates/formats/src/odt.rs +++ b/crates/formats/src/odt.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::io::{Cursor, Read, Write}; +use std::sync::Arc; use quick_xml::XmlVersion; use quick_xml::events::{BytesStart, Event}; @@ -1034,7 +1035,7 @@ struct FmtDelta { color: Option, background: Option, size: Option, - font: Option, + font: Option>, code: Option, } @@ -1340,7 +1341,7 @@ impl Styles { if model::is_mono(&fam) || get(&a, "style:font-pitch").is_some_and(|p| p.trim().eq_ignore_ascii_case("fixed")) { d.code = Some(true); } else if !fam.is_empty() { - d.font = Some(fam); + d.font = model::kept_font(&fam); } } } @@ -1446,7 +1447,7 @@ impl Styles { /// The document's default font: the Standard paragraph style's, else the default style's. fn doc_font(&self) -> Option { - self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.clone()).or_else(|| self.default_font.clone()) + self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.as_deref().map(str::to_string)).or_else(|| self.default_font.clone()) } fn chain<'a>(&'a self, name: &'a str) -> Vec<(&'a str, &'a StyleInfo)> { @@ -2053,7 +2054,7 @@ impl Body<'_> { "a" => { let mut f = self.cur_fmt(); if let Some(h) = get(a, "xlink:href").filter(|h| !h.is_empty()) { - f.link = Some(h.to_string()); + f.link = model::kept_link(h); } if let Some(s) = get(a, "text:style-name") { f = self.styles.text_fmt(s, &f); diff --git a/crates/formats/src/rtf.rs b/crates/formats/src/rtf.rs index 280b2feae2..7b06aa082f 100644 --- a/crates/formats/src/rtf.rs +++ b/crates/formats/src/rtf.rs @@ -19,6 +19,7 @@ //! tables. Group nesting and picture sizes are capped. use std::collections::HashMap; +use std::sync::Arc; use wordcraft_doc::props::VAlign; use wordcraft_doc::{Align, Document, Rgb}; @@ -306,11 +307,13 @@ impl Writer { "\\nosupersub" }); } - let font = |f: &Fmt| if f.code { Some(model::MONO_FONT.to_string()) } else { f.font.clone() }; + fn font(f: &Fmt) -> Option<&str> { + if f.code { Some(model::MONO_FONT) } else { f.font.as_deref() } + } let old_font = font(old); if old_font != font(cur) { // No font is the document's default font, font 0 (Normal's). - let i = old_font.map_or(0, |n| self.t.font(&n)); + let i = old_font.map_or(0, |n| self.t.font(n)); ctl.push_str(&format!("\\f{i}")); } let half_points = |s: Option| s.map(|s| (s * 2.0).round().clamp(2.0, 3276.0) as i32); @@ -1106,7 +1109,7 @@ struct Reader { st: St, stack: Vec, overflow: usize, - fonts: HashMap, + fonts: HashMap>, font_cur: i32, font_name: String, /// Code pages of the fonts whose charset names one (`\fcharset204` → 1251). @@ -1398,7 +1401,9 @@ impl Reader { fn commit_font(&mut self) { let name = std::mem::take(&mut self.font_name).trim().trim_end_matches(';').trim().to_string(); - if !name.is_empty() && self.fonts.len() < 4096 { + if let Some(name) = model::kept_font(&name).filter(|n| !n.is_empty()) + && self.fonts.len() < 4096 + { self.fonts.insert(self.font_cur, name); if let Some(cp) = self.font_cp_cur.take() { self.font_cps.insert(self.font_cur, cp); @@ -1626,7 +1631,8 @@ impl Reader { for c in s.chars() { if c == ';' { self.commit_font(); - } else { + } else if self.font_name.len() <= model::MAX_FONT_NAME { + // One past the limit is enough for `commit_font` to drop the name. self.font_name.push(c); } } @@ -1646,7 +1652,7 @@ impl Reader { for c in s.chars() { if c == ';' { self.commit_style(); - } else { + } else if self.style_name.len() < model::MAX_FONT_NAME { self.style_name.push(c); } } @@ -2329,7 +2335,7 @@ impl Reader { "fldrslt" => { let link = self.st.field.and_then(|i| self.fields.get(i)).and_then(|s| hyperlink(s)); if link.is_some() { - self.st.fmt.link = link; + self.st.fmt.link = link.as_deref().and_then(model::kept_link); } } _ => {} @@ -2807,7 +2813,7 @@ fn parse_full(bytes: &[u8]) -> Result { if blocks.is_empty() { blocks.push(FBlock::Para(Para::default())); } - let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).cloned(); + let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).map(|n| n.to_string()); Ok(Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers }) } diff --git a/crates/formats/src/tests.rs b/crates/formats/src/tests.rs index ca6b6f9926..5ef26d5f19 100644 --- a/crates/formats/src/tests.rs +++ b/crates/formats/src/tests.rs @@ -749,7 +749,7 @@ fn code_spans_and_links_around_backticks_read_as_before() { let flow = crate::markdown::parse("\\``e` `` a ` b `` then ``` and `c` [x `]` y](http://a.org) [p ` q](http://b.org) ``f"); let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) }; let spans: Vec<(String, bool, Option)> = - p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.clone())) } else { None }).collect(); + p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.as_deref().map(str::to_string))) } else { None }).collect(); let code: Vec<&str> = spans.iter().filter(|s| s.1).map(|s| s.0.as_str()).collect(); assert_eq!(code, ["e", "a ` b", "c", "]"], "{spans:?}"); let linked = |url: &str| spans.iter().filter(|s| s.2.as_deref() == Some(url)).map(|s| s.0.as_str()).collect::(); @@ -766,7 +766,7 @@ fn latex_href_keeps_ampersand() { .iter() .filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }) .flat_map(|p| p.inlines.iter()) - .filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.clone()) } else { None }) + .filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.as_deref().map(str::to_string)) } else { None }) .collect(); assert_eq!(links, [Some("https://e.com/a?b=1&c=2".to_string())]); @@ -775,7 +775,7 @@ fn latex_href_keeps_ampersand() { .iter() .filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }) .flat_map(|p| p.inlines.iter()) - .filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.clone())) } else { None }) + .filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.as_deref().map(str::to_string))) } else { None }) .collect(); let url = "https://e.com/a?b=1&c=2".to_string(); assert_eq!(texts, [(url.clone(), Some(url))]); @@ -789,7 +789,7 @@ fn brackets_do_not_turn_later_links_into_text() { let links = |md: &str| { let flow = crate::markdown::parse(md); let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) }; - let mut urls: Vec = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.clone() } else { None }).collect(); + let mut urls: Vec = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); urls.dedup(); urls }; @@ -875,3 +875,73 @@ fn hidden_drawings_are_left_out_of_exports() { // Two pictures, one hidden: one comes out. assert_eq!(crate::html::export(&d).matches(" Vec<(String, model::Fmt)> { + flow.blocks + .iter() + .filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }) + .flat_map(|p| p.inlines.iter()) + .filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.clone())) } else { None }) + .collect() +} + +/// A font name or link target too long for any real one is dropped, its text kept: every run +/// carried its own copy, so one long name times many runs was gigabytes (#3713). +#[test] +fn overlong_font_names_and_links_are_dropped() { + let long = "x".repeat(1 << 20); + let rtf = format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}}}\\f1 {}}}", "{\\b a}a".repeat(1000)); + let flow = crate::rtf::parse(rtf.as_bytes()).unwrap(); + let sp = spans(&flow); + assert!(sp.iter().map(|s| s.0.len()).sum::() >= 2000); + assert!(sp.iter().all(|(_, f)| f.font.is_none()), "an overlong RTF font name is no font"); + // A real one still is. + let flow = crate::rtf::parse(b"{\\rtf1{\\fonttbl{\\f0 Arial;}{\\f1 Georgia;}}\\f1 a}").unwrap(); + assert_eq!(spans(&flow)[0].1.font.as_deref(), Some("Georgia")); + + let html = format!("

kept too

"); + let sp = spans(&crate::html::parse(&html)); + assert!(sp.iter().any(|(t, _)| t.contains("kept")) && sp.iter().any(|(t, _)| t.contains("too"))); + assert!(sp.iter().all(|(_, f)| f.font.is_none() && f.link.is_none()), "{:?}", sp.iter().map(|s| &s.0).collect::>()); + + let sp = spans(&crate::markdown::parse(&format!("[kept](https://e.com/{long})"))); + assert_eq!(sp.len(), 1); + assert_eq!((sp[0].0.as_str(), sp[0].1.link.is_none()), ("kept", true)); +} + +/// Spans of one link share its target, and so do references to one definition: a long +/// definition and many references no longer copy the destination into each (#3713). +#[test] +fn spans_share_their_link_target() { + let dest = format!("https://e.com/{}", "x".repeat(4000)); + let md = format!("{}\n\n[r]: {dest}\n", "[a][r] **[b][r]** ".repeat(10_000)); + let sp = spans(&crate::markdown::parse(&md)); + let links: Vec<_> = sp.iter().filter_map(|(_, f)| f.link.as_ref()).collect(); + assert_eq!(links.len(), 20_000); + assert!(links.iter().all(|l| Arc::ptr_eq(l, links[0])), "one copy of the destination"); + + let html = format!("{}", "aa".repeat(1000)); + let sp = spans(&crate::html::parse(&html)); + assert!(sp.len() >= 2000); + let first = sp[0].1.link.clone().unwrap(); + assert!(sp.iter().all(|(_, f)| f.link.as_ref().is_some_and(|l| Arc::ptr_eq(l, &first)))); +} + +/// The document's runs own their link and font strings; past a budget further runs keep their +/// text but lose them, so a flow of shared names can't expand into gigabytes (#3713). +#[test] +fn run_link_and_font_copies_are_budgeted() { + let link: Arc = format!("https://e.com/{}", "x".repeat(model::MAX_LINK - 20)).into(); + let mut p = model::Para::default(); + for i in 0..20_000 { + let f = model::Fmt { bold: i % 2 == 0, link: Some(link.clone()), ..Default::default() }; + p.push_text("a", &f); + } + let doc = model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() }); + let paras: Vec<_> = doc.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p) } else { None }).collect(); + assert_eq!(paras.iter().map(|p| p.text.len()).sum::(), 20_000, "all text kept"); + let runs: Vec<_> = paras.iter().flat_map(|p| p.runs.iter()).collect(); + let kept: usize = runs.iter().filter_map(|r| r.props.link.as_ref()).map(String::len).sum(); + assert!(kept > 0 && kept <= 64 << 20, "{kept}"); +} From 9bee82220e89abdcf35e5631db28bd9cd73af53c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:24:47 +0100 Subject: [PATCH 069/112] Findings 40, 47, 52, 53, 57, 58, 65, 67, 74, 77 (#3692, #3705, #3713, Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/file-format-parity.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c2b352424f..2f58bcbe16 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity with Microsoft Word -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; importers bound hostile expansion: font names, links, ODS repeats, LaTeX wrappers, Markdown delimiters, #3692/#3713; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats Every format Word opens or saves, what WordCraft does with it, and how it's tested. Fidelity is the share of a typical real-world file's content and formatting that survives (read: shows correctly; @@ -121,7 +121,7 @@ Measured by grepping `crates/docx/src/read` and `src/write` for the OOXML elemen | `w:fitText` | ❌ | ❌ | East Asian layout | | Glossary document (building blocks) | ✅ | ✅ | `w:docPart` entries (name, style, gallery, category, types, behaviours, description, guid, body) are read into `Document::glossary` and offered by Insert › Quick Parts and AutoText; `save` with `"in": "document"` stores one in the file. An untouched glossary is written back byte for byte with its own parts (styles, settings, fonts…); an edited one, or one whose parts weren't kept, is written from the model as `word/glossary1.xml` with copies of the document's styles and numbering. The glossary's own styles, settings and fonts, text boxes inside entries, and the notes and comments of a selection saved into the document don't survive an edit. Tests: `crates/docx/tests/it/glossary.rs` | | `w:altChunk`, sub-documents | ❌ | ❌ | | -| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard | +| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard. Text importers drop font names over 256 bytes and links over 8 KiB, share a link or font between spans, and cap the runs' copies of them at 64 MiB (#3713) | | Document paragraph defaults (`w:pPrDefault`) | ✅ | ✅ | Omission and explicit empty defaults remain distinct across repeated saves; nonempty defaults and later default-spacing edits are written | ## Saving many content controls @@ -268,6 +268,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio | 2026-10-25 | trivial | Repeated DOCX style IDs: repeated save/reopen regression for whole-definition replacement. | | 2026-10-25 | trivial | Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`) | | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats | 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) | | 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | From d8501a5aeb58a85418e70bc9a8663945c2477241 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:47:09 +0100 Subject: [PATCH 070/112] #3713: rustfmt Finding 58 follow-up: formatting only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/markdown.rs | 3 ++- crates/formats/src/tests.rs | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index e5346cca7e..7970729e3d 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -2227,7 +2227,8 @@ mod tests { ] { let out = md(l); let back = inl(&out); - let got: Vec<_> = back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); + let got: Vec<_> = + back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); assert_eq!(got, [l.to_string()], "{l:?} → {out:?}"); } // Ordinary destinations are written as before. diff --git a/crates/formats/src/tests.rs b/crates/formats/src/tests.rs index 5ef26d5f19..16eb2afcac 100644 --- a/crates/formats/src/tests.rs +++ b/crates/formats/src/tests.rs @@ -748,8 +748,11 @@ fn code_spans_and_links_around_backticks_read_as_before() { // inside a code span in link text, and link text with a backtick that nothing closes. let flow = crate::markdown::parse("\\``e` `` a ` b `` then ``` and `c` [x `]` y](http://a.org) [p ` q](http://b.org) ``f"); let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) }; - let spans: Vec<(String, bool, Option)> = - p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.as_deref().map(str::to_string))) } else { None }).collect(); + let spans: Vec<(String, bool, Option)> = p + .inlines + .iter() + .filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.as_deref().map(str::to_string))) } else { None }) + .collect(); let code: Vec<&str> = spans.iter().filter(|s| s.1).map(|s| s.0.as_str()).collect(); assert_eq!(code, ["e", "a ` b", "c", "]"], "{spans:?}"); let linked = |url: &str| spans.iter().filter(|s| s.2.as_deref() == Some(url)).map(|s| s.0.as_str()).collect::(); @@ -789,7 +792,8 @@ fn brackets_do_not_turn_later_links_into_text() { let links = |md: &str| { let flow = crate::markdown::parse(md); let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) }; - let mut urls: Vec = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); + let mut urls: Vec = + p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect(); urls.dedup(); urls }; @@ -900,7 +904,8 @@ fn overlong_font_names_and_links_are_dropped() { let flow = crate::rtf::parse(b"{\\rtf1{\\fonttbl{\\f0 Arial;}{\\f1 Georgia;}}\\f1 a}").unwrap(); assert_eq!(spans(&flow)[0].1.font.as_deref(), Some("Georgia")); - let html = format!("

kept too

"); + let html = + format!("

kept too

"); let sp = spans(&crate::html::parse(&html)); assert!(sp.iter().any(|(t, _)| t.contains("kept")) && sp.iter().any(|(t, _)| t.contains("too"))); assert!(sp.iter().all(|(_, f)| f.font.is_none() && f.link.is_none()), "{:?}", sp.iter().map(|s| &s.0).collect::>()); From 582f4118764eb3981c7208f8775e1b3b519570e9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 19:56:02 +0100 Subject: [PATCH 071/112] =?UTF-8?q?#3692:=20Markdown=20export=20wraps=20li?= =?UTF-8?q?nk=20destinations=20with=20any=20Unicode=20whitespace=20in=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit review: only an ASCII space triggered the angle brackets, but the reader ends a bare destination at any whitespace (U+00A0, U+3000), so such a target read back as plain text. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/formats/src/markdown.rs | 2 +- crates/formats/src/tests.rs | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index 7970729e3d..cbf23326ab 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -1466,7 +1466,7 @@ fn link_target(l: &str) -> String { } out.push(c); } - if l.contains([' ', '(', ')', '<', '>']) { format!("<{out}>") } else { out } + if l.contains(|c: char| c.is_whitespace() || matches!(c, '(' | ')' | '<' | '>')) { format!("<{out}>") } else { out } } /// Whether `rest` (the text after a `&`) starts like an entity reference (`amp;`, `#106;`, diff --git a/crates/formats/src/tests.rs b/crates/formats/src/tests.rs index 16eb2afcac..dc23445347 100644 --- a/crates/formats/src/tests.rs +++ b/crates/formats/src/tests.rs @@ -867,6 +867,21 @@ fn exported_links_keep_only_safe_targets() { } } +/// A link target with any whitespace (a no-break or ideographic space too) is written in `<…>`, +/// so the reader, which ends a bare destination at the first whitespace, reads it back whole. +#[test] +fn markdown_link_targets_with_unicode_spaces_read_back() { + for l in ["https://e.com/a\u{a0}b", "https://e.com/a\u{3000}b", "https://e.com/a b"] { + let mut d = Document::new(); + let mut p = Paragraph::with_text("", CharProps::default()); + p.insert_text(0, "go", &CharProps { link: Some(l.to_string()), ..Default::default() }).unwrap(); + d.body = vec![para_block(p)]; + let md = String::from_utf8(export("md", &d).unwrap().unwrap()).unwrap(); + let sp = spans(&crate::markdown::parse(&md)); + assert_eq!(sp.iter().find(|(t, _)| t == "go").and_then(|(_, f)| f.link.as_deref()), Some(l), "md: {md}"); + } +} + #[test] fn hidden_drawings_are_left_out_of_exports() { let mut d = wordcraft_doc::Document::from_text("x"); From 41de47398a5d19da9d19d7e09e1f79619bf29b01 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 03:31:20 +0100 Subject: [PATCH 072/112] =?UTF-8?q?#3713:=20review=20=E2=80=94=20ODS=20rep?= =?UTF-8?q?eated=20cells=20are=20budgeted=20only=20for=20the=20copies=20th?= =?UTF-8?q?e=20grid=20can=20still=20keep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With one cell slot left, a long value repeated a thousand times was refused for all its copies' text although Grid::push keeps only the first, which fits. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/formats/src/sheet.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/formats/src/sheet.rs b/crates/formats/src/sheet.rs index 392b425c06..97cf1f8680 100644 --- a/crates/formats/src/sheet.rs +++ b/crates/formats/src/sheet.rs @@ -979,9 +979,10 @@ fn ods_read(xml: &[u8], want: Option) -> Result<(Vec, Vec"#, + MAX_COLS - 1 + )); + let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap(); + assert_eq!(rows.len(), full + 2); + assert_eq!(rows.last(), Some(&vec![wide])); + assert_eq!(rows.iter().map(Vec::len).sum::(), MAX_CELLS); + } } From 2c4f4669c605dd053342b2db8064b55239ae99fa Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:08:38 +0100 Subject: [PATCH 073/112] =?UTF-8?q?#3713:=20review=20=E2=80=94=20importer?= =?UTF-8?q?=20limits=20that=20drop=20fonts,=20links,=20style=20names=20or?= =?UTF-8?q?=20sheet=20rows=20log=20a=20warning=20once=20per=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every limit this branch added that leaves data out now says so: - font names over 256 bytes and links over 8 KiB (HTML, ODT, RTF, LaTeX, Markdown), and RTF style names cut to 256 bytes; - runs' links and fonts past the 64 MiB run-string budget; - spreadsheet rows refused for being over the text budget (ODS and Grid::push), once per sheet. model::dropped logs each kind once per imported document; each importer's entry point starts a new document with begin_import. Tests capture the warnings (crate::test_log) and check each is logged once, again for the next document, and not at all when nothing is dropped. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/formats/src/html.rs | 3 ++ crates/formats/src/latex.rs | 1 + crates/formats/src/lib.rs | 2 + crates/formats/src/markdown.rs | 5 +++ crates/formats/src/model.rs | 73 +++++++++++++++++++++++++++++++--- crates/formats/src/odt.rs | 1 + crates/formats/src/rtf.rs | 3 ++ crates/formats/src/sheet.rs | 32 ++++++++++++++- crates/formats/src/test_log.rs | 36 +++++++++++++++++ crates/formats/src/tests.rs | 39 +++++++++++++++++- 10 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 crates/formats/src/test_log.rs diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index 4970855ed4..f1eb1693cf 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -1588,17 +1588,20 @@ fn collapse_ws(s: &str) -> String { /// Parse HTML into a document. pub fn import(s: &str) -> Document { + model::begin_import(); model::to_doc(&parse(s)) } /// [`import`], loading pictures that aren't `data:` URIs through `images`. pub fn import_with(s: &str, images: ImageLoader) -> Document { + model::begin_import(); model::to_doc(&parse_with(s, images)) } /// [`import_with`] onto `page` (its size and margins), which tables size to: HTML gives no page /// size, so the caller picks it (the region's paper, #1381). pub fn import_on(s: &str, images: ImageLoader, page: &wordcraft_doc::SectionProps) -> Document { + model::begin_import(); model::to_doc_in(&parse_with(s, images), page.clone()) } diff --git a/crates/formats/src/latex.rs b/crates/formats/src/latex.rs index 18d807fdd1..a79f3a9f02 100644 --- a/crates/formats/src/latex.rs +++ b/crates/formats/src/latex.rs @@ -1057,6 +1057,7 @@ fn math_seq(t: &[MTok], pos: &mut usize, depth: usize, in_opt: bool) -> String { /// Import a LaTeX source file. pub fn import(bytes: &[u8]) -> Document { + model::begin_import(); let mut d = model::to_doc(&parse(&crate::txt::decode(bytes.get(..MAX_SOURCE).unwrap_or(bytes)))); d.ensure_nonempty(); d diff --git a/crates/formats/src/lib.rs b/crates/formats/src/lib.rs index 6678b5c954..8af0d2a503 100644 --- a/crates/formats/src/lib.rs +++ b/crates/formats/src/lib.rs @@ -91,5 +91,7 @@ pub fn export_with(ext: &str, doc: &Document, opts: &ExportOptions) -> Option Document { + model::begin_import(); model::to_doc(&parse(text)) } /// [`import`] onto `page`: Markdown gives no page size (#1381). pub fn import_on(text: &str, page: &wordcraft_doc::SectionProps) -> Document { + model::begin_import(); model::to_doc_in(&parse(text), page.clone()) } @@ -1032,6 +1034,9 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec { let mut f = base.clone(); // A destination too long to keep leaves its text unlinked. f.link = (dest.len() <= model::MAX_LINK).then_some(dest); + if f.link.is_none() { + model::dropped(model::Dropped::Link); + } for x in parse_inlines(&inner, &f, depth + 1, refs) { nodes.push(Node::Inl(x)); } diff --git a/crates/formats/src/model.rs b/crates/formats/src/model.rs index 156269b1be..61ae4aea5a 100644 --- a/crates/formats/src/model.rs +++ b/crates/formats/src/model.rs @@ -30,14 +30,72 @@ pub const MAX_LINK: usize = 8 << 10; /// lose their link and font rather than multiply a small file into gigabytes (#3713). const MAX_RUN_STRINGS: usize = 64 << 20; -/// `name` as a span's font, unless it is longer than [`MAX_FONT_NAME`]. +/// `name` as a span's font, unless it is longer than [`MAX_FONT_NAME`] (then reported, see +/// [`dropped`]). pub fn kept_font(name: &str) -> Option> { - (name.len() <= MAX_FONT_NAME).then(|| name.into()) + let kept = (name.len() <= MAX_FONT_NAME).then(|| name.into()); + if kept.is_none() { + dropped(Dropped::Font); + } + kept } -/// `target` as a span's link, unless it is longer than [`MAX_LINK`]. +/// `target` as a span's link, unless it is longer than [`MAX_LINK`] (then reported, see +/// [`dropped`]). pub fn kept_link(target: &str) -> Option> { - (target.len() <= MAX_LINK).then(|| target.into()) + let kept = (target.len() <= MAX_LINK).then(|| target.into()); + if kept.is_none() { + dropped(Dropped::Link); + } + kept +} + +/// What an importer leaves out to stay within a limit (review of #3873: no limit drops data +/// without saying so). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Dropped { + /// A font name over [`MAX_FONT_NAME`]. + Font, + /// A link target over [`MAX_LINK`]. + Link, + /// Runs' links and fonts past [`MAX_RUN_STRINGS`]. + RunStrings, + /// A style name cut to [`MAX_FONT_NAME`]. + StyleName, +} + +impl Dropped { + fn message(self) -> String { + match self { + Dropped::Font => format!("import: font names longer than {MAX_FONT_NAME} bytes are dropped; that text keeps the surrounding font"), + Dropped::Link => format!("import: links longer than {MAX_LINK} bytes are dropped; their text is kept"), + Dropped::RunStrings => { + format!("import: the document's runs carry more than {MAX_RUN_STRINGS} bytes of links and font names; later runs lose them") + } + Dropped::StyleName => format!("import: style names longer than {MAX_FONT_NAME} bytes are cut short"), + } + } +} + +thread_local! { + /// The kinds of [`Dropped`] already reported for the document being imported on this thread. + /// Per thread because a document is parsed and built on one, and the parsers drop things + /// from places that have no log of their own to hand. + static REPORTED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Log that the importer left something of `kind` out: once per kind per imported document +/// ([`begin_import`]), not once per item. +pub(crate) fn dropped(kind: Dropped) { + let bit = 1u8 << (kind as u8); + if REPORTED.with(|r| r.replace(r.get() | bit)) & bit == 0 { + log::warn!("{}", kind.message()); + } +} + +/// Start a new document's reports: each importer's entry point calls this first. +pub(crate) fn begin_import() { + REPORTED.with(|r| r.set(0)); } /// Character formatting the text formats can express. @@ -580,7 +638,12 @@ impl<'a> Builder<'a> { let n = cp.link.as_ref().map_or(0, String::len).saturating_add(cp.font.as_ref().map_or(0, String::len)); match self.run_strings.checked_add(n).filter(|&t| t <= MAX_RUN_STRINGS) { Some(t) => self.run_strings = t, - None => (cp.link, cp.font) = (None, None), + None => { + if cp.link.is_some() || cp.font.is_some() { + dropped(Dropped::RunStrings); + } + (cp.link, cp.font) = (None, None); + } } cp } diff --git a/crates/formats/src/odt.rs b/crates/formats/src/odt.rs index 6be9608110..472507f94f 100644 --- a/crates/formats/src/odt.rs +++ b/crates/formats/src/odt.rs @@ -2627,6 +2627,7 @@ fn parse_tracked(bytes: &[u8]) -> Result<(Flow, SectionProps, Option, Opti /// Parse an `.odt` package into a document. pub fn import(bytes: &[u8]) -> Result { + model::begin_import(); let (flow, last, track, font, parts) = parse_tracked(bytes)?; let mut doc = model::to_doc(&flow); if font.is_some() { diff --git a/crates/formats/src/rtf.rs b/crates/formats/src/rtf.rs index 7b06aa082f..2a9503fbd6 100644 --- a/crates/formats/src/rtf.rs +++ b/crates/formats/src/rtf.rs @@ -1654,6 +1654,8 @@ impl Reader { self.commit_style(); } else if self.style_name.len() < model::MAX_FONT_NAME { self.style_name.push(c); + } else { + model::dropped(model::Dropped::StyleName); } } } @@ -2856,6 +2858,7 @@ fn column_grid(rows: &mut [Vec], rights: &[Vec]) -> Vec { /// Parse RTF bytes into a document. pub fn import(bytes: &[u8]) -> Result { + model::begin_import(); let Parsed { flow, last_section, default_font, headers_footers, .. } = parse_full(bytes)?; let mut doc = model::to_doc_in(&flow, last_section); // Each section's own headers and footers; those it leaves out stay linked to the previous. diff --git a/crates/formats/src/sheet.rs b/crates/formats/src/sheet.rs index 97cf1f8680..e3ff85f298 100644 --- a/crates/formats/src/sheet.rs +++ b/crates/formats/src/sheet.rs @@ -343,6 +343,8 @@ struct Grid { rows: Vec>, cells: usize, text: TextBudget, + /// A row has been left out for its text (and logged, once per sheet). + refused: bool, } impl Grid { @@ -350,6 +352,15 @@ impl Grid { self.rows.len() >= MAX_ROWS || self.cells >= MAX_CELLS || self.text.0 >= MAX_TEXT_BYTES } + /// Note that a row is left out because its text is over what is left of [`MAX_TEXT_BYTES`]: + /// logged once per sheet, not once per row. + fn refuse(&mut self) { + if !self.refused { + self.refused = true; + log::warn!("sheet: rows with more text than the {MAX_TEXT_BYTES} byte budget has left are left out of the recipient list"); + } + } + /// Keep `row` (trailing blanks trimmed) `times` times, as far as the caps allow. fn push(&mut self, mut row: Vec, times: u64) { while row.last().is_some_and(String::is_empty) { @@ -364,6 +375,7 @@ impl Grid { let keep = row.len().min(MAX_CELLS.saturating_sub(self.cells)); let r = row.get(..keep).unwrap_or_default(); if !self.text.take(r.iter().map(String::len).sum()) { + self.refuse(); return; } self.cells = self.cells.saturating_add(r.len()); @@ -989,6 +1001,7 @@ fn ods_read(xml: &[u8], want: Option) -> Result<(Vec, Vec MAX_TEXT_BYTES.saturating_sub(grid.text.0) { cells.clear(); *budget = None; + grid.refuse(); return; } cells.resize(cells.len() + blanks, String::new()); @@ -1274,8 +1287,25 @@ mod tests { let book = ods(&format!( r#"next"# )); - let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap(); + let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap()); assert_eq!(rows, vec![vec!["next".to_string()]]); + // The row left out is said to be (review of #3873). + assert_eq!(warned.iter().filter(|w| w.contains("are left out of the recipient list")).count(), 1, "{warned:?}"); + } + + #[test] + fn rows_over_the_text_budget_are_logged_once_per_sheet() { + let wide = "€".repeat(MAX_CELL_CHARS); + let over = format!( + r#""# + ); + let book = ods(&format!("{over}{over}{over}next")); + let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap()); + assert_eq!(rows, vec![vec!["next".to_string()]]); + assert_eq!(warned.iter().filter(|w| w.contains("recipient list")).count(), 1, "one warning, not one per row: {warned:?}"); + let book = ods("fine"); + let (_, warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap()); + assert!(warned.is_empty(), "{warned:?}"); } #[test] diff --git a/crates/formats/src/test_log.rs b/crates/formats/src/test_log.rs new file mode 100644 index 0000000000..745ce21fe1 --- /dev/null +++ b/crates/formats/src/test_log.rs @@ -0,0 +1,36 @@ +//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that +//! a limit which leaves something out says so. + +use std::cell::RefCell; + +struct Capture; + +thread_local! { + static LINES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +impl log::Log for Capture { + fn enabled(&self, m: &log::Metadata) -> bool { + m.level() <= log::Level::Warn + } + + fn log(&self, r: &log::Record) { + if self.enabled(r.metadata()) { + LINES.with(|l| l.borrow_mut().push(r.args().to_string())); + } + } + + fn flush(&self) {} +} + +static CAPTURE: Capture = Capture; + +/// `f`'s result and the warnings logged on this thread while it ran. +pub(crate) fn warnings(f: impl FnOnce() -> T) -> (T, Vec) { + // Only this logger is ever installed in a test binary; a second call keeps it. + let _ = log::set_logger(&CAPTURE); + log::set_max_level(log::LevelFilter::Warn); + LINES.with(|l| l.borrow_mut().clear()); + let out = f(); + (out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut()))) +} diff --git a/crates/formats/src/tests.rs b/crates/formats/src/tests.rs index dc23445347..bc2442d80a 100644 --- a/crates/formats/src/tests.rs +++ b/crates/formats/src/tests.rs @@ -958,10 +958,47 @@ fn run_link_and_font_copies_are_budgeted() { let f = model::Fmt { bold: i % 2 == 0, link: Some(link.clone()), ..Default::default() }; p.push_text("a", &f); } - let doc = model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() }); + model::begin_import(); + let (doc, warned) = crate::test_log::warnings(|| model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() })); let paras: Vec<_> = doc.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p) } else { None }).collect(); assert_eq!(paras.iter().map(|p| p.text.len()).sum::(), 20_000, "all text kept"); let runs: Vec<_> = paras.iter().flat_map(|p| p.runs.iter()).collect(); let kept: usize = runs.iter().filter_map(|r| r.props.link.as_ref()).map(String::len).sum(); assert!(kept > 0 && kept <= 64 << 20, "{kept}"); + // The runs that lose theirs are said to, once for the document (review of #3873). + assert_eq!(warned.iter().filter(|w| w.contains("later runs lose them")).count(), 1, "{warned:?}"); +} + +/// Every limit that leaves a font name, link or style name out says so, once per kind per +/// document rather than once per item (review of #3873; the tracker complains of silent loss). +#[test] +fn dropped_names_and_links_are_logged_once_per_document() { + let long = "x".repeat(model::MAX_LINK + 1); + let html = format!( + "

a b c

" + ); + let count = |w: &[String], what: &str| w.iter().filter(|l| l.contains(what)).count(); + for _ in 0..2 { + // Each import is its own document: the second is reported too. + let (doc, warned) = crate::test_log::warnings(|| crate::html::import(&html)); + assert!(doc.body.len() == 1, "the text is kept"); + assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}"); + assert_eq!(count(&warned, "links longer"), 1, "{warned:?}"); + } + let (_, warned) = crate::test_log::warnings(|| crate::html::import("

fine

")); + assert!(warned.is_empty(), "nothing left out, nothing said: {warned:?}"); + + let md = format!("[a](https://e.com/{long}) [b](https://e.com/{long})"); + let (_, warned) = crate::test_log::warnings(|| crate::markdown::import(&md)); + assert_eq!(count(&warned, "links longer"), 1, "{warned:?}"); + + let rtf = + format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}{{\\f2 {long};}}}}{{\\stylesheet{{\\s1 {long};}}{{\\s2 {long};}}}}\\f1 a\\f2 b}}"); + let (_, warned) = crate::test_log::warnings(|| crate::rtf::import(rtf.as_bytes()).unwrap()); + assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}"); + assert_eq!(count(&warned, "style names longer"), 1, "{warned:?}"); + + let tex = format!("\\href{{https://e.com/{long}}}{{a}} \\url{{https://e.com/{long}}}"); + let (_, warned) = crate::test_log::warnings(|| crate::latex::import(tex.as_bytes())); + assert_eq!(count(&warned, "links longer"), 1, "{warned:?}"); } From 124243b1becc8056b23c8ddb210d7fa193420f31 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 13:20:09 +0100 Subject: [PATCH 074/112] Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- docs/file-format-parity.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 2f58bcbe16..1ba787f062 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -271,6 +271,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio > **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats | 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) | | 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | +| 2026-10-22 | trivial | Importer hardening (#3692, #3705, #3713, #3714): spreadsheet header numbering and ODS repeated cells, LaTeX wrapper nesting and verbatim ends, Markdown delimiter runs, blank-span trimming and link destinations, and font/link copies per run are bounded or escaped; normal documents unchanged | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | | 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) | | 2026-10-23 | trivial | DOCX: TOC styles Word stores under ids in another language (`Sumrio1` named `toc 1`, `CabealhodoSumrio` named `TOC Heading`) read as `TOC1`…`TOC9` and `TOCHeading` unless the file uses those ids, so Update Table builds the entries in the document's own TOC styles (#3891); a TOC whose heading ends a section (entries in columns after a continuous break) is written and read as the engine's TOC, with the page-number tab at the column's edge (#3889) | From 22df7caf4deac7dfd4e43a81713b72de8898d18f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 14:50:14 +0100 Subject: [PATCH 075/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20formats?= =?UTF-8?q?=20timing=20tests=20get=20a=20margin=20a=20busy=20CI=20machine?= =?UTF-8?q?=20can't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit many_formatted_leading_breaks_trim_quickly failed at its 2 s bound on a machine at load ~50 (it takes about a second there). Bounds now sit at least 10x over the fixed code's time on a loaded machine and far under the old code's on the same input (measured by running the pre-fix code, capped at 400 s): - HTML leading breaks: 2 s -> 20 s; ~1 s now, old trim ran past 6 minutes. - LaTeX nested multicolumn: 2 s -> 10 s; under 1 s now, old unwrapping took 213 s. - Markdown delimiters: 2 s -> 10 s; under 1 s now, old backwards search past 6 minutes. - Para::trim blank spans: 1 s -> 10 s; 0.16 s now, old Vec::remove(0) past 6 minutes. - repeated sheet headers: 2 s -> 10 s; 20 ms now, old numbering past 6 minutes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/formats/src/html.rs | 3 ++- crates/formats/src/latex.rs | 3 ++- crates/formats/src/markdown.rs | 3 ++- crates/formats/src/model.rs | 3 ++- crates/formats/src/sheet.rs | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index f1eb1693cf..a3d76578c4 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -2435,7 +2435,8 @@ mod tests { let html = format!("

{}x

", "

".repeat(100_000)); let t = std::time::Instant::now(); let f = parse(&html); - assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + // About a second on a loaded machine; trimming one span at a time ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed()); let texts: Vec = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect(); assert_eq!(texts, ["x"]); } diff --git a/crates/formats/src/latex.rs b/crates/formats/src/latex.rs index a79f3a9f02..d2c35f3cbb 100644 --- a/crates/formats/src/latex.rs +++ b/crates/formats/src/latex.rs @@ -3264,7 +3264,8 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after. let src = format!("\\begin{{tabular}}{{l}}{}x{}\\\\\\end{{tabular}}", "\\multicolumn{1}{c}{".repeat(n), "}".repeat(n)); let t = std::time::Instant::now(); let f = parse(&src); - assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + // Under a second on a loaded machine; unwrapping every wrapper took 213 s. + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); assert!(f.blocks.iter().any(|b| matches!(b, FBlock::Table(_)))); // One or two wrappers, as written in real tables, are still read. let f = parse("\\begin{tabular}{ll}\\multicolumn{2}{c}{\\multirow{2}{*}{Both}}\\\\\\end{tabular}"); diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index 56d631e7af..9196c1968a 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -2203,7 +2203,8 @@ mod tests { let src = "*_".repeat(200_000); let t = std::time::Instant::now(); let out = inl(&src); - assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + // Under a second on a loaded machine; the backwards search ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); let text: String = out.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect(); assert!(text.chars().all(|c| matches!(c, '*' | '_')), "only delimiters come out"); // Literal delimiters still take the link of the text before them. diff --git a/crates/formats/src/model.rs b/crates/formats/src/model.rs index 61ae4aea5a..ccc1f1c0cb 100644 --- a/crates/formats/src/model.rs +++ b/crates/formats/src/model.rs @@ -1669,7 +1669,8 @@ mod tests { p.push_text(" x ", &Fmt::default()); let t = std::time::Instant::now(); p.trim(); - assert!(t.elapsed() < std::time::Duration::from_secs(1), "{:?}", t.elapsed()); + // A fraction of a second; `Vec::remove(0)` per span ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); assert_eq!(p.inlines, vec![Inline::Text("x".into(), Fmt::default())]); // The first span left keeps its formatting and loses only its leading blanks. let mut p = Para::default(); diff --git a/crates/formats/src/sheet.rs b/crates/formats/src/sheet.rs index e3ff85f298..f7f300812e 100644 --- a/crates/formats/src/sheet.rs +++ b/crates/formats/src/sheet.rs @@ -1252,7 +1252,8 @@ mod tests { let row = vec!["Name".to_string(); 5_000]; let t = std::time::Instant::now(); let names = field_names(&row); - assert!(t.elapsed() < std::time::Duration::from_secs(2), "{:?}", t.elapsed()); + // Milliseconds; the old numbering ran past 6 minutes. + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); assert_eq!(names.len(), 5_000); assert_eq!(names[0], "Name"); assert_eq!(names[4_999], "Name 5000"); From 5155d2c18c0d482377cb222c06fef64f6ea28ca5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:39:46 +0100 Subject: [PATCH 076/112] #3703: WOFF and WOFF2 sizes are checked before the decoder allocates from them Findings 39 and 70. 39: wuff inflates each WOFF 1.0 table into a buffer sized from the directory's origLength, before anything checks it against our 128 MB cap, so a tiny file could ask for gigabytes. The directory's table sizes are now summed first and a file over MAX_DECODED_BYTES is TooLarge. 70: wuff's WOFF2 glyf reconstruction allocates a glyph's points from its nPoints counts and checks the 65536-point limit only afterwards. The brotli stream is now decompressed by us (brotli-decompressor, already in the tree through wuff) and every transformed glyf table's per-glyph point totals are checked before wuff unpacks them; a glyph over the limit (which wuff would refuse anyway) is refused up front. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + Cargo.toml | 2 + crates/fonts/Cargo.toml | 1 + crates/fonts/src/fontdb/user.rs | 214 ++++++++++++++++++++++++++++++-- 4 files changed, 211 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfcbb47b57..e88ebb28c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6755,6 +6755,7 @@ dependencies = [ name = "wordcraft-fonts" version = "0.11.0" dependencies = [ + "brotli-decompressor", "harfrust 0.13.3", "kurbo", "log", diff --git a/Cargo.toml b/Cargo.toml index 165eeb1bbd..6bdbf2ef9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,8 @@ harfrust = "0.13" # WOFF / WOFF2 web fonts added by the user (#582): pure Rust (MIT); its brotli and zlib decoders # (brotli-decompressor, flate2) are already in the tree. wuff = "0.2.9" +# WOFF2's brotli stream, decoded ahead of wuff so its glyph point counts are checked first (#3703). +brotli-decompressor = "5" vello_cpu = "0.2" image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "gif", "bmp", "tiff"] } craft-pdf = { git = "https://github.com/storytold/craft-libs", rev = "17744430be327d0257496a6a31c8d38fe20a6756", version = "=0.1.0", default-features = false, features = ["write"] } diff --git a/crates/fonts/Cargo.toml b/crates/fonts/Cargo.toml index 9107da44f1..8c0322a7d6 100644 --- a/crates/fonts/Cargo.toml +++ b/crates/fonts/Cargo.toml @@ -13,6 +13,7 @@ kurbo = { workspace = true } skrifa = { workspace = true } harfrust = { workspace = true } wuff = { workspace = true } +brotli-decompressor = { workspace = true } unicode-bidi = { workspace = true } unicode-script = { workspace = true } log = { workspace = true } diff --git a/crates/fonts/src/fontdb/user.rs b/crates/fonts/src/fontdb/user.rs index bc7ce6a6dd..7e0ead7ed7 100644 --- a/crates/fonts/src/fontdb/user.rs +++ b/crates/fonts/src/fontdb/user.rs @@ -149,8 +149,24 @@ pub fn decode_font_file(bytes: &[u8]) -> Result, UserFontError> { } let decoded = match sniff(bytes).ok_or(UserFontError::NotAFont)? { FontFormat::TrueType | FontFormat::OpenType | FontFormat::Collection => return Ok(bytes.to_vec()), - FontFormat::Woff => unwrap_web_font(|| wuff::decompress_woff1(bytes))?, - FontFormat::Woff2 => unwrap_web_font(|| wuff::decompress_woff2(bytes))?, + FontFormat::Woff => { + check_woff1_size(bytes)?; + unwrap_web_font(|| wuff::decompress_woff1(bytes))? + } + FontFormat::Woff2 => { + let mut too_many_points = false; + let unwrapped = unwrap_web_font(|| { + wuff::decompress_woff2_with_custom_brotli(bytes, &mut |data, size| { + let stream = unbrotli(data, size)?; + too_many_points = !woff2_points_fit(bytes, &stream); + if too_many_points { Err(TOO_MANY_POINTS.into()) } else { Ok(stream) } + }) + }); + if too_many_points { + return Err(UserFontError::BadWebFont(TOO_MANY_POINTS.into())); + } + unwrapped? + } }; if decoded.len() > MAX_DECODED_BYTES { return Err(UserFontError::TooLarge { bytes: decoded.len() }); @@ -161,6 +177,126 @@ pub fn decode_font_file(bytes: &[u8]) -> Result, UserFontError> { } } +/// WOFF 1.0: the font its table directory describes must fit [`MAX_DECODED_BYTES`]. The decoder +/// sizes each table's buffer from the directory's `origLength` before inflating it (#3703). +/// A short header or directory is left to the decoder to report. +fn check_woff1_size(bytes: &[u8]) -> Result<(), UserFontError> { + let u32_at = |at: usize| bytes.get(at..at.saturating_add(4)).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes); + let tables = bytes.get(12..14).and_then(|n| n.try_into().ok()).map_or(0, |n| usize::from(u16::from_be_bytes(n))); + let mut total = 12 + 16 * tables; + for i in 0..tables { + let entry = 44 + 20 * i; + let (Some(compressed), Some(orig)) = (u32_at(entry + 8), u32_at(entry + 12)) else { break }; + total = total.saturating_add((compressed.max(orig) as usize).saturating_add(3) & !3); + } + if total > MAX_DECODED_BYTES { + return Err(UserFontError::TooLarge { bytes: total }); + } + Ok(()) +} + +/// Why a WOFF2 file whose glyph has more points than a glyph may is refused. +const TOO_MANY_POINTS: &str = "a glyph has more than 65536 points"; + +/// WOFF2's brotli stream, which must decompress to exactly `size` bytes (the decoder has checked +/// `size` against its own cap and the compression ratio). +fn unbrotli(data: &[u8], size: usize) -> Result, Box> { + use std::io::Read; + let mut out = Vec::with_capacity(size); + brotli_decompressor::Decompressor::new(data, 4096).take(size as u64 + 1).read_to_end(&mut out)?; + if out.len() != size { + return Err("the compressed data doesn't have the size the directory gives".into()); + } + Ok(out) +} + +/// Does every glyph in the WOFF2 file's transformed `glyf` tables have at most 65536 points, as +/// the decoder requires? It allocates a glyph's points from its counts before checking them, so a +/// small file could claim gigabytes (#3703). `stream` is the decompressed table data; anything +/// that doesn't parse is refused too (the decoder would refuse it). +fn woff2_points_fit(file: &[u8], stream: &[u8]) -> bool { + woff2_transformed_glyf(file, stream).is_some_and(|tables| tables.iter().all(|t| glyf_points_fit(t).unwrap_or(false))) +} + +/// The transformed `glyf` tables in `stream`, found from the WOFF2 table directory in `file` +/// (tables follow each other in directory order). +fn woff2_transformed_glyf<'a>(file: &[u8], stream: &'a [u8]) -> Option> { + let tables = u16::from_be_bytes(file.get(12..14)?.try_into().ok()?); + let (mut at, mut offset, mut glyf) = (48usize, 0usize, Vec::new()); + for _ in 0..tables { + let flags = *file.get(at)?; + at += 1; + let tag: Option<&[u8]> = if flags & 63 == 63 { + at += 4; + Some(file.get(at - 4..at)?) + } else { + None + }; + let is = |index: u8, name: &[u8]| flags & 63 == index || tag == Some(name); + let (is_glyf, is_loca) = (is(10, b"glyf"), is(11, b"loca")); + let orig = base128(file, &mut at)?; + // glyf and loca are transformed at version 0, other tables at any other version. + let transformed = if is_glyf || is_loca { flags >> 6 == 0 } else { flags >> 6 != 0 }; + let len = if transformed { base128(file, &mut at)? } else { orig }; + let end = offset.checked_add(len)?; + if is_glyf && transformed { + glyf.push(stream.get(offset..end)?); + } + offset = end; + } + Some(glyf) +} + +/// WOFF2's UIntBase128. +fn base128(b: &[u8], at: &mut usize) -> Option { + let mut v: u32 = 0; + for i in 0..5 { + let byte = *b.get(*at)?; + *at += 1; + if (i == 0 && byte == 0x80) || v >> 25 != 0 { + return None; + } + v = v << 7 | u32::from(byte & 0x7f); + if byte & 0x80 == 0 { + return usize::try_from(v).ok(); + } + } + None +} + +/// Does every glyph of the transformed `glyf` table `t` have at most 65536 points? +fn glyf_points_fit(t: &[u8]) -> Option { + let u32_at = |at: usize| t.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes).map(|v| v as usize); + let glyphs = usize::from(u16::from_be_bytes(t.get(4..6)?.try_into().ok()?)); + let contours_end = 36usize.checked_add(u32_at(8)?)?; + let contours = t.get(36..contours_end)?; + let points = t.get(contours_end..contours_end.checked_add(u32_at(12)?)?)?; + let mut at = 0; + for g in 0..glyphs { + let n = i16::from_be_bytes(contours.get(2 * g..2 * g + 2)?.try_into().ok()?); + let mut total = 0usize; + for _ in 0..n.max(0) { + // 255UInt16 + let code = *points.get(at)?; + at += 1; + let mut byte = || { + at += 1; + points.get(at - 1).copied().map(usize::from) + }; + total += match code { + 253 => byte()? << 8 | byte()?, + 254 => byte()? + 253 * 2, + 255 => byte()? + 253, + v => usize::from(v), + }; + if total > 1 << 16 { + return Some(false); + } + } + } + Some(true) +} + /// Run a WOFF/WOFF2 decoder. The decoder checks its input, but it is third-party code reading /// hostile bytes: a panic in it is caught (native) and reported as a damaged file. fn unwrap_web_font(decode: impl FnOnce() -> Result, wuff::WuffErr>) -> Result, UserFontError> { @@ -564,25 +700,89 @@ pub(crate) mod tests { base128(data.len() as u32, &mut dir); stream.extend_from_slice(data); } - let compressed = brotli_stored(&stream); + let sfnt: usize = 12 + 16 * t.len() + t.iter().map(|(_, d)| d.len().next_multiple_of(4)).sum::(); + woff2_file(&font[..4], t.len(), &dir, &stream, sfnt) + } + + /// A WOFF2 file: header, the table directory `dir` of `n` tables, `stream` in stored brotli blocks. + fn woff2_file(flavor: &[u8], n: usize, dir: &[u8], stream: &[u8], sfnt: usize) -> Vec { + let compressed = brotli_stored(stream); let mut out = Vec::new(); out.extend_from_slice(b"wOF2"); - out.extend_from_slice(&font[..4]); + out.extend_from_slice(flavor); let total = (48 + dir.len() + compressed.len()).next_multiple_of(4); out.extend_from_slice(&(total as u32).to_be_bytes()); - out.extend_from_slice(&(t.len() as u16).to_be_bytes()); + out.extend_from_slice(&(n as u16).to_be_bytes()); out.extend_from_slice(&0u16.to_be_bytes()); - let sfnt: usize = 12 + 16 * t.len() + t.iter().map(|(_, d)| d.len().next_multiple_of(4)).sum::(); out.extend_from_slice(&(sfnt as u32).to_be_bytes()); out.extend_from_slice(&(compressed.len() as u32).to_be_bytes()); out.extend_from_slice(&[0, 1, 0, 0]); out.extend_from_slice(&[0; 20]); - out.extend(dir); + out.extend_from_slice(dir); out.extend(compressed); out.resize(total, 0); out } + /// A WOFF2 file holding a transformed `glyf` (and `loca`) whose one glyph has two contours of + /// `points` points each (and no coordinates: the decoder finds it damaged in the end). + fn woff2_glyph_with(points: u16) -> Vec { + let total = 2 * points as usize; + let mut glyf = vec![0, 0, 0, 0, 0, 1, 0, 0]; // reserved, optionFlags, numGlyphs 1, indexFormat + // nContour, nPoints, flag, glyph, composite, bbox and instruction stream sizes. + for size in [2, 6, total, 0, 0, 4, 0] { + glyf.extend_from_slice(&(size as u32).to_be_bytes()); + } + glyf.extend_from_slice(&2i16.to_be_bytes()); + let [hi, lo] = points.to_be_bytes(); + glyf.extend_from_slice(&[253, hi, lo, 253, hi, lo]); // 255UInt16 in its word form + glyf.resize(glyf.len() + total, 1); // on-curve flags + glyf.extend_from_slice(&[0; 4]); // no explicit bounding boxes + let mut dir = vec![10]; // glyf, transform version 0 + base128(4096, &mut dir); + base128(glyf.len() as u32, &mut dir); + dir.push(11); // loca, transformed away + base128(8, &mut dir); + base128(0, &mut dir); + woff2_file(&[0, 1, 0, 0], 2, &dir, &glyf, 12 + 32 + 4096 + 8) + } + + /// A WOFF 1.0 file whose one table is a zlib stream of nothing that claims to inflate to `orig` + /// bytes. + fn woff1_claiming(orig: u32) -> Vec { + let mut out = b"wOFF\0\x01\0\0".to_vec(); + out.extend_from_slice(&72u32.to_be_bytes()); // length + out.extend_from_slice(&1u16.to_be_bytes()); // numTables + out.extend_from_slice(&0u16.to_be_bytes()); + out.extend_from_slice(&orig.saturating_add(28).to_be_bytes()); // totalSfntSize + out.extend_from_slice(&[0, 1, 0, 0]); + out.extend_from_slice(&[0; 20]); + out.extend_from_slice(b"glyf"); + for v in [64, 8, orig, 0] { + out.extend_from_slice(&u32::to_be_bytes(v)); // offset, compLength, origLength, checksum + } + out.extend_from_slice(&[0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01]); + out + } + + #[test] + fn a_woff_claiming_a_huge_table_is_refused_before_inflating() { + // #3703: the decoder sizes a table's buffer from the directory before inflating it. + for orig in [200 << 20, u32::MAX] { + assert!(matches!(decode_font_file(&woff1_claiming(orig)), Err(UserFontError::TooLarge { .. })), "{orig}"); + } + // A small claim reaches the decoder, which finds the data doesn't inflate to it. + assert!(matches!(decode_font_file(&woff1_claiming(64)), Err(UserFontError::BadWebFont(_)))); + } + + #[test] + fn a_woff2_glyph_with_too_many_points_is_refused_before_unpacking() { + // #3703: the decoder allocates a glyph's points from these counts before checking them. + assert_eq!(decode_font_file(&woff2_glyph_with(40_000)), Err(UserFontError::BadWebFont(TOO_MANY_POINTS.into()))); + // Within the limit the scan passes the file on, to the decoder. + assert!(matches!(decode_font_file(&woff2_glyph_with(3)), Err(UserFontError::BadWebFont(e)) if e != TOO_MANY_POINTS)); + } + #[test] fn an_added_font_is_listed_resolved_and_measured() { let db = db(); From 6c6326486f573f9b737e4b90f55dfbf5aaace49e Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:40:48 +0100 Subject: [PATCH 077/112] #3692: font names are debug-quoted in log warnings Finding 46: install_font logged the caller-supplied file name raw when the font couldn't be stored, and the desktop log writes line breaks as they come, so a name could forge extra log records. The name is now logged with {:?} (escaped and quoted), and so are the font-derived family names in the two glyph-outline warnings, which come from a document's own fonts. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/fontdb.rs | 4 ++-- crates/fonts/src/fontdb/user.rs | 41 ++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 8402846909..973c7427da 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -194,13 +194,13 @@ impl FontFace { self.location(), HintingOptions { engine: Engine::Interpreter, target: Target::Mono }, ) - .map_err(|e| log::warn!("Cannot initialize outline interpreter for {}: {e}", self.family)) + .map_err(|e| log::warn!("Cannot initialize outline interpreter for {:?}: {e}", self.family)) .ok() }); let settings = hinter.as_ref().map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false)); if let Err(e) = g.draw(settings, &mut pen) { - log::warn!("Cannot draw glyph {gid} in {}: {e}", self.family); + log::warn!("Cannot draw glyph {gid} in {:?}: {e}", self.family); return BezPath::new(); } } diff --git a/crates/fonts/src/fontdb/user.rs b/crates/fonts/src/fontdb/user.rs index 7e0ead7ed7..09bbf84708 100644 --- a/crates/fonts/src/fontdb/user.rs +++ b/crates/fonts/src/fontdb/user.rs @@ -366,7 +366,8 @@ impl FontDb { None } Err(e) => { - log::warn!("couldn't store the font {name} in {}: {e}", dir.display()); + // Debug-quoted: a line break in the supplied name must not start a forged log record. + log::warn!("couldn't store the font {name:?} in {}: {e}", dir.display()); Some(format!("added for this session only: couldn't store it in {}: {e}", dir.display())) } }, @@ -903,6 +904,44 @@ pub(crate) mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// The messages logged while this test binary runs. + fn logged() -> &'static std::sync::Mutex> { + static LOG: std::sync::OnceLock>> = std::sync::OnceLock::new(); + struct Capture; + impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + fn log(&self, r: &log::Record) { + if let Some(l) = LOG.get() { + l.lock().unwrap().push(r.args().to_string()); + } + } + fn flush(&self) {} + } + LOG.get_or_init(|| { + let _ = log::set_logger(&Capture); + log::set_max_level(log::LevelFilter::Warn); + std::sync::Mutex::new(Vec::new()) + }) + } + + #[test] + fn a_supplied_name_is_logged_escaped() { + // #3692: a line break in the name must not start a forged record in the log file. + let log = logged(); + let not_a_folder = temp_dir("not-a-folder"); + std::fs::write(¬_a_folder, b"x").unwrap(); + let db = db(); + db.load_user_fonts(¬_a_folder); + let r = db.install_font("a\nERROR forged record.ttf", &renamed("SourceSans3-Regular.ttf", FAMILY)).unwrap(); + assert!(r.warning.is_some(), "the font can't be stored"); + let lines = log.lock().unwrap(); + let line = lines.iter().find(|l| l.contains("forged record")).unwrap(); + assert!(!line.contains('\n'), "{line}"); + let _ = std::fs::remove_file(¬_a_folder); + } + #[test] fn stored_names_are_safe_and_unique() { let ttf = renamed("SourceSans3-Regular.ttf", FAMILY); From ce7e4ea28eeb9710dff3bec754ef4a137fe5c1ab Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:43:28 +0100 Subject: [PATCH 078/112] #3716: a font file's faces are capped before they are built Finding 59: enumerate_faces took a collection's member count straight from its header (a TTC can list one table directory millions of times) and font_faces read every fvar named instance (up to 65535, with a linear style dedupe and a name-table lookup each). Every entry became a FontFace kept for the session, and add_fonts then checked each new face against every face in the database. Now a file yields at most 256 faces (MAX_FILE_FACES; user fonts ask for one more so they can still be refused as "more than 256 faces"), named instances are read to the same cap, a face with the family and style of an earlier one in the file is skipped (repeated collection offsets), and add_fonts dedupes against an indexed set. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/fontdb.rs | 109 ++++++++++++++++++++++++++--- crates/fonts/src/fontdb/user.rs | 2 +- crates/fonts/src/tests_sysfonts.rs | 2 +- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 973c7427da..e3111a3c86 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -825,7 +825,8 @@ fn font_faces(f: &skrifa::FontRef<'_>, index: u32) -> Vec { let east_asian = east_asian_font(f); let axes = f.axes(); let mut named: Vec = Vec::new(); - for ni in f.named_instances().iter() { + // At most one more than a file may add, so a caller can tell there are more (#3716). + for ni in f.named_instances().iter().take(MAX_FILE_FACES + 1) { let Some(style) = name(f, &[ni.subfamily_name_id()]) else { continue }; if named.iter().any(|n| n.style.eq_ignore_ascii_case(&style)) { continue; @@ -854,15 +855,25 @@ fn font_faces(f: &skrifa::FontRef<'_>, index: u32) -> Vec { }] } -/// Parse every face in `data` (a font file or collection); a variable font yields one face per -/// named instance. -fn enumerate_faces(data: &[u8]) -> Vec { +/// Most faces one font file adds (collection members, variable fonts' named instances). +const MAX_FILE_FACES: usize = 256; + +/// Parse the faces in `data` (a font file or collection), at most `limit` of them; a variable +/// font yields one face per named instance. A face with the family and style of an earlier one +/// is left out (a collection can list one font many times, #3716). +fn enumerate_faces(data: &[u8], limit: usize) -> Vec { let count = match FileRef::new(data) { Ok(FileRef::Font(_)) => 1, - Ok(FileRef::Collection(c)) => c.len(), + Ok(FileRef::Collection(c)) => c.len().min(u32::try_from(limit).unwrap_or(u32::MAX)), Err(_) => 0, }; - (0..count).filter_map(|i| skrifa::FontRef::from_index(data, i).ok().map(|f| font_faces(&f, i))).flatten().collect() + let mut seen = std::collections::HashSet::new(); + (0..count) + .filter_map(|i| skrifa::FontRef::from_index(data, i).ok().map(|f| font_faces(&f, i))) + .flatten() + .filter(|f| seen.insert((f.family.to_ascii_lowercase(), f.style.to_ascii_lowercase()))) + .take(limit) + .collect() } /// The compiled-in Source Sans 3 Regular, for a database that somehow has no faces at all. @@ -986,7 +997,7 @@ impl FontDb { let others = crate::CRAFT_FONTS.iter().filter(|f| !f.scripts.contains(&"Jpan")); let craft = crate::japanese_document_fonts().into_iter().chain(others).map(|f| f.bytes); for data in BUNDLED.iter().copied().chain(craft) { - for f in enumerate_faces(data) { + for f in enumerate_faces(data, MAX_FILE_FACES) { if let Some(mut face) = make_face(FontBytes::Static(data), f.index, f.family, f.style, f.legacy, f.coords) { face.local_names = f.local; faces.push(Arc::new(face)); @@ -1173,7 +1184,7 @@ impl FontDb { for (bytes, name) in files { let data = Arc::new(bytes); let name: Option = name.map(|n| n.trim().chars().take(MAX_FAMILY_NAME).collect()).filter(|n: &String| !n.is_empty()); - for f in enumerate_faces(&data) { + for f in enumerate_faces(&data, MAX_FILE_FACES) { if let Some(mut face) = make_face(FontBytes::Owned(data.clone()), f.index, f.family, f.style, f.legacy, f.coords) { face.local_names = f.local; if let Some(n) = name.as_deref().filter(|n| !face.named(n) && !face.local_names.iter().any(|l| l.name.eq_ignore_ascii_case(n))) { @@ -1185,9 +1196,12 @@ impl FontDb { } } let mut faces = self.faces.write().unwrap_or_else(std::sync::PoisonError::into_inner); + // Indexed: a document's fonts may add hundreds of faces to thousands (#3716). + let key = |f: &FontFace| (f.family.to_ascii_lowercase(), f.style.to_ascii_lowercase()); + let mut present: std::collections::HashSet<_> = faces.iter().map(|g| key(g)).collect(); let mut added = 0; for f in new { - if !faces.iter().any(|g| g.family.eq_ignore_ascii_case(&f.family) && g.style.eq_ignore_ascii_case(&f.style)) { + if present.insert(key(&f)) { faces.push(f); added += 1; } @@ -1387,7 +1401,9 @@ impl FontDb { (best.path.clone(), best.family.clone(), best.style.clone()) }; let data = Arc::new(read_fallback_file(&path)?); - let found = enumerate_faces(&data).into_iter().find(|f| f.family.eq_ignore_ascii_case(&fam) && f.style.eq_ignore_ascii_case(&style))?; + let found = enumerate_faces(&data, MAX_FILE_FACES) + .into_iter() + .find(|f| f.family.eq_ignore_ascii_case(&fam) && f.style.eq_ignore_ascii_case(&style))?; let mut face = make_face(FontBytes::Owned(data), found.index, found.family, found.style, found.legacy, found.coords)?; face.local_names = found.local; Some(Arc::new(face)) @@ -2127,4 +2143,77 @@ mod tests { assert!(!db.has_family("NotAFont-Bold")); assert_eq!(db.face("Source Serif 4", "Regular").style, "Regular"); } + + /// A variable font holding just `name` and `fvar` tables: family "Var Test", one `wght` axis + /// and `instances` named instances, each with its own style name. + fn variable_font(instances: u16) -> Vec { + let mut names: Vec<(u16, String)> = vec![(1, "Var Test".into()), (2, "Regular".into())]; + names.extend((0..instances).map(|i| (256 + i, format!("Style {i}")))); + let mut name = vec![0, 0]; + name.extend_from_slice(&(names.len() as u16).to_be_bytes()); + name.extend_from_slice(&(6 + 12 * names.len() as u16).to_be_bytes()); + let mut strings = Vec::new(); + for (id, text) in &names { + let utf16: Vec = text.encode_utf16().flat_map(u16::to_be_bytes).collect(); + for v in [3, 1, 0x409, *id, utf16.len() as u16, strings.len() as u16] { + name.extend_from_slice(&v.to_be_bytes()); + } + strings.extend(utf16); + } + name.extend(strings); + let mut fvar = Vec::new(); + for v in [1u16, 0, 16, 2, 1, 20, instances, 8] { + fvar.extend_from_slice(&v.to_be_bytes()); + } + fvar.extend_from_slice(b"wght"); + for v in [100u32 << 16, 400 << 16, 900 << 16] { + fvar.extend_from_slice(&v.to_be_bytes()); + } + fvar.extend_from_slice(&[0, 0, 0, 2]); // flags, axisNameId + for i in 0..instances { + fvar.extend_from_slice(&(256 + i).to_be_bytes()); + fvar.extend_from_slice(&[0, 0]); + fvar.extend_from_slice(&(400u32 << 16).to_be_bytes()); + } + mini_font(&[(*b"fvar", fvar), (*b"name", name)]) + } + + /// A collection listing `font` `members` times. + fn collection_of(font: &[u8], members: u32) -> Vec { + let start = 12 + 4 * members; + let mut out = b"ttcf\0\x01\0\0".to_vec(); + out.extend_from_slice(&members.to_be_bytes()); + for _ in 0..members { + out.extend_from_slice(&start.to_be_bytes()); + } + let mut font = font.to_vec(); + let tables = u16::from_be_bytes([font[4], font[5]]) as usize; + for i in 0..tables { + let at = 12 + 16 * i + 8; + let offset = u32::from_be_bytes(font[at..at + 4].try_into().unwrap()) + start; + font[at..at + 4].copy_from_slice(&offset.to_be_bytes()); + } + out.extend(font); + out + } + + #[test] + fn named_instances_and_collection_members_are_capped() { + // #3716: a file's faces are capped before they are built, however the file multiplies them. + let font = variable_font(300); + let faces = font_faces(&skrifa::FontRef::new(&font).unwrap(), 0); + assert_eq!(faces.len(), MAX_FILE_FACES + 1, "one more than the cap, so a caller can tell"); + assert_eq!(faces[0].style, "Style 0"); + assert_eq!(enumerate_faces(&font, MAX_FILE_FACES).len(), MAX_FILE_FACES); + // A million members, all the same font: read once, and no more than the cap. + let started = std::time::Instant::now(); + let ttc = collection_of(BUNDLED[0], 1_000_000); + let found = enumerate_faces(&ttc, MAX_FILE_FACES); + assert_eq!(found.iter().map(|f| (f.family.as_str(), f.style.as_str())).collect::>(), [(FALLBACK_FAMILY, "Regular")]); + let db = FontDb::with_font_dirs(vec![]); + assert_eq!(db.add_font(collection_of(&font, 1_000_000)), MAX_FILE_FACES); + // A font the user adds is refused instead. + assert!(matches!(db.install_font("v.ttf", &variable_font(300)), Err(user::UserFontError::Unusable(e)) if e.contains("more than 256"))); + assert!(started.elapsed() < std::time::Duration::from_secs(2), "{:?}", started.elapsed()); + } } diff --git a/crates/fonts/src/fontdb/user.rs b/crates/fonts/src/fontdb/user.rs index 09bbf84708..b81f89d693 100644 --- a/crates/fonts/src/fontdb/user.rs +++ b/crates/fonts/src/fontdb/user.rs @@ -403,7 +403,7 @@ impl FontDb { if FileRef::new(&sfnt).is_err() { return Err(UserFontError::Unusable("not a valid OpenType file".into())); } - let found = enumerate_faces(&sfnt); + let found = enumerate_faces(&sfnt, MAX_FACES_PER_FILE + 1); if found.is_empty() { return Err(UserFontError::Unusable("no face with a family name".into())); } diff --git a/crates/fonts/src/tests_sysfonts.rs b/crates/fonts/src/tests_sysfonts.rs index 5d9c965df8..5a0912c4b2 100644 --- a/crates/fonts/src/tests_sysfonts.rs +++ b/crates/fonts/src/tests_sysfonts.rs @@ -270,7 +270,7 @@ fn the_scan_lists_a_variable_fonts_named_instances_as_loading_it_does() { return; }; let data = std::fs::read(path).unwrap(); - let loaded: Vec<(String, String, String)> = enumerate_faces(&data).into_iter().map(|f| (f.family, f.style, f.legacy)).collect(); + let loaded: Vec<(String, String, String)> = enumerate_faces(&data, MAX_FILE_FACES).into_iter().map(|f| (f.family, f.style, f.legacy)).collect(); assert_eq!(names(path), loaded); if loaded.len() > 1 { // Instances other than Regular, Bold and Italic have names of their own. From b246ea166abe2bd8230bc9abacda0d07e5f28d5e Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:48:05 +0100 Subject: [PATCH 079/112] #3692: COLR gradient stops and clip paths count against a per-glyph budget Finding 61: Recorder copied every gradient's stop list (and every clip glyph's outline) per op, capped only by op count, so a compact COLR graph reusing one long stop list could expand to billions of stops. Stops and path elements now share MAX_PAYLOAD (65,536) per glyph, charged before copying; past it the glyph falls back to its outline and no further clip outlines are built. The render cache charges each glyph's real ops, stops and path bytes (ops_bytes) instead of a flat 64 bytes per op. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/color.rs | 73 ++++++++++++++++++++++++++++++-- crates/render/src/color_glyph.rs | 2 +- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/crates/fonts/src/color.rs b/crates/fonts/src/color.rs index 897f31c96b..1f03641b68 100644 --- a/crates/fonts/src/color.rs +++ b/crates/fonts/src/color.rs @@ -18,6 +18,9 @@ use crate::FontFace; /// Upper bound on the ops one colour glyph may expand to. pub const MAX_OPS: usize = 20_000; +/// Upper bound on the gradient stops and clip-path elements one colour glyph may copy in total: a +/// paint graph can reference one long stop list or outline from every op. +pub const MAX_PAYLOAD: usize = 1 << 16; /// Upper bound on a bitmap glyph's width and height in pixels. pub const MAX_BITMAP_SIDE: u32 = 2048; @@ -175,12 +178,26 @@ pub fn bitmap_placement(bg: &skrifa::bitmap::BitmapGlyph<'_>, upem: f64) -> Opti finite.then_some(ColorBitmap { data, width: w, height: h, rect, ppem: bg.ppem_y }) } +/// Approximate heap bytes held by `ops`, counting each op's copied stops and path elements. +pub fn ops_bytes(ops: &[ColorOp]) -> usize { + let payload = |op: &ColorOp| match op { + ColorOp::PushClip(p) => p.elements().len().saturating_mul(std::mem::size_of::()), + ColorOp::Fill(ColorPaint::Linear { stops, .. } | ColorPaint::Radial { stops, .. } | ColorPaint::Sweep { stops, .. }) => { + stops.len().saturating_mul(std::mem::size_of::()) + } + _ => 0, + }; + ops.iter().fold(ops.len().saturating_mul(std::mem::size_of::()), |n, op| n.saturating_add(payload(op))) +} + /// A [`ColorPainter`] that records the paint calls as [`ColorOp`]s, resolving palette colours and -/// clip glyphs. Stops recording (and [`Recorder::finish`] gives None) past [`MAX_OPS`]. +/// clip glyphs. Stops recording (and [`Recorder::finish`] gives None) past [`MAX_OPS`] or +/// [`MAX_PAYLOAD`]. pub struct Recorder<'a, F: Fn(GlyphId) -> BezPath> { palette: &'a [[u8; 4]], outline: F, ops: Vec, + payload: usize, overflow: bool, } @@ -188,7 +205,7 @@ impl<'a, F: Fn(GlyphId) -> BezPath> Recorder<'a, F> { /// A recorder with palette `palette` (straight RGBA) and clip outlines (font units, y up) from /// `outline`. pub fn new(palette: &'a [[u8; 4]], outline: F) -> Self { - Recorder { palette, outline, ops: vec![], overflow: false } + Recorder { palette, outline, ops: vec![], payload: 0, overflow: false } } /// The recorded ops, None when the glyph was too big or its pushes and pops don't balance. @@ -218,6 +235,15 @@ impl<'a, F: Fn(GlyphId) -> BezPath> Recorder<'a, F> { self.ops.push(op); } + /// Charge `n` stops or path elements to the glyph's payload; false (and overflow) past the budget. + fn charge(&mut self, n: usize) -> bool { + self.payload = self.payload.saturating_add(n); + if self.payload > MAX_PAYLOAD { + self.overflow = true; + } + !self.overflow + } + fn color(&self, palette_index: u16, alpha: f32) -> GlyphColor { let alpha = if alpha.is_finite() { alpha.clamp(0.0, 1.0) } else { 1.0 }; if palette_index == 0xFFFF { @@ -249,8 +275,13 @@ impl BezPath> ColorPainter for Recorder<'_, F> { self.push(ColorOp::PopTransform); } fn push_clip_glyph(&mut self, glyph_id: GlyphId) { + if self.overflow { + return; + } let p = (self.outline)(glyph_id); - self.push(ColorOp::PushClip(p)); + if self.charge(p.elements().len()) { + self.push(ColorOp::PushClip(p)); + } } fn push_clip_box(&mut self, b: BoundingBox) { let r = Rect::new(b.x_min as f64, b.y_min as f64, b.x_max as f64, b.y_max as f64); @@ -260,6 +291,15 @@ impl BezPath> ColorPainter for Recorder<'_, F> { self.push(ColorOp::PopClip); } fn fill(&mut self, brush: Brush<'_>) { + let stops = match &brush { + Brush::Solid { .. } => 0, + Brush::LinearGradient { color_stops, .. } | Brush::RadialGradient { color_stops, .. } | Brush::SweepGradient { color_stops, .. } => { + color_stops.len() + } + }; + if !self.charge(stops) { + return; + } let paint = match brush { Brush::Solid { palette_index, alpha } => ColorPaint::Solid(self.color(palette_index, alpha)), Brush::LinearGradient { p0, p1, color_stops, extend } => { @@ -337,6 +377,33 @@ mod tests { assert!(r.finish().is_none()); } + #[test] + fn gradient_stops_and_clip_paths_share_a_payload_budget() { + // A few hundred fills reusing one long stop list would copy it per fill, far under MAX_OPS. + let stops = vec![ColorStop { offset: 0.5, palette_index: 0, alpha: 1.0 }; 4096]; + let fills = MAX_PAYLOAD / stops.len() + 1; + assert!(fills < MAX_OPS); + let outlines = std::cell::Cell::new(0); + let count = |g| { + outlines.set(outlines.get() + 1); + square(g) + }; + let mut r = Recorder::new(&[], count); + for _ in 0..fills { + r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad }); + } + assert!(r.ops.len() < fills, "stops past the budget are not copied"); + r.push_clip_glyph(GlyphId::new(1)); + r.pop_clip(); + assert_eq!(outlines.get(), 0, "no clip outline is built once the budget is spent"); + assert!(r.finish().is_none()); + // Within budget nothing changes, and the byte estimate counts the copied stops. + let mut r = Recorder::new(&[], square); + r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad }); + let ops = r.finish().unwrap(); + assert!(ops_bytes(&ops) >= stops.len() * std::mem::size_of::()); + } + fn bitmap(origin: skrifa::bitmap::Origin, data: skrifa::bitmap::BitmapData<'_>) -> skrifa::bitmap::BitmapGlyph<'_> { skrifa::bitmap::BitmapGlyph { data, diff --git a/crates/render/src/color_glyph.rs b/crates/render/src/color_glyph.rs index 8b169d9b6c..9a77c1a7a6 100644 --- a/crates/render/src/color_glyph.rs +++ b/crates/render/src/color_glyph.rs @@ -23,7 +23,7 @@ pub(crate) enum Drawable { impl Drawable { fn bytes(&self) -> usize { match self { - Drawable::Layers(ops) => ops.len().saturating_mul(64), + Drawable::Layers(ops) => wordcraft_fonts::color::ops_bytes(ops), Drawable::Bitmap { pixmap, .. } => pixmap.width() as usize * pixmap.height() as usize * 4, } } From bcce02ab369c4a29864dc9ee0e472ea8b050fe05 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:56:36 +0100 Subject: [PATCH 080/112] #3692: confined automation adds or removes fonts only when the font store is inside its write root Finding 44: font.add stored the file in the user font folder and font.remove deleted stored files directly, bypassing the automation write root (#769). Both now check the user font folder against the write policy in force before touching the database or the disk; with no folder (headless) a font still lasts for the session only. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engine/src/cmd/font.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/engine/src/cmd/font.rs b/crates/engine/src/cmd/font.rs index dfb7a48166..6160f84713 100644 --- a/crates/engine/src/cmd/font.rs +++ b/crates/engine/src/cmd/font.rs @@ -43,6 +43,15 @@ fn read_font_file(path: &str) -> Result, CmdError> { crate::io::read_file_capped(path, wordcraft_fonts::MAX_FONT_FILE_BYTES as u64, "font file").map_err(CmdError::Failed) } +/// Adding or removing a font writes to the user font folder: automation confined to a write root +/// (#769) may do that only when the folder is inside it. +fn check_font_store(dir: Option<&std::path::Path>) -> Result<(), CmdError> { + match dir { + Some(dir) => crate::file_access::check_write(&dir.to_string_lossy()).map_err(CmdError::Failed), + None => Ok(()), + } +} + fn add(s: &mut Session, v: &Value) -> CmdResult { if p::str(v, "url").is_some() { return Err(CmdError::Failed( @@ -71,6 +80,7 @@ fn add(s: &mut Session, v: &Value) -> CmdResult { s.push_ui_request(json!({"open": "addFont"})); return Ok(json!({"pending": true})); }; + check_font_store(FontDb::global().user_font_dir().as_deref())?; let r = FontDb::global().install_font(&name, &bytes).map_err(|e| CmdError::Failed(e.to_string()))?; // Text already set in the family (shown in a fallback until now) takes the new font. s.relayout(); @@ -100,8 +110,33 @@ fn list(_: &mut Session, v: &Value) -> CmdResult { fn remove(s: &mut Session, v: &Value) -> CmdResult { let family = p::str(v, "family").filter(|f| !f.trim().is_empty()).ok_or_else(|| CmdError::Params("family is required".into()))?; + check_font_store(FontDb::global().user_font_dir().as_deref())?; let gone = FontDb::global().remove_user_font(family.trim()).map_err(|e| CmdError::Failed(e.to_string()))?; s.relayout(); s.status = format!("Removed {}", family.trim()); Ok(json!({"removed": gone.iter().map(font_json).collect::>()})) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn confined_automation_changes_only_a_font_store_inside_its_write_root() { + let root = std::env::temp_dir().join(format!("wordcraft-font-store-{}", std::process::id())); + let outside = std::env::temp_dir().join(format!("wordcraft-font-store-out-{}", std::process::id())); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + let write = crate::file_access::AutomationRoots::new(None, Some(&root)).unwrap(); + let read_only = crate::file_access::AutomationRoots::new(Some(&root), None).unwrap(); + crate::file_access::confine(write.as_ref(), || { + assert!(check_font_store(None).is_ok(), "no folder: the font lasts for the session"); + assert!(check_font_store(Some(&root.join("fonts"))).is_ok()); + assert!(check_font_store(Some(&outside)).is_err()); + }); + crate::file_access::confine(read_only.as_ref(), || assert!(check_font_store(Some(&root)).is_err())); + assert!(check_font_store(Some(&outside)).is_ok(), "unconfined: the person at the keyboard"); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&outside); + } +} From c7148d6ba9ba5e955a79723b269af3a809ca4edb Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 17:00:58 +0100 Subject: [PATCH 081/112] #3692: hint-reliant fonts with huge zones are drawn unhinted, and a slow hinted glyph turns hinting off Finding 62 (partial): skrifa 0.47 caps a hinting run at 1M instructions but not the points each moves (SHZ, IUP... walk a whole zone), and a font opts into the interpreter by its family name. Faces asking for a twilight zone over 1,024 points or a stack over 4,096 values get no interpreter; a glyph whose hinting buffer passes 256 KiB is drawn unhinted; and (native) one hinted glyph over 50 ms turns hinting off for its face. Residual: a face within the limits can still spend about 1M x zone size point moves on one glyph (and its prep program) before the strike lands; the full fix is a point-work budget in skrifa or an isolated worker with a deadline. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/fontdb.rs | 93 +++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 11 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index e3111a3c86..818f086d09 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex, RwLock}; use kurbo::BezPath; use skrifa::instance::{Location, LocationRef, Size}; -use skrifa::outline::{DrawSettings, Engine, HintingInstance, HintingOptions, OutlinePen, Target}; +use skrifa::outline::{DrawSettings, Engine, Hinting, HintingInstance, HintingOptions, OutlinePen, Target}; use skrifa::raw::FileRef; use skrifa::string::StringId; use skrifa::{GlyphId, MetadataProvider}; @@ -88,8 +88,24 @@ pub struct FontFace { bmp: std::sync::OnceLock>, // Some legacy CJK fonts assemble their contours with TrueType instructions. outline_hinter: std::sync::OnceLock>, + /// Set once a hinted glyph ran too long: the face is drawn unhinted from then on. + hinting_off: std::sync::atomic::AtomicBool, } +/// Limits on what a face's TrueType instructions may run over (#3692). skrifa's interpreter budget +/// counts instructions, not the points each one moves (SHZ, IUP… walk a whole zone), so a font with +/// huge zones turns a short program into billions of point moves. Real fonts stay far below: a +/// twilight zone of tens to hundreds of points, a stack of up to ~2,000 values, a few thousand +/// points a glyph. Past them, glyphs are drawn unhinted. +const MAX_HINT_TWILIGHT: u16 = 1024; +const MAX_HINT_STACK: u16 = 4096; +/// skrifa's buffer for one hinted glyph (points, stack, twilight zone, CVT and storage). +const MAX_HINT_GLYPH_BYTES: usize = 256 * 1024; +/// A hinted glyph slower than this turns hinting off for its face. A face under the limits above +/// can still take seconds for one glyph: this keeps it to one. Native only (no clock on wasm). +#[cfg(not(target_arch = "wasm32"))] +const HINT_STRIKE: std::time::Duration = std::time::Duration::from_millis(50); + /// A cheap `Copy` handle to a face. Faces are never unloaded, so the handle lives for the rest of /// the process; unlike cloning an `Arc`, copying it touches no shared reference count (which made /// parallel composition scale negatively). @@ -181,11 +197,39 @@ impl FontFace { if let Some(f) = self.skrifa() && let Some(g) = f.outline_glyphs().get(GlyphId::new(gid)) { - let hinter = self.outline_hinter.get_or_init(|| { + let hinter = + self.hinter().filter(|_| !self.hinting_off.load(Ordering::Relaxed) && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES); + #[cfg(not(target_arch = "wasm32"))] + let started = std::time::Instant::now(); + let settings = hinter.map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false)); + let drawn = g.draw(settings, &mut pen); + #[cfg(not(target_arch = "wasm32"))] + if hinter.is_some() && started.elapsed() > HINT_STRIKE && !self.hinting_off.swap(true, Ordering::Relaxed) { + log::warn!("Hinting glyph {gid} in {:?} took {:?}: drawing the face unhinted", self.family, started.elapsed()); + } + if let Err(e) = drawn { + log::warn!("Cannot draw glyph {gid} in {:?}: {e}", self.family); + return BezPath::new(); + } + } + pen.0 + } + /// The face's TrueType interpreter, for the fonts that need one to assemble their contours and + /// whose zones are within the limits ([`MAX_HINT_TWILIGHT`]). + fn hinter(&self) -> Option<&HintingInstance> { + self.outline_hinter + .get_or_init(|| { + use skrifa::raw::TableProvider; + let f = self.skrifa()?; let outlines = f.outline_glyphs(); if !outlines.require_interpreter() { return None; } + let zones = f.maxp().ok().map(|m| (m.max_twilight_points().unwrap_or(0), m.max_stack_elements().unwrap_or(0))); + if !zones.is_some_and(|(twilight, stack)| twilight <= MAX_HINT_TWILIGHT && stack <= MAX_HINT_STACK) { + log::warn!("{:?} asks for hinting zones of {zones:?}: drawing it unhinted", self.family); + return None; + } // At UPEM pixels, hinted coordinates remain in font units. This // executes contour assembly without tying the cache to view zoom. HintingInstance::new( @@ -196,15 +240,8 @@ impl FontFace { ) .map_err(|e| log::warn!("Cannot initialize outline interpreter for {:?}: {e}", self.family)) .ok() - }); - let settings = - hinter.as_ref().map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false)); - if let Err(e) = g.draw(settings, &mut pen) { - log::warn!("Cannot draw glyph {gid} in {:?}: {e}", self.family); - return BezPath::new(); - } - } - pen.0 + }) + .as_ref() } /// Is `family` this face's typographic, legacy, full or PostScript name? pub fn named(&self, family: &str) -> bool { @@ -935,6 +972,7 @@ fn make_face(bytes: FontBytes, index: u32, family: String, style: String, legacy index, bmp: std::sync::OnceLock::new(), outline_hinter: std::sync::OnceLock::new(), + hinting_off: Default::default(), }) } @@ -2197,6 +2235,39 @@ mod tests { out } + /// JetBrains Mono renamed "MingLiU JBMono" (a name the interpreter is required for), with its + /// `maxp` twilight-zone size set to `twilight`. + fn hint_reliant_font(twilight: u16) -> Vec { + let mut data = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/fonts/JetBrainsMono-Regular.ttf")).unwrap(); + let utf16 = |s: &str| s.encode_utf16().flat_map(u16::to_be_bytes).collect::>(); + for (from, to) in [(utf16("JetBrains Mono"), utf16("MingLiU JBMono")), (b"JetBrains Mono".to_vec(), b"MingLiU JBMono".to_vec())] { + while let Some(at) = data.windows(from.len()).position(|w| w == from) { + data[at..at + to.len()].copy_from_slice(&to); + } + } + let tables = u16::from_be_bytes([data[4], data[5]]) as usize; + let entry = (0..tables).map(|i| 12 + 16 * i).find(|e| &data[*e..e + 4] == b"maxp").unwrap(); + let maxp = u32::from_be_bytes(data[entry + 8..entry + 12].try_into().unwrap()) as usize; + data[maxp + 16..maxp + 18].copy_from_slice(&twilight.to_be_bytes()); + data + } + + #[test] + fn a_font_asking_for_huge_hinting_zones_is_drawn_unhinted() { + // #3692: skrifa's budget counts instructions, not the points each one moves. + let db = FontDb::with_font_dirs(vec![]); + db.add_font(hint_reliant_font(660)); + let face = db.face("MingLiU JBMono", "Regular"); + assert_eq!(face.family, "MingLiU JBMono"); + assert!(face.hinter().is_some(), "a real font's zones are hinted"); + assert!(!face.glyph_outline(face.glyph_for('a')).elements().is_empty()); + let db = FontDb::with_font_dirs(vec![]); + db.add_font(hint_reliant_font(u16::MAX)); + let face = db.face("MingLiU JBMono", "Regular"); + assert!(face.hinter().is_none(), "a huge twilight zone is not hinted"); + assert!(!face.glyph_outline(face.glyph_for('a')).elements().is_empty(), "drawn unhinted"); + } + #[test] fn named_instances_and_collection_members_are_capped() { // #3716: a file's faces are capped before they are built, however the file multiplies them. From 0f1a127e63c70a4d16bcd3c673dc7f031e4adbb3 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 17:01:42 +0100 Subject: [PATCH 082/112] Findings 39, 70, 59, 46, 61, 44, 62: record the new font limits where the matching parity rows and the MCP write-root list live. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/file-format-parity.md | 3 ++- docs/layout-parity.md | 5 +++-- docs/mcp.md | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c2b352424f..bb96a62c6b 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -111,7 +111,7 @@ Measured by grepping `crates/docx/src/read` and `src/write` for the OOXML elemen | Native Word ink (`w14:contentPart`, InkML) | 🟡 | 🟡 | Drawing-level ink kept opaque with its InkML part, relationship and content type; direct run-level `w:contentPart` and run-level `mc:AlternateContent` keep the native branch and VML picture fallback. Direct content parts remain direct through repeated save/reopen cycles, without a native rendering. Drawing-level ink shows the preview or a missing-picture box. Ink members of groups (`wpg:wgp`, nested groups too) and drawing canvases (`wpc:wpc`), bare or in `mc:AlternateContent` with a `pic:pic` fallback, keep their markup, part, relationship and place among the members; they are written at the member's current position in the group and drawn from their fallback picture. Such a group can't be ungrouped yet (Ungroup is unavailable, so the ink is never left in a form it can't be saved in). Their part is dropped with the group (no orphan part); ink whose part is missing or isn't InkML gives way to its fallback, else is dropped. No native stroke rendering, eraser, conversion or replay | | 3D models | ❌ | ❌ | Dropped on read, not preserved on save | | Equations (OMML `m:oMath`) | ✅ | ✅ | #191; manual breaks (`m:brk`) and the settings' `m:mathPr` wrapping options (#326). Every ECMA-376 §22.1 element is read and laid out (pinned by `read::math::tests::every_omml_element_keeps_its_content`); equations straight in the body or a cell (no `w:p`), behind `mc:AlternateContent Requires="m"`, or with runs inside hyperlinks, simple fields, content controls and tracked insertions inside the equation keep their content; `w:sym` and Symbol-font text in math become Unicode (#1273). Not yet: Equation Editor 3.0 / MathType OLE objects show their WMF preview only (text in #1994), `m:mathFont` (Cambria Math is always asked for), `m:dispDef`/`m:defJc`/`m:lMargin`, tracked deletions shown struck through inside equations | -| Themes, font table, settings, compatibility mode | ✅ | ✅ | `w:allowSpaceOfSameStyleInTable` read and written in schema order; absent/false default off, enabled flag retained in modes 11/12/14/15. Embedded fonts (`w:embedRegular`, `w:embedBold`…) lay out and export the text set in them for the session (#1176), subsets Word embeds ("embed only the characters used", a made-up family such as `___WRD_EMBED_SUB_46` inside) found by the font table's name (#2240); the font table, its relationships and the embedded font parts are written back unchanged (#770); attached template (`w:attachedTemplate` and its external relationship) and `w:linkStyles` round-trip (#377). `w:removePersonalInformation` and `w:removeDateAndTime` round-trip and are honoured on save (#1062): comment and revision authors written as "Author" (initials "A"), no `dc:creator`/`cp:lastModifiedBy`, no annotation dates; the open document keeps its values (`file.privacyOptions` sets them, `file.privacy` reads them; File › Options › Save › Privacy for this document). The template path comes from the file, so opening never reads it: styles update from it only on an explicit Update Styles. Settings the model doesn't hold (other `w:compatSetting`s, rsids, document variables, proofing state, colour scheme mapping…) survive a save: the writer merges into the file's own `settings.xml` ([below](#settings-preservation)); `w:latentStyles`, `w:default` on localized default styles and lists' `w:nsid`/`w:tmpl`/`w:multiLevelType` round-trip | +| Themes, font table, settings, compatibility mode | ✅ | ✅ | `w:allowSpaceOfSameStyleInTable` read and written in schema order; absent/false default off, enabled flag retained in modes 11/12/14/15. Embedded fonts (`w:embedRegular`, `w:embedBold`…) lay out and export the text set in them for the session (#1176), subsets Word embeds ("embed only the characters used", a made-up family such as `___WRD_EMBED_SUB_46` inside) found by the font table's name (#2240); an embedded font file yields at most 256 faces (collection members and named instances, #3716); the font table, its relationships and the embedded font parts are written back unchanged (#770); attached template (`w:attachedTemplate` and its external relationship) and `w:linkStyles` round-trip (#377). `w:removePersonalInformation` and `w:removeDateAndTime` round-trip and are honoured on save (#1062): comment and revision authors written as "Author" (initials "A"), no `dc:creator`/`cp:lastModifiedBy`, no annotation dates; the open document keeps its values (`file.privacyOptions` sets them, `file.privacy` reads them; File › Options › Save › Privacy for this document). The template path comes from the file, so opening never reads it: styles update from it only on an explicit Update Styles. Settings the model doesn't hold (other `w:compatSetting`s, rsids, document variables, proofing state, colour scheme mapping…) survive a save: the writer merges into the file's own `settings.xml` ([below](#settings-preservation)); `w:latentStyles`, `w:default` on localized default styles and lists' `w:nsid`/`w:tmpl`/`w:multiLevelType` round-trip | | Custom XML parts, document properties, bibliography sources | 🟡 | 🟡 | Custom XML stores reached from the package/main document preserve opaque data, properties and relationships; binding evaluation remains absent. Custom properties and bibliography sources round-trip (#532). Word's own bibliography file (`b:Sources`, `Sources.xml`) imports into Manage Sources, sharing the reader of a document's own bibliography part (#2640) | | Section text direction (`w:textDirection` in `w:sectPr`, §17.6.20) | ✅ | ✅ | `tbRl`, `btLr` (and `tbRlV`/`tbLrV`, read as `tbRl`); `lrTbV` and unknown values read as horizontal (#491) | | Section vertical alignment (`w:vAlign` in `w:sectPr`, §17.6.23) | ✅ | ✅ | `top`, `center`, `bottom` and `both` (Justified, read as Center before #1320); unknown values read as top, top writes nothing (`crates/docx/tests/it/section_valign.rs`). ODT/RTF don't carry it | @@ -270,6 +270,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | | 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) | | 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | +| 2026-10-22 | trivial | An embedded font file yields at most 256 faces however many collection members or named instances it lists (#3716) | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | | 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) | | 2026-10-23 | trivial | DOCX: TOC styles Word stores under ids in another language (`Sumrio1` named `toc 1`, `CabealhodoSumrio` named `TOC Heading`) read as `TOC1`…`TOC9` and `TOCHeading` unless the file uses those ids, so Update Table builds the entries in the document's own TOC styles (#3891); a TOC whose heading ends a section (entries in columns after a continuous break) is written and read as the engine's TOC, with the page-number tab at the column's edge (#3889) | diff --git a/docs/layout-parity.md b/docs/layout-parity.md index d35ce0a1cd..32e6b82ceb 100644 --- a/docs/layout-parity.md +++ b/docs/layout-parity.md @@ -23,7 +23,7 @@ Exact line spacing keeps the run font baseline around tall equations; automatic | Feature | State | Evidence / gap | Hours | |---|---|---|---| | 2026-10-16 | trivial | Split-cell content retains its resolved top margin and border band on continuation pages, matching original Word Mac16.113.2 controls. Synthetic regressions cover margins, borders, line spacing and image frames. | -| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. | — | +| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. A font asking for a twilight zone over 1,024 points or a stack over 4,096 values, a glyph past 256 KiB of hinting buffer, or (desktop) a face whose glyph took over 50 ms to hint is drawn unhinted (#3692). | — | | Subpoint table grid columns | 🟡 | Preserve positive grid widths rather than imposing a text-box minimum; ECMA-376 §17.4.16. Native Word for Mac 16.113.2 build26092012 confirms narrow trailing grid units. `layout::tests::subpoint_grid_keeps_widths`, `tiny_grid_scales_to_width`. Tiny totals normalize without overflowing the scale factor. Zero and invalid widths retain their fallback. Border painting remains separate. | n/a | | Line breaking (UAX #14 with Word's exceptions: no break after `/`, a break after `-` before a digit) | ✅ | `layout::tests::no_line_break_right_after_a_slash`, `layout::tests::a_hyphen_before_digits_is_a_break_opportunity_like_word`, `layout::tests::a_bookmark_is_no_line_break_opportunity` | — | | A break before a slash that starts a word (" /word" wraps whole; not after a no-break space) | 🟡 | `layout::tests::a_word_after_a_slash_wraps_whole`, `layout::tests::a_no_break_space_before_a_slash_still_holds`; behaviour from the #2963 report, not compared with Word yet | — | @@ -35,7 +35,7 @@ Exact line spacing keeps the run font baseline around tall equations; automatic | Missing Thai document fonts (TH Sarabun New, Angsana New, Cordia New, Browallia New, the UPC families, Leelawadee): one Thai substitute face for the whole run, Latin letters, digits and ฿ included, so the run keeps one font at one size as Word's Cordia New substitution does | ✅ | `fonts::word::tests::a_missing_thai_font_resolves_to_an_installed_thai_face`, `fonts::fontdb::tests::a_thai_run_with_latin_falls_back_to_one_face`, `layout::tests_complex::a_missing_thai_font_draws_the_whole_run_in_one_thai_face`, `layout::tests_complex::a_thai_substitute_without_latin_hands_the_run_to_one_face_covering_it`, `fonts::fontdb::tests_sysfonts::a_run_no_preferred_face_covers_whole_loads_an_installed_font_that_does` (#1549). Windows: Cordia New, then Leelawadee UI, Tahoma, Noto Sans Thai; macOS: Thonburi (Sathu for the serif families); Linux: Noto Sans Thai, then the TLWG fonts. The first listed face with Latin letters and digits as well as Thai wins. Other Thai names (`TH …`, `…UPC`, names with `Thai`) use the same lists. When no listed face is installed, or the one found lacks the run's Thai or Latin letters, the run moves to one installed face covering its Thai and Latin text if there is one, even when a preferred Thai family without Latin letters (Noto Sans Thai) is installed; that face's whole family is loaded, so regular text keeps the Regular face. Only Thai fonts: a missing Latin font keeps Latin text in its own substitute, and a run with its own complex-script font keeps it for the Thai letters, as Word does | — | | Distributed / Thai distributed alignment | 🟡 | read; layout untested | 1–2 | | Default font metrics (Aptos, Calibri, Cambria…) | 🟡 | Missing Arial Unicode MS tries Arial then the Arial substitutes, case-insensitively (`fonts::word::tests::unicode_arial_is_proportional`); glyph coverage and exact Word line height can differ. Carlito/Caladea/Liberation are metric-compatible for Calibri/Cambria/Arial/Times/Courier; **Aptos (Word's default since 2024) has no metric-compatible open substitute**, so new Word documents wrap differently ([typing-parity.md](typing-parity.md) Known gaps). Needs an owner decision (commission or license a metric-matched font) | 6–10 + owner | -| Colour emoji and colour fonts on screen and in page renders: COLR v0/v1 paint graphs (solid fills, linear, radial and sweep gradients, composite modes) and bitmap strikes (Apple Color Emoji `sbix`, Noto Color Emoji `CBDT`) drawn in the font's colours at the layout's position and size; a dark page doesn't invert them, the same as pictures | 🟡 | `fonts::color` (skrifa), `render::color_glyph`; `render::tests_color_glyphs::emoji_are_drawn_in_colour` and `emoji_keep_their_colours_on_a_dark_page` (system font, skip without one), `render::color_glyph::tests`, `fonts::color::tests` (#1269). CBDT is verified on a real page with Noto Color Emoji; COLR v0/v1 and sbix are implemented to the OpenType spec and tested with synthetic data, but not yet seen on a real font (Segoe UI Emoji, COLRv1 Noto, Apple Color Emoji). PDF export already drew them through krilla | 1–2 | +| Colour emoji and colour fonts on screen and in page renders: COLR v0/v1 paint graphs (solid fills, linear, radial and sweep gradients, composite modes) and bitmap strikes (Apple Color Emoji `sbix`, Noto Color Emoji `CBDT`) drawn in the font's colours at the layout's position and size; a dark page doesn't invert them, the same as pictures | 🟡 | `fonts::color` (skrifa), `render::color_glyph`; `render::tests_color_glyphs::emoji_are_drawn_in_colour` and `emoji_keep_their_colours_on_a_dark_page` (system font, skip without one), `render::color_glyph::tests`, `fonts::color::tests` (#1269). A glyph copies at most 65,536 gradient stops and clip-path elements in all, else it falls back to its outline, and the cache charges what it keeps (#3692). CBDT is verified on a real page with Noto Color Emoji; COLR v0/v1 and sbix are implemented to the OpenType spec and tested with synthetic data, but not yet seen on a real font (Segoe UI Emoji, COLRv1 Noto, Apple Color Emoji). PDF export already drew them through krilla | 1–2 | | Full font names in document font requests | ✅ | Exact English (or first available) OpenType name ID4 lookup in loaded faces and the native lazy catalog; family and legacy-family matches keep priority. No style-suffix guessing; named variable-instance aliases remain unsupported | 0 | | Symbol fonts (Symbol including its PCL6 alias, Wingdings, Wingdings 2/3, Webdings) without the proprietary font: each character, stored as U+F020–U+F0FF or a plain 8-bit code, is drawn as its Unicode equivalent from a face that has it; PDF text and copies read as those symbols | ✅ | `doc::symbol_fonts` tables (Adobe Symbol table and Unicode L2/12-368), `layout::tests::wingdings_text_draws_and_reads_as_its_unicode_symbols` (#776). Shapes come from Noto Sans Symbols / Symbols 2, DejaVu Sans or Segoe UI Symbol, so they differ from Word's; Wingdings 0xFF (not in Unicode) still has no symbol. A line keeps the symbol font's own height (Symbol, Wingdings: the ascent and descent Word's PDFs give them), so a Symbol bullet makes its line as tall as in Word, `layout::tests::a_symbol_bullet_sizes_its_line_by_the_symbol_font`. Apple's one-em Symbol takes the same height, as in Word for Mac (`fonts::word::tests::a_one_em_symbol_face_takes_windows_symbols_line`) | 0 | | List bullets stored as a private-use character (U+E000–U+F8FF) in a font that isn't installed and isn't a mapped symbol-font code: drawn as the level's default bullet (• ○ ▪), not as whatever an installed icon font has at that code | ✅ | `layout::tests::private_use_bullet_no_font_draws_shows_the_levels_default_bullet` (#1683: U+F081 showed Font Awesome's Twitter logo) | — | @@ -273,6 +273,7 @@ layout suite are pending; this is not a new Word/native-IME acceptance claim. | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | | 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged | | 2026-10-22 | trivial | Layout lookups linear in clipped cells, authors, sections, header paragraphs and bidi offsets (#3692, #3706) | +| 2026-10-22 | trivial | Hostile fonts (#3692): hint-reliant fonts with huge hinting zones, or a slow hinted glyph, are drawn unhinted; a colour glyph's gradient stops and clip paths share one budget | | 2026-10-22 | trivial | Group connector rerouting preserves displayed free and unresolved endpoints under nonuniform scaling while resolved ends keep their connection-site path; degenerate dimensions and unrepresentable fallback display spans are guarded; synthetic model/engine coverage only, readiness estimates unchanged | | 2026-10-21 | trivial | A word with no break opportunity in it, laid out beside a floating object, no longer lays out quadratically: every word is measured once, before the line is broken (#3412) | | 2026-10-23 | trivial | An empty paragraph whose mark is hidden is laid out as nothing, like a deleted one: no room, no list number (still counted, as in Word) and no shading, in the body and in table cells, headers and text boxes (#3982) | diff --git a/docs/mcp.md b/docs/mcp.md index 634691776a..f44ecd9aba 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -70,7 +70,8 @@ config works across the suite) keep the agent to the folders you give it: from beside that file, so they are inside the read root too (a link that leads out is not followed). - Every file a command writes must lie inside the **write root**: `save_document`, `file.save`, `file.saveAs`, `file.rename` (the file and its new name), `file.exportPdf`, `file.exportPng`, `file.printPdf`, - `file.saveTemplate`, and `mailings.finish` with a `path`. + `file.saveTemplate`, and `mailings.finish` with a `path`. `font.add` and `font.remove` change the + user font folder, so they are refused when it is outside the write root (#3692). - Read and write access are separate. Give the same folder twice for both; a root left out grants none of its access (with only `--automation-read-root`, nothing can be written to disk). Without either flag nothing changes. From bbe1dad1e7e643b44fc755793b49a2217046746a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 21:07:11 +0100 Subject: [PATCH 083/112] #3692: hinting turns off after three slow glyphs, not one CodeRabbit review of finding 62's fix: one hinted draw slowed by a busy machine (scheduler pause, page fault) switched hinting off for the face for the rest of the process, leaving hint-reliant faces with timing-dependent missing strokes. A strike counter now drops hinting only at the third slow glyph. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/fontdb.rs | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 818f086d09..f554b1720f 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -88,8 +88,8 @@ pub struct FontFace { bmp: std::sync::OnceLock>, // Some legacy CJK fonts assemble their contours with TrueType instructions. outline_hinter: std::sync::OnceLock>, - /// Set once a hinted glyph ran too long: the face is drawn unhinted from then on. - hinting_off: std::sync::atomic::AtomicBool, + /// Hinted glyphs that ran too long; at [`MAX_HINT_STRIKES`] the face is drawn unhinted. + hint_strikes: std::sync::atomic::AtomicU8, } /// Limits on what a face's TrueType instructions may run over (#3692). skrifa's interpreter budget @@ -101,10 +101,13 @@ const MAX_HINT_TWILIGHT: u16 = 1024; const MAX_HINT_STACK: u16 = 4096; /// skrifa's buffer for one hinted glyph (points, stack, twilight zone, CVT and storage). const MAX_HINT_GLYPH_BYTES: usize = 256 * 1024; -/// A hinted glyph slower than this turns hinting off for its face. A face under the limits above -/// can still take seconds for one glyph: this keeps it to one. Native only (no clock on wasm). +/// A hinted glyph slower than this is a strike against its face; [`MAX_HINT_STRIKES`] of them turn +/// hinting off. A face under the limits above can still take seconds for one glyph: this keeps it +/// to a few, while one draw slowed by a busy machine doesn't drop hinting. Native only (no clock +/// on wasm). #[cfg(not(target_arch = "wasm32"))] const HINT_STRIKE: std::time::Duration = std::time::Duration::from_millis(50); +const MAX_HINT_STRIKES: u8 = 3; /// A cheap `Copy` handle to a face. Faces are never unloaded, so the handle lives for the rest of /// the process; unlike cloning an `Arc`, copying it touches no shared reference count (which made @@ -190,6 +193,11 @@ impl FontFace { pub fn id(&self) -> u32 { self.id } + /// Counts a slow hinted draw; true when it is the one that turns hinting off. + #[cfg(not(target_arch = "wasm32"))] + fn strike_hinting(&self) -> bool { + self.hint_strikes.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| (n < MAX_HINT_STRIKES).then_some(n + 1)) == Ok(MAX_HINT_STRIKES - 1) + } /// Glyph `gid`'s outline in font units, y-down (flipped), uncached: for faces the database /// doesn't keep (font list previews); others go through [`FontDb::outline`]'s cache. pub fn glyph_outline(&self, gid: u32) -> BezPath { @@ -197,14 +205,15 @@ impl FontFace { if let Some(f) = self.skrifa() && let Some(g) = f.outline_glyphs().get(GlyphId::new(gid)) { - let hinter = - self.hinter().filter(|_| !self.hinting_off.load(Ordering::Relaxed) && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES); + let hinter = self.hinter().filter(|_| { + self.hint_strikes.load(Ordering::Relaxed) < MAX_HINT_STRIKES && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES + }); #[cfg(not(target_arch = "wasm32"))] let started = std::time::Instant::now(); let settings = hinter.map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false)); let drawn = g.draw(settings, &mut pen); #[cfg(not(target_arch = "wasm32"))] - if hinter.is_some() && started.elapsed() > HINT_STRIKE && !self.hinting_off.swap(true, Ordering::Relaxed) { + if hinter.is_some() && started.elapsed() > HINT_STRIKE && self.strike_hinting() { log::warn!("Hinting glyph {gid} in {:?} took {:?}: drawing the face unhinted", self.family, started.elapsed()); } if let Err(e) = drawn { @@ -972,7 +981,7 @@ fn make_face(bytes: FontBytes, index: u32, family: String, style: String, legacy index, bmp: std::sync::OnceLock::new(), outline_hinter: std::sync::OnceLock::new(), - hinting_off: Default::default(), + hint_strikes: Default::default(), }) } @@ -1975,6 +1984,15 @@ mod tests_sysfonts; #[cfg(test)] mod tests { + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn one_slow_hinted_glyph_does_not_turn_hinting_off() { + let db = super::FontDb::with_font_dirs(vec![]); + let face = db.face("Anything", "Regular"); + let strikes: Vec = (0..5).map(|_| face.strike_hinting()).collect(); + assert_eq!(strikes, [false, false, true, false, false]); + assert_eq!(face.hint_strikes.load(std::sync::atomic::Ordering::Relaxed), super::MAX_HINT_STRIKES); + } #[test] #[ignore = "requires a locally licensed kaiu.ttf; set WORDCRAFT_TEST_KAI_FONT"] fn kai_contours_include_complete_character_body() { From e541a6f62fb6395fe1038b4877f3ec1535c753e7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 21:57:45 +0100 Subject: [PATCH 084/112] #3692: hinting turns off only when slow draws are a real share of a face's draws Ultrareview finding 5: the slow-draw count only grew, so three draws slowed by a busy machine at any point in a session switched hinting off until restart, breaking hint-reliant faces (MingLiU, DFKai-SB). Hinting now stops only when at least three draws were slow and they are at least one in 16 of the face's hinted draws: a face slow throughout still stops at its third glyph, and slow glyphs spread among fast ones are still bounded. Counting interpreter work instead of wall-clock stays with #3827. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/fonts/src/fontdb.rs | 68 ++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index f554b1720f..9b4ef93721 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -88,8 +88,13 @@ pub struct FontFace { bmp: std::sync::OnceLock>, // Some legacy CJK fonts assemble their contours with TrueType instructions. outline_hinter: std::sync::OnceLock>, - /// Hinted glyphs that ran too long; at [`MAX_HINT_STRIKES`] the face is drawn unhinted. - hint_strikes: std::sync::atomic::AtomicU8, + /// Hinted draws so far, those that ran too long, and whether that turned hinting off + /// ([`FontFace::count_hinted`]). + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + hint_draws: std::sync::atomic::AtomicU32, + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + hint_slow: std::sync::atomic::AtomicU32, + hinting_off: std::sync::atomic::AtomicBool, } /// Limits on what a face's TrueType instructions may run over (#3692). skrifa's interpreter budget @@ -101,13 +106,18 @@ const MAX_HINT_TWILIGHT: u16 = 1024; const MAX_HINT_STACK: u16 = 4096; /// skrifa's buffer for one hinted glyph (points, stack, twilight zone, CVT and storage). const MAX_HINT_GLYPH_BYTES: usize = 256 * 1024; -/// A hinted glyph slower than this is a strike against its face; [`MAX_HINT_STRIKES`] of them turn -/// hinting off. A face under the limits above can still take seconds for one glyph: this keeps it -/// to a few, while one draw slowed by a busy machine doesn't drop hinting. Native only (no clock -/// on wasm). +/// A hinted glyph slower than this is a strike against its face. A face under the limits above can +/// still take seconds for one glyph, so [`MIN_HINT_STRIKES`] strikes that are also at least one +/// in [`HINT_SLOW_RATIO`] of its hinted draws turn hinting off: a face that is slow throughout +/// stops after a few glyphs, while the odd draw slowed by a busy machine leaves a face hinted. +/// ponytail: wall-clock, not interpreter work (skrifa has no work bound yet, #3827). Native only +/// (no clock on wasm). #[cfg(not(target_arch = "wasm32"))] const HINT_STRIKE: std::time::Duration = std::time::Duration::from_millis(50); -const MAX_HINT_STRIKES: u8 = 3; +#[cfg(not(target_arch = "wasm32"))] +const MIN_HINT_STRIKES: u32 = 3; +#[cfg(not(target_arch = "wasm32"))] +const HINT_SLOW_RATIO: u32 = 16; /// A cheap `Copy` handle to a face. Faces are never unloaded, so the handle lives for the rest of /// the process; unlike cloning an `Arc`, copying it touches no shared reference count (which made @@ -193,10 +203,16 @@ impl FontFace { pub fn id(&self) -> u32 { self.id } - /// Counts a slow hinted draw; true when it is the one that turns hinting off. + /// Counts a hinted draw, `slow` when it ran past [`HINT_STRIKE`]; true when it is the one that + /// turns hinting off. #[cfg(not(target_arch = "wasm32"))] - fn strike_hinting(&self) -> bool { - self.hint_strikes.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| (n < MAX_HINT_STRIKES).then_some(n + 1)) == Ok(MAX_HINT_STRIKES - 1) + fn count_hinted(&self, slow: bool) -> bool { + let draws = self.hint_draws.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if !slow { + return false; + } + let slow = self.hint_slow.fetch_add(1, Ordering::Relaxed).saturating_add(1); + slow >= MIN_HINT_STRIKES && slow.saturating_mul(HINT_SLOW_RATIO) >= draws && !self.hinting_off.swap(true, Ordering::Relaxed) } /// Glyph `gid`'s outline in font units, y-down (flipped), uncached: for faces the database /// doesn't keep (font list previews); others go through [`FontDb::outline`]'s cache. @@ -205,15 +221,14 @@ impl FontFace { if let Some(f) = self.skrifa() && let Some(g) = f.outline_glyphs().get(GlyphId::new(gid)) { - let hinter = self.hinter().filter(|_| { - self.hint_strikes.load(Ordering::Relaxed) < MAX_HINT_STRIKES && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES - }); + let hinter = + self.hinter().filter(|_| !self.hinting_off.load(Ordering::Relaxed) && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES); #[cfg(not(target_arch = "wasm32"))] let started = std::time::Instant::now(); let settings = hinter.map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false)); let drawn = g.draw(settings, &mut pen); #[cfg(not(target_arch = "wasm32"))] - if hinter.is_some() && started.elapsed() > HINT_STRIKE && self.strike_hinting() { + if hinter.is_some() && self.count_hinted(started.elapsed() > HINT_STRIKE) { log::warn!("Hinting glyph {gid} in {:?} took {:?}: drawing the face unhinted", self.family, started.elapsed()); } if let Err(e) = drawn { @@ -981,7 +996,9 @@ fn make_face(bytes: FontBytes, index: u32, family: String, style: String, legacy index, bmp: std::sync::OnceLock::new(), outline_hinter: std::sync::OnceLock::new(), - hint_strikes: Default::default(), + hint_draws: Default::default(), + hint_slow: Default::default(), + hinting_off: Default::default(), }) } @@ -1986,12 +2003,21 @@ mod tests_sysfonts; mod tests { #[test] #[cfg(not(target_arch = "wasm32"))] - fn one_slow_hinted_glyph_does_not_turn_hinting_off() { - let db = super::FontDb::with_font_dirs(vec![]); - let face = db.face("Anything", "Regular"); - let strikes: Vec = (0..5).map(|_| face.strike_hinting()).collect(); - assert_eq!(strikes, [false, false, true, false, false]); - assert_eq!(face.hint_strikes.load(std::sync::atomic::Ordering::Relaxed), super::MAX_HINT_STRIKES); + fn occasional_slow_hinted_glyphs_do_not_turn_hinting_off() { + let off = |pattern: &[bool]| { + let face = + super::make_face(super::FontBytes::Static(super::BUNDLED[0]), 0, "T".into(), "Regular".into(), "T".into(), Vec::new()).unwrap(); + pattern.iter().position(|&slow| face.count_hinted(slow)) + }; + // A face slow throughout stops at the third strike. + assert_eq!(off(&[true; 5]), Some(2)); + // Three draws slowed by a busy machine among many: hinting stays on. + let mut busy = vec![false; 100]; + busy.extend([true; 3]); + assert_eq!(off(&busy), None); + // Slow glyphs spread among fast ones still stop it once they are one in 16. + let spread: Vec = (0..64).map(|i| i % 8 == 7).collect(); + assert_eq!(off(&spread), Some(23)); } #[test] #[ignore = "requires a locally licensed kaiu.ttf; set WORDCRAFT_TEST_KAI_FONT"] From d69c386f8fecd4031a5eaa362d54679cf70febe5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:51:42 +0100 Subject: [PATCH 085/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20a=20font'?= =?UTF-8?q?s=20temporary=20file=20is=20created=20afresh,=20so=20a=20link?= =?UTF-8?q?=20planted=20at=20its=20name=20is=20not=20written=20through?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/fonts/src/fontdb/user.rs | 40 +++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/fonts/src/fontdb/user.rs b/crates/fonts/src/fontdb/user.rs index b81f89d693..d31feebf46 100644 --- a/crates/fonts/src/fontdb/user.rs +++ b/crates/fonts/src/fontdb/user.rs @@ -553,8 +553,20 @@ fn write_unique(dir: &Path, name: &str, bytes: &[u8]) -> std::io::Result { + std::fs::remove_file(&tmp)?; + create()? + } + r => r?, + }; + let written = std::io::Write::write_all(&mut file, bytes); + drop(file); + if let Err(e) = written.and_then(|()| std::fs::rename(&tmp, &path)) { let _ = std::fs::remove_file(&tmp); return Err(e); } @@ -950,4 +962,28 @@ pub(crate) mod tests { assert_eq!(stored_name("-rf.otf", b"OTTO"), "font-rf.otf"); assert_eq!(unique_name("a.ttf", |n| n == "a.ttf" || n == "a-2.ttf"), "a-3.ttf"); } + + #[cfg(unix)] + #[test] + fn a_link_planted_at_the_temporary_name_is_not_written_through() { + // #3703 review: the temporary file is created afresh, so a planted link can't carry the + // font bytes to a file outside the folder. + let root = std::env::temp_dir().join(format!("wordcraft-font-part-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let dir = root.join("fonts"); + std::fs::create_dir_all(&dir).unwrap(); + let outside = root.join("outside.txt"); + std::fs::write(&outside, b"keep me").unwrap(); + std::os::unix::fs::symlink(&outside, dir.join(".a.ttf.part")).unwrap(); + let path = write_unique(&dir, "a.ttf", b"font bytes").unwrap(); + assert_eq!(std::fs::read(&outside).unwrap(), b"keep me"); + assert_eq!(path, dir.join("a.ttf")); + assert_eq!(std::fs::read(&path).unwrap(), b"font bytes"); + assert!(!std::fs::symlink_metadata(&path).unwrap().file_type().is_symlink()); + assert!(std::fs::symlink_metadata(dir.join(".a.ttf.part")).is_err(), "the temporary name is gone"); + // A leftover from a crash doesn't block the next add either. + std::fs::write(dir.join(".b.ttf.part"), b"half a font").unwrap(); + assert_eq!(std::fs::read(write_unique(&dir, "b.ttf", b"whole").unwrap()).unwrap(), b"whole"); + let _ = std::fs::remove_dir_all(&root); + } } From 68d45318bbc976a9c358316ee0254f3ef6b3f05a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:51:43 +0100 Subject: [PATCH 086/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20a=20colou?= =?UTF-8?q?r=20glyph's=20clip=20outline=20is=20drawn=20within=20the=20payl?= =?UTF-8?q?oad=20left,=20refused=20before=20it=20is=20built=20whole?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/fonts/src/color.rs | 45 ++++++++++++++++------- crates/fonts/src/fontdb.rs | 74 +++++++++++++++++++++++++++++++++----- 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/crates/fonts/src/color.rs b/crates/fonts/src/color.rs index 1f03641b68..3e0a35641d 100644 --- a/crates/fonts/src/color.rs +++ b/crates/fonts/src/color.rs @@ -127,10 +127,10 @@ impl FontFace { if let Some(cg) = ColorGlyphCollection::new(&f).get(glyph) { let palette: Vec<[u8; 4]> = ColorPalettes::new(&f).get(0).map(|p| p.colors().iter().map(|c| [c.red, c.green, c.blue, c.alpha]).collect()).unwrap_or_default(); - let mut rec = Recorder::new(&palette, |g: GlyphId| { - let mut p = self.glyph_outline(g.to_u32()); + let mut rec = Recorder::new(&palette, |g: GlyphId, room: usize| { + let mut p = self.glyph_outline_capped(g.to_u32(), room)?; p.apply_affine(Affine::FLIP_Y); - p + Some(p) }); if cg.paint(self.location(), &mut rec).is_ok() && let Some(ops) = rec.finish() @@ -193,7 +193,7 @@ pub fn ops_bytes(ops: &[ColorOp]) -> usize { /// A [`ColorPainter`] that records the paint calls as [`ColorOp`]s, resolving palette colours and /// clip glyphs. Stops recording (and [`Recorder::finish`] gives None) past [`MAX_OPS`] or /// [`MAX_PAYLOAD`]. -pub struct Recorder<'a, F: Fn(GlyphId) -> BezPath> { +pub struct Recorder<'a, F: Fn(GlyphId, usize) -> Option> { palette: &'a [[u8; 4]], outline: F, ops: Vec, @@ -201,9 +201,9 @@ pub struct Recorder<'a, F: Fn(GlyphId) -> BezPath> { overflow: bool, } -impl<'a, F: Fn(GlyphId) -> BezPath> Recorder<'a, F> { +impl<'a, F: Fn(GlyphId, usize) -> Option> Recorder<'a, F> { /// A recorder with palette `palette` (straight RGBA) and clip outlines (font units, y up) from - /// `outline`. + /// `outline`, which is given the path elements left in the budget and gives None past them. pub fn new(palette: &'a [[u8; 4]], outline: F) -> Self { Recorder { palette, outline, ops: vec![], payload: 0, overflow: false } } @@ -267,7 +267,7 @@ fn pt(p: skrifa::raw::types::Point) -> Point { Point::new(p.x as f64, p.y as f64) } -impl BezPath> ColorPainter for Recorder<'_, F> { +impl Option> ColorPainter for Recorder<'_, F> { fn push_transform(&mut self, t: Transform) { self.push(ColorOp::PushTransform(Affine::new([t.xx, t.yx, t.xy, t.yy, t.dx, t.dy].map(|v| v as f64)))); } @@ -278,7 +278,11 @@ impl BezPath> ColorPainter for Recorder<'_, F> { if self.overflow { return; } - let p = (self.outline)(glyph_id); + // Built within what is left of the budget: a huge outline is refused while it is drawn. + let Some(p) = (self.outline)(glyph_id, MAX_PAYLOAD.saturating_sub(self.payload)) else { + self.overflow = true; + return; + }; if self.charge(p.elements().len()) { self.push(ColorOp::PushClip(p)); } @@ -326,8 +330,8 @@ impl BezPath> ColorPainter for Recorder<'_, F> { mod tests { use super::*; - fn square(_: GlyphId) -> BezPath { - kurbo::Shape::to_path(&Rect::new(0.0, 0.0, 10.0, 10.0), 0.1) + fn square(_: GlyphId, room: usize) -> Option { + Some(kurbo::Shape::to_path(&Rect::new(0.0, 0.0, 10.0, 10.0), 0.1)).filter(|p| p.elements().len() <= room) } #[test] @@ -384,9 +388,9 @@ mod tests { let fills = MAX_PAYLOAD / stops.len() + 1; assert!(fills < MAX_OPS); let outlines = std::cell::Cell::new(0); - let count = |g| { + let count = |g, room| { outlines.set(outlines.get() + 1); - square(g) + square(g, room) }; let mut r = Recorder::new(&[], count); for _ in 0..fills { @@ -404,6 +408,23 @@ mod tests { assert!(ops_bytes(&ops) >= stops.len() * std::mem::size_of::()); } + #[test] + fn a_clip_outline_is_drawn_within_the_payload_left() { + // #3716 review: the outline is told the room left, so an oversized one is never built whole. + let stops = vec![ColorStop { offset: 0.5, palette_index: 0, alpha: 1.0 }; MAX_PAYLOAD - 3]; + let rooms = std::cell::RefCell::new(vec![]); + let seen = |g, room| { + rooms.borrow_mut().push(room); + square(g, room) + }; + let mut r = Recorder::new(&[], seen); + r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad }); + r.push_clip_glyph(GlyphId::new(1)); + r.pop_clip(); + assert_eq!(*rooms.borrow(), [3]); + assert!(r.finish().is_none(), "a square needs more than the 3 elements left"); + } + fn bitmap(origin: skrifa::bitmap::Origin, data: skrifa::bitmap::BitmapData<'_>) -> skrifa::bitmap::BitmapGlyph<'_> { skrifa::bitmap::BitmapGlyph { data, diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 9b4ef93721..931655903f 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -217,7 +217,16 @@ impl FontFace { /// Glyph `gid`'s outline in font units, y-down (flipped), uncached: for faces the database /// doesn't keep (font list previews); others go through [`FontDb::outline`]'s cache. pub fn glyph_outline(&self, gid: u32) -> BezPath { - let mut pen = FlipPen(BezPath::new()); + self.draw_outline(gid, usize::MAX).path + } + /// [`FontFace::glyph_outline`], or None when the outline has more than `max_elements` path + /// elements: drawing stops copying them at the limit, so a huge glyph is never built whole. + pub fn glyph_outline_capped(&self, gid: u32, max_elements: usize) -> Option { + let pen = self.draw_outline(gid, max_elements); + (!pen.over).then_some(pen.path) + } + fn draw_outline(&self, gid: u32, max_elements: usize) -> FlipPen { + let mut pen = FlipPen { path: BezPath::new(), room: max_elements, over: false }; if let Some(f) = self.skrifa() && let Some(g) = f.outline_glyphs().get(GlyphId::new(gid)) { @@ -233,10 +242,10 @@ impl FontFace { } if let Err(e) = drawn { log::warn!("Cannot draw glyph {gid} in {:?}: {e}", self.family); - return BezPath::new(); + pen.path = BezPath::new(); } } - pen.0 + pen } /// The face's TrueType interpreter, for the fonts that need one to assemble their contours and /// whose zones are within the limits ([`MAX_HINT_TWILIGHT`]). @@ -1974,23 +1983,55 @@ impl FontDb { } } -struct FlipPen(BezPath); +/// Collects an outline flipped to y-down, keeping at most `room` path elements (`over` once it +/// had to drop one). +struct FlipPen { + path: BezPath, + room: usize, + over: bool, +} + +impl FlipPen { + /// Take one element's room; false (and `over`) when there is none left. + fn take(&mut self) -> bool { + match self.room.checked_sub(1) { + Some(room) if !self.over => { + self.room = room; + true + } + _ => { + self.over = true; + false + } + } + } +} impl OutlinePen for FlipPen { fn move_to(&mut self, x: f32, y: f32) { - self.0.move_to((x as f64, -y as f64)); + if self.take() { + self.path.move_to((x as f64, -y as f64)); + } } fn line_to(&mut self, x: f32, y: f32) { - self.0.line_to((x as f64, -y as f64)); + if self.take() { + self.path.line_to((x as f64, -y as f64)); + } } fn quad_to(&mut self, cx0: f32, cy0: f32, x: f32, y: f32) { - self.0.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64)); + if self.take() { + self.path.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64)); + } } fn curve_to(&mut self, cx0: f32, cy0: f32, cx1: f32, cy1: f32, x: f32, y: f32) { - self.0.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64)); + if self.take() { + self.path.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64)); + } } fn close(&mut self) { - self.0.close_path(); + if self.take() { + self.path.close_path(); + } } } @@ -2051,6 +2092,21 @@ mod tests { } } + #[test] + fn a_capped_outline_is_refused_while_it_is_drawn() { + // #3716 review: a colour glyph's clip outline stops growing at the payload left. + let db = FontDb::with_font_dirs(Vec::new()); + db.set_system_fallback(false); + let face = db.face(FALLBACK_FAMILY, "Regular"); + let gid = face.glyph_for('g'); + let whole = face.glyph_outline(gid); + let n = whole.elements().len(); + assert!(n > 4, "{n}"); + assert_eq!(face.glyph_outline_capped(gid, n).as_ref().map(BezPath::elements), Some(whole.elements())); + assert!(face.glyph_outline_capped(gid, n - 1).is_none()); + assert!(face.glyph_outline_capped(gid, 0).is_none()); + } + #[test] fn run_fallback_is_cached_per_char_set() { let db = FontDb::with_font_dirs(Vec::new()); From 87d3ea5363750be994c5813453b8bf8581123e11 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:52:13 +0100 Subject: [PATCH 087/112] Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- docs/layout-parity.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/layout-parity.md b/docs/layout-parity.md index 32e6b82ceb..c82dad91b8 100644 --- a/docs/layout-parity.md +++ b/docs/layout-parity.md @@ -23,7 +23,7 @@ Exact line spacing keeps the run font baseline around tall equations; automatic | Feature | State | Evidence / gap | Hours | |---|---|---|---| | 2026-10-16 | trivial | Split-cell content retains its resolved top margin and border band on continuation pages, matching original Word Mac16.113.2 controls. Synthetic regressions cover margins, borders, line spacing and image frames. | -| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. A font asking for a twilight zone over 1,024 points or a stack over 4,096 values, a glyph past 256 KiB of hinting buffer, or (desktop) a face whose glyph took over 50 ms to hint is drawn unhinted (#3692). | — | +| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. A font asking for a twilight zone over 1,024 points or a stack over 4,096 values, or a glyph past 256 KiB of hinting buffer, is drawn unhinted; on desktop, once at least three of a face's hinted draws have taken over 50 ms and those slow draws are at least 1/16 of its hinted draws, its later glyphs are drawn unhinted (#3692). | — | | Subpoint table grid columns | 🟡 | Preserve positive grid widths rather than imposing a text-box minimum; ECMA-376 §17.4.16. Native Word for Mac 16.113.2 build26092012 confirms narrow trailing grid units. `layout::tests::subpoint_grid_keeps_widths`, `tiny_grid_scales_to_width`. Tiny totals normalize without overflowing the scale factor. Zero and invalid widths retain their fallback. Border painting remains separate. | n/a | | Line breaking (UAX #14 with Word's exceptions: no break after `/`, a break after `-` before a digit) | ✅ | `layout::tests::no_line_break_right_after_a_slash`, `layout::tests::a_hyphen_before_digits_is_a_break_opportunity_like_word`, `layout::tests::a_bookmark_is_no_line_break_opportunity` | — | | A break before a slash that starts a word (" /word" wraps whole; not after a no-break space) | 🟡 | `layout::tests::a_word_after_a_slash_wraps_whole`, `layout::tests::a_no_break_space_before_a_slash_still_holds`; behaviour from the #2963 report, not compared with Word yet | — | @@ -273,7 +273,7 @@ layout suite are pending; this is not a new Word/native-IME acceptance claim. | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | | 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged | | 2026-10-22 | trivial | Layout lookups linear in clipped cells, authors, sections, header paragraphs and bidi offsets (#3692, #3706) | -| 2026-10-22 | trivial | Hostile fonts (#3692): hint-reliant fonts with huge hinting zones, or a slow hinted glyph, are drawn unhinted; a colour glyph's gradient stops and clip paths share one budget | +| 2026-10-22 | trivial | Hostile fonts (#3692): hint-reliant fonts with huge hinting zones are drawn unhinted, and a face draws its later glyphs unhinted once at least three of its hinted draws took over 50 ms and those are at least 1/16 of its hinted draws; a colour glyph's gradient stops and clip paths share one budget | | 2026-10-22 | trivial | Group connector rerouting preserves displayed free and unresolved endpoints under nonuniform scaling while resolved ends keep their connection-site path; degenerate dimensions and unrepresentable fallback display spans are guarded; synthetic model/engine coverage only, readiness estimates unchanged | | 2026-10-21 | trivial | A word with no break opportunity in it, laid out beside a floating object, no longer lays out quadratically: every word is measured once, before the line is broken (#3412) | | 2026-10-23 | trivial | An empty paragraph whose mark is hidden is laid out as nothing, like a deleted one: no room, no list number (still counted, as in Word) and no shading, in the body and in table cells, headers and text boxes (#3982) | From 57e9e0f2441fb8cbcc60c26c8bd1266a43254565 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 05:19:20 +0100 Subject: [PATCH 088/112] =?UTF-8?q?#3703:=20review=20=E2=80=94=20font=20st?= =?UTF-8?q?ore=20adds=20run=20one=20at=20a=20time,=20so=20racing=20adds=20?= =?UTF-8?q?never=20share=20a=20temporary=20or=20a=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit on #3869: two concurrent install_font calls could pick the same destination and temporary, and the leftover-unlink retry could remove the other call's open temporary. write_unique now holds a process-wide lock. In-process only (noted in a ponytail comment). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/fonts/src/fontdb/user.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/fonts/src/fontdb/user.rs b/crates/fonts/src/fontdb/user.rs index d31feebf46..c67fb6a213 100644 --- a/crates/fonts/src/fontdb/user.rs +++ b/crates/fonts/src/fontdb/user.rs @@ -549,6 +549,11 @@ fn unique_name(name: &str, taken: impl Fn(&str) -> bool) -> String { /// Write `bytes` to a new file in `dir` named `name` (or a numbered variant when that exists), /// through a temporary file so a crash never leaves half a font. fn write_unique(dir: &Path, name: &str, bytes: &[u8]) -> std::io::Result { + // One store at a time: two adds racing for the same name would otherwise pick the same file + // and unlink each other's temporary. ponytail: in-process only; two app instances adding the + // same name at the same instant can still collide (the later rename wins). + static STORE: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _held = STORE.lock().unwrap_or_else(std::sync::PoisonError::into_inner); std::fs::create_dir_all(dir)?; let name = unique_name(name, |n| dir.join(n).exists()); let path = dir.join(&name); @@ -986,4 +991,26 @@ pub(crate) mod tests { assert_eq!(std::fs::read(write_unique(&dir, "b.ttf", b"whole").unwrap()).unwrap(), b"whole"); let _ = std::fs::remove_dir_all(&root); } + + #[test] + fn fonts_stored_at_once_under_one_name_each_get_their_own_file() { + // #3703 review: concurrent adds of one name never share a temporary or a destination. + let dir = std::env::temp_dir().join(format!("wordcraft-font-race-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let paths: Vec<_> = (0..8u8) + .map(|i| { + let dir = dir.clone(); + std::thread::spawn(move || (i, write_unique(&dir, "a.ttf", &[i; 64]).unwrap())) + }) + .collect::>() + .into_iter() + .map(|t| t.join().unwrap()) + .collect(); + let distinct: std::collections::HashSet<_> = paths.iter().map(|(_, p)| p.clone()).collect(); + assert_eq!(distinct.len(), 8); + for (i, p) in &paths { + assert_eq!(std::fs::read(p).unwrap(), vec![*i; 64], "{}", p.display()); + } + let _ = std::fs::remove_dir_all(&dir); + } } From b059da2d4e52ad40a9cb54f8de6e65d08308f737 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:09:43 +0100 Subject: [PATCH 089/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20docs:=20w?= =?UTF-8?q?hy=20the=20hinting=20breaker=20is=20timed,=20and=20what=20brotl?= =?UTF-8?q?i-decompressor=20brings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hinting breaker stays wall-clock: skrifa's interpreter budget is fixed, counts instructions rather than the points each one moves, and reports no work count, so there is nothing deterministic to bound. The comment now says the result can depend on the machine by design, and that only faces needing the interpreter for their contours are hinted at all. Cargo.toml notes that brotli-decompressor is the crate and version wuff already uses: pure Rust, forbid(unsafe_code) by default, wasm-ready, BSD-3-Clause/MIT. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- Cargo.toml | 2 ++ crates/fonts/src/fontdb.rs | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6bdbf2ef9c..ee319c50ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,8 @@ harfrust = "0.13" # (brotli-decompressor, flate2) are already in the tree. wuff = "0.2.9" # WOFF2's brotli stream, decoded ahead of wuff so its glyph point counts are checked first (#3703). +# The same crate and version wuff already decodes it with (no new code in the build): pure Rust, +# `forbid(unsafe_code)` in its default build, no_std-capable (builds for wasm), BSD-3-Clause/MIT. brotli-decompressor = "5" vello_cpu = "0.2" image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "gif", "bmp", "tiff"] } diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 931655903f..7da5d927c2 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -110,8 +110,15 @@ const MAX_HINT_GLYPH_BYTES: usize = 256 * 1024; /// still take seconds for one glyph, so [`MIN_HINT_STRIKES`] strikes that are also at least one /// in [`HINT_SLOW_RATIO`] of its hinted draws turn hinting off: a face that is slow throughout /// stops after a few glyphs, while the odd draw slowed by a busy machine leaves a face hinted. -/// ponytail: wall-clock, not interpreter work (skrifa has no work bound yet, #3827). Native only -/// (no clock on wasm). +/// +/// Wall-clock, not interpreter work: skrifa's own budget (a million instructions a program, not +/// settable) counts instructions, not the points each one moves, and it exposes no count of the +/// work done, so time is the only measure there is (#3827). Whether such a face stays hinted can +/// therefore depend on the machine, by design: only faces that need the interpreter to assemble +/// their contours (legacy CJK fonts such as DFKai-SB) are hinted at all, and those typically hint +/// a glyph in microseconds, so 50 ms is far above an honest face on a loaded machine; a face +/// that trips it is drawn unhinted (its outlines possibly misassembled) rather than freezing the +/// app. Native only (no clock on wasm). #[cfg(not(target_arch = "wasm32"))] const HINT_STRIKE: std::time::Duration = std::time::Duration::from_millis(50); #[cfg(not(target_arch = "wasm32"))] From 2d9bcb6b1b36d6f8a8c71c77b1e438b20f1ce492 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 14:44:44 +0100 Subject: [PATCH 090/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20fonts=20t?= =?UTF-8?q?iming=20test=20gets=20a=20margin=20a=20busy=20CI=20machine=20ca?= =?UTF-8?q?n't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit named_instances_and_collection_members_are_capped: 2 s -> 10 s. The face counts in the test (one face for a million identical members, MAX_FILE_FACES from add_font) are what catch the old code, which built a face per member; the time (under 0.1 s on a loaded machine, so 10 s is over 100x) only backs up "read once". Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/fonts/src/fontdb.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/fonts/src/fontdb.rs b/crates/fonts/src/fontdb.rs index 7da5d927c2..2ccb4cc744 100644 --- a/crates/fonts/src/fontdb.rs +++ b/crates/fonts/src/fontdb.rs @@ -2392,6 +2392,8 @@ mod tests { assert_eq!(db.add_font(collection_of(&font, 1_000_000)), MAX_FILE_FACES); // A font the user adds is refused instead. assert!(matches!(db.install_font("v.ttf", &variable_font(300)), Err(user::UserFontError::Unusable(e)) if e.contains("more than 256"))); - assert!(started.elapsed() < std::time::Duration::from_secs(2), "{:?}", started.elapsed()); + // The face counts above are what tell the old code (a face per member) apart; the time + // (under 0.1 s on a loaded machine) only backs up "read once", with room for a busy CI. + assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed()); } } From 2337ff75fa7b77987ec4bc7c0e8bcf2321ef3094 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:03:09 +0100 Subject: [PATCH 091/112] #3723: zip entries that fail their CRC check are charged to the package total Finding 43. Package::open_limited dropped an entry whose read failed before adding the bytes it had already inflated to the running total, so many entries with a wrong CRC (checked only after the whole stream inflates) could each inflate up to the part limit for free. The bytes are now charged first and the size check runs before the read result is looked at. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/package.rs | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/docx/src/package.rs b/crates/docx/src/package.rs index f98a3a709f..bc143ac740 100644 --- a/crates/docx/src/package.rs +++ b/crates/docx/src/package.rs @@ -223,15 +223,17 @@ impl Package { let room = max_part.min(max_total.saturating_sub(total)); let mut buf = Vec::with_capacity(f.size().min(16 * 1024 * 1024) as usize); let read = (&mut f).take(room + 1).read_to_end(&mut buf); + // Charged whether or not the read succeeds: an entry inflates in full before its CRC + // is checked, so failed entries would otherwise inflate without limit (#3723). + total = total.saturating_add(buf.len() as u64); + if buf.len() as u64 > room { + return Err(DocxError::Limit(format!("part {name} decompresses past the size limit"))); + } if let Err(e) = read { // A corrupt entry: keep going unless it's a part we can't do without. log::warn!("docx: zip entry {name} unreadable: {e}"); continue; } - if buf.len() as u64 > room { - return Err(DocxError::Limit(format!("part {name} decompresses past the size limit"))); - } - total = total.saturating_add(buf.len() as u64); files.insert(name, Arc::new(buf)); } Ok(Package { files }) @@ -429,6 +431,26 @@ mod tests { assert!(Package::open_limited(&z, 10_000_000, 10_000_000).is_ok()); } + #[test] + fn entries_that_fail_their_check_still_count_against_the_total() { + // Each entry inflates in full before its CRC is checked; a failed one was dropped without + // being charged, so any number of them inflated (#3723). + let names: Vec = (0..20).map(|i| format!("p{i}.bin")).collect(); + let entries: Vec<(&str, Vec)> = names.iter().map(|n| (n.as_str(), vec![0u8; 1_000_000])).collect(); + let mut z = zip_of(&entries); + let mut patched = 0; + for sig in [[0x50u8, 0x4B, 0x01, 0x02], [0x50, 0x4B, 0x03, 0x04]] { + let crc_at = if sig[2] == 1 { 16 } else { 14 }; + let found: Vec = z.windows(4).enumerate().filter(|(_, w)| *w == sig).map(|(i, _)| i).collect(); + for pos in found { + z[pos + crc_at] ^= 0xFF; + patched += 1; + } + } + assert_eq!(patched, 40); + assert!(matches!(Package::open_limited(&z, 2_000_000, 4_000_000), Err(DocxError::Limit(_)))); + } + #[test] fn lying_declared_size_is_rejected_or_bounded() { let mut z = zip_of(&[("a.xml", b"".to_vec())]); From 12afc7541100d2b2461daed800893710df6ec5a9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:04:19 +0100 Subject: [PATCH 092/112] Finding 42. read_comments checked each candidate id for a comment whose w:id isn't a number against every id mapped so far, so a comments part of many non-numeric ids took time quadratic in their count (49,000 comments: 2.4 s, now 0.35 s in a debug build). The ids mapped before the part are collected into a set once; ids given in the part are already in `entries`. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/read/mod.rs | 6 +++++- crates/docx/tests/it/malformed.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/docx/src/read/mod.rs b/crates/docx/src/read/mod.rs index 414993b97c..3902f81003 100644 --- a/crates/docx/src/read/mod.rs +++ b/crates/docx/src/read/mod.rs @@ -1363,6 +1363,10 @@ impl Reader<'_> { let mut by_para: HashMap = HashMap::new(); let mut next_id = 0u32; let mut entries: BTreeMap = BTreeMap::new(); + // Ids given before this part, looked up rather than scanned for every generated id: a + // scan per comment made many non-numeric ids quadratic (#3692). Ids given here are in + // `entries`. + let given: HashSet = self.comment_map.values().copied().collect(); for c in root.children("w:comment") { if self.doc.parts.len() >= MAX_PARTS { break; @@ -1374,7 +1378,7 @@ impl Reader<'_> { let id = match file_id.parse::() { Ok(v) if !entries.contains_key(&v) => v, _ => { - while entries.contains_key(&next_id) || self.comment_map.values().any(|v| *v == next_id) { + while entries.contains_key(&next_id) || given.contains(&next_id) { next_id += 1; } next_id diff --git a/crates/docx/tests/it/malformed.rs b/crates/docx/tests/it/malformed.rs index 67eedf78d5..c2496de54c 100644 --- a/crates/docx/tests/it/malformed.rs +++ b/crates/docx/tests/it/malformed.rs @@ -351,3 +351,32 @@ fn a_repeating_difat_is_refused_before_cfb_opens_it() { assert!(!wordcraft_docx::is_encrypted(&f)); assert!(matches!(wordcraft_docx::decrypt(&f, None), Err(e) if e.to_string().contains("FAT sectors in a file of"))); } + +#[test] +fn comments_with_non_numeric_ids_open_in_linear_time() { + // Each generated id was checked against every id given so far, so a comments part with many + // ids that aren't numbers took time quadratic in their count (#3692). + const RELS_NS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; + const COMMENTS_REL: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments"; + let n = 49_000; + let mut comments = format!(""); + for i in 0..n { + comments.push_str(&format!(r#""#)); + } + comments.push_str(""); + let doc = format!("body"); + let rels = format!(r#""#); + let z = zip(&[ + ("_rels/.rels", ROOT_RELS.as_bytes()), + ("word/document.xml", doc.as_bytes()), + ("word/_rels/document.xml.rels", rels.as_bytes()), + ("word/comments.xml", comments.as_bytes()), + ]); + let t = std::time::Instant::now(); + let d = read(&z).unwrap(); + let took = t.elapsed(); + assert_eq!(d.comments.len(), n); + let ids: std::collections::HashSet<_> = d.comments.keys().collect(); + assert_eq!(ids.len(), n, "every comment has its own id"); + assert!(took.as_secs_f32() < 2.0, "opening took {took:?}"); +} From 4c5930dac1c2f01756bd8aa5a2dabe9157aad085 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:05:00 +0100 Subject: [PATCH 093/112] #3692: embedded fonts are copied once per file and within a byte budget Finding 51. embedded_fonts copied a part's bytes for every font-table reference, so the 64 references allowed to one 256 MB part could make 16 GB of copies on open. A reference repeating one already taken (same part, key and name) is skipped, and copies stop at MAX_FONT_BYTES (one part's limit), checked before each copy is made. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/fonts.rs | 69 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/crates/docx/src/fonts.rs b/crates/docx/src/fonts.rs index f63930b571..7329c122af 100644 --- a/crates/docx/src/fonts.rs +++ b/crates/docx/src/fonts.rs @@ -4,6 +4,8 @@ //! back. [`embedded_fonts`] hands their font files to whoever lays the document out, undoing the //! obfuscation Word applies to them (ECMA-376 Part 1, 17.8.1). +use std::collections::HashSet; + use wordcraft_doc::Document; use crate::package::{EMBEDDED_PARTS, EmbeddedManifest, FONT_TABLE_PART}; @@ -11,6 +13,11 @@ use crate::package::{EMBEDDED_PARTS, EmbeddedManifest, FONT_TABLE_PART}; /// Most font files taken from one document. const MAX_FONTS: usize = 64; +/// Most bytes of font files copied out of one document: one part's limit, however many +/// references the font table makes (each reference is a copy; 64 of them to one large part +/// were 64 copies, #3692). +const MAX_FONT_BYTES: u64 = crate::package::MAX_PART; + /// The font table's references to embedded font files, one per style. const EMBEDS: [&str; 4] = ["w:embedRegular", "w:embedBold", "w:embedItalic", "w:embedBoldItalic"]; @@ -28,6 +35,11 @@ pub struct EmbeddedFont { /// fonts: that is up to the font database. Empty for a document read from anything but a package /// with a font table. pub fn embedded_fonts(doc: &Document) -> Vec { + fonts_within(doc, MAX_FONT_BYTES) +} + +/// [`embedded_fonts`], copying at most `budget` bytes of font files. +fn fonts_within(doc: &Document, mut budget: u64) -> Vec { let Some(table) = doc.passthrough.get(FONT_TABLE_PART).and_then(|b| crate::xml::parse(b).ok()) else { return Vec::new(); }; @@ -36,6 +48,8 @@ pub fn embedded_fonts(doc: &Document) -> Vec { return Vec::new(); }; let mut out = Vec::new(); + // A reference repeating one already taken (same file, key and name) gives the same font. + let mut taken: HashSet<(&str, [u8; 16], &str)> = HashSet::new(); let embeds = table .children("w:font") .flat_map(|f| f.els().filter(|e| EMBEDS.contains(&e.name.as_str())).map(move |e| (f.attr("w:name").unwrap_or(""), e))); @@ -61,6 +75,15 @@ pub fn embedded_fonts(doc: &Document) -> Vec { }, None => [0; 16], }; + if !taken.insert((rel.target.as_str(), key, name)) { + continue; + } + // Checked before the copy is made. + let Some(left) = budget.checked_sub(bytes.len() as u64) else { + log::warn!("docx: embedded fonts past {MAX_FONT_BYTES} bytes; {} is not loaded", rel.target); + continue; + }; + budget = left; let mut data = bytes.to_vec(); for (i, b) in data.iter_mut().take(32).enumerate() { *b ^= key.get(i % 16).copied().unwrap_or(0); @@ -83,3 +106,49 @@ fn font_key(guid: &str) -> Option<[u8; 16]> { } Some(key) } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + /// A document whose font table makes `refs` references (`(font name, part)`) to font parts + /// `parts` (`(name, size)`). + fn doc_with(refs: &[(&str, &str)], parts: &[(&str, usize)]) -> Document { + let mut table = String::from( + r#""#, + ); + let mut manifest = format!("P\t{FONT_TABLE_PART}\tapplication/xml\n"); + for (i, (font, part)) in refs.iter().enumerate() { + table.push_str(&format!(r#""#)); + manifest.push_str(&format!("R\t{FONT_TABLE_PART}\trId{i}\tfont\t{part}\tI\n")); + } + table.push_str(""); + let mut doc = Document::default(); + doc.passthrough.insert(FONT_TABLE_PART.into(), Arc::new(table.into_bytes())); + doc.passthrough.insert(EMBEDDED_PARTS.into(), Arc::new(manifest.into_bytes())); + for (part, size) in parts { + doc.passthrough.insert((*part).into(), Arc::new(vec![7u8; *size])); + } + doc + } + + #[test] + fn a_font_referenced_again_is_copied_once() { + // Each reference was its own copy: the 64 references allowed to one 256 MB part were + // 16 GB of copies (#3692). + let refs = vec![("Body", "word/fonts/a.odttf"); MAX_FONTS]; + let fonts = embedded_fonts(&doc_with(&refs, &[("word/fonts/a.odttf", 1000)])); + assert_eq!(fonts.len(), 1); + } + + #[test] + fn font_copies_stop_at_the_byte_budget() { + let refs: Vec<(String, &str)> = (0..MAX_FONTS).map(|i| (format!("Font {i}"), "word/fonts/a.odttf")).collect(); + let refs: Vec<(&str, &str)> = refs.iter().map(|(n, p)| (n.as_str(), *p)).collect(); + let doc = doc_with(&refs, &[("word/fonts/a.odttf", 1000)]); + assert_eq!(fonts_within(&doc, 3500).len(), 3, "three copies fit"); + assert_eq!(embedded_fonts(&doc).len(), MAX_FONTS, "well within the real budget"); + } +} From 639964b0ede80144081da1e6c3e418989dcf36ab Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:06:09 +0100 Subject: [PATCH 094/112] #3692: a glossary is written from the model when the privacy settings ask for changes Finding 72. An unchanged glossary was carried byte for byte, judged by a fingerprint of its entries that leaves out the document's privacy settings, so a save with "remove personal information" or "remove date and time" kept the authors and dates of the glossary's tracked changes. With either setting on, the glossary is now written from the model, which applies them. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/write/mod.rs | 5 ++++- crates/docx/tests/it/glossary.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/crates/docx/src/write/mod.rs b/crates/docx/src/write/mod.rs index b6a66186ad..427c0df3ea 100644 --- a/crates/docx/src/write/mod.rs +++ b/crates/docx/src/write/mod.rs @@ -272,7 +272,10 @@ pub(crate) fn write_parts(doc: &Document, flavor: Flavor) -> ResultAdded clause"#; + let bytes = package( + &[("rId7", GLOSSARY_REL, "glossary/document.xml")], + &[ + ("[Content_Types].xml", content_types(&[("/word/glossary/document.xml", GLOSSARY_CT)])), + ("word/glossary/document.xml", glossary_xml(tracked)), + ], + ); + let doc = wordcraft_docx::read(&bytes).unwrap(); + let as_read = save(&doc); + let kept = text(&as_read, &glossary_target(&as_read).expect("a glossary")); + assert!(kept.contains("Quill Ramsey"), "the fixture's glossary is kept verbatim: {kept}"); + for (personal, dates) in [(true, false), (false, true), (true, true)] { + let mut private = doc.clone(); + private.settings.remove_personal_info = personal; + private.settings.remove_date_time = dates; + let files = save(&private); + let g = text(&files, &glossary_target(&files).expect("a glossary")); + assert!(g.contains("Added clause"), "{g}"); + assert_eq!(g.contains("Quill Ramsey"), !personal, "{g}"); + assert_eq!(g.contains("2026-03-04T05:06:07Z"), !dates, "{g}"); + } +} + /// The relationships part of a part in `word/`. fn glossary_rels(part: &str) -> String { let (dir, file) = part.rsplit_once('/').unwrap(); From 1955793aa637bd8017606f6ba15e162a8d2f3415 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:07:57 +0100 Subject: [PATCH 095/112] #3648: a chart part is written within the chart limits whatever the model Finding 76. chart_xml trusted its ChartSpec to be sanitized, but a .json document is deserialised without ChartSpec::sanitize, and every series repeats every category, so a chart past the limits multiplied into a gigabyte-sized part on save. chart_xml now writes a bounded, sanitized copy: only the series and points the limits keep are copied before sanitizing. (Sanitizing on .json open, and SmartArt, are in the engine and not covered here.) Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/chart_spec.rs | 48 ++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/crates/docx/src/chart_spec.rs b/crates/docx/src/chart_spec.rs index 698db44c0d..33de668900 100644 --- a/crates/docx/src/chart_spec.rs +++ b/crates/docx/src/chart_spec.rs @@ -6,7 +6,7 @@ //! [`spec_of`] turns a chart part back into the model only when writing that model again gives //! the same markup, so a chart with anything the model leaves out stays as it was read. -use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, LegendPos}; +use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES}; use wordcraft_doc::graphic::GraphicItem; use wordcraft_doc::props::Rgb; @@ -23,9 +23,10 @@ pub(crate) const CHART_URI: &str = "http://schemas.openxmlformats.org/drawingml/ const CAT_AX: &str = "500000001"; const VAL_AX: &str = "500000002"; -/// The chart part for `spec`: a complete XML document. `spec` should be sanitized -/// ([`ChartSpec::sanitize`]); a value that isn't finite is written as a gap either way. +/// The chart part for `spec`: a complete XML document, written as [`ChartSpec::sanitize`] leaves +/// it (a value that isn't finite is a gap). pub fn chart_xml(spec: &ChartSpec) -> String { + let spec = &bounded(spec); let mut w = W::new(); w.open("c:chartSpace", &[("xmlns:c", NS_C), ("xmlns:a", NS_A), ("xmlns:r", NS_R)]); w.empty("c:roundedCorners", &[("val", "0")]); @@ -79,6 +80,26 @@ pub fn chart_xml(spec: &ChartSpec) -> String { w.s } +/// `spec` held to the chart limits. A model deserialised as is (a `.json` document) never went +/// through [`ChartSpec::sanitize`], and every series repeats every category, so its size would +/// multiply (#3648). Only what the limits keep is copied before sanitizing. +fn bounded(spec: &ChartSpec) -> ChartSpec { + ChartSpec { + kind: spec.kind, + categories: spec.categories.iter().take(MAX_POINTS).cloned().collect(), + series: spec + .series + .iter() + .take(MAX_SERIES) + .map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() }) + .collect(), + title: spec.title.clone(), + legend: spec.legend, + data_labels: spec.data_labels, + } + .sanitized() +} + /// What `spec` draws as, `w` × `h` points, in the colours of `theme` (series take the accents in /// turn). Empty when the chart has no series. pub fn chart_items(spec: &ChartSpec, theme: &[Rgb], w: f32, h: f32) -> Vec { @@ -421,6 +442,27 @@ mod tests { e.els().map(|c| c.name.as_str()).collect() } + #[test] + fn a_chart_past_the_limits_is_written_within_them() { + // A model deserialised as is (a `.json` document) never went through `sanitize`, and every + // series repeats every category: 300 series of 4,500 points wrote 2.7 million points (#3648). + use wordcraft_doc::chart::MAX_CELLS; + let spec = ChartSpec { + kind: ChartType::Column, + categories: (0..4500).map(|i| i.to_string()).collect(), + series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(), + title: None, + legend: None, + data_labels: false, + }; + let x = chart_xml(&spec); + let series = x.matches("").count(); + let points = x.matches(" Date: Thu, 22 Oct 2026 16:10:52 +0100 Subject: [PATCH 096/112] #3742: kept XML is inlined in Flat OPC only when it is one well-formed element Finding 55: flat.rs xml_fragment inlined kept part bytes into pkg:xmlData after a prefix check only, so a part whose text closed the wrapper () wrote parts of its own into the saved .xml - e.g. a second core properties part. Now the bytes are inlined only when quick-xml reads them as exactly one balanced element with nothing but whitespace, comments and PIs around it; anything else is written as pkg:binaryData and still round-trips. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/flat.rs | 89 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/crates/docx/src/flat.rs b/crates/docx/src/flat.rs index 6c3b0bcef6..5cb44a0637 100644 --- a/crates/docx/src/flat.rs +++ b/crates/docx/src/flat.rs @@ -437,7 +437,46 @@ fn xml_fragment<'a>(ct: &str, bytes: &'a [u8]) -> Option<&'a str> { Some(rest) => rest.split_once("?>").map(|(_, r)| r)?.trim_start(), None => s, }; - (s.starts_with('<') && !s.contains(" bool { + let mut r = quick_xml::Reader::from_str(s); + let (mut depth, mut roots) = (0usize, 0usize); + loop { + let attrs_ok = |e: &BytesStart| e.attributes().all(|a| a.is_ok()); + match r.read_event() { + Ok(Event::Start(e)) => { + if !attrs_ok(&e) || (depth == 0 && roots > 0) { + return false; + } + roots += usize::from(depth == 0); + depth += 1; + } + Ok(Event::Empty(e)) => { + if !attrs_ok(&e) || (depth == 0 && roots > 0) { + return false; + } + roots += usize::from(depth == 0); + } + Ok(Event::End(_)) => match depth.checked_sub(1) { + Some(d) => depth = d, + None => return false, + }, + Ok(Event::Text(t)) if depth == 0 => { + if !t.iter().all(u8::is_ascii_whitespace) { + return false; + } + } + Ok(Event::CData(_) | Event::GeneralRef(_)) if depth == 0 => return false, + Ok(Event::Decl(_) | Event::DocType(_)) | Err(_) => return false, + Ok(Event::Eof) => return depth == 0 && roots == 1, + Ok(_) => {} + } + } } #[cfg(test)] @@ -494,6 +533,54 @@ mod tests { assert!(matches!(unpack(package(xml::MAX_DEPTH + 1).as_bytes()), Err(DocxError::Limit(_)))); } + #[test] + fn kept_xml_that_would_break_out_of_its_part_is_written_in_base64() { + // A part kept from a file is written back as it was read; inlined as is, markup that + // closes the wrapper adds parts of its own (here a second core properties part, read + // before the real one) (#3742). + let evil = format!( + "Injected", + "http://schemas.openxmlformats.org/package/2006/metadata/core-properties" + ); + let w_ns = "xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\""; + let rel = |id: &str, kind: &str, target: &str| format!(""); + let rels = format!( + "{}{}", + rel("rId1", crate::package::rt::OFFICE_DOC, "word/document.xml"), + rel("rId2", crate::package::rt::CUSTOM_XML, "customXml/item1.xml") + ); + let types = ""; + let z = crate::package::zip_entries(&[ + ("[Content_Types].xml".into(), types.as_bytes().to_vec()), + ("_rels/.rels".into(), rels.into_bytes()), + ("word/document.xml".into(), format!("").into_bytes()), + ("customXml/item1.xml".into(), evil.clone().into_bytes()), + ]) + .unwrap(); + let mut doc = crate::read(&z).unwrap(); + assert!(doc.passthrough.values().any(|b| b.as_slice() == evil.as_bytes()), "the part is kept as read"); + doc.core.title = "Real".into(); + let flat = write(&doc).unwrap(); + let text = String::from_utf8_lossy(&flat); + assert_eq!(text.matches("pkg:name=\"/docProps/core.xml\"").count(), 1, "{text}"); + let back = read(&flat).unwrap(); + assert_eq!(back.core.title, "Real"); + assert!(back.passthrough.values().any(|b| b.as_slice() == evil.as_bytes()), "the part survives, in base64"); + } + + #[test] + fn only_one_well_formed_element_is_inlined() { + let ct = "application/xml"; + for ok in ["", "\nt & \n", ""] { + assert!(xml_fragment(ct, ok.as_bytes()).is_some(), "{ok}"); + } + for bad in + ["", "", "", "", "text", "&", "", "", ""] + { + assert!(xml_fragment(ct, bad.as_bytes()).is_none(), "{bad}"); + } + } + #[test] fn repeated_and_nameless_parts_count_against_the_size_limits() { let (mut files, mut types, mut total) = (BTreeMap::new(), Vec::new(), 0u64); From 60a64acec376ee05fee84b00b73834edd8659fc6 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 16:13:28 +0100 Subject: [PATCH 097/112] #3704: a part named in several casings is kept, cached and written once Finding 73: Package lookup matched part names case-insensitively, but relationship targets kept the spelling they were written in, so every casing of one part became its own media entry, graphic parse and passthrough key, and a save wrote a full copy per spelling (64 spellings of a 64 KB picture: 64 media parts). Package now keeps a lowercase index, Package::canonical returns the stored spelling, and rels() resolves internal targets to it, so every downstream cache is keyed by the one part; get_shared uses the index instead of a linear scan. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/package.rs | 39 +++++++++++++++++++----- crates/docx/tests/it/passthrough.rs | 46 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/crates/docx/src/package.rs b/crates/docx/src/package.rs index bc143ac740..ddf106d747 100644 --- a/crates/docx/src/package.rs +++ b/crates/docx/src/package.rs @@ -191,6 +191,9 @@ pub fn rel_is(t: &str, full: &str) -> bool { /// picture, an embedded object) is handed over rather than copied. pub struct Package { files: BTreeMap>>, + /// Lowercased part name → its stored spelling (the first in name order): part names are + /// case-insensitive (ECMA-376 Part 2 §6.2.2.3), so every spelling is one part. + lower: HashMap, } impl Package { @@ -236,13 +239,30 @@ impl Package { } files.insert(name, Arc::new(buf)); } - Ok(Package { files }) + Ok(Package::new(files)) } /// A package from parts already in memory (a Flat OPC file, a converted document), by part /// name without the leading `/`. pub fn from_parts(files: BTreeMap>) -> Package { - Package { files: files.into_iter().map(|(name, bytes)| (name, Arc::new(bytes))).collect() } + Package::new(files.into_iter().map(|(name, bytes)| (name, Arc::new(bytes))).collect()) + } + + fn new(files: BTreeMap>>) -> Package { + let mut lower = HashMap::new(); + for name in files.keys() { + lower.entry(name.to_ascii_lowercase()).or_insert_with(|| name.clone()); + } + Package { files, lower } + } + + /// The stored spelling of part `name` (leading `/` ignored; case-insensitive fallback). + pub fn canonical(&self, name: &str) -> Option<&str> { + let n = name.trim_start_matches('/'); + match self.files.get_key_value(n) { + Some((k, _)) => Some(k), + None => self.lower.get(&n.to_ascii_lowercase()).map(String::as_str), + } } /// Part bytes by name (leading `/` ignored; case-insensitive fallback). @@ -253,11 +273,7 @@ impl Package { /// [`Package::get`], as the shared bytes themselves: for a part the document keeps whole, which /// then costs no copy (a picture document's pictures used to be held twice while it opened). pub fn get_shared(&self, name: &str) -> Option<&Arc>> { - let n = name.trim_start_matches('/'); - if let Some(v) = self.files.get(n) { - return Some(v); - } - self.files.iter().find(|(k, _)| k.eq_ignore_ascii_case(n)).map(|(_, v)| v) + self.canonical(name).and_then(|n| self.files.get(n)) } /// [`Package::xml`] for the main document part: malformed XML is an error, not a truncated read. @@ -291,7 +307,14 @@ impl Package { for r in root.els().filter(|e| e.local() == "Relationship") { let (Some(id), Some(target)) = (r.attr("Id"), r.attr("Target")) else { continue }; let external = r.attr("TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")); - let resolved = if external { target.to_string() } else { resolve(dir, target) }; + // Internal targets name the part as stored, so a part reached under several spellings + // is kept, cached and written once (#3704). + let resolved = if external { + target.to_string() + } else { + let path = resolve(dir, target); + self.canonical(&path).map_or(path, str::to_string) + }; rels.list.push(Rel { id: id.to_string(), kind: r.attr("Type").unwrap_or("").to_string(), target: resolved, external }); } rels diff --git a/crates/docx/tests/it/passthrough.rs b/crates/docx/tests/it/passthrough.rs index 3f29bd50f1..65d4e3495e 100644 --- a/crates/docx/tests/it/passthrough.rs +++ b/crates/docx/tests/it/passthrough.rs @@ -357,3 +357,49 @@ fn extended_properties_are_kept_without_words_counts_and_personal_ones_follow_th assert!(!private.contains("Company") && !private.contains("Manager"), "{private}"); assert!(private.contains("")); } + +/// `name` with its letters' case flipped by the bits of `i`. +fn spelling(name: &str, i: u32) -> String { + let mut bit = 0; + name.chars() + .map(|c| { + if !c.is_ascii_alphabetic() { + return c; + } + bit += 1; + if i >> (bit - 1) & 1 == 1 { c.to_ascii_uppercase() } else { c.to_ascii_lowercase() } + }) + .collect() +} + +#[test] +fn a_part_reached_under_many_spellings_is_kept_and_written_once() { + // Part names are case-insensitive (ECMA-376 Part 2 §6.2.2.3): 64 relationships naming one picture + // and one unmodelled part in 64 casings are two parts, not 128 copies on save (#3704). + let ns = r#"xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture""#; + let n = 64; + let mut body = String::new(); + let mut doc_rels = Vec::new(); + for i in 0..n { + body.push_str(&format!( + r#""#, + i + 1 + )); + doc_rels.push((format!("rIdP{i}"), format!("{REL}/image"), spelling("media/image1.png", i))); + doc_rels.push((format!("rIdU{i}"), "urn:example:future-relationship".to_string(), spelling("unknownPart.xml", i))); + } + let doc_rels: Vec<(&str, &str, &str)> = doc_rels.iter().map(|(a, b, c)| (a.as_str(), b.as_str(), c.as_str())).collect(); + let picture = format!("\u{89}PNG\r\n\u{1a}\n{}", "0".repeat(64 * 1024)); + let unknown = format!(r#"{}"#, "x".repeat(64 * 1024)); + let ct = CT.replace(" Date: Thu, 22 Oct 2026 16:13:47 +0100 Subject: [PATCH 098/112] Progress log row and the file-format parity Hostile input line for findings 42, 43, 51, 55, 72, 73 and 76. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/file-format-parity.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c2b352424f..82dafdb426 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -121,7 +121,7 @@ Measured by grepping `crates/docx/src/read` and `src/write` for the OOXML elemen | `w:fitText` | ❌ | ❌ | East Asian layout | | Glossary document (building blocks) | ✅ | ✅ | `w:docPart` entries (name, style, gallery, category, types, behaviours, description, guid, body) are read into `Document::glossary` and offered by Insert › Quick Parts and AutoText; `save` with `"in": "document"` stores one in the file. An untouched glossary is written back byte for byte with its own parts (styles, settings, fonts…); an edited one, or one whose parts weren't kept, is written from the model as `word/glossary1.xml` with copies of the document's styles and numbering. The glossary's own styles, settings and fonts, text boxes inside entries, and the notes and comments of a selection saved into the document don't survive an edit. Tests: `crates/docx/tests/it/glossary.rs` | | `w:altChunk`, sub-documents | ❌ | ❌ | | -| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard | +| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard. Failed zip entries count against the size limit, case variants of a part name are one part, embedded fonts and charts are written within budgets, and Flat OPC inlines kept XML only when it is one well-formed element (2026-10-22) | | Document paragraph defaults (`w:pPrDefault`) | ✅ | ✅ | Omission and explicit empty defaults remain distinct across repeated saves; nonempty defaults and later default-spacing edits are written | ## Saving many content controls @@ -270,6 +270,7 @@ Repeated DOCX style IDs use the last complete definition. The original regressio | 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged | | 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) | | 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) | +| 2026-10-22 | trivial | Hostile input: DOCX package, comment, font, chart and Flat OPC hardening (#3692, #3723, #3704, #3742, #3648) | | 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) | | 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) | | 2026-10-23 | trivial | DOCX: TOC styles Word stores under ids in another language (`Sumrio1` named `toc 1`, `CabealhodoSumrio` named `TOC Heading`) read as `TOC1`…`TOC9` and `TOCHeading` unless the file uses those ids, so Update Table builds the entries in the document's own TOC styles (#3891); a TOC whose heading ends a section (entries in columns after a continuous break) is written and read as the engine's TOC, with the page-number tab at the column's edge (#3889) | From 6dc062d132820268f338fc25a177d708f0c7f4d9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 22:04:09 +0100 Subject: [PATCH 099/112] #3692: a private save keeps the glossary's parts, without names and dates Ultrareview finding 1. With "remove personal information" or "remove date and time" set, the glossary was always written from the model (01c78899), which holds only the entries the reader took: one without a name, past the entry cap or with a bad content type was lost, and with none left the whole glossary part went. The kept parts are now written with the same names and dates removed that the writer leaves out of the main document (annotation authors become "Author", comment initials "A", annotation and UTC dates go; a people part's authors and presence too), rewritten with quick-xml and matched by namespace. A part that can't be rewritten (malformed, UTF-16) falls back to the model, so the privacy guarantee holds either way. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/docx/src/write/embed.rs | 113 ++++++++++++++++++++++++++++++- crates/docx/src/write/mod.rs | 13 ++-- crates/docx/tests/it/glossary.rs | 69 +++++++++++++++++++ 3 files changed, 189 insertions(+), 6 deletions(-) diff --git a/crates/docx/src/write/embed.rs b/crates/docx/src/write/embed.rs index 6e04956143..869d91a401 100644 --- a/crates/docx/src/write/embed.rs +++ b/crates/docx/src/write/embed.rs @@ -48,6 +48,8 @@ pub(crate) struct EmbedWriter<'d> { pub parts: Vec<(String, Vec, String, PartRels)>, /// Leave out the main document's kept glossary (it's written from the model instead). pub drop_glossary: bool, + /// Kept parts written with these bytes instead of their own (see [`Self::scrub_glossary`]). + scrubbed: HashMap>, } impl<'d> EmbedWriter<'d> { @@ -93,6 +95,7 @@ impl<'d> EmbedWriter<'d> { markup_bytes: MAX_MARKUP_BYTES, parts: Vec::new(), drop_glossary: false, + scrubbed: HashMap::new(), } } @@ -230,6 +233,39 @@ impl<'d> EmbedWriter<'d> { }) } + /// Have the main document's kept glossary, and the parts it reaches, written without the + /// personal information (`names`) and dates the privacy settings remove (#3692). False when a + /// part's markup can't be rewritten: the glossary is then written from the model. + pub fn scrub_glossary(&mut self, names: bool, dates: bool) -> bool { + let Some(roots) = self.passthrough.get(crate::package::UNMODELLED_ROOTS) else { return true }; + let mut todo: Vec = String::from_utf8_lossy(roots) + .lines() + .take(MAX_EMBEDDED_RELS * 2) + .filter_map(|line| match line.split('\t').collect::>().as_slice() { + [source, kind, target] if *source == "D" && rel_is(kind, rt::GLOSSARY) => Some(target.to_string()), + _ => None, + }) + .collect(); + let mut seen: HashSet = HashSet::new(); + while let Some(p) = todo.pop() { + if seen.len() >= MAX_EMBEDDED_PARTS || !seen.insert(p.clone()) { + continue; + } + let (Some(info), Some(bytes)) = (self.manifest.parts.get(&p), self.passthrough.get(&p)) else { continue }; + todo.extend(info.rels.iter().filter(|r| !r.external).map(|r| r.target.clone())); + if !(info.content_type.ends_with("xml") || p.to_ascii_lowercase().ends_with(".xml")) { + continue; + } + match scrub_private(bytes, names, dates) { + Some(b) => { + self.scrubbed.insert(p, b); + } + None => return false, + } + } + true + } + /// What copying the parts reached from `roots` costs against the copy budget. fn tree_cost(&self, roots: &[&str]) -> u64 { let mut seen: HashSet<&str> = HashSet::new(); @@ -283,8 +319,9 @@ impl<'d> EmbedWriter<'d> { rels.add_with_id(&r.id, &r.kind, &relative(&new, &target), false); } let ct = if info.content_type.is_empty() { "application/octet-stream".to_string() } else { info.content_type }; + let bytes = self.scrubbed.get(&orig).cloned().unwrap_or_else(|| bytes.to_vec()); written.push(self.parts.len()); - self.parts.push((new, bytes.to_vec(), ct, rels)); + self.parts.push((new, bytes, ct, rels)); } Some(name) } @@ -640,6 +677,80 @@ fn patch_rel_ids(bytes: &[u8], ids: &HashMap) -> Vec { out } +/// `bytes` (kept XML) without the personal information and dates the privacy settings remove, as +/// the writer leaves them out of the parts it writes: annotation authors become "Author" and +/// comment initials "A" (`names`); annotation dates and UTC dates go (`dates`); and with `names`, +/// a people part's authors become "Author" and their presence information goes. Attributes are +/// matched by namespace, so any prefix is caught. `None` when the markup can't be rewritten +/// (malformed, or not UTF-8). +fn scrub_private(bytes: &[u8], names: bool, dates: bool) -> Option> { + use std::borrow::Cow; + + use quick_xml::events::Event; + use quick_xml::events::attributes::Attribute; + use quick_xml::name::{QName, ResolveResult}; + use quick_xml::reader::NsReader; + use quick_xml::writer::Writer; + + const WML: &[u8] = b"http://schemas.openxmlformats.org/wordprocessingml/2006/main"; + const WML_STRICT: &[u8] = b"http://purl.oclc.org/ooxml/wordprocessingml/main"; + const W15: &[u8] = b"http://schemas.microsoft.com/office/word/2012/wordml"; + // UTF-16 (or other wide encodings) would pass through unread. + if bytes.contains(&0) || bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) { + return None; + } + let ns_is = |r: &ResolveResult, ns: &[u8]| matches!(r, ResolveResult::Bound(n) if n.as_ref() == ns); + let wml = |r: &ResolveResult| ns_is(r, WML) || ns_is(r, WML_STRICT); + let mut r = NsReader::from_reader(bytes); + r.config_mut().check_end_names = false; + let mut out = Writer::new(Vec::with_capacity(bytes.len())); + let mut buf = Vec::new(); + let mut skip = Vec::new(); + loop { + buf.clear(); + let (res, ev) = r.read_resolved_event_into(&mut buf).ok()?; + let (in_wml, in_w15) = (wml(&res), ns_is(&res, W15)); + let ev = match ev { + Event::Eof => break, + Event::Start(ref e) | Event::Empty(ref e) if names && in_w15 && e.local_name().as_ref() == b"presenceInfo" => { + if matches!(ev, Event::Start(_)) { + let end = e.name().as_ref().to_vec(); + r.read_to_end_into(QName(&end), &mut skip).ok()?; + skip.clear(); + } + continue; + } + Event::Start(ref e) | Event::Empty(ref e) => { + let mut el = e.to_owned(); + el.clear_attributes(); + for a in e.attributes().with_checks(false) { + let a = a.ok()?; + let (ares, local) = r.resolver_mut().resolve_attribute(a.key); + // An unprefixed attribute of a WordprocessingML element is read as its own. + let wml_attr = wml(&ares) || (matches!(ares, ResolveResult::Unbound) && in_wml); + let value: Option<&[u8]> = match local.as_ref() { + b"author" if names && (wml_attr || ns_is(&ares, W15)) => Some(super::ANONYMOUS_AUTHOR.as_bytes()), + b"initials" if names && wml_attr => Some(super::ANONYMOUS_INITIALS.as_bytes()), + b"date" if dates && wml_attr => None, + b"dateUtc" if dates => None, + _ => { + el.push_attribute(a); + continue; + } + }; + if let Some(v) = value { + el.push_attribute(Attribute { key: a.key, value: Cow::Borrowed(v) }); + } + } + if matches!(ev, Event::Start(_)) { Event::Start(el) } else { Event::Empty(el) } + } + ev => ev, + }; + out.write_event(ev).ok()?; + } + Some(out.into_inner()) +} + /// A VML `style` with the object's size, rotation and flips, and its offset when it floats /// (`position:absolute`). fn vml_style_with(style: &str, w: f32, h: f32, float: &Float) -> String { diff --git a/crates/docx/src/write/mod.rs b/crates/docx/src/write/mod.rs index 427c0df3ea..36a46de645 100644 --- a/crates/docx/src/write/mod.rs +++ b/crates/docx/src/write/mod.rs @@ -272,10 +272,13 @@ pub(crate) fn write_parts(doc: &Document, flavor: Flavor) -> Result Vec { /// The author the saved file gives a comment or tracked change when the document removes personal /// information on save (`w:removePersonalInformation`), as Word does. pub(crate) const ANONYMOUS_AUTHOR: &str = "Author"; -const ANONYMOUS_INITIALS: &str = "A"; +pub(crate) const ANONYMOUS_INITIALS: &str = "A"; /// The `w:author` written for an annotation by `author`: "Author" when the document removes /// personal information on save, "Unknown" when it has none. diff --git a/crates/docx/tests/it/glossary.rs b/crates/docx/tests/it/glossary.rs index 3d1521550f..8d7b9f0bcd 100644 --- a/crates/docx/tests/it/glossary.rs +++ b/crates/docx/tests/it/glossary.rs @@ -262,6 +262,75 @@ fn an_unchanged_glossary_follows_the_privacy_settings() { } } +#[test] +fn a_private_save_keeps_the_glossary_entries_the_model_passed_over() { + // The privacy settings had the glossary written from the model, which holds only the + // entries the reader took: one without a name was lost, and with no other entry the whole + // glossary went. The kept parts are now written without the names and dates instead. + let nameless = r#"Nameless entrynoted"#; + // The comments part binds WordprocessingML to another prefix: names are matched by namespace. + let comments = format!( + r#"A note"# + ); + let bytes = package( + &[("rId7", GLOSSARY_REL, "glossary/document.xml")], + &[ + ( + "[Content_Types].xml", + content_types(&[ + ("/word/glossary/document.xml", GLOSSARY_CT), + ("/word/glossary/comments.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"), + ]), + ), + ("word/glossary/document.xml", glossary_xml(nameless)), + ("word/glossary/_rels/document.xml.rels", rels(&[("rId1", &format!("{REL}/comments"), "comments.xml")])), + ("word/glossary/comments.xml", comments), + ], + ); + let doc = wordcraft_docx::read(&bytes).unwrap(); + assert!(doc.glossary.is_empty(), "the reader passes over an entry without a name"); + for (personal, dates) in [(true, false), (false, true), (true, true)] { + let mut private = doc.clone(); + private.settings.remove_personal_info = personal; + private.settings.remove_date_time = dates; + let files = save(&private); + let target = glossary_target(&files).expect("the glossary survives"); + let g = text(&files, &target); + assert!(g.contains("Nameless entry"), "{g}"); + assert_eq!(g.contains("Quill Ramsey"), !personal, "{g}"); + assert_eq!(g.contains("2026-03-04T05:06:07Z"), !dates, "{g}"); + let c = files.iter().find(|(n, _)| n.starts_with("word/glossary/comments")).map(|(_, b)| String::from_utf8_lossy(b).into_owned()); + let c = c.expect("the glossary's comments survive"); + assert!(c.contains("A note"), "{c}"); + assert_eq!(c.contains("Quill Ramsey"), !personal, "{c}"); + assert_eq!(c.contains("x:initials=\"QR\""), !personal, "{c}"); + assert_eq!(c.contains("2026-03-04T05:06:07Z"), !dates, "{c}"); + } +} + +#[test] +fn a_kept_glossary_that_cant_be_rewritten_privately_is_written_from_the_model() { + // UTF-16 markup isn't rewritten in place: the model, which applies the settings, is written. + let tracked = r#"Added clause"#; + let utf16: Vec = [0xFF, 0xFE].into_iter().chain(glossary_xml(tracked).encode_utf16().flat_map(u16::to_le_bytes)).collect(); + let bytes = zip(&[ + ("_rels/.rels", rels(&[("rIdMain", &format!("{REL}/officeDocument"), "word/document.xml")]).into_bytes()), + ( + "word/document.xml", + format!(r#"Body"#).into_bytes(), + ), + ("word/_rels/document.xml.rels", rels(&[("rId7", GLOSSARY_REL, "glossary/document.xml")]).into_bytes()), + ("[Content_Types].xml", content_types(&[("/word/glossary/document.xml", GLOSSARY_CT)]).into_bytes()), + ("word/glossary/document.xml", utf16), + ]); + let mut doc = wordcraft_docx::read(&bytes).unwrap(); + assert_eq!(doc.glossary.len(), 1); + doc.settings.remove_personal_info = true; + let files = save(&doc); + let g = text(&files, &glossary_target(&files).expect("a glossary")); + assert!(g.contains("Added clause") && !g.contains("Quill Ramsey"), "{g}"); +} + /// The relationships part of a part in `word/`. fn glossary_rels(part: &str) -> String { let (dir, file) = part.rsplit_once('/').unwrap(); From eb2f0dab464e042529037492840daca47c108795 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:51:49 +0100 Subject: [PATCH 100/112] =?UTF-8?q?#3742:=20review=20=E2=80=94=20a=20kept?= =?UTF-8?q?=20XML=20part=20with=20a=20malformed=20comment=20goes=20out=20i?= =?UTF-8?q?n=20base64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docx/src/flat.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/docx/src/flat.rs b/crates/docx/src/flat.rs index 5cb44a0637..a11b69f156 100644 --- a/crates/docx/src/flat.rs +++ b/crates/docx/src/flat.rs @@ -445,6 +445,7 @@ fn xml_fragment<'a>(ct: &str, bytes: &'a [u8]) -> Option<&'a str> { /// close the wrapper and add parts of its own, so it goes out as `pkg:binaryData` (#3742). fn one_element(s: &str) -> bool { let mut r = quick_xml::Reader::from_str(s); + r.config_mut().check_comments = true; let (mut depth, mut roots) = (0usize, 0usize); loop { let attrs_ok = |e: &BytesStart| e.attributes().all(|a| a.is_ok()); @@ -574,9 +575,19 @@ mod tests { for ok in ["", "\nt & \n", ""] { assert!(xml_fragment(ct, ok.as_bytes()).is_some(), "{ok}"); } - for bad in - ["", "", "", "", "text", "&", "", "", ""] - { + for bad in [ + "", + "", + "", + "", + "text", + "&", + "", + "", + "", + "", + "", + ] { assert!(xml_fragment(ct, bad.as_bytes()).is_none(), "{bad}"); } } From c14e18cfa83a8096a4ed6a1d2c447fe1e67f73ce Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:51:49 +0100 Subject: [PATCH 101/112] =?UTF-8?q?#3704:=20review=20=E2=80=94=20zip=20ent?= =?UTF-8?q?ries=20whose=20names=20differ=20only=20in=20case=20keep=20one?= =?UTF-8?q?=20stored=20spelling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docx/src/package.rs | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/crates/docx/src/package.rs b/crates/docx/src/package.rs index ddf106d747..9ced85c4d6 100644 --- a/crates/docx/src/package.rs +++ b/crates/docx/src/package.rs @@ -248,21 +248,26 @@ impl Package { Package::new(files.into_iter().map(|(name, bytes)| (name, Arc::new(bytes))).collect()) } - fn new(files: BTreeMap>>) -> Package { + fn new(mut files: BTreeMap>>) -> Package { let mut lower = HashMap::new(); for name in files.keys() { lower.entry(name.to_ascii_lowercase()).or_insert_with(|| name.clone()); } + // Entries whose names differ only in case are one part: keep the first spelling, so every + // spelling of the name reaches the same stored entry (#3704). + files.retain(|name, _| { + let kept = lower.get(&name.to_ascii_lowercase()) == Some(name); + if !kept { + log::warn!("docx: dropping zip entry {name}, a second spelling of a part"); + } + kept + }); Package { files, lower } } - /// The stored spelling of part `name` (leading `/` ignored; case-insensitive fallback). + /// The stored spelling of part `name` (leading `/` ignored; case-insensitive). pub fn canonical(&self, name: &str) -> Option<&str> { - let n = name.trim_start_matches('/'); - match self.files.get_key_value(n) { - Some((k, _)) => Some(k), - None => self.lower.get(&n.to_ascii_lowercase()).map(String::as_str), - } + self.lower.get(&name.trim_start_matches('/').to_ascii_lowercase()).map(String::as_str) } /// Part bytes by name (leading `/` ignored; case-insensitive fallback). @@ -445,6 +450,17 @@ mod tests { zip_entries(&e).unwrap() } + #[test] + fn case_colliding_entries_are_one_part() { + let z = zip_of(&[("word/media/Image.png", b"upper".to_vec()), ("word/media/image.png", b"lower".to_vec())]); + let p = Package::open(&z).unwrap(); + for spelling in ["word/media/Image.png", "word/media/image.png", "/WORD/MEDIA/IMAGE.PNG"] { + assert_eq!(p.canonical(spelling), Some("word/media/Image.png"), "{spelling}"); + assert_eq!(p.get(spelling), Some(&b"upper"[..]), "{spelling}"); + } + assert_eq!(p.files.len(), 1); + } + #[test] fn decompression_bomb_is_rejected() { let z = zip_of(&[("word/document.xml", vec![b' '; 2_000_000])]); From 5819b1ca62a61ce39e6d2bcb291e873ab69be142 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:51:49 +0100 Subject: [PATCH 102/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20comment-i?= =?UTF-8?q?d=20test=20checks=20the=20ids,=20not=20a=20wall-clock=20deadlin?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docx/tests/it/malformed.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/docx/tests/it/malformed.rs b/crates/docx/tests/it/malformed.rs index c2496de54c..a714f27b36 100644 --- a/crates/docx/tests/it/malformed.rs +++ b/crates/docx/tests/it/malformed.rs @@ -372,11 +372,8 @@ fn comments_with_non_numeric_ids_open_in_linear_time() { ("word/_rels/document.xml.rels", rels.as_bytes()), ("word/comments.xml", comments.as_bytes()), ]); - let t = std::time::Instant::now(); let d = read(&z).unwrap(); - let took = t.elapsed(); assert_eq!(d.comments.len(), n); let ids: std::collections::HashSet<_> = d.comments.keys().collect(); assert_eq!(ids.len(), n, "every comment has its own id"); - assert!(took.as_secs_f32() < 2.0, "opening took {took:?}"); } From 364688a8c3b2ef63d0bc392a0b324ca185c2d92c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:08:28 +0100 Subject: [PATCH 103/112] =?UTF-8?q?#3704:=20review=20=E2=80=94=20case-vari?= =?UTF-8?q?ant=20zip=20names=20keep=20the=20spelling=20a=20relationship=20?= =?UTF-8?q?names;=20chart,=20font=20and=20glossary=20limits=20log=20once?= =?UTF-8?q?=20per=20save?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folding zip entries whose names differ only in case now keeps the spelling a relationship part targets, and the first in byte order only when no relationship names any of them. The fold logs one warning per package, not one per entry. Limits that leave data out now say so, once per document or save: - a chart past the chart limits on save (series or points dropped); - embedded fonts past the byte budget (one warning with the count); - a kept glossary that can't be cleaned of personal information and is written from the model instead. Tests capture the warnings (crate::test_log) and check each is logged once. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/docx/src/chart_spec.rs | 13 +++++- crates/docx/src/fonts.rs | 14 +++++- crates/docx/src/lib.rs | 2 + crates/docx/src/package.rs | 77 +++++++++++++++++++++++++++----- crates/docx/src/test_log.rs | 36 +++++++++++++++ crates/docx/src/write/embed.rs | 51 ++++++++++++++++++++- crates/docx/tests/it/glossary.rs | 30 ++++++++++++- 7 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 crates/docx/src/test_log.rs diff --git a/crates/docx/src/chart_spec.rs b/crates/docx/src/chart_spec.rs index 33de668900..4097002b6b 100644 --- a/crates/docx/src/chart_spec.rs +++ b/crates/docx/src/chart_spec.rs @@ -26,7 +26,18 @@ const VAL_AX: &str = "500000002"; /// The chart part for `spec`: a complete XML document, written as [`ChartSpec::sanitize`] leaves /// it (a value that isn't finite is a gap). pub fn chart_xml(spec: &ChartSpec) -> String { - let spec = &bounded(spec); + chart_part(spec).0 +} + +/// [`chart_xml`], and whether the chart limits left series or points of `spec` out of it. +pub(crate) fn chart_part(spec: &ChartSpec) -> (String, bool) { + let kept = bounded(spec); + let points = spec.series.iter().map(|s| s.values.len()).max().unwrap_or(0).max(spec.categories.len()); + let cut = kept.series.len() < spec.series.len() || kept.points() < points; + (write_chart(&kept), cut) +} + +fn write_chart(spec: &ChartSpec) -> String { let mut w = W::new(); w.open("c:chartSpace", &[("xmlns:c", NS_C), ("xmlns:a", NS_A), ("xmlns:r", NS_R)]); w.empty("c:roundedCorners", &[("val", "0")]); diff --git a/crates/docx/src/fonts.rs b/crates/docx/src/fonts.rs index 7329c122af..66e92ae0b8 100644 --- a/crates/docx/src/fonts.rs +++ b/crates/docx/src/fonts.rs @@ -40,6 +40,7 @@ pub fn embedded_fonts(doc: &Document) -> Vec { /// [`embedded_fonts`], copying at most `budget` bytes of font files. fn fonts_within(doc: &Document, mut budget: u64) -> Vec { + let budget_total = budget; let Some(table) = doc.passthrough.get(FONT_TABLE_PART).and_then(|b| crate::xml::parse(b).ok()) else { return Vec::new(); }; @@ -50,6 +51,7 @@ fn fonts_within(doc: &Document, mut budget: u64) -> Vec { let mut out = Vec::new(); // A reference repeating one already taken (same file, key and name) gives the same font. let mut taken: HashSet<(&str, [u8; 16], &str)> = HashSet::new(); + let mut over_budget = 0usize; let embeds = table .children("w:font") .flat_map(|f| f.els().filter(|e| EMBEDS.contains(&e.name.as_str())).map(move |e| (f.attr("w:name").unwrap_or(""), e))); @@ -80,7 +82,7 @@ fn fonts_within(doc: &Document, mut budget: u64) -> Vec { } // Checked before the copy is made. let Some(left) = budget.checked_sub(bytes.len() as u64) else { - log::warn!("docx: embedded fonts past {MAX_FONT_BYTES} bytes; {} is not loaded", rel.target); + over_budget += 1; continue; }; budget = left; @@ -90,6 +92,10 @@ fn fonts_within(doc: &Document, mut budget: u64) -> Vec { } out.push(EmbeddedFont { name: name.to_string(), data }); } + if over_budget > 0 { + // Once for the document, not once per font. + log::warn!("docx: embedded fonts past the {budget_total} byte budget; {over_budget} of them are not loaded"); + } out } @@ -148,7 +154,11 @@ mod tests { let refs: Vec<(String, &str)> = (0..MAX_FONTS).map(|i| (format!("Font {i}"), "word/fonts/a.odttf")).collect(); let refs: Vec<(&str, &str)> = refs.iter().map(|(n, p)| (n.as_str(), *p)).collect(); let doc = doc_with(&refs, &[("word/fonts/a.odttf", 1000)]); - assert_eq!(fonts_within(&doc, 3500).len(), 3, "three copies fit"); + let (fonts, warned) = crate::test_log::warnings(|| fonts_within(&doc, 3500)); + assert_eq!(fonts.len(), 3, "three copies fit"); + let budget: Vec<_> = warned.iter().filter(|w| w.contains("byte budget")).collect(); + assert_eq!(budget.len(), 1, "one warning for the document, not one per font: {warned:?}"); + assert!(budget.iter().all(|w| w.contains(&format!("{} of them", MAX_FONTS - 3))), "{budget:?}"); assert_eq!(embedded_fonts(&doc).len(), MAX_FONTS, "well within the real budget"); } } diff --git a/crates/docx/src/lib.rs b/crates/docx/src/lib.rs index 27cb0e6da1..72cfdebfa9 100644 --- a/crates/docx/src/lib.rs +++ b/crates/docx/src/lib.rs @@ -16,6 +16,8 @@ mod fonts; mod package; mod read; mod smart_art; +#[cfg(test)] +mod test_log; mod units; mod wordml2003; mod wordml2003_write; diff --git a/crates/docx/src/package.rs b/crates/docx/src/package.rs index 9ced85c4d6..a7511fd2bc 100644 --- a/crates/docx/src/package.rs +++ b/crates/docx/src/package.rs @@ -1,6 +1,7 @@ //! OPC package access: bounded zip reading, relationships, part names. -use std::collections::{BTreeMap, HashMap}; +use std::collections::hash_map::Entry; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::{Cursor, Read, Write}; use std::sync::Arc; @@ -249,19 +250,31 @@ impl Package { } fn new(mut files: BTreeMap>>) -> Package { - let mut lower = HashMap::new(); + let mut lower: HashMap = HashMap::new(); + let mut collides = false; for name in files.keys() { - lower.entry(name.to_ascii_lowercase()).or_insert_with(|| name.clone()); - } - // Entries whose names differ only in case are one part: keep the first spelling, so every - // spelling of the name reaches the same stored entry (#3704). - files.retain(|name, _| { - let kept = lower.get(&name.to_ascii_lowercase()) == Some(name); - if !kept { - log::warn!("docx: dropping zip entry {name}, a second spelling of a part"); + match lower.entry(name.to_ascii_lowercase()) { + Entry::Occupied(_) => collides = true, + Entry::Vacant(v) => { + v.insert(name.clone()); + } } - kept - }); + } + if !collides { + return Package { files, lower }; + } + // Entries whose names differ only in case are one part, so every spelling of the name + // reaches the same stored entry (#3704). The spelling kept is the one a relationship + // names, so the part the document points at is the one read; the first in byte order only + // when no relationship names any of them. + let named = rel_targets(&files); + for name in files.keys().rev().filter(|n| named.contains(n.as_str())) { + lower.insert(name.to_ascii_lowercase(), name.clone()); + } + let before = files.len(); + files.retain(|name, _| lower.get(&name.to_ascii_lowercase()) == Some(name)); + let dropped = before.saturating_sub(files.len()); + log::warn!("docx: {dropped} zip entries are other spellings of parts already read; they are left out"); Package { files, lower } } @@ -394,6 +407,24 @@ fn split_dir(part: &str) -> (&str, &str) { } } +/// Every internal part name the relationship parts in `files` target, spelled as they spell it. +fn rel_targets(files: &BTreeMap>>) -> HashSet { + let mut named = HashSet::new(); + for (name, bytes) in files { + // `dir/_rels/file.rels` relates `dir/file`, so its targets resolve against `dir`. + let Some(at) = name.rfind("_rels/").filter(|_| name.to_ascii_lowercase().ends_with(".rels")) else { continue }; + let dir = name.get(..at).unwrap_or("").trim_end_matches('/'); + let Ok(root) = xml::parse(bytes) else { continue }; + for r in root.els().filter(|e| e.local() == "Relationship") { + let external = r.attr("TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")); + if let (Some(target), false) = (r.attr("Target"), external) { + named.insert(resolve(dir, target)); + } + } + } + named +} + /// Resolve a relative target against a source directory into a package path. pub fn resolve(dir: &str, target: &str) -> String { let target = target.split('#').next().unwrap_or(target); @@ -461,6 +492,28 @@ mod tests { assert_eq!(p.files.len(), 1); } + #[test] + fn the_spelling_a_relationship_names_is_kept() { + // The first spelling in byte order ("Image.png") was kept even when the document's + // relationship named the other one, so the picture shown was not the one it pointed at. + let rels = r#""#; + let z = zip_of(&[ + ("word/_rels/document.xml.rels", rels.as_bytes().to_vec()), + ("word/media/Image.png", b"upper".to_vec()), + ("word/media/image.png", b"lower".to_vec()), + ("word/media/Other.png", b"first".to_vec()), + ("word/media/other.png", b"second".to_vec()), + ]); + let (p, warned) = crate::test_log::warnings(|| Package::open(&z).unwrap()); + assert_eq!(p.canonical("word/media/IMAGE.png"), Some("word/media/image.png")); + assert_eq!(p.get("word/media/Image.png"), Some(&b"lower"[..])); + assert_eq!(p.get("word/media/OTHER.PNG"), Some(&b"first"[..]), "no relationship names it: first in byte order"); + assert_eq!(p.files.len(), 3); + let folded: Vec<_> = warned.iter().filter(|w| w.contains("other spellings")).collect(); + assert_eq!(folded.len(), 1, "one warning for the package: {warned:?}"); + assert!(folded.iter().all(|w| w.contains("2 zip entries")), "{folded:?}"); + } + #[test] fn decompression_bomb_is_rejected() { let z = zip_of(&[("word/document.xml", vec![b' '; 2_000_000])]); diff --git a/crates/docx/src/test_log.rs b/crates/docx/src/test_log.rs new file mode 100644 index 0000000000..745ce21fe1 --- /dev/null +++ b/crates/docx/src/test_log.rs @@ -0,0 +1,36 @@ +//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that +//! a limit which leaves something out says so. + +use std::cell::RefCell; + +struct Capture; + +thread_local! { + static LINES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +impl log::Log for Capture { + fn enabled(&self, m: &log::Metadata) -> bool { + m.level() <= log::Level::Warn + } + + fn log(&self, r: &log::Record) { + if self.enabled(r.metadata()) { + LINES.with(|l| l.borrow_mut().push(r.args().to_string())); + } + } + + fn flush(&self) {} +} + +static CAPTURE: Capture = Capture; + +/// `f`'s result and the warnings logged on this thread while it ran. +pub(crate) fn warnings(f: impl FnOnce() -> T) -> (T, Vec) { + // Only this logger is ever installed in a test binary; a second call keeps it. + let _ = log::set_logger(&CAPTURE); + log::set_max_level(log::LevelFilter::Warn); + LINES.with(|l| l.borrow_mut().clear()); + let out = f(); + (out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut()))) +} diff --git a/crates/docx/src/write/embed.rs b/crates/docx/src/write/embed.rs index 869d91a401..405b901e4f 100644 --- a/crates/docx/src/write/embed.rs +++ b/crates/docx/src/write/embed.rs @@ -50,6 +50,8 @@ pub(crate) struct EmbedWriter<'d> { pub drop_glossary: bool, /// Kept parts written with these bytes instead of their own (see [`Self::scrub_glossary`]). scrubbed: HashMap>, + /// A chart past the chart limits has been written (and the loss logged, once per save). + chart_cut: bool, } impl<'d> EmbedWriter<'d> { @@ -96,6 +98,7 @@ impl<'d> EmbedWriter<'d> { parts: Vec::new(), drop_glossary: false, scrubbed: HashMap::new(), + chart_cut: false, } } @@ -260,7 +263,12 @@ impl<'d> EmbedWriter<'d> { Some(b) => { self.scrubbed.insert(p, b); } - None => return false, + None => { + log::warn!( + "docx: the glossary's part {p} can't be cleaned of personal information; the glossary is written from its entries, without what WordCraft doesn't model" + ); + return false; + } } } true @@ -445,7 +453,12 @@ impl super::Writer<'_> { /// related from the story part through `rels`. pub(super) fn chart_graphic(&mut self, spec: &wordcraft_doc::chart::ChartSpec, rels: &mut PartRels) -> String { let name = self.embeds.fresh("word/charts/chart", ".xml"); - let bytes = crate::chart_spec::chart_xml(spec).into_bytes(); + let (xml, cut) = crate::chart_spec::chart_part(spec); + if cut && !self.embeds.chart_cut { + self.embeds.chart_cut = true; + log::warn!("docx: a chart has more series or points than a chart part holds; the rest are left out of the file"); + } + let bytes = xml.into_bytes(); self.embeds.parts.push((name.clone(), bytes, crate::chart_spec::CHART_CT.to_string(), PartRels::default())); let id = rels.add(crate::package::rt::CHART, &relative(STORY_PART, &name), false); let mut w = xml::W::default(); @@ -820,6 +833,40 @@ fn valid_part_name(name: &str) -> String { mod tests { use super::*; + #[test] + fn charts_cut_to_the_limits_on_save_say_so_once() { + // A model deserialised as is can hold more than a chart part keeps; the save left the rest + // out without a word (review of #3872). + use std::sync::Arc; + use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, MAX_SERIES}; + use wordcraft_doc::graphic::{Graphic, GraphicKind}; + use wordcraft_doc::{CharProps, InlineObject, Pos}; + let chart = |series: usize| { + let spec = ChartSpec { + kind: ChartType::Column, + categories: vec!["a".into(), "b".into()], + series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(), + title: None, + legend: None, + data_labels: false, + }; + let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() }; + InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) } + }; + let save = |charts: Vec| { + let mut doc = wordcraft_doc::Document::new(); + for c in charts { + doc.insert_object(&Pos::body(0, 0), c, &CharProps::default()).unwrap(); + } + crate::test_log::warnings(|| crate::write(&doc).unwrap()).1 + }; + let cut = |w: &Vec| w.iter().filter(|w| w.contains("chart")).count(); + let warned = save(vec![chart(MAX_SERIES + 1), chart(MAX_SERIES + 5), chart(2)]); + assert_eq!(cut(&warned), 1, "one warning for the save, not one per chart: {warned:?}"); + let warned = save(vec![chart(3)]); + assert_eq!(cut(&warned), 0, "nothing left out: {warned:?}"); + } + #[test] fn data_part_drawing_ids_are_rewritten_once() { let ids: HashMap = [("rId9".to_string(), "rId2".to_string()), ("rId2".to_string(), "rId3".to_string())].into_iter().collect(); diff --git a/crates/docx/tests/it/glossary.rs b/crates/docx/tests/it/glossary.rs index 8d7b9f0bcd..92e07a6514 100644 --- a/crates/docx/tests/it/glossary.rs +++ b/crates/docx/tests/it/glossary.rs @@ -326,9 +326,37 @@ fn a_kept_glossary_that_cant_be_rewritten_privately_is_written_from_the_model() let mut doc = wordcraft_docx::read(&bytes).unwrap(); assert_eq!(doc.glossary.len(), 1); doc.settings.remove_personal_info = true; - let files = save(&doc); + let (files, warned) = warnings(|| save(&doc)); let g = text(&files, &glossary_target(&files).expect("a glossary")); assert!(g.contains("Added clause") && !g.contains("Quill Ramsey"), "{g}"); + // What the model doesn't hold is lost: the save says so (review of #3872). + assert_eq!(warned.iter().filter(|w| w.contains("written from its entries")).count(), 1, "{warned:?}"); +} + +/// `f`'s result and the warnings logged on this thread while it ran. +fn warnings(f: impl FnOnce() -> T) -> (T, Vec) { + use std::cell::RefCell; + struct Capture; + thread_local! { + static LINES: RefCell> = const { RefCell::new(Vec::new()) }; + } + impl log::Log for Capture { + fn enabled(&self, m: &log::Metadata) -> bool { + m.level() <= log::Level::Warn + } + fn log(&self, r: &log::Record) { + if self.enabled(r.metadata()) { + LINES.with(|l| l.borrow_mut().push(r.args().to_string())); + } + } + fn flush(&self) {} + } + static CAPTURE: Capture = Capture; + let _ = log::set_logger(&CAPTURE); + log::set_max_level(log::LevelFilter::Warn); + LINES.with(|l| l.borrow_mut().clear()); + let out = f(); + (out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut()))) } /// The relationships part of a part in `word/`. From 01a75b99b1bf644f1b77ae7bad3607ee1c64f745 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:05:47 +0100 Subject: [PATCH 104/112] #3692: add_media looks up duplicates by content instead of scanning every image Finding 45 (CWE-407): Document::add_media compared each new image with every stored one, so an import with many distinct small images did quadratic work (40k images: 213 s before, 0.26 s after). A MediaIndex cache (not saved, ignored by ==) maps bytes to their first key, is rebuilt when media changed size behind its back, and every hit is checked against media. Generated image{n} names also skip taken names once instead of on every add. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/doc/src/lib.rs | 74 +++++++++++++++++++++++++++++++++++++---- crates/doc/src/tests.rs | 38 +++++++++++++++++++++ 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index 75fa27ac47..4301d03792 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -859,6 +859,50 @@ pub struct Document { /// The document's own building blocks (its glossary document), in file order. #[serde(skip_serializing_if = "Vec::is_empty")] pub glossary: Vec, + /// Lookup cache for [`Document::add_media`]; not part of the document (never saved, ignored + /// by `==`). + #[serde(skip)] + pub media_index: MediaIndex, +} + +/// [`Document::media`] by content, so adding media is linear in the bytes added rather than a +/// scan of every earlier image (CWE-407). Rebuilt whenever `media` changed size behind its back; +/// a hit is always checked against `media`, so a stale entry can at worst miss a duplicate. +#[derive(Clone, Default)] +pub struct MediaIndex { + by_bytes: std::collections::HashMap>, String>, + /// `media.len()` when the index last matched it. + len: usize, + /// The number after the last generated `image{n}` key, so taken names are probed once. + next: usize, +} + +impl MediaIndex { + fn rebuild(&mut self, media: &BTreeMap>>) { + self.by_bytes.clear(); + // The first key (in key order) of equal bytes wins, as a scan would find it. + for (k, v) in media.iter().rev() { + self.by_bytes.insert(v.clone(), k.clone()); + } + self.len = media.len(); + self.next = 0; + } + + fn find(&self, media: &BTreeMap>>, bytes: &Vec) -> Option { + self.by_bytes.get(bytes).filter(|k| media.get(*k).is_some_and(|v| **v == *bytes)).cloned() + } +} + +impl PartialEq for MediaIndex { + fn eq(&self, _: &Self) -> bool { + true + } +} + +impl std::fmt::Debug for MediaIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("MediaIndex") + } } impl Default for Document { @@ -887,6 +931,7 @@ impl Document { media: BTreeMap::new(), passthrough: BTreeMap::new(), glossary: Vec::new(), + media_index: MediaIndex::default(), } } @@ -1563,17 +1608,32 @@ impl Document { /// Add a media item; returns its key (deduplicated by content). pub fn add_media(&mut self, bytes: Vec, ext: &str) -> String { - if let Some((k, _)) = self.media.iter().find(|(_, v)| v.as_slice() == bytes.as_slice()) { - return k.clone(); + let ix = &mut self.media_index; + if ix.len != self.media.len() { + ix.rebuild(&self.media); } - let n = self.media.len() + 1; - let mut key = format!("image{n}.{ext}"); - let mut i = n; + if let Some(k) = ix.find(&self.media, &bytes) { + return k; + } + if ix.by_bytes.contains_key(&bytes) { + // Stale (`media` was edited directly without changing its size): look again. + ix.rebuild(&self.media); + if let Some(k) = ix.find(&self.media, &bytes) { + return k; + } + } + let n = self.media.len().saturating_add(1); + let mut i = n.max(ix.next); + let mut key = format!("image{i}.{ext}"); while self.media.contains_key(&key) { - i += 1; + i = i.saturating_add(1); key = format!("image{i}.{ext}"); } - self.media.insert(key.clone(), Arc::new(bytes)); + let bytes = Arc::new(bytes); + self.media.insert(key.clone(), bytes.clone()); + ix.by_bytes.entry(bytes).or_insert_with(|| key.clone()); + ix.len = self.media.len(); + ix.next = i.saturating_add(1); key } diff --git a/crates/doc/src/tests.rs b/crates/doc/src/tests.rs index 18e69c8325..6097a52ed8 100644 --- a/crates/doc/src/tests.rs +++ b/crates/doc/src/tests.rs @@ -90,6 +90,44 @@ fn parts_and_media() { assert_ne!(k1, k3); } +#[test] +fn many_distinct_media_add_in_linear_time() { + // An import with tens of thousands of small distinct images (same length, differing only at + // the end) used to compare each against every earlier one. + let mut d = Document::new(); + let t = std::time::Instant::now(); + let n = 40_000u32; + for i in 0..n { + let mut bytes = vec![7u8; 256]; + bytes.extend(i.to_le_bytes()); + d.add_media(bytes, "png"); + } + assert_eq!(d.media.len(), n as usize); + // Duplicates still find their first copy, including media inserted directly. + let mut first = vec![7u8; 256]; + first.extend(0u32.to_le_bytes()); + assert_eq!(d.add_media(first, "png"), "image1.png"); + d.media.insert("direct.png".into(), Arc::new(vec![9, 9])); + assert_eq!(d.add_media(vec![9, 9], "png"), "direct.png"); + assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + // The lookup cache isn't part of the document. + let mut e = d.clone(); + e.media_index = MediaIndex::default(); + assert_eq!(d, e); +} + +#[test] +fn media_keys_skip_taken_names_once() { + let mut d = Document::new(); + // A file whose media already use the names add_media would pick next. + for i in 2..2002u32 { + d.media.insert(format!("image{i}.png"), Arc::new(i.to_le_bytes().to_vec())); + } + assert_eq!(d.add_media(vec![0], "png"), "image2002.png"); + assert_eq!(d.add_media(vec![1], "png"), "image2003.png"); + assert_eq!(d.add_media(vec![2], "png"), "image2004.png"); +} + #[test] fn free_ids_skip_taken_ones_instead_of_overflowing() { assert_eq!(free_id(None, std::iter::empty(), 1), Some(1)); From acfc3d5041fec075a632b32675bdd70ce0263864 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:07:33 +0100 Subject: [PATCH 105/112] #3692: function-name lookahead in linear math stops at the longest name Finding 48 (CWE-407): P::word scanned and copied the rest of an alphabetic run for every letter that didn't start a function name, so a 20,000-letter equation did ~200 million steps outside the parser's step budget (5 parses: 8.3 s before, well under 0.1 s after). The scan now stops one past the longest name in FUNCTION_NAMES (computed at compile time); a longer run is never a name, as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/doc/src/math.rs | 17 +++++++++++++++++ crates/doc/src/math_linear.rs | 18 ++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/crates/doc/src/math.rs b/crates/doc/src/math.rs index d620303484..650f92079d 100644 --- a/crates/doc/src/math.rs +++ b/crates/doc/src/math.rs @@ -1095,4 +1095,21 @@ mod tests { let deep = "√".repeat(3000); let _ = parse_linear(&deep); } + + #[test] + fn a_long_run_of_letters_parses_in_linear_time() { + // Each letter that doesn't start a function name used to rescan (and copy) the rest of + // the run, so a 20,000-letter equation did 200 million steps. + let t = std::time::Instant::now(); + for _ in 0..5 { + let e = parse_linear(&"x".repeat(30_000)); + assert!(matches!(e.as_slice(), [MNode::Run(r)] if r.text.len() == 20_000), "{:?}", e.len()); + } + assert!(t.elapsed().as_secs_f32() < 1.0, "{:?}", t.elapsed()); + // Function names are still found at the start of a run, and only as whole runs. + assert!(matches!(parse_linear("sin x").as_slice(), [MNode::Func { .. }])); + assert!(matches!(parse_linear("arcsin x").as_slice(), [MNode::Func { .. }])); + assert!(matches!(parse_linear("sinx").as_slice(), [MNode::Run(_)])); + assert!(matches!(parse_linear("arcsinhx").as_slice(), [MNode::Run(_)])); + } } diff --git a/crates/doc/src/math_linear.rs b/crates/doc/src/math_linear.rs index 1820c248d1..3f077f3778 100644 --- a/crates/doc/src/math_linear.rs +++ b/crates/doc/src/math_linear.rs @@ -18,6 +18,18 @@ pub const FUNCTION_NAMES: &[&str] = &[ "tanh", ]; +/// Length of the longest of [`FUNCTION_NAMES`]. +const MAX_NAME: usize = { + let (mut max, mut i) = (0, 0); + while i < FUNCTION_NAMES.len() { + if FUNCTION_NAMES[i].len() > max { + max = FUNCTION_NAMES[i].len(); + } + i += 1; + } + max +}; + /// Function names whose subscripts go underneath (`lim_(n→∞)`). const LIMIT_NAMES: &[&str] = &["lim", "liminf", "limsup", "max", "min", "sup", "inf", "det", "gcd", "Pr"]; @@ -410,10 +422,12 @@ impl P { fn word(&mut self, stops: &[char]) -> Arg { let start = self.i; let mut end = start; - while self.c.get(end).is_some_and(char::is_ascii_alphabetic) { + // Look no further than the longest name (and one past it, to see the run is longer): a + // scan to the end of every run made a long run of letters quadratic (CWE-407). + while end - start <= MAX_NAME && self.c.get(end).is_some_and(char::is_ascii_alphabetic) { end += 1; } - let word: String = self.c.get(start..end).unwrap_or(&[]).iter().collect(); + let word: String = if end - start > MAX_NAME { String::new() } else { self.c.get(start..end).unwrap_or(&[]).iter().collect() }; if !FUNCTION_NAMES.contains(&word.as_str()) { self.i = start + 1; return vec![text(self.c.get(start).copied().unwrap_or(' '))]; From 3a1c91f2d4af88f607f60f6580298cb9cb50343f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:19:09 +0100 Subject: [PATCH 106/112] #3614: crates/doc walks a paragraph's objects in one pass Finding 71 (CWE-407): loops of object_offsets + object_at recounted the objects before every offset, so walking a paragraph's objects was quadratic (40k markers: 360 s for the regression test before, 0.2 s after). Paragraph::objects_with_offsets yields (offset, object) in one pass, and every such walk in crates/doc uses it: the field scan (field_ranges, run on every docx load), control events, bookmarks, perm ranges, text_box_anchor, past_field_ends, typing_pos and the markers a range delete keeps. Also: perm_ranges keeps open ranges per id instead of searching a list for each end; kept_control_markers pairs each container with its table wraps once (was once per marker) and probes a set instead of every range. Part of #3614: the engine, docx reader and UI callers it lists live outside crates/doc and still use object_at in loops; they can move to objects_with_offsets separately. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/doc/src/control.rs | 64 +++++++++++++++++++++++++------------ crates/doc/src/edit.rs | 22 +++++++++---- crates/doc/src/fields.rs | 13 +++++--- crates/doc/src/lib.rs | 32 +++++++++---------- crates/doc/src/para.rs | 11 +++++++ crates/doc/src/tests.rs | 67 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 160 insertions(+), 49 deletions(-) diff --git a/crates/doc/src/control.rs b/crates/doc/src/control.rs index b9eb2c6a99..1396ae39d9 100644 --- a/crates/doc/src/control.rs +++ b/crates/doc/src/control.rs @@ -14,6 +14,8 @@ //! outside what's deleted (so deleting across a control's edge empties it rather than breaking //! it); writers call [`Document::balance_controls`] so a saved file always has whole controls. +use std::collections::{BTreeMap, BTreeSet}; + use serde::{Deserialize, Serialize}; use crate::para::OBJ; @@ -312,10 +314,10 @@ pub fn control_events(bl: &Blocks) -> Vec> { if !p.objects.iter().any(|o| matches!(o, InlineObject::ControlStart { .. } | InlineObject::ControlEnd)) { continue; } - for off in p.object_offsets() { - match p.object_at(off) { - Some(InlineObject::ControlStart { control }) => out.push(ControlEvent::Open { block: i, off: Some(off), control }), - Some(InlineObject::ControlEnd) => out.push(ControlEvent::Close { block: i, off: Some(off) }), + for (off, o) in p.objects_with_offsets() { + match o { + InlineObject::ControlStart { control } => out.push(ControlEvent::Open { block: i, off: Some(off), control }), + InlineObject::ControlEnd => out.push(ControlEvent::Close { block: i, off: Some(off) }), _ => {} } } @@ -639,15 +641,21 @@ impl Document { out.push((r.end.path.clone(), r.end.off)); } } - // Markers paired with a table wrap: their partner is never deleted with them. + // Markers paired with a table wrap: their partner is never deleted with them. Each lookup + // is a set probe, and each container is paired once, not once per marker (#3614). + let in_ranges: BTreeSet<(&Path, usize)> = ranges.iter().flat_map(|r| [(&r.start.path, r.start.off), (&r.end.path, r.end.off)]).collect(); + let mut table_paired: BTreeMap<&[u32], BTreeSet<(usize, usize)>> = BTreeMap::new(); for path in &paths { let Some(p) = self.para(a.story, path) else { continue }; - for off in p.object_offsets() { - if !inside(path, off) || !matches!(p.object_at(off), Some(InlineObject::ControlStart { .. } | InlineObject::ControlEnd)) { + for (off, o) in p.objects_with_offsets() { + if !inside(path, off) + || !matches!(o, InlineObject::ControlStart { .. } | InlineObject::ControlEnd) + || in_ranges.contains(&(path, off)) + { continue; } - let paired = ranges.iter().any(|r| (r.start.path == *path && r.start.off == off) || (r.end.path == *path && r.end.off == off)); - if !paired && self.paired_with_table(a.story, path, off) { + let paired = table_paired.entry(path.parent()).or_insert_with(|| self.table_paired_markers(a.story, path)); + if paired.contains(&(path.last(), off)) { out.push((path.clone(), off)); } } @@ -657,16 +665,26 @@ impl Document { out } - /// Whether the marker at `path`/`off` is paired with a table wrap's open or close. - fn paired_with_table(&self, s: StoryRef, path: &Path, off: usize) -> bool { - let Some(bl) = self.container(s, path) else { return false }; + /// (block, offset) of the paragraph markers in `path`'s container that are paired with a + /// table wrap's open or close. + fn table_paired_markers(&self, s: StoryRef, path: &Path) -> BTreeSet<(usize, usize)> { + let Some(bl) = self.container(s, path) else { return BTreeSet::new() }; let events = control_events(bl); let (pairs, _) = pair_events(&events); - let me = events.iter().position(|e| e.block() == path.last() && e.off() == Some(off)); - let Some(me) = me else { return false }; - pairs.iter().any(|(o, c, _)| { - (*o == me && events.get(*c).is_some_and(|e| e.off().is_none())) || (*c == me && events.get(*o).is_some_and(|e| e.off().is_none())) - }) + let mut out = BTreeSet::new(); + for (o, c, _) in pairs { + let (Some(open), Some(close)) = (events.get(o), events.get(c)) else { continue }; + match (open.off(), close.off()) { + (Some(off), None) => { + out.insert((open.block(), off)); + } + (None, Some(off)) => { + out.insert((close.block(), off)); + } + _ => {} + } + } + out } /// Where typing at `p` should go: a caret at the start of a paragraph, before the start of @@ -674,8 +692,11 @@ impl Document { /// end, goes inside it too (the control is around the whole paragraph). pub fn typing_pos(&self, p: &Pos) -> Pos { let Some(para) = self.para_at(p) else { return p.clone() }; - let is_start = |off: usize| matches!(para.object_at(off), Some(InlineObject::ControlStart { control }) if control.block); - let is_end = |off: usize| matches!(para.object_at(off), Some(InlineObject::ControlEnd)); + // The objects once, looked up by offset (object_at per step rescans the text). + let objs: Vec<(usize, &InlineObject)> = para.objects_with_offsets().collect(); + let obj = |off: usize| objs.binary_search_by_key(&off, |(o, _)| *o).ok().and_then(|k| objs.get(k)).map(|(_, o)| *o); + let is_start = |off: usize| matches!(obj(off), Some(InlineObject::ControlStart { control }) if control.block); + let is_end = |off: usize| matches!(obj(off), Some(InlineObject::ControlEnd)); let w = OBJ.len_utf8(); let mut off = p.off; // Only block starts before the caret (or none) and block starts right after it. @@ -688,8 +709,9 @@ impl Document { } if p.off == para.len() && off >= w && is_end(off - w) { // Step back over the trailing ends that close block-level controls. - let ranges = self.control_ranges(p.story); - while off >= w && is_end(off - w) && ranges.iter().any(|r| r.end.path == p.path && r.end.off == off - w && r.control.block) { + let block_ends: BTreeSet = + self.control_ranges(p.story).into_iter().filter(|r| r.end.path == p.path && r.control.block).map(|r| r.end.off).collect(); + while off >= w && is_end(off - w) && block_ends.contains(&(off - w)) { off -= w; } return Pos { off, ..p.clone() }; diff --git a/crates/doc/src/edit.rs b/crates/doc/src/edit.rs index 9107139aff..456e2d3286 100644 --- a/crates/doc/src/edit.rs +++ b/crates/doc/src/edit.rs @@ -485,13 +485,21 @@ impl Document { let (ia, ib) = (a.path.last(), b.path.last()); // Kept markers of the paragraphs in between move to the join. let mut moved: Vec<(InlineObject, CharProps)> = Vec::new(); - for (path, off) in &kept { - if *path != a.path - && *path != b.path - && let Some(p) = self.para(a.story, path) - && let Some(o) = p.object_at(*off) - { - moved.push((o.clone(), p.props_of_char(*off).clone())); + // `kept` is sorted: one walk of each paragraph's objects and runs (#3614). + for group in kept.chunk_by(|x, y| x.0 == y.0) { + let Some((path, _)) = group.first() else { continue }; + if *path == a.path || *path == b.path { + continue; + } + let Some(p) = self.para(a.story, path) else { continue }; + let mut runs = p.run_ranges().peekable(); + for (off, o) in p.objects_with_offsets() { + if group.binary_search_by_key(&off, |(_, k)| *k).is_err() { + continue; + } + while runs.next_if(|(r, _)| r.end <= off).is_some() {} + let props = runs.peek().filter(|(r, _)| r.contains(&off)).map_or(&p.mark, |(_, c)| *c); + moved.push((o.clone(), props.clone())); } } { diff --git a/crates/doc/src/fields.rs b/crates/doc/src/fields.rs index 29b050e603..fec8900fb7 100644 --- a/crates/doc/src/fields.rs +++ b/crates/doc/src/fields.rs @@ -43,12 +43,12 @@ fn scan(story: StoryRef, bl: &Blocks, prefix: &mut Vec, depth: usize, out: match &**b { Block::Para(p) => { let path = Path(prefix.clone()); - for off in p.object_offsets() { - match p.object_at(off) { - Some(InlineObject::FieldStart { instr, locked, .. }) => { + for (off, o) in p.objects_with_offsets() { + match o { + InlineObject::FieldStart { instr, locked, .. } => { open.push((instr.clone(), *locked, Pos::new(story, path.clone(), off))); } - Some(InlineObject::FieldEnd) => match open.pop() { + InlineObject::FieldEnd => match open.pop() { Some((instr, locked, start)) => { let depth = open.len(); out.ranges.push(FieldRange { instr, locked, start, end: Pos::new(story, path.clone(), off), depth }); @@ -99,7 +99,10 @@ impl Document { pub fn past_field_ends(&self, p: &Pos) -> Pos { let Some(para) = self.para_at(p) else { return p.clone() }; let mut off = p.off; - while matches!(para.object_at(off), Some(InlineObject::FieldEnd)) { + for (at, o) in para.objects_with_offsets().skip_while(|(at, _)| *at < p.off) { + if at != off || !matches!(o, InlineObject::FieldEnd) { + break; + } off += OBJ.len_utf8(); } Pos { off, ..p.clone() } diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index 4301d03792..6fa852080a 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -1147,8 +1147,8 @@ impl Document { for story in stories { for path in self.para_paths(story) { let Some(p) = self.para(story, &path) else { continue }; - for off in p.object_offsets() { - if p.object_at(off).is_some_and(|o| o.text_boxes().contains(&part)) { + for (off, o) in p.objects_with_offsets() { + if o.text_boxes().contains(&part) { return Some(Pos { story, path, off: off + para::OBJ.len_utf8() }); } } @@ -1668,22 +1668,22 @@ impl Document { let mut out: Vec = Vec::new(); let stories = std::iter::once(StoryRef::Body).chain(self.parts.keys().map(|k| StoryRef::Part(*k))); for story in stories { - let mut open: Vec = Vec::new(); + // Open ranges by id, latest last (an end closes the latest open range of its id). + let mut open: BTreeMap> = BTreeMap::new(); for path in self.para_paths(story) { let Some(p) = self.para(story, &path) else { continue }; - for off in p.object_offsets() { - match p.object_at(off) { - Some(InlineObject::PermStart { id, group, editor }) => { + for (off, o) in p.objects_with_offsets() { + match o { + InlineObject::PermStart { id, group, editor } => { let start = Pos { story, path: path.clone(), off: off + para::OBJ.len_utf8() }; - open.push(out.len()); + open.entry(*id).or_default().push(out.len()); out.push(PermRange { id: *id, group: group.clone(), editor: editor.clone(), start: start.clone(), end: start }); } - Some(InlineObject::PermEnd { id }) => { - if let Some(k) = open.iter().rposition(|i| out.get(*i).is_some_and(|r| r.id == *id)) { - let i = open.remove(k); - if let Some(r) = out.get_mut(i) { - r.end = Pos { story, path: path.clone(), off }; - } + InlineObject::PermEnd { id } => { + if let Some(i) = open.get_mut(id).and_then(Vec::pop) + && let Some(r) = out.get_mut(i) + { + r.end = Pos { story, path: path.clone(), off }; } } _ => {} @@ -1692,7 +1692,7 @@ impl Document { } // A range left open runs to the end of its story. let end = self.end_of(story); - for i in open { + for i in open.into_values().flatten() { if let Some(r) = out.get_mut(i) { r.end = end.clone(); } @@ -1706,8 +1706,8 @@ impl Document { let mut out = Vec::new(); for path in self.para_paths(StoryRef::Body) { if let Some(p) = self.para(StoryRef::Body, &path) { - for off in p.object_offsets() { - if let Some(InlineObject::BookmarkStart { name }) = p.object_at(off) { + for (off, o) in p.objects_with_offsets() { + if let InlineObject::BookmarkStart { name } = o { out.push((name.clone(), Pos { story: StoryRef::Body, path: path.clone(), off })); } } diff --git a/crates/doc/src/para.rs b/crates/doc/src/para.rs index 9add66c65a..94738db5d5 100644 --- a/crates/doc/src/para.rs +++ b/crates/doc/src/para.rs @@ -1089,6 +1089,12 @@ impl Paragraph { pub fn object_offsets(&self) -> Vec { self.text.char_indices().filter(|(_, c)| *c == OBJ).map(|(i, _)| i).collect() } + /// (byte offset, object) for every object, in text order, in one pass. Prefer it to + /// [`Paragraph::object_offsets`] + [`Paragraph::object_at`], which counts the objects before + /// each offset again, so a loop over a paragraph's objects is quadratic (#3614). + pub fn objects_with_offsets(&self) -> impl Iterator + '_ { + self.text.char_indices().filter(|(_, c)| *c == OBJ).map(|(i, _)| i).zip(self.objects.iter()) + } /// Insert plain text (U+FFFC is dropped) at `off` with `props`. Returns the inserted length. pub fn insert_text(&mut self, off: usize, s: &str, props: &CharProps) -> Result { @@ -1559,6 +1565,11 @@ mod tests { assert!(matches!(p.object_at(0), Some(InlineObject::BookmarkStart { .. }))); let off = p.object_offsets()[1]; assert!(matches!(p.object_at(off), Some(InlineObject::Field { .. }))); + // The one-pass walk agrees with offsets + object_at. + let one_pass: Vec<(usize, &InlineObject)> = p.objects_with_offsets().collect(); + let each: Vec<(usize, &InlineObject)> = p.object_offsets().into_iter().filter_map(|o| Some((o, p.object_at(o)?))).collect(); + assert_eq!(one_pass, each); + assert_eq!(one_pass.len(), 2); assert_eq!(p.plain_text(), "a1b"); p.delete(0, 3).unwrap(); assert_eq!(p.objects.len(), 1); diff --git a/crates/doc/src/tests.rs b/crates/doc/src/tests.rs index 6097a52ed8..06833ec13b 100644 --- a/crates/doc/src/tests.rs +++ b/crates/doc/src/tests.rs @@ -561,3 +561,70 @@ fn custom_props_set_get_remove() { assert!(!d.remove_custom_prop("other")); assert_eq!(d.custom_props.len(), 1); } + +/// A paragraph that is nothing but `objs` (built directly: inserting one at a time is slow too). +fn para_of_objects(objs: Vec) -> Paragraph { + let mut p = Paragraph::new(); + p.text = para::OBJ.to_string().repeat(objs.len()); + p.runs = vec![Run { len: p.text.len(), props: CharProps::default() }]; + p.objects = objs; + p +} + +#[test] +fn object_walks_are_linear_in_the_paragraph() { + // #3614: loops of object_offsets + object_at recounted the objects before each one, so a + // paragraph of 40,000 markers took seconds per walk. + let n = 40_000u32; + let w = para::OBJ.len_utf8(); + let mut objs: Vec = Vec::new(); + for i in 0..n { + objs.push(InlineObject::BookmarkStart { name: format!("b{i}") }); + objs.push(fs("ADDIN x")); + objs.push(InlineObject::PermStart { id: i, group: "everyone".into(), editor: String::new() }); + } + // Ranges close in the order they opened (the worst case for a backwards search). + for i in 0..n { + objs.push(InlineObject::FieldEnd); + objs.push(InlineObject::PermEnd { id: i }); + } + let mut d = Document::new(); + d.body = vec![Arc::new(Block::Para(para_of_objects(objs)))]; + let t = std::time::Instant::now(); + assert_eq!(d.bookmarks().len(), n as usize); + assert_eq!(d.field_ranges(StoryRef::Body).len(), n as usize); + let perms = d.perm_ranges(); + assert_eq!(perms.len(), n as usize); + assert!(perms.iter().all(|r| r.end.off > r.start.off), "every range closed by its own end"); + assert_eq!(d.text_box_anchor(7), None); + // A caret before a long row of field ends moves past all of them. + let first_end = 3 * n as usize * w; + let p = d.past_field_ends(&Pos::body(0, first_end)); + assert_eq!(p.off, first_end + w); + assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + + let ends: Vec = (0..n).map(|_| InlineObject::FieldEnd).collect(); + d.body = vec![Arc::new(Block::Para(para_of_objects(ends)))]; + let t = std::time::Instant::now(); + assert_eq!(d.past_field_ends(&Pos::body(0, 0)).off, n as usize * w); + // Typing at the end of a row of block-level control starts. + let block = ContentControl { block: true, ..Default::default() }; + let starts: Vec = (0..n).map(|_| InlineObject::ControlStart { control: Box::new(block.clone()) }).collect(); + d.body = vec![Arc::new(Block::Para(para_of_objects(starts)))]; + let end = Pos::body(0, n as usize * w); + assert_eq!(d.typing_pos(&end), end); + assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); +} + +#[test] +fn deleting_across_many_lone_control_markers_is_not_quadratic() { + let n = 1_500; + let starts: Vec = (0..n).map(|_| InlineObject::ControlStart { control: Box::default() }).collect(); + let mut d = Document::from_text("before\n\nafter"); + d.body[1] = Arc::new(Block::Para(para_of_objects(starts))); + let t = std::time::Instant::now(); + let at = d.delete_range(&Pos::body(0, 3), &Pos::body(2, 2)).unwrap(); + assert_eq!(at, Pos::body(0, 3)); + assert_eq!(d.plain_text(StoryRef::Body), "befter"); + assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); +} From f235208630086e35b26a7846f55bc756a35a0561 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:20:42 +0100 Subject: [PATCH 107/112] #3692: pasted text boxes share what they need not copy, within a byte budget Finding 80 (CWE-400): adopting a fragment deep-copied a box's source story once per text box showing it, including boxes inside the story being copied, so 4096 boxes (the story budget) pointing at one large story copied it 4096 times. Now: a box showing a story it is inside gets no story (a cycle); blocks without text boxes are shared with the source (copy-on-write Arcs) instead of copied; and a paste adopts at most MAX_PASTED_BOX_BYTES (32 MiB, estimated text + runs + objects) of box content, charged before copying, after which later boxes keep no story, as when the story budget runs out. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/doc/src/edit.rs | 119 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 11 deletions(-) diff --git a/crates/doc/src/edit.rs b/crates/doc/src/edit.rs index 456e2d3286..e9bc75f6a2 100644 --- a/crates/doc/src/edit.rs +++ b/crates/doc/src/edit.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; -use crate::para::InlineObject; +use crate::para::{InlineObject, Run}; use crate::props::{CharProps, ParaProps}; use crate::{Block, Blocks, DocError, Document, Paragraph, Part, PartKind, Path, Pos, Result, para_block}; use crate::{Table, table::MAX_TABLE_DEPTH}; @@ -188,6 +188,33 @@ fn fit_tables(b: Block, room: usize, out: &mut Vec) { const MAX_BOX_DEPTH: usize = 8; /// Most text box stories one paste may create (hostile documents can fan out). const MAX_PASTED_BOXES: usize = 4096; +/// Most text box content (estimated bytes, see [`block_cost`]) one paste may adopt: thousands of +/// boxes can show the same large story, and each needs a story of its own. +const MAX_PASTED_BOX_BYTES: usize = 32 << 20; + +/// What a paste may still spend on text box stories. +struct BoxBudget { + stories: usize, + bytes: usize, + /// The source stories being copied, innermost last: a box inside one of them showing it + /// again is a cycle. + active: Vec, +} + +/// Rough size of a block's content, and whether it holds text boxes (and so must be copied). +fn block_cost(b: &Block) -> (usize, bool) { + let (mut cost, mut boxes) = (std::mem::size_of::(), false); + each_para(b, 0, &mut |p| { + let items = p + .runs + .len() + .saturating_mul(std::mem::size_of::()) + .saturating_add(p.objects.len().saturating_mul(std::mem::size_of::())); + cost = cost.saturating_add(p.text.len()).saturating_add(items); + boxes |= box_ids(p).next().is_some(); + }); + (cost, boxes) +} /// Call `f` on every paragraph of a block, in document order, however deep its tables nest. The /// walk keeps its own stack instead of recursing, so it is complete: a text box it doesn't reach @@ -752,7 +779,7 @@ impl Document { /// Give every text box in `blocks` a new story of its own, copied from `parts` (or, for a /// fragment without them, from this document's text box of that id). A box whose story can't /// be found keeps no story rather than showing an unrelated one. - fn adopt_text_boxes(&mut self, blocks: &mut [Block], parts: &BTreeMap, depth: usize, budget: &mut usize) { + fn adopt_text_boxes(&mut self, blocks: &mut [Block], parts: &BTreeMap, depth: usize, budget: &mut BoxBudget) { let mut any = false; for b in blocks.iter() { each_para(b, 0, &mut |p| any |= box_ids(p).next().is_some()); @@ -778,18 +805,40 @@ impl Document { } } - /// A new text box story copied from `old`; its own boxes get new stories too. - fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap, depth: usize, budget: &mut usize) -> Option { - if depth >= MAX_BOX_DEPTH || *budget == 0 { + /// A new text box story copied from `old`; its own boxes get new stories too. Blocks without + /// text boxes are shared with the source (copy-on-write), the others copied. A box showing a + /// story it is inside, or past the paste's budget, gets none (CWE-400). + fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap, depth: usize, budget: &mut BoxBudget) -> Option { + if depth >= MAX_BOX_DEPTH || budget.stories == 0 || budget.active.contains(&old) { return None; } let src = parts.get(&old).or_else(|| self.parts.get(&old).filter(|p| p.kind == PartKind::TextBox))?; - let mut blocks: Vec = src.blocks.iter().map(|b| (**b).clone()).collect(); + // Charge the whole story before copying any of it; once a story doesn't fit, the paste + // adopts no more (as when it runs out of stories). + let costs: Vec<(usize, bool)> = src.blocks.iter().map(|b| block_cost(b)).collect(); + let cost = costs.iter().fold(0usize, |a, (c, _)| a.saturating_add(*c)); + if cost > budget.bytes { + budget.bytes = 0; + return None; + } + budget.bytes -= cost; + budget.stories -= 1; + let src_blocks = src.blocks.clone(); // The copy keeps its text direction and alignment, not its link (a copy stands alone). let body = crate::TextBody { next: None, ..src.body }; - *budget -= 1; - self.adopt_text_boxes(&mut blocks, parts, depth + 1, budget); - let id = self.try_add_part(PartKind::TextBox, blocks.into_iter().map(Arc::new).collect()).ok()?; + budget.active.push(old); + let mut blocks: Blocks = Vec::with_capacity(src_blocks.len()); + for (b, (_, boxes)) in src_blocks.into_iter().zip(costs) { + if boxes { + let mut copy = (*b).clone(); + self.adopt_text_boxes(std::slice::from_mut(&mut copy), parts, depth + 1, budget); + blocks.push(Arc::new(copy)); + } else { + blocks.push(b); + } + } + budget.active.pop(); + let id = self.try_add_part(PartKind::TextBox, blocks).ok()?; if let Some(p) = self.parts.get_mut(&id) { p.body = body; } @@ -811,7 +860,7 @@ impl Document { return Err(DocError::BadPath(pos.path.to_string())); } // Pasted text boxes get stories of their own (never shared with the copy's source). - let mut budget = MAX_PASTED_BOXES; + let mut budget = BoxBudget { stories: MAX_PASTED_BOXES, bytes: MAX_PASTED_BOX_BYTES, active: Vec::new() }; self.adopt_text_boxes(&mut blocks, &frag.parts, 0, &mut budget); // Single paragraph: inline insert keeping runs. if blocks.len() == 1 @@ -1204,7 +1253,55 @@ mod tests { let before = d.parts.len(); d.insert_fragment(&Pos::body(0, 0), &f).unwrap(); let added = d.parts.len() - before; - assert!((1..=MAX_BOX_DEPTH).contains(&added), "{added}"); + // The copy's inner box would show the story being copied: a cycle, so it gets none. + assert_eq!(added, 1); + } + + /// A fragment whose body is one paragraph of `refs` text boxes all showing story 1. + fn fan_out(story: Vec>, refs: usize) -> Fragment { + let mut p = Paragraph::new(); + for _ in 0..refs { + p.insert_object(0, box_shape(Some(1)), &CharProps::default()).unwrap(); + } + let mut parts = BTreeMap::new(); + parts.insert(1, crate::Part { kind: PartKind::TextBox, blocks: story, body: Default::default() }); + Fragment { blocks: vec![Block::Para(p)], parts } + } + + #[test] + fn pasted_boxes_share_the_blocks_they_need_not_copy() { + let big = para_block(Paragraph::with_text(&"x".repeat(100_000), CharProps::default())); + let f = fan_out(vec![big.clone()], 3); + let mut d = doc(&["x"]); + d.insert_fragment(&Pos::body(0, 1), &f).unwrap(); + let got = boxes_in(&d, &Path::top(0)); + assert_eq!(got.len(), 3); + let ids: Vec = got.into_iter().flatten().collect(); + assert_eq!(ids.len(), 3, "each box has a story"); + assert!(ids.windows(2).all(|w| w[0] != w[1]), "stories of their own: {ids:?}"); + for id in ids { + let b = d.parts[&id].blocks.first().unwrap(); + assert!(Arc::ptr_eq(b, &big), "no text box inside: shared, not copied"); + } + } + + #[test] + fn pasted_box_copies_are_capped_in_bytes() { + // Story 1 is a 1 MiB paragraph that holds a text box itself, so each box showing it needs + // its own copy; 64 of them used to copy 64 MiB. + let mut p = Paragraph::with_text(&"x".repeat(1 << 20), CharProps::default()); + p.insert_object(0, box_shape(Some(999)), &CharProps::default()).unwrap(); + let f = fan_out(vec![para_block(p)], 64); + let mut d = doc(&["x"]); + let before = d.parts.len(); + d.insert_fragment(&Pos::body(0, 1), &f).unwrap(); + let added = d.parts.len() - before; + let copied: usize = d.parts.values().flat_map(|p| p.blocks.iter()).map(|b| if let Block::Para(p) = &**b { p.text.len() } else { 0 }).sum(); + assert!((1..64).contains(&added), "{added}"); + assert!(copied <= MAX_PASTED_BOX_BYTES, "{copied}"); + // The boxes past the cap keep no story rather than a shared one. + let got = boxes_in(&d, &Path::top(0)); + assert_eq!(got.iter().flatten().count(), added); } #[test] From 722a6ab1c01c5202131048f8cf9fd88228715e7f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 18:28:44 +0100 Subject: [PATCH 108/112] Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/gaps.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/gaps.md b/docs/gaps.md index 8e424edc86..42a5ffef7e 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -45,6 +45,8 @@ Exact line spacing keeps the run font baseline around tall equations; automatic Legacy Word imports (#2445): interactive Save for `.doc`/`.dot` uses the existing DOCX Save As dialog. Cancel and write failures retain the draft and original path; Save before Close continues only after a successful supported-format save. This narrows save reliability under gap #3; binary `.doc` writing remains unsupported. Late Save As answers are rejected before writing if automation has replaced the document. +Document model hangs (#3614, #3692): paragraph object walks, image deduplication, linear-equation letter runs and text box paste no longer do quadratic work or unbounded copies on hostile files. This narrows gap #3; the engine, DOCX reader and UI loops named in #3614 remain. + Active footnote and endnote text stays readable while editing (#2545). This fixes a display-context defect; the broader fidelity, accessibility and readiness gates below are unchanged. Russian language-aware sorting now keeps ё with е instead of after я, with the diaeresis as a secondary difference. Composed and decomposed ё and й compare equally. Fixed synthetic collation tests cover order, case and index headings; this does not establish full Cyrillic or CLDR collation parity. @@ -161,6 +163,7 @@ Small follow-ups from the 2026-10-25 field, table and AutoFormat batch (#4232): | 2026-10-24 | trivial | Exact-spacing table contact uses the specified line height in row selection and word skipping; large/small mark regressions in modes 11/12/14/15 (#3977); shape estimates unchanged | | 2026-10-24 | trivial | Floating tables count exact line-bottom contact; shapes keep strict overlap (#3977); original boundary/skip regressions in modes 11/12/14/15, expected Word rule from cyril265; readiness estimates unchanged | | 2026-10-22 | trivial | Gap 3: proofing robustness — linear affix rule walks, bounded dictionary reads, linear Russian hyphenation, Turkish word cap (#3613 part, #3661, #3657, #3625); readiness estimates unchanged | +| 2026-10-22 | trivial | Gap #3 note: document model quadratic scans and text box paste copies bounded (#3692, part of #3614) | | 2026-10-22 | trivial | Gap 10: group connector rerouting preserves rotated/flipped free and unresolved endpoints under nonuniform scaling, with safe degenerate-dimension and display-size-cap handling; routing around shapes and readiness estimates unchanged | | 2026-10-21 | trivial | The ranked gap list held two rows each for #19, #20 and #21; the stale copies are gone, so each gap appears once, and the #20 row carries the measured figure (#3411) | | 2026-10-21 | trivial | #20 narrowed: a document with many relationships opens in the time its part takes (#3411) | From 72cbbe096e4bde9c0333bcc62f4217f7eef9cb74 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 01:54:48 +0100 Subject: [PATCH 109/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20the=20lon?= =?UTF-8?q?g-letter-run=20math=20test=20asserts=20results=20only,=20no=20w?= =?UTF-8?q?all-clock=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/doc/src/math.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/doc/src/math.rs b/crates/doc/src/math.rs index 650f92079d..5e930df7f7 100644 --- a/crates/doc/src/math.rs +++ b/crates/doc/src/math.rs @@ -1099,13 +1099,12 @@ mod tests { #[test] fn a_long_run_of_letters_parses_in_linear_time() { // Each letter that doesn't start a function name used to rescan (and copy) the rest of - // the run, so a 20,000-letter equation did 200 million steps. - let t = std::time::Instant::now(); + // the run, so a 20,000-letter equation did 200 million steps. (No wall-clock assert: a + // loaded runner would fail it; a regression shows as a slow test.) for _ in 0..5 { let e = parse_linear(&"x".repeat(30_000)); assert!(matches!(e.as_slice(), [MNode::Run(r)] if r.text.len() == 20_000), "{:?}", e.len()); } - assert!(t.elapsed().as_secs_f32() < 1.0, "{:?}", t.elapsed()); // Function names are still found at the start of a run, and only as whole runs. assert!(matches!(parse_linear("sin x").as_slice(), [MNode::Func { .. }])); assert!(matches!(parse_linear("arcsin x").as_slice(), [MNode::Func { .. }])); From 26187afac23d2cb8c690a6a40f1cf88dabea6a76 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 02:44:56 +0100 Subject: [PATCH 110/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20pasted=20?= =?UTF-8?q?text=20box=20copies=20are=20charged=20for=20every=20payload=20t?= =?UTF-8?q?heir=20objects=20copy=20(group=20members,=20equations,=20kept?= =?UTF-8?q?=20XML),=20not=20only=20the=20object=20slots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/doc/src/edit.rs | 105 ++++++++++++++++++++++++++++++++++++++++- crates/doc/src/math.rs | 18 +++++++ 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/crates/doc/src/edit.rs b/crates/doc/src/edit.rs index e9bc75f6a2..40c48daced 100644 --- a/crates/doc/src/edit.rs +++ b/crates/doc/src/edit.rs @@ -210,12 +210,78 @@ fn block_cost(b: &Block) -> (usize, bool) { .len() .saturating_mul(std::mem::size_of::()) .saturating_add(p.objects.len().saturating_mul(std::mem::size_of::())); - cost = cost.saturating_add(p.text.len()).saturating_add(items); + let objects = p.objects.iter().fold(0usize, |n, o| n.saturating_add(object_cost(o))); + cost = cost.saturating_add(p.text.len()).saturating_add(items).saturating_add(objects); boxes |= box_ids(p).next().is_some(); }); (cost, boxes) } +/// Rough heap bytes a clone of `o` copies beyond its own storage: its strings and lists, and its +/// group members however deep. What it shares through `Arc` (pictures' embedded objects, charts, +/// freeforms, wrap polygons, gradients) is not copied, so not counted. +fn object_cost(o: &InlineObject) -> usize { + let float = |f: &crate::para::Float| f.alt.text.len().saturating_add(f.alt.title.len()); + let mut cost = 0usize; + let mut todo = vec![o]; + while let Some(o) = todo.pop() { + let add = match o { + InlineObject::Image { media, alt, float: f, .. } => media.len().saturating_add(alt.len()).saturating_add(float(f)), + InlineObject::Graphic { alt, float: f, .. } => alt.len().saturating_add(float(f)), + InlineObject::Shape { float: f, extra, .. } => { + let warp = extra.warp.as_ref().map_or(0, |w| { + let adj = w.adj.iter().map(|(name, _)| std::mem::size_of::<(String, i64)>().saturating_add(name.len())); + adj.fold(w.preset.len(), usize::saturating_add) + }); + float(f).saturating_add(warp) + } + InlineObject::Group { float: f, children, .. } => { + todo.extend(children.iter().map(|c| &c.obj)); + children.len().saturating_mul(std::mem::size_of::()).saturating_add(float(f)) + } + InlineObject::Field { instr, result, form, .. } => instr.len().saturating_add(result.len()).saturating_add(form.len()), + InlineObject::FieldStart { instr, .. } => instr.len(), + InlineObject::ControlStart { control } => std::mem::size_of::().saturating_add(serialized_len(control)), + InlineObject::NoteRef { custom, .. } => custom.len(), + InlineObject::BookmarkStart { name } | InlineObject::BookmarkEnd { name } => name.len(), + InlineObject::PermStart { group, editor, .. } => group.len().saturating_add(editor.len()), + InlineObject::Equation { linear, math, .. } => linear.len().saturating_add(math.heap_bytes()), + InlineObject::Ruby { base, ruby, lang, props, .. } => { + [base.len(), ruby.len(), lang.len(), std::mem::size_of::(), serialized_len(props)] + .into_iter() + .fold(0, usize::saturating_add) + } + InlineObject::Opaque { format, xml, text } => format.len().saturating_add(xml.len()).saturating_add(text.len()), + InlineObject::FieldEnd + | InlineObject::ControlEnd + | InlineObject::CommentStart { .. } + | InlineObject::CommentEnd { .. } + | InlineObject::PermEnd { .. } + | InlineObject::PositionalTab { .. } => 0, + }; + cost = cost.saturating_add(add); + } + cost +} + +/// Length of `v` written as JSON: at least the bytes of every string and list it owns. For the +/// nested formatting and control types, which share nothing through `Arc`. +fn serialized_len(v: &T) -> usize { + struct Count(usize); + impl std::io::Write for Count { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(b.len()); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut n = Count(0); + let _ = serde_json::to_writer(&mut n, v); + n.0 +} + /// Call `f` on every paragraph of a block, in document order, however deep its tables nest. The /// walk keeps its own stack instead of recursing, so it is complete: a text box it doesn't reach /// is shown nowhere, which pruning relies on (#1339). `_depth` is unused (callers keep passing it). @@ -809,7 +875,8 @@ impl Document { /// text boxes are shared with the source (copy-on-write), the others copied. A box showing a /// story it is inside, or past the paste's budget, gets none (CWE-400). fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap, depth: usize, budget: &mut BoxBudget) -> Option { - if depth >= MAX_BOX_DEPTH || budget.stories == 0 || budget.active.contains(&old) { + // A spent byte budget stops before costing the story again (that walks its payloads). + if depth >= MAX_BOX_DEPTH || budget.stories == 0 || budget.bytes == 0 || budget.active.contains(&old) { return None; } let src = parts.get(&old).or_else(|| self.parts.get(&old).filter(|p| p.kind == PartKind::TextBox))?; @@ -1304,6 +1371,40 @@ mod tests { assert_eq!(got.iter().flatten().count(), added); } + #[test] + fn pasted_box_copies_count_the_objects_they_copy() { + // #3614 review: a story holding a text box and a 1 MiB kept-for-round-trip object costs + // the object's XML too, not only its slot: 64 copies used to copy 64 MiB of it. + let mut p = Paragraph::with_text("x", CharProps::default()); + p.insert_object(0, box_shape(Some(999)), &CharProps::default()).unwrap(); + let opaque = InlineObject::Opaque { format: "docx".into(), xml: "x".repeat(1 << 20), text: String::new() }; + p.insert_object(0, opaque, &CharProps::default()).unwrap(); + let f = fan_out(vec![para_block(p)], 64); + let mut d = doc(&["x"]); + let before = d.parts.len(); + d.insert_fragment(&Pos::body(0, 1), &f).unwrap(); + let added = d.parts.len() - before; + let xml = |o: &InlineObject| if let InlineObject::Opaque { xml, .. } = o { xml.len() } else { 0 }; + let copied: usize = + d.parts.values().flat_map(|p| p.blocks.iter()).map(|b| if let Block::Para(p) = &**b { p.objects.iter().map(xml).sum() } else { 0 }).sum(); + assert!((1..64).contains(&added), "{added}"); + assert!(copied <= MAX_PASTED_BOX_BYTES, "{copied}"); + // Nested group members and equations count too. + let member = |obj| crate::para::GroupChild { x: 0.0, y: 0.0, obj }; + let deep = InlineObject::Opaque { format: String::new(), xml: "y".repeat(5000), text: String::new() }; + let inner = + InlineObject::Group { w: 1.0, h: 1.0, float: Default::default(), ch_w: 1.0, ch_h: 1.0, children: vec![member(deep)], canvas: None }; + let outer = + InlineObject::Group { w: 1.0, h: 1.0, float: Default::default(), ch_w: 1.0, ch_h: 1.0, children: vec![member(inner)], canvas: None }; + assert!(object_cost(&outer) >= 5000); + let eq = InlineObject::Equation { + linear: String::new(), + display: false, + math: crate::math::Math { nodes: crate::math::parse_linear(&"z".repeat(5000)), ..Default::default() }, + }; + assert!(object_cost(&eq) >= 5000); + } + #[test] fn tables_in_paths() { let mut d = doc(&["before", "after"]); diff --git a/crates/doc/src/math.rs b/crates/doc/src/math.rs index 5e930df7f7..f4ee463810 100644 --- a/crates/doc/src/math.rs +++ b/crates/doc/src/math.rs @@ -41,6 +41,24 @@ impl Math { pub fn is_empty(&self) -> bool { self.nodes.is_empty() } + + /// Rough heap bytes a clone copies: every node, its argument lists and its run text. + /// Iterative, so it costs at most the size of the structure. + pub(crate) fn heap_bytes(&self) -> usize { + let mut n = self.omml.len(); + let mut todo: Vec<&MNode> = self.nodes.iter().collect(); + while let Some(node) = todo.pop() { + n = n.saturating_add(std::mem::size_of::()); + if let MNode::Run(r) = node { + n = n.saturating_add(r.text.len()).saturating_add(r.font.as_ref().map_or(0, String::len)); + } + each_arg(node, &mut |a| { + n = n.saturating_add(std::mem::size_of::()); + todo.extend(a); + }); + } + n + } } /// Display equation justification (`m:oMathParaPr/m:jc`). From bec112a4b8b2a0f265b15d53ef347032f407599c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:19:57 +0100 Subject: [PATCH 111/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20pasted=20?= =?UTF-8?q?text=20boxes=20skip=20only=20the=20box=20over=20the=20budget,?= =?UTF-8?q?=20and=20say=20so;=20media=5Findex=20private;=20kept=20markers'?= =?UTF-8?q?=20order=20pinned?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A text box whose story doesn't fit what is left of the paste's byte budget gets no story, but later boxes with smaller stories are still copied while the budget lasts (it used to zero the budget). Each source story is costed once per paste, so skipping a big story many times doesn't walk it again. The paste logs one warning with the number of boxes left empty. wordcraft-doc now depends on log. - Document::media_index is a private field and MediaIndex a private type: a lookup cache, not API. The one struct-update literal outside the crate (a formats test) builds the document field by field. - kept_control_markers documents that delete_range relies on its sorted, deduplicated output; delete_range sorts it if it ever isn't, and control_markers_kept_are_sorted pins it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- Cargo.lock | 1 + crates/doc/Cargo.toml | 1 + crates/doc/src/control.rs | 2 + crates/doc/src/edit.rs | 87 ++++++++++++++++++++++++++++------ crates/doc/src/lib.rs | 6 ++- crates/doc/src/test_log.rs | 36 ++++++++++++++ crates/doc/src/tests.rs | 20 ++++++++ crates/formats/src/markdown.rs | 3 +- 8 files changed, 138 insertions(+), 18 deletions(-) create mode 100644 crates/doc/src/test_log.rs diff --git a/Cargo.lock b/Cargo.lock index cfcbb47b57..5f19b3527a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6671,6 +6671,7 @@ name = "wordcraft-doc" version = "0.11.0" dependencies = [ "base64 0.23.1", + "log", "proptest", "serde", "serde_json", diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml index 5a6d6622e4..aff28f8d91 100644 --- a/crates/doc/Cargo.toml +++ b/crates/doc/Cargo.toml @@ -16,6 +16,7 @@ test-support = [] [dependencies] wordcraft-geom = { workspace = true } base64 = { workspace = true } +log = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/doc/src/control.rs b/crates/doc/src/control.rs index 1396ae39d9..31ff6814af 100644 --- a/crates/doc/src/control.rs +++ b/crates/doc/src/control.rs @@ -660,6 +660,8 @@ impl Document { } } } + // Sorted and without repeats: `delete_range` groups this by paragraph and binary-searches + // each group (`control_markers_kept_are_sorted` pins it). out.sort(); out.dedup(); out diff --git a/crates/doc/src/edit.rs b/crates/doc/src/edit.rs index 40c48daced..1c5b877666 100644 --- a/crates/doc/src/edit.rs +++ b/crates/doc/src/edit.rs @@ -199,6 +199,17 @@ struct BoxBudget { /// The source stories being copied, innermost last: a box inside one of them showing it /// again is a cycle. active: Vec, + /// Each source story's cost ([`block_cost`]), worked out once per paste: a story too big for + /// what is left is not walked again for every box that shows it. + costs: BTreeMap, + /// Boxes left without a story because the paste's budget couldn't hold theirs. + skipped: usize, +} + +impl BoxBudget { + fn new() -> BoxBudget { + BoxBudget { stories: MAX_PASTED_BOXES, bytes: MAX_PASTED_BOX_BYTES, active: Vec::new(), costs: BTreeMap::new(), skipped: 0 } + } } /// Rough size of a block's content, and whether it holds text boxes (and so must be copied). @@ -567,7 +578,13 @@ impl Document { } // Content control markers whose partner is outside the range stay (the control is // emptied, not broken). - let kept = self.kept_control_markers(&a, &b); + let mut kept = self.kept_control_markers(&a, &b); + // The walk below groups `kept` by paragraph (`chunk_by`) and looks offsets up + // (`binary_search`), so it must be sorted; `kept_control_markers` returns it so, and this + // keeps a change there from quietly losing markers. + if !kept.is_sorted() { + kept.sort(); + } let kept_in = |path: &Path| -> Vec { kept.iter().filter(|(p, _)| p == path).map(|(_, o)| *o).collect() }; if a.path == b.path { self.para_mut(a.story, &a.path)?.delete_except(a.off, b.off, &kept_in(&a.path))?; @@ -873,29 +890,34 @@ impl Document { /// A new text box story copied from `old`; its own boxes get new stories too. Blocks without /// text boxes are shared with the source (copy-on-write), the others copied. A box showing a - /// story it is inside, or past the paste's budget, gets none (CWE-400). + /// story it is inside, or whose story the paste's budget can't hold, gets none (CWE-400); + /// only that box: a later box with a smaller story is still copied while the budget lasts. fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap, depth: usize, budget: &mut BoxBudget) -> Option { - // A spent byte budget stops before costing the story again (that walks its payloads). - if depth >= MAX_BOX_DEPTH || budget.stories == 0 || budget.bytes == 0 || budget.active.contains(&old) { + if depth >= MAX_BOX_DEPTH || budget.active.contains(&old) { + return None; + } + if budget.stories == 0 { + budget.skipped = budget.skipped.saturating_add(1); return None; } let src = parts.get(&old).or_else(|| self.parts.get(&old).filter(|p| p.kind == PartKind::TextBox))?; - // Charge the whole story before copying any of it; once a story doesn't fit, the paste - // adopts no more (as when it runs out of stories). - let costs: Vec<(usize, bool)> = src.blocks.iter().map(|b| block_cost(b)).collect(); - let cost = costs.iter().fold(0usize, |a, (c, _)| a.saturating_add(*c)); - if cost > budget.bytes { - budget.bytes = 0; + // Charge the whole story before copying any of it. Its cost is worked out once per paste + // (that walks its payloads), however many boxes show it. + let cost = *budget.costs.entry(old).or_insert_with(|| src.blocks.iter().fold(0usize, |a, b| a.saturating_add(block_cost(b).0))); + let Some(left) = budget.bytes.checked_sub(cost) else { + budget.skipped = budget.skipped.saturating_add(1); return None; - } - budget.bytes -= cost; + }; + budget.bytes = left; budget.stories -= 1; let src_blocks = src.blocks.clone(); // The copy keeps its text direction and alignment, not its link (a copy stands alone). let body = crate::TextBody { next: None, ..src.body }; budget.active.push(old); let mut blocks: Blocks = Vec::with_capacity(src_blocks.len()); - for (b, (_, boxes)) in src_blocks.into_iter().zip(costs) { + for b in src_blocks { + let mut boxes = false; + each_para(&b, 0, &mut |p| boxes |= box_ids(p).next().is_some()); if boxes { let mut copy = (*b).clone(); self.adopt_text_boxes(std::slice::from_mut(&mut copy), parts, depth + 1, budget); @@ -927,8 +949,15 @@ impl Document { return Err(DocError::BadPath(pos.path.to_string())); } // Pasted text boxes get stories of their own (never shared with the copy's source). - let mut budget = BoxBudget { stories: MAX_PASTED_BOXES, bytes: MAX_PASTED_BOX_BYTES, active: Vec::new() }; + let mut budget = BoxBudget::new(); self.adopt_text_boxes(&mut blocks, &frag.parts, 0, &mut budget); + if budget.skipped > 0 { + // Once per paste, not once per box (review of #3874). + log::warn!( + "paste: {} text boxes are left empty; their stories are past what one paste copies ({MAX_PASTED_BOXES} stories, {MAX_PASTED_BOX_BYTES} bytes)", + budget.skipped + ); + } // Single paragraph: inline insert keeping runs. if blocks.len() == 1 && let Some(Block::Para(src)) = blocks.first() @@ -1352,6 +1381,30 @@ mod tests { } } + #[test] + fn a_box_over_the_budget_leaves_later_smaller_boxes_their_stories() { + // Review of #3874: once one story didn't fit, every later box in the paste was left + // empty, however small its story. Only the box that doesn't fit is. + let big = para_block(Paragraph::with_text(&"x".repeat(100_000), CharProps::default())); + let small = para_block(Paragraph::with_text("small", CharProps::default())); + let mut p = Paragraph::new(); + for story in [2, 1, 2, 1] { + p.insert_object(0, box_shape(Some(story)), &CharProps::default()).unwrap(); + } + let mut parts = BTreeMap::new(); + parts.insert(1, crate::Part { kind: PartKind::TextBox, blocks: vec![big], body: Default::default() }); + parts.insert(2, crate::Part { kind: PartKind::TextBox, blocks: vec![small], body: Default::default() }); + let mut blocks = vec![Block::Para(p)]; + let mut d = doc(&["x"]); + let mut budget = BoxBudget { bytes: 50_000, ..BoxBudget::new() }; + d.adopt_text_boxes(&mut blocks, &parts, 0, &mut budget); + let Some(Block::Para(p)) = blocks.first() else { panic!() }; + let got: Vec> = p.objects.iter().map(|o| if let InlineObject::Shape { story, .. } = o { *story } else { None }).collect(); + assert_eq!(got.iter().map(Option::is_some).collect::>(), [false, true, false, true], "{got:?}"); + assert_eq!(budget.skipped, 2); + assert_eq!(budget.costs.len(), 2, "each story costed once"); + } + #[test] fn pasted_box_copies_are_capped_in_bytes() { // Story 1 is a 1 MiB paragraph that holds a text box itself, so each box showing it needs @@ -1361,8 +1414,12 @@ mod tests { let f = fan_out(vec![para_block(p)], 64); let mut d = doc(&["x"]); let before = d.parts.len(); - d.insert_fragment(&Pos::body(0, 1), &f).unwrap(); + let (_, warned) = crate::test_log::warnings(|| d.insert_fragment(&Pos::body(0, 1), &f).unwrap()); let added = d.parts.len() - before; + // The boxes left empty are said to be, once for the paste (review of #3874). + let empty: Vec<_> = warned.iter().filter(|w| w.contains("text boxes are left empty")).collect(); + assert_eq!(empty.len(), 1, "{warned:?}"); + assert!(empty[0].starts_with(&format!("paste: {} text boxes", 64 - added)), "{empty:?}"); let copied: usize = d.parts.values().flat_map(|p| p.blocks.iter()).map(|b| if let Block::Para(p) = &**b { p.text.len() } else { 0 }).sum(); assert!((1..64).contains(&added), "{added}"); assert!(copied <= MAX_PASTED_BOX_BYTES, "{copied}"); diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index 6fa852080a..ada156750e 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -862,14 +862,14 @@ pub struct Document { /// Lookup cache for [`Document::add_media`]; not part of the document (never saved, ignored /// by `==`). #[serde(skip)] - pub media_index: MediaIndex, + media_index: MediaIndex, } /// [`Document::media`] by content, so adding media is linear in the bytes added rather than a /// scan of every earlier image (CWE-407). Rebuilt whenever `media` changed size behind its back; /// a hit is always checked against `media`, so a stale entry can at worst miss a duplicate. #[derive(Clone, Default)] -pub struct MediaIndex { +struct MediaIndex { by_bytes: std::collections::HashMap>, String>, /// `media.len()` when the index last matched it. len: usize, @@ -1889,5 +1889,7 @@ fn story_word_count_of(bl: &Blocks, numbering: &Numbering, styles: &styles::Styl total } +#[cfg(test)] +mod test_log; #[cfg(test)] mod tests; diff --git a/crates/doc/src/test_log.rs b/crates/doc/src/test_log.rs new file mode 100644 index 0000000000..745ce21fe1 --- /dev/null +++ b/crates/doc/src/test_log.rs @@ -0,0 +1,36 @@ +//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that +//! a limit which leaves something out says so. + +use std::cell::RefCell; + +struct Capture; + +thread_local! { + static LINES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +impl log::Log for Capture { + fn enabled(&self, m: &log::Metadata) -> bool { + m.level() <= log::Level::Warn + } + + fn log(&self, r: &log::Record) { + if self.enabled(r.metadata()) { + LINES.with(|l| l.borrow_mut().push(r.args().to_string())); + } + } + + fn flush(&self) {} +} + +static CAPTURE: Capture = Capture; + +/// `f`'s result and the warnings logged on this thread while it ran. +pub(crate) fn warnings(f: impl FnOnce() -> T) -> (T, Vec) { + // Only this logger is ever installed in a test binary; a second call keeps it. + let _ = log::set_logger(&CAPTURE); + log::set_max_level(log::LevelFilter::Warn); + LINES.with(|l| l.borrow_mut().clear()); + let out = f(); + (out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut()))) +} diff --git a/crates/doc/src/tests.rs b/crates/doc/src/tests.rs index 06833ec13b..16ba0a2675 100644 --- a/crates/doc/src/tests.rs +++ b/crates/doc/src/tests.rs @@ -628,3 +628,23 @@ fn deleting_across_many_lone_control_markers_is_not_quadratic() { assert_eq!(d.plain_text(StoryRef::Body), "befter"); assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); } + +#[test] +fn control_markers_kept_are_sorted() { + // `delete_range` groups the kept markers by paragraph and binary-searches each group, so + // `kept_control_markers` must hand them over sorted and without repeats (review of #3874). + let start = || InlineObject::ControlStart { control: Box::default() }; + let mut d = Document::from_text("before"); + // Controls open and close across paragraphs: 1 closes the one 0 opens, 3 the one 2 opens… + for i in 0..12 { + let objs = if i % 2 == 0 { vec![start(), start()] } else { vec![InlineObject::ControlEnd, InlineObject::ControlEnd] }; + d.body.push(Arc::new(Block::Para(para_of_objects(objs)))); + } + d.body.push(Arc::new(Block::Para(Paragraph::with_text("after", CharProps::default())))); + let w = para::OBJ.len_utf8(); + for (a, b) in [(Pos::body(2, w), Pos::body(11, w)), (Pos::body(1, w), Pos::body(12, w)), (Pos::body(3, w), Pos::body(8, 0))] { + let kept = d.kept_control_markers(&a, &b); + assert!(!kept.is_empty(), "{a:?}..{b:?}"); + assert!(kept.windows(2).all(|p| p[0] < p[1]), "sorted, no repeats: {kept:?}"); + } +} diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index e483cc928d..a74a5556c3 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -2067,7 +2067,8 @@ mod tests { use wordcraft_doc::{CharProps, Document, Paragraph, StoryRef, para_block}; let cases = ["Shopping\n- milk\n# not a heading", "a\n1. x", "a\n2) x", "a\n+ x", "a\n---", "a\n***", "a\n==", "a\n--", "a\n> q"]; for s in cases { - let d = Document { body: vec![para_block(Paragraph::with_text(s, CharProps::default()))], ..Default::default() }; + let mut d = Document::new(); + d.body = vec![para_block(Paragraph::with_text(s, CharProps::default()))]; let md = export(&d); let back = import(&md); assert_eq!(back.body.len(), 1, "{s:?} -> {md:?}"); From def2b9bd3123bb68491010f005c1d6a5a039814d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 14:26:27 +0100 Subject: [PATCH 112/112] =?UTF-8?q?#3692:=20review=20=E2=80=94=20doc=20tim?= =?UTF-8?q?ing=20tests=20get=20a=20margin=20a=20busy=20CI=20machine=20can'?= =?UTF-8?q?t=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 2 s failed on loaded runners. Each bound is now 10 s, at least 10x the fixed code's time and well under the old code's on the same input (measured by running the pre-fix code under the same load): - many distinct media: 0.5 s now; the old scan of every image took 131 s. - object walks: under 0.3 s now; the old walks took 360 s (#3614). - delete across lone control markers: 3,000 markers (was 1,500); 1 ms now, the old delete took 18 s at 1,500 and 114 s at 3,000. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/doc/src/tests.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/doc/src/tests.rs b/crates/doc/src/tests.rs index 16ba0a2675..67256fd78c 100644 --- a/crates/doc/src/tests.rs +++ b/crates/doc/src/tests.rs @@ -109,7 +109,8 @@ fn many_distinct_media_add_in_linear_time() { assert_eq!(d.add_media(first, "png"), "image1.png"); d.media.insert("direct.png".into(), Arc::new(vec![9, 9])); assert_eq!(d.add_media(vec![9, 9], "png"), "direct.png"); - assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + // About half a second on a loaded machine; the old scan took over two minutes. + assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed()); // The lookup cache isn't part of the document. let mut e = d.clone(); e.media_index = MediaIndex::default(); @@ -601,7 +602,8 @@ fn object_walks_are_linear_in_the_paragraph() { let first_end = 3 * n as usize * w; let p = d.past_field_ends(&Pos::body(0, first_end)); assert_eq!(p.off, first_end + w); - assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + // Under 0.3 s on a loaded machine; the old walks took minutes (360 s for the first test). + assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed()); let ends: Vec = (0..n).map(|_| InlineObject::FieldEnd).collect(); d.body = vec![Arc::new(Block::Para(para_of_objects(ends)))]; @@ -613,12 +615,13 @@ fn object_walks_are_linear_in_the_paragraph() { d.body = vec![Arc::new(Block::Para(para_of_objects(starts)))]; let end = Pos::body(0, n as usize * w); assert_eq!(d.typing_pos(&end), end); - assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed()); } #[test] fn deleting_across_many_lone_control_markers_is_not_quadratic() { - let n = 1_500; + // A millisecond now; the old delete took 18 s at 1,500 markers and 114 s at 3,000. + let n = 3_000; let starts: Vec = (0..n).map(|_| InlineObject::ControlStart { control: Box::default() }).collect(); let mut d = Document::from_text("before\n\nafter"); d.body[1] = Arc::new(Block::Para(para_of_objects(starts))); @@ -626,7 +629,7 @@ fn deleting_across_many_lone_control_markers_is_not_quadratic() { let at = d.delete_range(&Pos::body(0, 3), &Pos::body(2, 2)).unwrap(); assert_eq!(at, Pos::body(0, 3)); assert_eq!(d.plain_text(StoryRef::Body), "befter"); - assert!(t.elapsed().as_secs_f32() < 2.0, "{:?}", t.elapsed()); + assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed()); } #[test]